From e6d1592492a3a379186bfb02bd0f4eda0669c0d5 Mon Sep 17 00:00:00 2001 From: Dimitry Andric Date: Tue, 20 Aug 2019 20:50:12 +0000 Subject: Vendor import of stripped llvm trunk r366426 (just before the release_90 branch point): https://llvm.org/svn/llvm-project/llvm/trunk@366426 --- LICENSE.TXT | 261 +- include/llvm-c/Analysis.h | 8 +- include/llvm-c/BitReader.h | 8 +- include/llvm-c/BitWriter.h | 8 +- include/llvm-c/Comdat.h | 8 +- include/llvm-c/Core.h | 223 +- include/llvm-c/DataTypes.h | 8 +- include/llvm-c/DebugInfo.h | 135 +- include/llvm-c/Disassembler.h | 8 +- include/llvm-c/DisassemblerTypes.h | 8 +- include/llvm-c/Error.h | 10 +- include/llvm-c/ErrorHandling.h | 8 +- include/llvm-c/ExecutionEngine.h | 8 +- include/llvm-c/IRReader.h | 8 +- include/llvm-c/Initialization.h | 8 +- include/llvm-c/LinkTimeOptimizer.h | 7 +- include/llvm-c/Linker.h | 8 +- include/llvm-c/Object.h | 163 +- include/llvm-c/OptRemarks.h | 204 - include/llvm-c/OrcBindings.h | 8 +- include/llvm-c/Remarks.h | 329 + include/llvm-c/Support.h | 8 +- include/llvm-c/Target.h | 12 +- include/llvm-c/TargetMachine.h | 8 +- include/llvm-c/Transforms/AggressiveInstCombine.h | 8 +- include/llvm-c/Transforms/Coroutines.h | 8 +- include/llvm-c/Transforms/IPO.h | 8 +- include/llvm-c/Transforms/InstCombine.h | 8 +- include/llvm-c/Transforms/PassManagerBuilder.h | 8 +- include/llvm-c/Transforms/Scalar.h | 8 +- include/llvm-c/Transforms/Utils.h | 11 +- include/llvm-c/Transforms/Vectorize.h | 8 +- include/llvm-c/Types.h | 13 +- include/llvm-c/lto.h | 52 +- include/llvm/ADT/APFloat.h | 18 +- include/llvm/ADT/APInt.h | 18 +- include/llvm/ADT/APSInt.h | 25 +- include/llvm/ADT/AllocatorList.h | 7 +- include/llvm/ADT/Any.h | 7 +- include/llvm/ADT/ArrayRef.h | 15 +- include/llvm/ADT/BitVector.h | 7 +- include/llvm/ADT/BitmaskEnum.h | 7 +- include/llvm/ADT/BreadthFirstIterator.h | 9 +- include/llvm/ADT/CachedHashString.h | 7 +- include/llvm/ADT/DAGDeltaAlgorithm.h | 7 +- include/llvm/ADT/DeltaAlgorithm.h | 7 +- include/llvm/ADT/DenseMap.h | 15 +- include/llvm/ADT/DenseMapInfo.h | 23 +- include/llvm/ADT/DenseSet.h | 9 +- include/llvm/ADT/DepthFirstIterator.h | 7 +- include/llvm/ADT/EpochTracker.h | 7 +- include/llvm/ADT/EquivalenceClasses.h | 7 +- include/llvm/ADT/FoldingSet.h | 7 +- include/llvm/ADT/FunctionExtras.h | 7 +- include/llvm/ADT/GraphTraits.h | 7 +- include/llvm/ADT/Hashing.h | 9 +- include/llvm/ADT/ImmutableList.h | 11 +- include/llvm/ADT/ImmutableMap.h | 7 +- include/llvm/ADT/ImmutableSet.h | 7 +- include/llvm/ADT/IndexedMap.h | 7 +- include/llvm/ADT/IntEqClasses.h | 7 +- include/llvm/ADT/IntervalMap.h | 7 +- include/llvm/ADT/IntrusiveRefCntPtr.h | 7 +- include/llvm/ADT/MapVector.h | 7 +- include/llvm/ADT/None.h | 7 +- include/llvm/ADT/Optional.h | 255 +- include/llvm/ADT/PackedVector.h | 7 +- include/llvm/ADT/PointerEmbeddedInt.h | 7 +- include/llvm/ADT/PointerIntPair.h | 27 +- include/llvm/ADT/PointerSumType.h | 7 +- include/llvm/ADT/PointerUnion.h | 482 +- include/llvm/ADT/PostOrderIterator.h | 7 +- include/llvm/ADT/PriorityQueue.h | 7 +- include/llvm/ADT/PriorityWorklist.h | 7 +- include/llvm/ADT/SCCIterator.h | 7 +- include/llvm/ADT/STLExtras.h | 108 +- include/llvm/ADT/ScopeExit.h | 7 +- include/llvm/ADT/ScopedHashTable.h | 7 +- include/llvm/ADT/Sequence.h | 7 +- include/llvm/ADT/SetOperations.h | 7 +- include/llvm/ADT/SetVector.h | 7 +- include/llvm/ADT/SmallBitVector.h | 7 +- include/llvm/ADT/SmallPtrSet.h | 7 +- include/llvm/ADT/SmallSet.h | 7 +- include/llvm/ADT/SmallString.h | 7 +- include/llvm/ADT/SmallVector.h | 48 +- include/llvm/ADT/SparseBitVector.h | 7 +- include/llvm/ADT/SparseMultiSet.h | 7 +- include/llvm/ADT/SparseSet.h | 7 +- include/llvm/ADT/Statistic.h | 7 +- include/llvm/ADT/StringExtras.h | 7 +- include/llvm/ADT/StringMap.h | 12 +- include/llvm/ADT/StringRef.h | 43 +- include/llvm/ADT/StringSet.h | 14 +- include/llvm/ADT/StringSwitch.h | 29 +- include/llvm/ADT/TinyPtrVector.h | 7 +- include/llvm/ADT/Triple.h | 85 +- include/llvm/ADT/Twine.h | 10 +- include/llvm/ADT/UniqueVector.h | 7 +- include/llvm/ADT/VariadicFunction.h | 9 +- include/llvm/ADT/bit.h | 17 +- include/llvm/ADT/edit_distance.h | 7 +- include/llvm/ADT/fallible_iterator.h | 243 + include/llvm/ADT/ilist.h | 16 +- include/llvm/ADT/ilist_base.h | 7 +- include/llvm/ADT/ilist_iterator.h | 7 +- include/llvm/ADT/ilist_node.h | 7 +- include/llvm/ADT/ilist_node_base.h | 7 +- include/llvm/ADT/ilist_node_options.h | 7 +- include/llvm/ADT/iterator.h | 7 +- include/llvm/ADT/iterator_range.h | 7 +- include/llvm/ADT/simple_ilist.h | 7 +- include/llvm/Analysis/AliasAnalysis.h | 239 +- include/llvm/Analysis/AliasAnalysisEvaluator.h | 7 +- include/llvm/Analysis/AliasSetTracker.h | 28 +- include/llvm/Analysis/AssumptionCache.h | 15 +- include/llvm/Analysis/BasicAliasAnalysis.h | 34 +- include/llvm/Analysis/BlockFrequencyInfo.h | 10 +- include/llvm/Analysis/BlockFrequencyInfoImpl.h | 33 +- include/llvm/Analysis/BranchProbabilityInfo.h | 7 +- include/llvm/Analysis/CFG.h | 32 +- include/llvm/Analysis/CFGPrinter.h | 7 +- include/llvm/Analysis/CFLAliasAnalysisUtils.h | 7 +- include/llvm/Analysis/CFLAndersAliasAnalysis.h | 10 +- include/llvm/Analysis/CFLSteensAliasAnalysis.h | 14 +- include/llvm/Analysis/CGSCCPassManager.h | 396 +- include/llvm/Analysis/CallGraph.h | 24 +- include/llvm/Analysis/CallGraphSCCPass.h | 7 +- include/llvm/Analysis/CallPrinter.h | 7 +- include/llvm/Analysis/CaptureTracking.h | 7 +- include/llvm/Analysis/CmpInstAnalysis.h | 7 +- include/llvm/Analysis/CodeMetrics.h | 16 +- include/llvm/Analysis/ConstantFolding.h | 22 +- include/llvm/Analysis/DOTGraphTraitsPass.h | 7 +- include/llvm/Analysis/DemandedBits.h | 7 +- include/llvm/Analysis/DependenceAnalysis.h | 11 +- include/llvm/Analysis/DivergenceAnalysis.h | 7 +- include/llvm/Analysis/DomPrinter.h | 7 +- include/llvm/Analysis/DomTreeUpdater.h | 309 + include/llvm/Analysis/DominanceFrontier.h | 7 +- include/llvm/Analysis/DominanceFrontierImpl.h | 7 +- include/llvm/Analysis/EHPersonalities.h | 7 +- include/llvm/Analysis/GlobalsModRef.h | 15 +- include/llvm/Analysis/GuardUtils.h | 30 +- include/llvm/Analysis/IVDescriptors.h | 28 +- include/llvm/Analysis/IVUsers.h | 7 +- .../llvm/Analysis/IndirectCallPromotionAnalysis.h | 7 +- include/llvm/Analysis/IndirectCallVisitor.h | 7 +- include/llvm/Analysis/InlineCost.h | 21 +- .../llvm/Analysis/InstructionPrecedenceTracking.h | 9 +- include/llvm/Analysis/InstructionSimplify.h | 33 +- include/llvm/Analysis/Interval.h | 7 +- include/llvm/Analysis/IntervalIterator.h | 7 +- include/llvm/Analysis/IntervalPartition.h | 7 +- include/llvm/Analysis/IteratedDominanceFrontier.h | 154 +- include/llvm/Analysis/LazyBlockFrequencyInfo.h | 7 +- include/llvm/Analysis/LazyBranchProbabilityInfo.h | 7 +- include/llvm/Analysis/LazyCallGraph.h | 32 +- include/llvm/Analysis/LazyValueInfo.h | 7 +- include/llvm/Analysis/LegacyDivergenceAnalysis.h | 7 +- include/llvm/Analysis/Lint.h | 7 +- include/llvm/Analysis/Loads.h | 29 +- include/llvm/Analysis/LoopAccessAnalysis.h | 13 +- include/llvm/Analysis/LoopAnalysisManager.h | 10 +- include/llvm/Analysis/LoopInfo.h | 249 +- include/llvm/Analysis/LoopInfoImpl.h | 85 +- include/llvm/Analysis/LoopIterator.h | 7 +- include/llvm/Analysis/LoopPass.h | 7 +- include/llvm/Analysis/LoopUnrollAnalyzer.h | 7 +- include/llvm/Analysis/MemoryBuiltins.h | 50 +- include/llvm/Analysis/MemoryDependenceAnalysis.h | 26 +- include/llvm/Analysis/MemoryLocation.h | 7 +- include/llvm/Analysis/MemorySSA.h | 49 +- include/llvm/Analysis/MemorySSAUpdater.h | 42 +- include/llvm/Analysis/ModuleSummaryAnalysis.h | 7 +- include/llvm/Analysis/MustExecute.h | 7 +- include/llvm/Analysis/ObjCARCAliasAnalysis.h | 16 +- include/llvm/Analysis/ObjCARCAnalysisUtils.h | 7 +- include/llvm/Analysis/ObjCARCInstKind.h | 11 +- include/llvm/Analysis/OptimizationRemarkEmitter.h | 11 +- include/llvm/Analysis/OrderedBasicBlock.h | 15 +- include/llvm/Analysis/OrderedInstructions.h | 7 +- include/llvm/Analysis/PHITransAddr.h | 7 +- include/llvm/Analysis/Passes.h | 7 +- include/llvm/Analysis/PhiValues.h | 7 +- include/llvm/Analysis/PostDominators.h | 7 +- include/llvm/Analysis/ProfileSummaryInfo.h | 16 +- include/llvm/Analysis/PtrUseVisitor.h | 11 +- include/llvm/Analysis/RegionInfo.h | 7 +- include/llvm/Analysis/RegionInfoImpl.h | 7 +- include/llvm/Analysis/RegionIterator.h | 7 +- include/llvm/Analysis/RegionPass.h | 7 +- include/llvm/Analysis/RegionPrinter.h | 7 +- include/llvm/Analysis/ScalarEvolution.h | 72 +- .../llvm/Analysis/ScalarEvolutionAliasAnalysis.h | 10 +- include/llvm/Analysis/ScalarEvolutionExpander.h | 17 +- include/llvm/Analysis/ScalarEvolutionExpressions.h | 156 +- .../llvm/Analysis/ScalarEvolutionNormalization.h | 7 +- include/llvm/Analysis/ScopedNoAliasAA.h | 16 +- include/llvm/Analysis/SparsePropagation.h | 15 +- include/llvm/Analysis/StackSafetyAnalysis.h | 7 +- include/llvm/Analysis/SyncDependenceAnalysis.h | 7 +- include/llvm/Analysis/SyntheticCountsUtils.h | 7 +- include/llvm/Analysis/TargetFolder.h | 11 +- include/llvm/Analysis/TargetLibraryInfo.def | 65 +- include/llvm/Analysis/TargetLibraryInfo.h | 14 +- include/llvm/Analysis/TargetTransformInfo.h | 235 +- include/llvm/Analysis/TargetTransformInfoImpl.h | 191 +- include/llvm/Analysis/Trace.h | 7 +- include/llvm/Analysis/TypeBasedAliasAnalysis.h | 19 +- include/llvm/Analysis/TypeMetadataUtils.h | 7 +- include/llvm/Analysis/Utils/Local.h | 7 +- include/llvm/Analysis/ValueLattice.h | 7 +- include/llvm/Analysis/ValueLatticeUtils.h | 7 +- include/llvm/Analysis/ValueTracking.h | 64 +- include/llvm/Analysis/VecFuncs.def | 250 + include/llvm/Analysis/VectorUtils.h | 81 +- include/llvm/AsmParser/Parser.h | 7 +- include/llvm/AsmParser/SlotMapping.h | 7 +- include/llvm/BinaryFormat/AMDGPUMetadataVerifier.h | 37 +- include/llvm/BinaryFormat/COFF.h | 14 +- include/llvm/BinaryFormat/Dwarf.def | 16 +- include/llvm/BinaryFormat/Dwarf.h | 11 +- include/llvm/BinaryFormat/DynamicTags.def | 28 + include/llvm/BinaryFormat/ELF.h | 88 +- include/llvm/BinaryFormat/ELFRelocs/ARM.def | 3 + include/llvm/BinaryFormat/ELFRelocs/PowerPC.def | 33 + include/llvm/BinaryFormat/MachO.def | 7 +- include/llvm/BinaryFormat/MachO.h | 25 +- include/llvm/BinaryFormat/Magic.h | 10 +- include/llvm/BinaryFormat/Minidump.h | 203 + include/llvm/BinaryFormat/MinidumpConstants.def | 107 + include/llvm/BinaryFormat/MsgPack.def | 7 +- include/llvm/BinaryFormat/MsgPack.h | 7 +- include/llvm/BinaryFormat/MsgPackDocument.h | 385 + include/llvm/BinaryFormat/MsgPackReader.h | 7 +- include/llvm/BinaryFormat/MsgPackTypes.h | 372 - include/llvm/BinaryFormat/MsgPackWriter.h | 7 +- include/llvm/BinaryFormat/Wasm.h | 65 +- include/llvm/BinaryFormat/WasmRelocs.def | 24 +- include/llvm/BinaryFormat/XCOFF.h | 145 + include/llvm/Bitcode/BitCodes.h | 185 - include/llvm/Bitcode/BitcodeAnalyzer.h | 103 + include/llvm/Bitcode/BitcodeReader.h | 9 +- include/llvm/Bitcode/BitcodeWriter.h | 7 +- include/llvm/Bitcode/BitcodeWriterPass.h | 7 +- include/llvm/Bitcode/BitstreamReader.h | 506 -- include/llvm/Bitcode/BitstreamWriter.h | 550 -- include/llvm/Bitcode/LLVMBitCodes.h | 42 +- include/llvm/Bitstream/BitCodes.h | 184 + include/llvm/Bitstream/BitstreamReader.h | 557 ++ include/llvm/Bitstream/BitstreamWriter.h | 547 ++ include/llvm/CodeGen/AccelTable.h | 31 +- include/llvm/CodeGen/Analysis.h | 27 +- include/llvm/CodeGen/AsmPrinter.h | 61 +- include/llvm/CodeGen/AsmPrinterHandler.h | 7 +- include/llvm/CodeGen/AtomicExpandUtils.h | 7 +- include/llvm/CodeGen/BasicTTIImpl.h | 250 +- include/llvm/CodeGen/BuiltinGCs.h | 7 +- include/llvm/CodeGen/CSEConfigBase.h | 28 + include/llvm/CodeGen/CalcSpillWeights.h | 7 +- include/llvm/CodeGen/CallingConvLower.h | 11 +- include/llvm/CodeGen/CommandFlags.inc | 13 +- include/llvm/CodeGen/CostTable.h | 7 +- include/llvm/CodeGen/DAGCombine.h | 7 +- include/llvm/CodeGen/DFAPacketizer.h | 7 +- include/llvm/CodeGen/DIE.h | 59 +- include/llvm/CodeGen/DIEValue.def | 8 +- include/llvm/CodeGen/DbgEntityHistoryCalculator.h | 93 +- include/llvm/CodeGen/DebugHandlerBase.h | 9 +- include/llvm/CodeGen/DwarfStringPoolEntry.h | 7 +- include/llvm/CodeGen/EdgeBundles.h | 7 +- include/llvm/CodeGen/ExecutionDomainFix.h | 7 +- include/llvm/CodeGen/ExpandReductions.h | 7 +- include/llvm/CodeGen/FastISel.h | 9 +- include/llvm/CodeGen/FaultMaps.h | 7 +- include/llvm/CodeGen/FunctionLoweringInfo.h | 65 +- include/llvm/CodeGen/GCMetadata.h | 7 +- include/llvm/CodeGen/GCMetadataPrinter.h | 7 +- include/llvm/CodeGen/GCStrategy.h | 7 +- include/llvm/CodeGen/GlobalISel/CSEInfo.h | 41 +- include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h | 7 +- include/llvm/CodeGen/GlobalISel/CallLowering.h | 137 +- include/llvm/CodeGen/GlobalISel/Combiner.h | 9 +- include/llvm/CodeGen/GlobalISel/CombinerHelper.h | 27 +- include/llvm/CodeGen/GlobalISel/CombinerInfo.h | 7 +- .../CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h | 7 +- .../llvm/CodeGen/GlobalISel/GISelChangeObserver.h | 18 +- include/llvm/CodeGen/GlobalISel/GISelWorkList.h | 50 +- include/llvm/CodeGen/GlobalISel/IRTranslator.h | 121 +- .../llvm/CodeGen/GlobalISel/InstructionSelect.h | 7 +- .../llvm/CodeGen/GlobalISel/InstructionSelector.h | 17 +- .../CodeGen/GlobalISel/InstructionSelectorImpl.h | 60 +- .../GlobalISel/LegalizationArtifactCombiner.h | 278 +- include/llvm/CodeGen/GlobalISel/Legalizer.h | 14 +- include/llvm/CodeGen/GlobalISel/LegalizerHelper.h | 120 +- include/llvm/CodeGen/GlobalISel/LegalizerInfo.h | 211 +- include/llvm/CodeGen/GlobalISel/Localizer.h | 22 +- include/llvm/CodeGen/GlobalISel/MIPatternMatch.h | 14 +- include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h | 361 +- include/llvm/CodeGen/GlobalISel/RegBankSelect.h | 16 +- include/llvm/CodeGen/GlobalISel/RegisterBank.h | 7 +- include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h | 53 +- include/llvm/CodeGen/GlobalISel/Types.h | 7 +- include/llvm/CodeGen/GlobalISel/Utils.h | 74 +- include/llvm/CodeGen/ISDOpcodes.h | 54 +- include/llvm/CodeGen/IntrinsicLowering.h | 11 +- include/llvm/CodeGen/LatencyPriorityQueue.h | 7 +- .../llvm/CodeGen/LazyMachineBlockFrequencyInfo.h | 7 +- include/llvm/CodeGen/LexicalScopes.h | 7 +- include/llvm/CodeGen/LinkAllAsmWriterComponents.h | 7 +- include/llvm/CodeGen/LinkAllCodegenComponents.h | 7 +- include/llvm/CodeGen/LiveInterval.h | 54 +- include/llvm/CodeGen/LiveIntervalUnion.h | 7 +- include/llvm/CodeGen/LiveIntervals.h | 16 +- include/llvm/CodeGen/LivePhysRegs.h | 7 +- include/llvm/CodeGen/LiveRangeEdit.h | 7 +- include/llvm/CodeGen/LiveRegMatrix.h | 7 +- include/llvm/CodeGen/LiveRegUnits.h | 7 +- include/llvm/CodeGen/LiveStacks.h | 7 +- include/llvm/CodeGen/LiveVariables.h | 7 +- include/llvm/CodeGen/LoopTraversal.h | 7 +- include/llvm/CodeGen/LowLevelType.h | 7 +- include/llvm/CodeGen/MIRParser/MIParser.h | 233 + include/llvm/CodeGen/MIRParser/MIRParser.h | 7 +- include/llvm/CodeGen/MIRPrinter.h | 9 +- include/llvm/CodeGen/MIRYamlMapping.h | 108 +- include/llvm/CodeGen/MachORelocation.h | 7 +- include/llvm/CodeGen/MachineBasicBlock.h | 28 +- include/llvm/CodeGen/MachineBlockFrequencyInfo.h | 7 +- .../llvm/CodeGen/MachineBranchProbabilityInfo.h | 7 +- include/llvm/CodeGen/MachineCombinerPattern.h | 7 +- include/llvm/CodeGen/MachineConstantPool.h | 7 +- include/llvm/CodeGen/MachineDominanceFrontier.h | 7 +- include/llvm/CodeGen/MachineDominators.h | 7 +- include/llvm/CodeGen/MachineFrameInfo.h | 14 +- include/llvm/CodeGen/MachineFunction.h | 76 +- include/llvm/CodeGen/MachineFunctionPass.h | 7 +- include/llvm/CodeGen/MachineInstr.h | 81 +- include/llvm/CodeGen/MachineInstrBuilder.h | 15 +- include/llvm/CodeGen/MachineInstrBundle.h | 15 +- include/llvm/CodeGen/MachineInstrBundleIterator.h | 7 +- include/llvm/CodeGen/MachineJumpTableInfo.h | 7 +- include/llvm/CodeGen/MachineLoopInfo.h | 7 +- include/llvm/CodeGen/MachineMemOperand.h | 26 +- include/llvm/CodeGen/MachineModuleInfo.h | 32 +- include/llvm/CodeGen/MachineModuleInfoImpls.h | 7 +- include/llvm/CodeGen/MachineOperand.h | 21 +- .../CodeGen/MachineOptimizationRemarkEmitter.h | 21 +- include/llvm/CodeGen/MachineOutliner.h | 13 +- include/llvm/CodeGen/MachinePassRegistry.h | 7 +- include/llvm/CodeGen/MachinePipeliner.h | 85 +- include/llvm/CodeGen/MachinePostDominators.h | 9 +- include/llvm/CodeGen/MachineRegionInfo.h | 7 +- include/llvm/CodeGen/MachineRegisterInfo.h | 20 +- include/llvm/CodeGen/MachineSSAUpdater.h | 7 +- include/llvm/CodeGen/MachineScheduler.h | 35 +- include/llvm/CodeGen/MachineTraceMetrics.h | 7 +- include/llvm/CodeGen/MacroFusion.h | 7 +- include/llvm/CodeGen/PBQP/CostAllocator.h | 7 +- include/llvm/CodeGen/PBQP/Graph.h | 7 +- include/llvm/CodeGen/PBQP/Math.h | 7 +- include/llvm/CodeGen/PBQP/ReductionRules.h | 7 +- include/llvm/CodeGen/PBQP/Solution.h | 7 +- include/llvm/CodeGen/PBQPRAConstraint.h | 9 +- include/llvm/CodeGen/ParallelCG.h | 7 +- include/llvm/CodeGen/Passes.h | 15 +- include/llvm/CodeGen/PreISelIntrinsicLowering.h | 7 +- include/llvm/CodeGen/PseudoSourceValue.h | 10 +- include/llvm/CodeGen/ReachingDefAnalysis.h | 9 +- include/llvm/CodeGen/RegAllocPBQP.h | 7 +- include/llvm/CodeGen/RegAllocRegistry.h | 35 +- include/llvm/CodeGen/Register.h | 60 + include/llvm/CodeGen/RegisterClassInfo.h | 7 +- include/llvm/CodeGen/RegisterPressure.h | 11 +- include/llvm/CodeGen/RegisterScavenging.h | 24 +- include/llvm/CodeGen/RegisterUsageInfo.h | 7 +- include/llvm/CodeGen/ResourcePriorityQueue.h | 7 +- include/llvm/CodeGen/RuntimeLibcalls.h | 7 +- include/llvm/CodeGen/SDNodeProperties.td | 7 +- include/llvm/CodeGen/ScheduleDAG.h | 31 +- include/llvm/CodeGen/ScheduleDAGInstrs.h | 23 +- include/llvm/CodeGen/ScheduleDAGMutation.h | 7 +- include/llvm/CodeGen/ScheduleDFS.h | 9 +- include/llvm/CodeGen/ScheduleHazardRecognizer.h | 7 +- include/llvm/CodeGen/SchedulerRegistry.h | 7 +- include/llvm/CodeGen/ScoreboardHazardRecognizer.h | 7 +- include/llvm/CodeGen/SelectionDAG.h | 139 +- include/llvm/CodeGen/SelectionDAGAddressAnalysis.h | 46 +- include/llvm/CodeGen/SelectionDAGISel.h | 15 +- include/llvm/CodeGen/SelectionDAGNodes.h | 185 +- include/llvm/CodeGen/SelectionDAGTargetInfo.h | 15 +- include/llvm/CodeGen/SlotIndexes.h | 87 +- include/llvm/CodeGen/StackMaps.h | 7 +- include/llvm/CodeGen/StackProtector.h | 13 +- include/llvm/CodeGen/SwiftErrorValueTracking.h | 110 + include/llvm/CodeGen/SwitchLoweringUtils.h | 297 + include/llvm/CodeGen/TailDuplicator.h | 7 +- include/llvm/CodeGen/TargetCallingConv.h | 23 +- include/llvm/CodeGen/TargetFrameLowering.h | 37 +- include/llvm/CodeGen/TargetInstrInfo.h | 46 +- include/llvm/CodeGen/TargetLowering.h | 375 +- .../llvm/CodeGen/TargetLoweringObjectFileImpl.h | 7 +- include/llvm/CodeGen/TargetOpcodes.h | 7 +- include/llvm/CodeGen/TargetPassConfig.h | 37 +- include/llvm/CodeGen/TargetRegisterInfo.h | 14 +- include/llvm/CodeGen/TargetSchedule.h | 7 +- include/llvm/CodeGen/TargetSubtargetInfo.h | 33 +- include/llvm/CodeGen/UnreachableBlockElim.h | 7 +- include/llvm/CodeGen/ValueTypes.h | 7 +- include/llvm/CodeGen/ValueTypes.td | 200 +- include/llvm/CodeGen/VirtRegMap.h | 17 +- include/llvm/CodeGen/WasmEHFuncInfo.h | 29 +- include/llvm/CodeGen/WinEHFuncInfo.h | 7 +- .../DebugInfo/CodeView/AppendingTypeTableBuilder.h | 7 +- include/llvm/DebugInfo/CodeView/CVRecord.h | 38 +- include/llvm/DebugInfo/CodeView/CVSymbolVisitor.h | 7 +- include/llvm/DebugInfo/CodeView/CVTypeVisitor.h | 11 +- include/llvm/DebugInfo/CodeView/CodeView.h | 29 +- include/llvm/DebugInfo/CodeView/CodeViewError.h | 7 +- include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h | 140 +- .../llvm/DebugInfo/CodeView/CodeViewRegisters.def | 210 +- .../llvm/DebugInfo/CodeView/CodeViewSymbols.def | 9 +- include/llvm/DebugInfo/CodeView/CodeViewTypes.def | 7 +- .../DebugInfo/CodeView/ContinuationRecordBuilder.h | 9 +- .../DebugInfo/CodeView/DebugChecksumsSubsection.h | 7 +- .../DebugInfo/CodeView/DebugCrossExSubsection.h | 7 +- .../DebugInfo/CodeView/DebugCrossImpSubsection.h | 9 +- .../DebugInfo/CodeView/DebugFrameDataSubsection.h | 7 +- .../CodeView/DebugInlineeLinesSubsection.h | 14 +- .../llvm/DebugInfo/CodeView/DebugLinesSubsection.h | 7 +- .../CodeView/DebugStringTableSubsection.h | 7 +- include/llvm/DebugInfo/CodeView/DebugSubsection.h | 7 +- .../DebugInfo/CodeView/DebugSubsectionRecord.h | 7 +- .../DebugInfo/CodeView/DebugSubsectionVisitor.h | 7 +- .../DebugInfo/CodeView/DebugSymbolRVASubsection.h | 7 +- .../DebugInfo/CodeView/DebugSymbolsSubsection.h | 7 +- .../DebugInfo/CodeView/DebugUnknownSubsection.h | 7 +- include/llvm/DebugInfo/CodeView/EnumTables.h | 9 +- include/llvm/DebugInfo/CodeView/Formatters.h | 7 +- include/llvm/DebugInfo/CodeView/FunctionId.h | 7 +- include/llvm/DebugInfo/CodeView/GUID.h | 7 +- .../DebugInfo/CodeView/GlobalTypeTableBuilder.h | 29 +- .../DebugInfo/CodeView/LazyRandomTypeCollection.h | 7 +- include/llvm/DebugInfo/CodeView/Line.h | 7 +- .../DebugInfo/CodeView/MergingTypeTableBuilder.h | 7 +- include/llvm/DebugInfo/CodeView/RecordName.h | 7 +- .../llvm/DebugInfo/CodeView/RecordSerialization.h | 10 +- .../llvm/DebugInfo/CodeView/SimpleTypeSerializer.h | 7 +- .../llvm/DebugInfo/CodeView/StringsAndChecksums.h | 7 +- .../llvm/DebugInfo/CodeView/SymbolDeserializer.h | 7 +- .../llvm/DebugInfo/CodeView/SymbolDumpDelegate.h | 7 +- include/llvm/DebugInfo/CodeView/SymbolDumper.h | 7 +- include/llvm/DebugInfo/CodeView/SymbolRecord.h | 66 +- .../llvm/DebugInfo/CodeView/SymbolRecordHelpers.h | 7 +- .../llvm/DebugInfo/CodeView/SymbolRecordMapping.h | 7 +- include/llvm/DebugInfo/CodeView/SymbolSerializer.h | 11 +- .../CodeView/SymbolVisitorCallbackPipeline.h | 7 +- .../DebugInfo/CodeView/SymbolVisitorCallbacks.h | 7 +- .../DebugInfo/CodeView/SymbolVisitorDelegate.h | 7 +- include/llvm/DebugInfo/CodeView/TypeCollection.h | 7 +- include/llvm/DebugInfo/CodeView/TypeDeserializer.h | 17 +- include/llvm/DebugInfo/CodeView/TypeDumpVisitor.h | 7 +- include/llvm/DebugInfo/CodeView/TypeHashing.h | 38 +- include/llvm/DebugInfo/CodeView/TypeIndex.h | 7 +- .../llvm/DebugInfo/CodeView/TypeIndexDiscovery.h | 7 +- include/llvm/DebugInfo/CodeView/TypeRecord.h | 7 +- .../llvm/DebugInfo/CodeView/TypeRecordHelpers.h | 7 +- .../llvm/DebugInfo/CodeView/TypeRecordMapping.h | 9 +- include/llvm/DebugInfo/CodeView/TypeStreamMerger.h | 7 +- .../llvm/DebugInfo/CodeView/TypeSymbolEmitter.h | 7 +- .../llvm/DebugInfo/CodeView/TypeTableCollection.h | 7 +- .../CodeView/TypeVisitorCallbackPipeline.h | 12 +- .../llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h | 7 +- include/llvm/DebugInfo/DIContext.h | 46 +- .../DebugInfo/DWARF/DWARFAbbreviationDeclaration.h | 7 +- .../llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h | 23 +- include/llvm/DebugInfo/DWARF/DWARFAddressRange.h | 13 +- include/llvm/DebugInfo/DWARF/DWARFAttribute.h | 21 +- include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h | 7 +- include/llvm/DebugInfo/DWARF/DWARFContext.h | 25 +- include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h | 7 +- include/llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h | 7 +- include/llvm/DebugInfo/DWARF/DWARFDebugAddr.h | 7 +- include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h | 7 +- include/llvm/DebugInfo/DWARF/DWARFDebugAranges.h | 11 +- include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h | 7 +- include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h | 7 +- include/llvm/DebugInfo/DWARF/DWARFDebugLine.h | 74 +- include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h | 11 +- include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h | 7 +- include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h | 7 +- include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h | 9 +- include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h | 11 +- include/llvm/DebugInfo/DWARF/DWARFDie.h | 7 +- include/llvm/DebugInfo/DWARF/DWARFExpression.h | 26 +- include/llvm/DebugInfo/DWARF/DWARFFormValue.h | 54 +- include/llvm/DebugInfo/DWARF/DWARFGdbIndex.h | 7 +- include/llvm/DebugInfo/DWARF/DWARFListTable.h | 11 +- include/llvm/DebugInfo/DWARF/DWARFObject.h | 7 +- include/llvm/DebugInfo/DWARF/DWARFRelocMap.h | 14 +- include/llvm/DebugInfo/DWARF/DWARFSection.h | 12 +- include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h | 7 +- include/llvm/DebugInfo/DWARF/DWARFUnit.h | 50 +- include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h | 7 +- include/llvm/DebugInfo/DWARF/DWARFVerifier.h | 7 +- include/llvm/DebugInfo/GSYM/FileEntry.h | 68 + include/llvm/DebugInfo/GSYM/FunctionInfo.h | 107 + include/llvm/DebugInfo/GSYM/InlineInfo.h | 78 + include/llvm/DebugInfo/GSYM/LineEntry.h | 48 + include/llvm/DebugInfo/GSYM/Range.h | 87 + include/llvm/DebugInfo/GSYM/StringTable.h | 54 + include/llvm/DebugInfo/MSF/IMSFFile.h | 7 +- include/llvm/DebugInfo/MSF/MSFBuilder.h | 7 +- include/llvm/DebugInfo/MSF/MSFCommon.h | 7 +- include/llvm/DebugInfo/MSF/MSFError.h | 7 +- include/llvm/DebugInfo/MSF/MappedBlockStream.h | 7 +- .../llvm/DebugInfo/PDB/ConcreteSymbolEnumerator.h | 7 +- include/llvm/DebugInfo/PDB/DIA/DIADataStream.h | 7 +- .../llvm/DebugInfo/PDB/DIA/DIAEnumDebugStreams.h | 7 +- include/llvm/DebugInfo/PDB/DIA/DIAEnumFrameData.h | 7 +- .../DebugInfo/PDB/DIA/DIAEnumInjectedSources.h | 7 +- .../llvm/DebugInfo/PDB/DIA/DIAEnumLineNumbers.h | 7 +- .../DebugInfo/PDB/DIA/DIAEnumSectionContribs.h | 7 +- .../llvm/DebugInfo/PDB/DIA/DIAEnumSourceFiles.h | 7 +- include/llvm/DebugInfo/PDB/DIA/DIAEnumSymbols.h | 7 +- include/llvm/DebugInfo/PDB/DIA/DIAEnumTables.h | 7 +- include/llvm/DebugInfo/PDB/DIA/DIAError.h | 7 +- include/llvm/DebugInfo/PDB/DIA/DIAFrameData.h | 7 +- include/llvm/DebugInfo/PDB/DIA/DIAInjectedSource.h | 9 +- include/llvm/DebugInfo/PDB/DIA/DIALineNumber.h | 7 +- include/llvm/DebugInfo/PDB/DIA/DIARawSymbol.h | 7 +- include/llvm/DebugInfo/PDB/DIA/DIASectionContrib.h | 7 +- include/llvm/DebugInfo/PDB/DIA/DIASession.h | 7 +- include/llvm/DebugInfo/PDB/DIA/DIASourceFile.h | 7 +- include/llvm/DebugInfo/PDB/DIA/DIASupport.h | 7 +- include/llvm/DebugInfo/PDB/DIA/DIATable.h | 7 +- include/llvm/DebugInfo/PDB/DIA/DIAUtils.h | 7 +- include/llvm/DebugInfo/PDB/GenericError.h | 9 +- include/llvm/DebugInfo/PDB/IPDBDataStream.h | 7 +- include/llvm/DebugInfo/PDB/IPDBEnumChildren.h | 7 +- include/llvm/DebugInfo/PDB/IPDBFrameData.h | 7 +- include/llvm/DebugInfo/PDB/IPDBInjectedSource.h | 13 +- include/llvm/DebugInfo/PDB/IPDBLineNumber.h | 7 +- include/llvm/DebugInfo/PDB/IPDBRawSymbol.h | 7 +- include/llvm/DebugInfo/PDB/IPDBSectionContrib.h | 7 +- include/llvm/DebugInfo/PDB/IPDBSession.h | 7 +- include/llvm/DebugInfo/PDB/IPDBSourceFile.h | 7 +- include/llvm/DebugInfo/PDB/IPDBTable.h | 7 +- .../DebugInfo/PDB/Native/DbiModuleDescriptor.h | 7 +- .../PDB/Native/DbiModuleDescriptorBuilder.h | 7 +- include/llvm/DebugInfo/PDB/Native/DbiModuleList.h | 7 +- include/llvm/DebugInfo/PDB/Native/DbiStream.h | 26 +- .../llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h | 7 +- include/llvm/DebugInfo/PDB/Native/EnumTables.h | 7 +- include/llvm/DebugInfo/PDB/Native/Formatters.h | 7 +- .../llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h | 7 +- include/llvm/DebugInfo/PDB/Native/GlobalsStream.h | 7 +- include/llvm/DebugInfo/PDB/Native/Hash.h | 7 +- include/llvm/DebugInfo/PDB/Native/HashTable.h | 92 +- .../DebugInfo/PDB/Native/ISectionContribVisitor.h | 7 +- include/llvm/DebugInfo/PDB/Native/InfoStream.h | 7 +- .../llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h | 7 +- .../DebugInfo/PDB/Native/InjectedSourceStream.h | 44 + .../llvm/DebugInfo/PDB/Native/ModuleDebugStream.h | 9 +- include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h | 9 +- .../DebugInfo/PDB/Native/NativeCompilandSymbol.h | 7 +- .../llvm/DebugInfo/PDB/Native/NativeEnumGlobals.h | 7 +- .../PDB/Native/NativeEnumInjectedSources.h | 43 + .../llvm/DebugInfo/PDB/Native/NativeEnumModules.h | 7 +- .../llvm/DebugInfo/PDB/Native/NativeEnumTypes.h | 7 +- .../llvm/DebugInfo/PDB/Native/NativeExeSymbol.h | 7 +- .../llvm/DebugInfo/PDB/Native/NativeRawSymbol.h | 7 +- include/llvm/DebugInfo/PDB/Native/NativeSession.h | 7 +- .../DebugInfo/PDB/Native/NativeSymbolEnumerator.h | 7 +- .../llvm/DebugInfo/PDB/Native/NativeTypeArray.h | 7 +- .../llvm/DebugInfo/PDB/Native/NativeTypeBuiltin.h | 7 +- include/llvm/DebugInfo/PDB/Native/NativeTypeEnum.h | 7 +- .../DebugInfo/PDB/Native/NativeTypeFunctionSig.h | 7 +- .../llvm/DebugInfo/PDB/Native/NativeTypePointer.h | 7 +- .../llvm/DebugInfo/PDB/Native/NativeTypeTypedef.h | 7 +- include/llvm/DebugInfo/PDB/Native/NativeTypeUDT.h | 7 +- .../llvm/DebugInfo/PDB/Native/NativeTypeVTShape.h | 7 +- include/llvm/DebugInfo/PDB/Native/PDBFile.h | 23 +- include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h | 9 +- include/llvm/DebugInfo/PDB/Native/PDBStringTable.h | 7 +- .../DebugInfo/PDB/Native/PDBStringTableBuilder.h | 7 +- include/llvm/DebugInfo/PDB/Native/PublicsStream.h | 7 +- include/llvm/DebugInfo/PDB/Native/RawConstants.h | 7 +- include/llvm/DebugInfo/PDB/Native/RawError.h | 7 +- include/llvm/DebugInfo/PDB/Native/RawTypes.h | 18 +- include/llvm/DebugInfo/PDB/Native/SymbolCache.h | 7 +- include/llvm/DebugInfo/PDB/Native/SymbolStream.h | 7 +- include/llvm/DebugInfo/PDB/Native/TpiHashing.h | 7 +- include/llvm/DebugInfo/PDB/Native/TpiStream.h | 7 +- .../llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h | 7 +- include/llvm/DebugInfo/PDB/PDB.h | 7 +- include/llvm/DebugInfo/PDB/PDBContext.h | 16 +- include/llvm/DebugInfo/PDB/PDBExtras.h | 13 +- include/llvm/DebugInfo/PDB/PDBSymDumper.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbol.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbolAnnotation.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbolBlock.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbolCompiland.h | 7 +- .../llvm/DebugInfo/PDB/PDBSymbolCompilandDetails.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbolCompilandEnv.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbolCustom.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbolData.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbolExe.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbolFunc.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h | 7 +- .../llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbolLabel.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbolThunk.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h | 7 +- .../llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbolTypeCustom.h | 7 +- .../llvm/DebugInfo/PDB/PDBSymbolTypeDimension.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbolTypeFriend.h | 7 +- .../llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h | 7 +- .../llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbolTypeManaged.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h | 7 +- .../llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h | 7 +- include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h | 7 +- .../llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h | 7 +- include/llvm/DebugInfo/PDB/PDBTypes.h | 76 +- include/llvm/DebugInfo/PDB/UDTLayout.h | 7 +- include/llvm/DebugInfo/Symbolize/DIPrinter.h | 20 +- .../llvm/DebugInfo/Symbolize/SymbolizableModule.h | 20 +- include/llvm/DebugInfo/Symbolize/Symbolize.h | 56 +- include/llvm/Demangle/Compiler.h | 93 - include/llvm/Demangle/Demangle.h | 15 +- include/llvm/Demangle/DemangleConfig.h | 99 + include/llvm/Demangle/ItaniumDemangle.h | 108 +- include/llvm/Demangle/MicrosoftDemangle.h | 53 +- include/llvm/Demangle/MicrosoftDemangleNodes.h | 33 +- include/llvm/Demangle/README.txt | 52 + include/llvm/Demangle/StringView.h | 21 +- include/llvm/Demangle/Utility.h | 18 +- include/llvm/ExecutionEngine/ExecutionEngine.h | 20 +- include/llvm/ExecutionEngine/GenericValue.h | 7 +- include/llvm/ExecutionEngine/Interpreter.h | 7 +- include/llvm/ExecutionEngine/JITEventListener.h | 7 +- .../llvm/ExecutionEngine/JITLink/EHFrameSupport.h | 80 + include/llvm/ExecutionEngine/JITLink/JITLink.h | 930 ++ .../ExecutionEngine/JITLink/JITLinkMemoryManager.h | 99 + include/llvm/ExecutionEngine/JITLink/MachO.h | 30 + .../llvm/ExecutionEngine/JITLink/MachO_x86_64.h | 63 + include/llvm/ExecutionEngine/JITSymbol.h | 34 +- include/llvm/ExecutionEngine/MCJIT.h | 7 +- include/llvm/ExecutionEngine/OProfileWrapper.h | 7 +- include/llvm/ExecutionEngine/ObjectCache.h | 7 +- .../ExecutionEngine/Orc/CompileOnDemandLayer.h | 52 +- include/llvm/ExecutionEngine/Orc/CompileUtils.h | 95 +- include/llvm/ExecutionEngine/Orc/Core.h | 228 +- include/llvm/ExecutionEngine/Orc/ExecutionUtils.h | 52 +- .../llvm/ExecutionEngine/Orc/GlobalMappingLayer.h | 7 +- include/llvm/ExecutionEngine/Orc/IRCompileLayer.h | 27 +- .../llvm/ExecutionEngine/Orc/IRTransformLayer.h | 26 +- .../llvm/ExecutionEngine/Orc/IndirectionUtils.h | 11 +- .../ExecutionEngine/Orc/JITTargetMachineBuilder.h | 7 +- include/llvm/ExecutionEngine/Orc/LLJIT.h | 230 +- include/llvm/ExecutionEngine/Orc/LambdaResolver.h | 34 +- include/llvm/ExecutionEngine/Orc/Layer.h | 7 +- .../llvm/ExecutionEngine/Orc/LazyEmittingLayer.h | 24 +- include/llvm/ExecutionEngine/Orc/LazyReexports.h | 7 +- include/llvm/ExecutionEngine/Orc/Legacy.h | 18 +- include/llvm/ExecutionEngine/Orc/NullResolver.h | 7 +- .../llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h | 165 + .../ExecutionEngine/Orc/ObjectTransformLayer.h | 23 +- include/llvm/ExecutionEngine/Orc/OrcABISupport.h | 7 +- include/llvm/ExecutionEngine/Orc/OrcError.h | 7 +- .../ExecutionEngine/Orc/OrcRemoteTargetClient.h | 7 +- .../ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h | 7 +- .../ExecutionEngine/Orc/OrcRemoteTargetServer.h | 13 +- .../llvm/ExecutionEngine/Orc/RPCSerialization.h | 93 +- include/llvm/ExecutionEngine/Orc/RPCUtils.h | 25 +- .../ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h | 50 +- include/llvm/ExecutionEngine/Orc/RawByteChannel.h | 7 +- .../llvm/ExecutionEngine/Orc/RemoteObjectLayer.h | 60 +- .../llvm/ExecutionEngine/Orc/SymbolStringPool.h | 85 +- .../llvm/ExecutionEngine/Orc/ThreadSafeModule.h | 7 +- include/llvm/ExecutionEngine/OrcMCJITReplacement.h | 7 +- include/llvm/ExecutionEngine/OrcV1Deprecation.h | 22 + include/llvm/ExecutionEngine/RTDyldMemoryManager.h | 7 +- include/llvm/ExecutionEngine/RuntimeDyld.h | 32 +- include/llvm/ExecutionEngine/RuntimeDyldChecker.h | 98 +- .../llvm/ExecutionEngine/SectionMemoryManager.h | 7 +- include/llvm/FuzzMutate/FuzzerCLI.h | 7 +- include/llvm/FuzzMutate/IRMutator.h | 7 +- include/llvm/FuzzMutate/OpDescriptor.h | 7 +- include/llvm/FuzzMutate/Operations.h | 7 +- include/llvm/FuzzMutate/Random.h | 7 +- include/llvm/FuzzMutate/RandomIRBuilder.h | 9 +- include/llvm/IR/Argument.h | 15 +- include/llvm/IR/AssemblyAnnotationWriter.h | 7 +- include/llvm/IR/Attributes.h | 27 +- include/llvm/IR/Attributes.td | 16 + include/llvm/IR/AutoUpgrade.h | 13 +- include/llvm/IR/BasicBlock.h | 17 +- include/llvm/IR/CFG.h | 11 +- include/llvm/IR/CFGDiff.h | 7 +- include/llvm/IR/CallSite.h | 311 +- include/llvm/IR/CallingConv.h | 7 +- include/llvm/IR/Comdat.h | 7 +- include/llvm/IR/Constant.h | 11 +- include/llvm/IR/ConstantFolder.h | 11 +- include/llvm/IR/ConstantRange.h | 191 +- include/llvm/IR/Constants.h | 7 +- include/llvm/IR/DIBuilder.h | 17 +- include/llvm/IR/DataLayout.h | 47 +- include/llvm/IR/DebugInfo.h | 7 +- include/llvm/IR/DebugInfoFlags.def | 17 +- include/llvm/IR/DebugInfoMetadata.h | 386 +- include/llvm/IR/DebugLoc.h | 7 +- include/llvm/IR/DerivedTypes.h | 107 +- include/llvm/IR/DerivedUser.h | 7 +- include/llvm/IR/DiagnosticHandler.h | 9 +- include/llvm/IR/DiagnosticInfo.h | 20 +- include/llvm/IR/DiagnosticPrinter.h | 7 +- include/llvm/IR/DomTreeUpdater.h | 257 - include/llvm/IR/Dominators.h | 7 +- include/llvm/IR/Function.h | 48 +- include/llvm/IR/GVMaterializer.h | 7 +- include/llvm/IR/GetElementPtrTypeIterator.h | 7 +- include/llvm/IR/GlobalAlias.h | 7 +- include/llvm/IR/GlobalIFunc.h | 7 +- include/llvm/IR/GlobalIndirectSymbol.h | 7 +- include/llvm/IR/GlobalObject.h | 7 +- include/llvm/IR/GlobalValue.h | 25 +- include/llvm/IR/GlobalVariable.h | 7 +- include/llvm/IR/IRBuilder.h | 340 +- include/llvm/IR/IRPrintingPasses.h | 7 +- include/llvm/IR/InlineAsm.h | 7 +- include/llvm/IR/InstIterator.h | 7 +- include/llvm/IR/InstVisitor.h | 20 +- include/llvm/IR/InstrTypes.h | 166 +- include/llvm/IR/Instruction.def | 146 +- include/llvm/IR/Instruction.h | 28 +- include/llvm/IR/Instructions.h | 487 +- include/llvm/IR/IntrinsicInst.h | 136 +- include/llvm/IR/Intrinsics.h | 37 +- include/llvm/IR/Intrinsics.td | 227 +- include/llvm/IR/IntrinsicsAArch64.td | 77 +- include/llvm/IR/IntrinsicsAMDGPU.td | 524 +- include/llvm/IR/IntrinsicsARM.td | 57 +- include/llvm/IR/IntrinsicsBPF.td | 7 +- include/llvm/IR/IntrinsicsHexagon.td | 513 +- include/llvm/IR/IntrinsicsMips.td | 277 +- include/llvm/IR/IntrinsicsNVVM.td | 465 +- include/llvm/IR/IntrinsicsPowerPC.td | 35 +- include/llvm/IR/IntrinsicsRISCV.td | 38 +- include/llvm/IR/IntrinsicsSystemZ.td | 66 +- include/llvm/IR/IntrinsicsWebAssembly.td | 45 +- include/llvm/IR/IntrinsicsX86.td | 1266 ++- include/llvm/IR/IntrinsicsXCore.td | 7 +- include/llvm/IR/LLVMContext.h | 42 +- include/llvm/IR/LegacyPassManager.h | 7 +- include/llvm/IR/LegacyPassManagers.h | 7 +- include/llvm/IR/LegacyPassNameParser.h | 7 +- include/llvm/IR/MDBuilder.h | 18 +- include/llvm/IR/Mangler.h | 7 +- include/llvm/IR/Metadata.def | 8 +- include/llvm/IR/Metadata.h | 7 +- include/llvm/IR/Module.h | 50 +- include/llvm/IR/ModuleSlotTracker.h | 7 +- include/llvm/IR/ModuleSummaryIndex.h | 253 +- include/llvm/IR/ModuleSummaryIndexYAML.h | 15 +- include/llvm/IR/NoFolder.h | 11 +- include/llvm/IR/OperandTraits.h | 7 +- include/llvm/IR/Operator.h | 14 +- include/llvm/IR/OptBisect.h | 43 +- include/llvm/IR/PassInstrumentation.h | 7 +- include/llvm/IR/PassManager.h | 14 +- include/llvm/IR/PassManagerInternal.h | 7 +- include/llvm/IR/PassTimingInfo.h | 28 +- include/llvm/IR/PatternMatch.h | 91 +- include/llvm/IR/PredIteratorCache.h | 7 +- include/llvm/IR/ProfileSummary.h | 10 +- include/llvm/IR/RemarkStreamer.h | 96 + include/llvm/IR/RuntimeLibcalls.def | 30 +- include/llvm/IR/SafepointIRVerifier.h | 19 +- include/llvm/IR/Statepoint.h | 149 +- include/llvm/IR/SymbolTableListTraits.h | 7 +- include/llvm/IR/TrackingMDRef.h | 7 +- include/llvm/IR/Type.h | 30 +- include/llvm/IR/TypeFinder.h | 7 +- include/llvm/IR/Use.h | 9 +- include/llvm/IR/UseListOrder.h | 7 +- include/llvm/IR/User.h | 7 +- include/llvm/IR/Value.def | 7 +- include/llvm/IR/Value.h | 69 +- include/llvm/IR/ValueHandle.h | 24 +- include/llvm/IR/ValueMap.h | 7 +- include/llvm/IR/ValueSymbolTable.h | 7 +- include/llvm/IR/Verifier.h | 7 +- include/llvm/IRReader/IRReader.h | 18 +- include/llvm/InitializePasses.h | 25 +- include/llvm/LTO/Caching.h | 11 +- include/llvm/LTO/Config.h | 35 +- include/llvm/LTO/LTO.h | 47 +- include/llvm/LTO/LTOBackend.h | 7 +- include/llvm/LTO/SummaryBasedOptimizations.h | 7 +- include/llvm/LTO/legacy/LTOCodeGenerator.h | 8 +- include/llvm/LTO/legacy/LTOModule.h | 18 +- include/llvm/LTO/legacy/ThinLTOCodeGenerator.h | 49 +- include/llvm/LTO/legacy/UpdateCompilerUsed.h | 7 +- include/llvm/LineEditor/LineEditor.h | 7 +- include/llvm/LinkAllIR.h | 7 +- include/llvm/LinkAllPasses.h | 13 +- include/llvm/Linker/IRMover.h | 7 +- include/llvm/Linker/Linker.h | 7 +- include/llvm/MC/ConstantPools.h | 9 +- include/llvm/MC/LaneBitmask.h | 7 +- include/llvm/MC/MCAsmBackend.h | 23 +- include/llvm/MC/MCAsmInfo.h | 24 +- include/llvm/MC/MCAsmInfoCOFF.h | 7 +- include/llvm/MC/MCAsmInfoDarwin.h | 7 +- include/llvm/MC/MCAsmInfoELF.h | 7 +- include/llvm/MC/MCAsmInfoWasm.h | 7 +- include/llvm/MC/MCAsmInfoXCOFF.h | 25 + include/llvm/MC/MCAsmLayout.h | 7 +- include/llvm/MC/MCAsmMacro.h | 7 +- include/llvm/MC/MCAssembler.h | 7 +- include/llvm/MC/MCCodeEmitter.h | 7 +- include/llvm/MC/MCCodePadder.h | 9 +- include/llvm/MC/MCCodeView.h | 7 +- include/llvm/MC/MCContext.h | 54 +- include/llvm/MC/MCDirectives.h | 8 +- include/llvm/MC/MCDisassembler/MCDisassembler.h | 25 +- .../llvm/MC/MCDisassembler/MCExternalSymbolizer.h | 7 +- include/llvm/MC/MCDisassembler/MCRelocationInfo.h | 7 +- include/llvm/MC/MCDisassembler/MCSymbolizer.h | 7 +- include/llvm/MC/MCDwarf.h | 91 +- include/llvm/MC/MCELFObjectWriter.h | 11 +- include/llvm/MC/MCELFStreamer.h | 7 +- include/llvm/MC/MCExpr.h | 28 +- include/llvm/MC/MCFixedLenDisassembler.h | 7 +- include/llvm/MC/MCFixup.h | 10 +- include/llvm/MC/MCFixupKindInfo.h | 7 +- include/llvm/MC/MCFragment.h | 7 +- include/llvm/MC/MCInst.h | 10 +- include/llvm/MC/MCInstBuilder.h | 7 +- include/llvm/MC/MCInstPrinter.h | 11 +- include/llvm/MC/MCInstrAnalysis.h | 7 +- include/llvm/MC/MCInstrDesc.h | 13 +- include/llvm/MC/MCInstrInfo.h | 7 +- include/llvm/MC/MCInstrItineraries.h | 7 +- include/llvm/MC/MCLabel.h | 7 +- include/llvm/MC/MCLinkerOptimizationHint.h | 7 +- include/llvm/MC/MCMachObjectWriter.h | 7 +- include/llvm/MC/MCObjectFileInfo.h | 14 +- include/llvm/MC/MCObjectStreamer.h | 10 +- include/llvm/MC/MCObjectWriter.h | 7 +- include/llvm/MC/MCParser/AsmCond.h | 7 +- include/llvm/MC/MCParser/AsmLexer.h | 7 +- include/llvm/MC/MCParser/MCAsmLexer.h | 7 +- include/llvm/MC/MCParser/MCAsmParser.h | 13 +- include/llvm/MC/MCParser/MCAsmParserExtension.h | 7 +- include/llvm/MC/MCParser/MCAsmParserUtils.h | 7 +- include/llvm/MC/MCParser/MCParsedAsmOperand.h | 7 +- include/llvm/MC/MCParser/MCTargetAsmParser.h | 27 +- include/llvm/MC/MCRegisterInfo.h | 7 +- include/llvm/MC/MCSchedule.h | 13 +- include/llvm/MC/MCSection.h | 9 +- include/llvm/MC/MCSectionCOFF.h | 9 +- include/llvm/MC/MCSectionELF.h | 9 +- include/llvm/MC/MCSectionMachO.h | 7 +- include/llvm/MC/MCSectionWasm.h | 27 +- include/llvm/MC/MCSectionXCOFF.h | 56 + include/llvm/MC/MCStreamer.h | 34 +- include/llvm/MC/MCSubtargetInfo.h | 71 +- include/llvm/MC/MCSymbol.h | 36 +- include/llvm/MC/MCSymbolCOFF.h | 7 +- include/llvm/MC/MCSymbolELF.h | 7 +- include/llvm/MC/MCSymbolMachO.h | 12 +- include/llvm/MC/MCSymbolWasm.h | 40 +- include/llvm/MC/MCSymbolXCOFF.h | 26 + include/llvm/MC/MCTargetOptions.h | 28 +- include/llvm/MC/MCTargetOptionsCommandFlags.inc | 18 +- include/llvm/MC/MCValue.h | 7 +- include/llvm/MC/MCWasmObjectWriter.h | 7 +- include/llvm/MC/MCWasmStreamer.h | 7 +- include/llvm/MC/MCWin64EH.h | 7 +- include/llvm/MC/MCWinCOFFObjectWriter.h | 7 +- include/llvm/MC/MCWinCOFFStreamer.h | 7 +- include/llvm/MC/MCWinEH.h | 7 +- include/llvm/MC/MCXCOFFObjectWriter.h | 41 + include/llvm/MC/MCXCOFFStreamer.h | 33 + include/llvm/MC/MachineLocation.h | 7 +- include/llvm/MC/SectionKind.h | 7 +- include/llvm/MC/StringTableBuilder.h | 7 +- include/llvm/MC/SubtargetFeature.h | 100 +- include/llvm/MCA/Context.h | 21 +- include/llvm/MCA/HWEventListener.h | 38 +- include/llvm/MCA/HardwareUnits/HardwareUnit.h | 7 +- include/llvm/MCA/HardwareUnits/LSUnit.h | 393 +- include/llvm/MCA/HardwareUnits/RegisterFile.h | 10 +- include/llvm/MCA/HardwareUnits/ResourceManager.h | 31 +- include/llvm/MCA/HardwareUnits/RetireControlUnit.h | 7 +- include/llvm/MCA/HardwareUnits/Scheduler.h | 138 +- include/llvm/MCA/InstrBuilder.h | 7 +- include/llvm/MCA/Instruction.h | 163 +- include/llvm/MCA/Pipeline.h | 7 +- include/llvm/MCA/SourceMgr.h | 7 +- include/llvm/MCA/Stages/DispatchStage.h | 13 +- include/llvm/MCA/Stages/EntryStage.h | 7 +- include/llvm/MCA/Stages/ExecuteStage.h | 20 +- include/llvm/MCA/Stages/InstructionTables.h | 7 +- include/llvm/MCA/Stages/MicroOpQueueStage.h | 88 + include/llvm/MCA/Stages/RetireStage.h | 7 +- include/llvm/MCA/Stages/Stage.h | 7 +- include/llvm/MCA/Support.h | 33 +- include/llvm/Object/Archive.h | 40 +- include/llvm/Object/ArchiveWriter.h | 10 +- include/llvm/Object/Binary.h | 22 +- include/llvm/Object/COFF.h | 20 +- include/llvm/Object/COFFImportFile.h | 24 +- include/llvm/Object/COFFModuleDefinition.h | 7 +- include/llvm/Object/CVDebugRecord.h | 7 +- include/llvm/Object/Decompressor.h | 7 +- include/llvm/Object/ELF.h | 158 +- include/llvm/Object/ELFObjectFile.h | 133 +- include/llvm/Object/ELFTypes.h | 9 +- include/llvm/Object/Error.h | 7 +- include/llvm/Object/IRObjectFile.h | 10 +- include/llvm/Object/IRSymtab.h | 25 +- include/llvm/Object/MachO.h | 76 +- include/llvm/Object/MachOUniversal.h | 7 +- include/llvm/Object/Minidump.h | 165 + include/llvm/Object/ModuleSymbolTable.h | 7 +- include/llvm/Object/ObjectFile.h | 90 +- include/llvm/Object/RelocVisitor.h | 351 - include/llvm/Object/RelocationResolver.h | 42 + include/llvm/Object/StackMapParser.h | 50 +- include/llvm/Object/SymbolSize.h | 7 +- include/llvm/Object/SymbolicFile.h | 14 +- include/llvm/Object/Wasm.h | 80 +- include/llvm/Object/WasmTraits.h | 7 +- include/llvm/Object/WindowsMachineFlag.h | 33 + include/llvm/Object/WindowsResource.h | 47 +- include/llvm/Object/XCOFFObjectFile.h | 268 + include/llvm/ObjectYAML/COFFYAML.h | 7 +- .../llvm/ObjectYAML/CodeViewYAMLDebugSections.h | 7 +- include/llvm/ObjectYAML/CodeViewYAMLSymbols.h | 7 +- include/llvm/ObjectYAML/CodeViewYAMLTypeHashing.h | 7 +- include/llvm/ObjectYAML/CodeViewYAMLTypes.h | 7 +- include/llvm/ObjectYAML/DWARFEmitter.h | 7 +- include/llvm/ObjectYAML/DWARFYAML.h | 7 +- include/llvm/ObjectYAML/ELFYAML.h | 152 +- include/llvm/ObjectYAML/MachOYAML.h | 7 +- include/llvm/ObjectYAML/MinidumpYAML.h | 239 + include/llvm/ObjectYAML/ObjectYAML.h | 9 +- include/llvm/ObjectYAML/WasmYAML.h | 69 +- include/llvm/ObjectYAML/XCOFFYAML.h | 71 + include/llvm/ObjectYAML/YAML.h | 10 +- include/llvm/Option/Arg.h | 31 +- include/llvm/Option/ArgList.h | 17 +- include/llvm/Option/OptParser.td | 7 +- include/llvm/Option/OptSpecifier.h | 7 +- include/llvm/Option/OptTable.h | 7 +- include/llvm/Option/Option.h | 12 +- include/llvm/Pass.h | 7 +- include/llvm/PassAnalysisSupport.h | 7 +- include/llvm/PassInfo.h | 7 +- include/llvm/PassRegistry.h | 7 +- include/llvm/PassSupport.h | 7 +- include/llvm/Passes/PassBuilder.h | 114 +- include/llvm/Passes/PassPlugin.h | 7 +- include/llvm/Passes/StandardInstrumentations.h | 9 +- .../llvm/ProfileData/Coverage/CoverageMapping.h | 7 +- .../ProfileData/Coverage/CoverageMappingReader.h | 17 +- .../ProfileData/Coverage/CoverageMappingWriter.h | 7 +- include/llvm/ProfileData/GCOV.h | 19 +- include/llvm/ProfileData/InstrProf.h | 109 +- include/llvm/ProfileData/InstrProfData.inc | 92 +- include/llvm/ProfileData/InstrProfReader.h | 56 +- include/llvm/ProfileData/InstrProfWriter.h | 38 +- include/llvm/ProfileData/ProfileCommon.h | 10 +- include/llvm/ProfileData/SampleProf.h | 50 +- include/llvm/ProfileData/SampleProfReader.h | 16 +- include/llvm/ProfileData/SampleProfWriter.h | 7 +- include/llvm/Remarks/Remark.h | 113 + include/llvm/Remarks/RemarkFormat.h | 33 + include/llvm/Remarks/RemarkParser.h | 77 + include/llvm/Remarks/RemarkSerializer.h | 68 + include/llvm/Remarks/RemarkStringTable.h | 59 + include/llvm/Support/AArch64TargetParser.def | 109 +- include/llvm/Support/AArch64TargetParser.h | 12 +- include/llvm/Support/AMDGPUMetadata.h | 39 +- include/llvm/Support/AMDHSAKernelDescriptor.h | 33 +- include/llvm/Support/ARMAttributeParser.h | 9 +- include/llvm/Support/ARMBuildAttributes.h | 13 +- include/llvm/Support/ARMEHABI.h | 7 +- include/llvm/Support/ARMTargetParser.def | 22 +- include/llvm/Support/ARMTargetParser.h | 19 +- include/llvm/Support/ARMWinEH.h | 11 +- include/llvm/Support/AlignOf.h | 7 +- include/llvm/Support/Allocator.h | 7 +- include/llvm/Support/ArrayRecycler.h | 7 +- include/llvm/Support/Atomic.h | 7 +- include/llvm/Support/AtomicOrdering.h | 7 +- include/llvm/Support/BinaryByteStream.h | 7 +- include/llvm/Support/BinaryItemStream.h | 7 +- include/llvm/Support/BinaryStream.h | 7 +- include/llvm/Support/BinaryStreamArray.h | 7 +- include/llvm/Support/BinaryStreamError.h | 7 +- include/llvm/Support/BinaryStreamReader.h | 19 +- include/llvm/Support/BinaryStreamRef.h | 7 +- include/llvm/Support/BinaryStreamWriter.h | 21 +- include/llvm/Support/BlockFrequency.h | 7 +- include/llvm/Support/BranchProbability.h | 35 +- include/llvm/Support/BuryPointer.h | 7 +- include/llvm/Support/CBindingWrapping.h | 9 +- include/llvm/Support/CFGUpdate.h | 7 +- include/llvm/Support/COM.h | 7 +- include/llvm/Support/CRC.h | 25 + include/llvm/Support/CachePruning.h | 7 +- include/llvm/Support/Capacity.h | 7 +- include/llvm/Support/Casting.h | 17 +- include/llvm/Support/CheckedArithmetic.h | 16 +- include/llvm/Support/Chrono.h | 13 +- include/llvm/Support/CodeGen.h | 20 +- include/llvm/Support/CodeGenCoverage.h | 7 +- include/llvm/Support/CommandLine.h | 137 +- include/llvm/Support/Compiler.h | 16 +- include/llvm/Support/Compression.h | 7 +- include/llvm/Support/ConvertUTF.h | 7 +- include/llvm/Support/CrashRecoveryContext.h | 7 +- include/llvm/Support/DJB.h | 7 +- include/llvm/Support/DOTGraphTraits.h | 9 +- include/llvm/Support/DataExtractor.h | 7 +- include/llvm/Support/DataTypes.h | 7 +- include/llvm/Support/Debug.h | 7 +- include/llvm/Support/DebugCounter.h | 7 +- include/llvm/Support/DynamicLibrary.h | 7 +- include/llvm/Support/Endian.h | 28 +- include/llvm/Support/EndianStream.h | 7 +- include/llvm/Support/Errc.h | 7 +- include/llvm/Support/Errno.h | 7 +- include/llvm/Support/Error.h | 60 +- include/llvm/Support/ErrorHandling.h | 7 +- include/llvm/Support/ErrorOr.h | 7 +- include/llvm/Support/FileCheck.h | 557 +- include/llvm/Support/FileOutputBuffer.h | 12 +- include/llvm/Support/FileSystem.h | 113 +- include/llvm/Support/FileUtilities.h | 7 +- include/llvm/Support/Format.h | 7 +- include/llvm/Support/FormatAdapters.h | 7 +- include/llvm/Support/FormatCommon.h | 9 +- include/llvm/Support/FormatProviders.h | 7 +- include/llvm/Support/FormatVariadic.h | 7 +- include/llvm/Support/FormatVariadicDetails.h | 7 +- include/llvm/Support/FormattedStream.h | 7 +- include/llvm/Support/GenericDomTree.h | 19 +- include/llvm/Support/GenericDomTreeConstruction.h | 333 +- .../Support/GenericIteratedDominanceFrontier.h | 209 + include/llvm/Support/GlobPattern.h | 7 +- include/llvm/Support/GraphWriter.h | 7 +- include/llvm/Support/Host.h | 7 +- include/llvm/Support/InitLLVM.h | 12 +- .../llvm/Support/ItaniumManglingCanonicalizer.h | 7 +- include/llvm/Support/JSON.h | 180 +- include/llvm/Support/JamCRC.h | 7 +- include/llvm/Support/KnownBits.h | 40 +- include/llvm/Support/LEB128.h | 15 +- include/llvm/Support/LineIterator.h | 7 +- include/llvm/Support/LockFileManager.h | 7 +- include/llvm/Support/LowLevelTypeImpl.h | 53 +- include/llvm/Support/MSVCErrorWorkarounds.h | 7 +- include/llvm/Support/MachineValueType.h | 287 +- include/llvm/Support/ManagedStatic.h | 32 +- include/llvm/Support/MathExtras.h | 42 +- include/llvm/Support/MemAlloc.h | 31 +- include/llvm/Support/Memory.h | 46 +- include/llvm/Support/MemoryBuffer.h | 13 +- include/llvm/Support/MipsABIFlags.h | 7 +- include/llvm/Support/Mutex.h | 7 +- include/llvm/Support/MutexGuard.h | 7 +- include/llvm/Support/NativeFormatting.h | 7 +- include/llvm/Support/OnDiskHashTable.h | 7 +- include/llvm/Support/Options.h | 7 +- include/llvm/Support/Parallel.h | 11 +- include/llvm/Support/Path.h | 7 +- include/llvm/Support/PluginLoader.h | 7 +- include/llvm/Support/PointerLikeTypeTraits.h | 7 +- include/llvm/Support/PrettyStackTrace.h | 21 +- include/llvm/Support/Printable.h | 7 +- include/llvm/Support/Process.h | 28 +- include/llvm/Support/Program.h | 7 +- include/llvm/Support/RWMutex.h | 7 +- include/llvm/Support/RandomNumberGenerator.h | 7 +- include/llvm/Support/Recycler.h | 7 +- include/llvm/Support/RecyclingAllocator.h | 7 +- include/llvm/Support/Regex.h | 7 +- include/llvm/Support/Registry.h | 13 +- include/llvm/Support/SHA1.h | 7 +- include/llvm/Support/SMLoc.h | 7 +- include/llvm/Support/SMTAPI.h | 447 + include/llvm/Support/SaveAndRestore.h | 7 +- include/llvm/Support/ScalableSize.h | 43 + include/llvm/Support/ScaledNumber.h | 11 +- include/llvm/Support/ScopedPrinter.h | 9 +- include/llvm/Support/Signals.h | 25 +- include/llvm/Support/Signposts.h | 43 + include/llvm/Support/SmallVectorMemoryBuffer.h | 7 +- include/llvm/Support/Solaris/sys/regset.h | 7 +- include/llvm/Support/SourceMgr.h | 9 +- include/llvm/Support/SpecialCaseList.h | 7 +- include/llvm/Support/StringPool.h | 7 +- include/llvm/Support/StringSaver.h | 7 +- include/llvm/Support/SwapByteOrder.h | 15 +- include/llvm/Support/SymbolRemappingReader.h | 7 +- include/llvm/Support/SystemUtils.h | 7 +- include/llvm/Support/TarWriter.h | 7 +- include/llvm/Support/TargetOpcodes.def | 70 +- include/llvm/Support/TargetParser.h | 14 +- include/llvm/Support/TargetRegistry.h | 18 +- include/llvm/Support/TargetSelect.h | 7 +- include/llvm/Support/TaskQueue.h | 7 +- include/llvm/Support/ThreadLocal.h | 7 +- include/llvm/Support/ThreadPool.h | 7 +- include/llvm/Support/Threading.h | 23 +- include/llvm/Support/TimeProfiler.h | 76 + include/llvm/Support/Timer.h | 14 +- include/llvm/Support/ToolOutputFile.h | 7 +- include/llvm/Support/TrailingObjects.h | 7 +- include/llvm/Support/TrigramIndex.h | 7 +- include/llvm/Support/TypeName.h | 7 +- include/llvm/Support/Unicode.h | 7 +- include/llvm/Support/UnicodeCharRanges.h | 7 +- include/llvm/Support/UniqueLock.h | 7 +- include/llvm/Support/Valgrind.h | 7 +- include/llvm/Support/VersionTuple.h | 7 +- include/llvm/Support/VirtualFileSystem.h | 34 +- include/llvm/Support/Watchdog.h | 7 +- include/llvm/Support/Win64EH.h | 7 +- include/llvm/Support/WindowsError.h | 7 +- include/llvm/Support/WithColor.h | 7 +- .../llvm/Support/X86DisassemblerDecoderCommon.h | 48 +- include/llvm/Support/X86TargetParser.def | 16 +- include/llvm/Support/YAMLParser.h | 7 +- include/llvm/Support/YAMLTraits.h | 83 +- include/llvm/Support/circular_raw_ostream.h | 7 +- include/llvm/Support/raw_os_ostream.h | 7 +- include/llvm/Support/raw_ostream.h | 9 +- include/llvm/Support/raw_sha1_ostream.h | 7 +- include/llvm/Support/thread.h | 7 +- include/llvm/Support/type_traits.h | 114 +- include/llvm/TableGen/Error.h | 7 +- include/llvm/TableGen/Main.h | 7 +- include/llvm/TableGen/Record.h | 100 +- include/llvm/TableGen/SearchableTable.td | 7 +- include/llvm/TableGen/SetTheory.h | 7 +- include/llvm/TableGen/StringMatcher.h | 7 +- include/llvm/TableGen/StringToOffsetTable.h | 7 +- include/llvm/TableGen/TableGenBackend.h | 9 +- include/llvm/Target/CodeGenCWrappers.h | 7 +- include/llvm/Target/GenericOpcodes.td | 195 +- include/llvm/Target/GlobalISel/RegisterBank.td | 7 +- .../llvm/Target/GlobalISel/SelectionDAGCompat.td | 23 +- include/llvm/Target/GlobalISel/Target.td | 7 +- include/llvm/Target/Target.td | 109 +- include/llvm/Target/TargetCallingConv.td | 21 +- include/llvm/Target/TargetInstrPredicate.td | 7 +- include/llvm/Target/TargetIntrinsicInfo.h | 7 +- include/llvm/Target/TargetItinerary.td | 7 +- include/llvm/Target/TargetLoweringObjectFile.h | 12 +- include/llvm/Target/TargetMachine.h | 39 +- include/llvm/Target/TargetOptions.h | 13 +- include/llvm/Target/TargetPfmCounters.td | 7 +- include/llvm/Target/TargetSchedule.td | 9 +- include/llvm/Target/TargetSelectionDAG.td | 178 +- include/llvm/Testing/Support/Annotations.h | 90 + include/llvm/Testing/Support/Error.h | 7 +- include/llvm/Testing/Support/SupportHelpers.h | 56 +- include/llvm/TextAPI/ELF/ELFStub.h | 7 +- include/llvm/TextAPI/ELF/TBEHandler.h | 7 +- include/llvm/TextAPI/MachO/Architecture.def | 38 + include/llvm/TextAPI/MachO/Architecture.h | 47 + include/llvm/TextAPI/MachO/ArchitectureSet.h | 159 + include/llvm/TextAPI/MachO/InterfaceFile.h | 436 + include/llvm/TextAPI/MachO/PackedVersion.h | 64 + include/llvm/TextAPI/MachO/Symbol.h | 96 + include/llvm/TextAPI/MachO/TextAPIReader.h | 34 + include/llvm/TextAPI/MachO/TextAPIWriter.h | 29 + .../llvm/ToolDrivers/llvm-dlltool/DlltoolDriver.h | 7 +- include/llvm/ToolDrivers/llvm-lib/LibDriver.h | 8 +- .../AggressiveInstCombine/AggressiveInstCombine.h | 7 +- include/llvm/Transforms/Coroutines.h | 7 +- include/llvm/Transforms/IPO.h | 11 +- include/llvm/Transforms/IPO/AlwaysInliner.h | 7 +- include/llvm/Transforms/IPO/ArgumentPromotion.h | 7 +- include/llvm/Transforms/IPO/Attributor.h | 789 ++ .../llvm/Transforms/IPO/CalledValuePropagation.h | 7 +- include/llvm/Transforms/IPO/ConstantMerge.h | 7 +- include/llvm/Transforms/IPO/CrossDSOCFI.h | 7 +- .../llvm/Transforms/IPO/DeadArgumentElimination.h | 7 +- include/llvm/Transforms/IPO/ElimAvailExtern.h | 7 +- include/llvm/Transforms/IPO/ForceFunctionAttrs.h | 7 +- include/llvm/Transforms/IPO/FunctionAttrs.h | 7 +- include/llvm/Transforms/IPO/FunctionImport.h | 7 +- include/llvm/Transforms/IPO/GlobalDCE.h | 7 +- include/llvm/Transforms/IPO/GlobalOpt.h | 7 +- include/llvm/Transforms/IPO/GlobalSplit.h | 7 +- include/llvm/Transforms/IPO/HotColdSplitting.h | 7 +- include/llvm/Transforms/IPO/InferFunctionAttrs.h | 7 +- include/llvm/Transforms/IPO/Inliner.h | 7 +- include/llvm/Transforms/IPO/Internalize.h | 13 +- include/llvm/Transforms/IPO/LowerTypeTests.h | 7 +- include/llvm/Transforms/IPO/PartialInlining.h | 7 +- include/llvm/Transforms/IPO/PassManagerBuilder.h | 30 +- include/llvm/Transforms/IPO/SCCP.h | 7 +- include/llvm/Transforms/IPO/SampleProfile.h | 7 +- include/llvm/Transforms/IPO/StripDeadPrototypes.h | 7 +- include/llvm/Transforms/IPO/ThinLTOBitcodeWriter.h | 7 +- include/llvm/Transforms/IPO/WholeProgramDevirt.h | 7 +- include/llvm/Transforms/InstCombine/InstCombine.h | 7 +- .../Transforms/InstCombine/InstCombineWorklist.h | 7 +- include/llvm/Transforms/Instrumentation.h | 52 +- .../Transforms/Instrumentation/AddressSanitizer.h | 143 + .../Transforms/Instrumentation/BoundsChecking.h | 7 +- .../llvm/Transforms/Instrumentation/CGProfile.h | 7 +- .../Instrumentation/ControlHeightReduction.h | 7 +- .../llvm/Transforms/Instrumentation/GCOVProfiler.h | 7 +- .../Instrumentation/HWAddressSanitizer.h | 41 + .../Transforms/Instrumentation/InstrOrderFile.h | 28 + .../Transforms/Instrumentation/InstrProfiling.h | 15 +- .../Transforms/Instrumentation/MemorySanitizer.h | 30 +- .../Instrumentation/PGOInstrumentation.h | 33 +- .../Transforms/Instrumentation/PoisonChecking.h | 25 + .../Transforms/Instrumentation/ThreadSanitizer.h | 9 +- include/llvm/Transforms/ObjCARC.h | 7 +- include/llvm/Transforms/Scalar.h | 38 +- include/llvm/Transforms/Scalar/ADCE.h | 7 +- .../Transforms/Scalar/AlignmentFromAssumptions.h | 7 +- include/llvm/Transforms/Scalar/BDCE.h | 7 +- include/llvm/Transforms/Scalar/CallSiteSplitting.h | 7 +- include/llvm/Transforms/Scalar/ConstantHoisting.h | 14 +- .../Transforms/Scalar/CorrelatedValuePropagation.h | 7 +- include/llvm/Transforms/Scalar/DCE.h | 7 +- .../llvm/Transforms/Scalar/DeadStoreElimination.h | 7 +- include/llvm/Transforms/Scalar/DivRemPairs.h | 7 +- include/llvm/Transforms/Scalar/EarlyCSE.h | 7 +- include/llvm/Transforms/Scalar/Float2Int.h | 7 +- include/llvm/Transforms/Scalar/GVN.h | 7 +- include/llvm/Transforms/Scalar/GVNExpression.h | 7 +- include/llvm/Transforms/Scalar/GuardWidening.h | 11 +- include/llvm/Transforms/Scalar/IVUsersPrinter.h | 7 +- include/llvm/Transforms/Scalar/IndVarSimplify.h | 7 +- .../Scalar/InductiveRangeCheckElimination.h | 7 +- include/llvm/Transforms/Scalar/InstSimplifyPass.h | 7 +- include/llvm/Transforms/Scalar/JumpThreading.h | 9 +- include/llvm/Transforms/Scalar/LICM.h | 19 +- .../Transforms/Scalar/LoopAccessAnalysisPrinter.h | 7 +- include/llvm/Transforms/Scalar/LoopDataPrefetch.h | 7 +- include/llvm/Transforms/Scalar/LoopDeletion.h | 7 +- include/llvm/Transforms/Scalar/LoopDistribute.h | 7 +- include/llvm/Transforms/Scalar/LoopFuse.h | 30 + .../llvm/Transforms/Scalar/LoopIdiomRecognize.h | 7 +- include/llvm/Transforms/Scalar/LoopInstSimplify.h | 7 +- .../llvm/Transforms/Scalar/LoopLoadElimination.h | 7 +- include/llvm/Transforms/Scalar/LoopPassManager.h | 7 +- include/llvm/Transforms/Scalar/LoopPredication.h | 7 +- include/llvm/Transforms/Scalar/LoopRotation.h | 7 +- include/llvm/Transforms/Scalar/LoopSimplifyCFG.h | 7 +- include/llvm/Transforms/Scalar/LoopSink.h | 7 +- .../llvm/Transforms/Scalar/LoopStrengthReduce.h | 7 +- .../llvm/Transforms/Scalar/LoopUnrollAndJamPass.h | 7 +- include/llvm/Transforms/Scalar/LoopUnrollPass.h | 31 +- include/llvm/Transforms/Scalar/LowerAtomic.h | 7 +- .../llvm/Transforms/Scalar/LowerExpectIntrinsic.h | 7 +- .../llvm/Transforms/Scalar/LowerGuardIntrinsic.h | 7 +- .../Transforms/Scalar/LowerWidenableCondition.h | 26 + .../llvm/Transforms/Scalar/MakeGuardsExplicit.h | 7 +- include/llvm/Transforms/Scalar/MemCpyOptimizer.h | 7 +- include/llvm/Transforms/Scalar/MergeICmps.h | 25 + .../llvm/Transforms/Scalar/MergedLoadStoreMotion.h | 7 +- include/llvm/Transforms/Scalar/NaryReassociate.h | 7 +- include/llvm/Transforms/Scalar/NewGVN.h | 7 +- .../Transforms/Scalar/PartiallyInlineLibCalls.h | 7 +- include/llvm/Transforms/Scalar/Reassociate.h | 16 +- .../Transforms/Scalar/RewriteStatepointsForGC.h | 7 +- include/llvm/Transforms/Scalar/SCCP.h | 7 +- include/llvm/Transforms/Scalar/SROA.h | 8 +- include/llvm/Transforms/Scalar/Scalarizer.h | 7 +- .../llvm/Transforms/Scalar/SimpleLoopUnswitch.h | 7 +- include/llvm/Transforms/Scalar/SimplifyCFG.h | 7 +- include/llvm/Transforms/Scalar/Sink.h | 7 +- .../llvm/Transforms/Scalar/SpeculateAroundPHIs.h | 7 +- .../llvm/Transforms/Scalar/SpeculativeExecution.h | 7 +- .../Transforms/Scalar/TailRecursionElimination.h | 7 +- .../llvm/Transforms/Scalar/WarnMissedTransforms.h | 7 +- include/llvm/Transforms/Utils.h | 7 +- .../llvm/Transforms/Utils/ASanStackFrameLayout.h | 7 +- include/llvm/Transforms/Utils/AddDiscriminators.h | 7 +- include/llvm/Transforms/Utils/BasicBlockUtils.h | 58 +- include/llvm/Transforms/Utils/BreakCriticalEdges.h | 7 +- include/llvm/Transforms/Utils/BuildLibCalls.h | 62 +- include/llvm/Transforms/Utils/BypassSlowDivision.h | 7 +- include/llvm/Transforms/Utils/CallPromotionUtils.h | 7 +- .../llvm/Transforms/Utils/CanonicalizeAliases.h | 7 +- include/llvm/Transforms/Utils/Cloning.h | 19 +- include/llvm/Transforms/Utils/CodeExtractor.h | 21 +- include/llvm/Transforms/Utils/CtorUtils.h | 7 +- .../llvm/Transforms/Utils/EntryExitInstrumenter.h | 7 +- include/llvm/Transforms/Utils/EscapeEnumerator.h | 7 +- include/llvm/Transforms/Utils/Evaluator.h | 7 +- include/llvm/Transforms/Utils/FunctionComparator.h | 7 +- .../llvm/Transforms/Utils/FunctionImportUtils.h | 12 +- include/llvm/Transforms/Utils/GlobalStatus.h | 7 +- include/llvm/Transforms/Utils/GuardUtils.h | 7 +- .../Utils/ImportedFunctionsInliningStatistics.h | 9 +- include/llvm/Transforms/Utils/IntegerDivision.h | 7 +- include/llvm/Transforms/Utils/LCSSA.h | 7 +- include/llvm/Transforms/Utils/LibCallsShrinkWrap.h | 7 +- include/llvm/Transforms/Utils/Local.h | 45 +- include/llvm/Transforms/Utils/LoopRotationUtils.h | 7 +- include/llvm/Transforms/Utils/LoopSimplify.h | 15 +- include/llvm/Transforms/Utils/LoopUtils.h | 45 +- include/llvm/Transforms/Utils/LoopVersioning.h | 7 +- include/llvm/Transforms/Utils/LowerInvoke.h | 7 +- include/llvm/Transforms/Utils/LowerMemIntrinsics.h | 9 +- include/llvm/Transforms/Utils/Mem2Reg.h | 7 +- include/llvm/Transforms/Utils/ModuleUtils.h | 24 +- include/llvm/Transforms/Utils/NameAnonGlobals.h | 7 +- include/llvm/Transforms/Utils/PredicateInfo.h | 7 +- include/llvm/Transforms/Utils/PromoteMemToReg.h | 7 +- include/llvm/Transforms/Utils/SSAUpdater.h | 11 +- include/llvm/Transforms/Utils/SSAUpdaterBulk.h | 7 +- include/llvm/Transforms/Utils/SSAUpdaterImpl.h | 7 +- include/llvm/Transforms/Utils/SanitizerStats.h | 7 +- include/llvm/Transforms/Utils/SimplifyIndVar.h | 7 +- include/llvm/Transforms/Utils/SimplifyLibCalls.h | 45 +- include/llvm/Transforms/Utils/SizeOpts.h | 34 + include/llvm/Transforms/Utils/SplitModule.h | 7 +- include/llvm/Transforms/Utils/SymbolRewriter.h | 7 +- .../llvm/Transforms/Utils/UnifyFunctionExitNodes.h | 7 +- include/llvm/Transforms/Utils/UnrollLoop.h | 48 +- include/llvm/Transforms/Utils/VNCoercion.h | 7 +- include/llvm/Transforms/Utils/ValueMapper.h | 7 +- include/llvm/Transforms/Vectorize.h | 12 +- .../Transforms/Vectorize/LoadStoreVectorizer.h | 7 +- .../Vectorize/LoopVectorizationLegality.h | 60 +- include/llvm/Transforms/Vectorize/LoopVectorize.h | 58 +- include/llvm/Transforms/Vectorize/SLPVectorizer.h | 9 +- .../llvm/WindowsManifest/WindowsManifestMerger.h | 7 +- include/llvm/WindowsResource/ResourceProcessor.h | 7 +- include/llvm/WindowsResource/ResourceScriptToken.h | 7 +- .../llvm/WindowsResource/ResourceScriptTokenList.h | 7 +- include/llvm/XRay/BlockIndexer.h | 7 +- include/llvm/XRay/BlockPrinter.h | 7 +- include/llvm/XRay/BlockVerifier.h | 7 +- include/llvm/XRay/FDRLogBuilder.h | 7 +- include/llvm/XRay/FDRRecordConsumer.h | 7 +- include/llvm/XRay/FDRRecordProducer.h | 7 +- include/llvm/XRay/FDRRecords.h | 7 +- include/llvm/XRay/FDRTraceExpander.h | 7 +- include/llvm/XRay/FDRTraceWriter.h | 7 +- include/llvm/XRay/FileHeaderReader.h | 7 +- include/llvm/XRay/Graph.h | 7 +- include/llvm/XRay/InstrumentationMap.h | 7 +- include/llvm/XRay/Profile.h | 7 +- include/llvm/XRay/RecordPrinter.h | 7 +- include/llvm/XRay/Trace.h | 7 +- include/llvm/XRay/XRayRecord.h | 7 +- include/llvm/XRay/YAMLXRayRecord.h | 7 +- include/llvm/module.modulemap | 5 +- lib/Analysis/AliasAnalysis.cpp | 136 +- lib/Analysis/AliasAnalysisEvaluator.cpp | 7 +- lib/Analysis/AliasAnalysisSummary.cpp | 18 +- lib/Analysis/AliasAnalysisSummary.h | 22 +- lib/Analysis/AliasSetTracker.cpp | 131 +- lib/Analysis/Analysis.cpp | 7 +- lib/Analysis/AssumptionCache.cpp | 35 +- lib/Analysis/BasicAliasAnalysis.cpp | 239 +- lib/Analysis/BlockFrequencyInfo.cpp | 12 +- lib/Analysis/BlockFrequencyInfoImpl.cpp | 18 +- lib/Analysis/BranchProbabilityInfo.cpp | 15 +- lib/Analysis/CFG.cpp | 83 +- lib/Analysis/CFGPrinter.cpp | 7 +- lib/Analysis/CFLAndersAliasAnalysis.cpp | 16 +- lib/Analysis/CFLGraph.h | 68 +- lib/Analysis/CFLSteensAliasAnalysis.cpp | 7 +- lib/Analysis/CGSCCPassManager.cpp | 13 +- lib/Analysis/CallGraph.cpp | 32 +- lib/Analysis/CallGraphSCCPass.cpp | 94 +- lib/Analysis/CallPrinter.cpp | 7 +- lib/Analysis/CaptureTracking.cpp | 39 +- lib/Analysis/CmpInstAnalysis.cpp | 7 +- lib/Analysis/CodeMetrics.cpp | 18 +- lib/Analysis/ConstantFolding.cpp | 1099 +-- lib/Analysis/CostModel.cpp | 7 +- lib/Analysis/Delinearization.cpp | 7 +- lib/Analysis/DemandedBits.cpp | 35 +- lib/Analysis/DependenceAnalysis.cpp | 51 +- lib/Analysis/DivergenceAnalysis.cpp | 7 +- lib/Analysis/DomPrinter.cpp | 7 +- lib/Analysis/DomTreeUpdater.cpp | 533 ++ lib/Analysis/DominanceFrontier.cpp | 7 +- lib/Analysis/EHPersonalities.cpp | 7 +- lib/Analysis/GlobalsModRef.cpp | 39 +- lib/Analysis/GuardUtils.cpp | 36 +- lib/Analysis/IVDescriptors.cpp | 33 +- lib/Analysis/IVUsers.cpp | 7 +- lib/Analysis/IndirectCallPromotionAnalysis.cpp | 7 +- lib/Analysis/InlineCost.cpp | 424 +- lib/Analysis/InstCount.cpp | 7 +- lib/Analysis/InstructionPrecedenceTracking.cpp | 11 +- lib/Analysis/InstructionSimplify.cpp | 713 +- lib/Analysis/Interval.cpp | 7 +- lib/Analysis/IntervalPartition.cpp | 7 +- lib/Analysis/IteratedDominanceFrontier.cpp | 110 - lib/Analysis/LazyBlockFrequencyInfo.cpp | 7 +- lib/Analysis/LazyBranchProbabilityInfo.cpp | 7 +- lib/Analysis/LazyCallGraph.cpp | 20 +- lib/Analysis/LazyValueInfo.cpp | 192 +- lib/Analysis/LegacyDivergenceAnalysis.cpp | 7 +- lib/Analysis/Lint.cpp | 15 +- lib/Analysis/Loads.cpp | 44 +- lib/Analysis/LoopAccessAnalysis.cpp | 94 +- lib/Analysis/LoopAnalysisManager.cpp | 14 +- lib/Analysis/LoopInfo.cpp | 353 +- lib/Analysis/LoopPass.cpp | 20 +- lib/Analysis/LoopUnrollAnalyzer.cpp | 7 +- lib/Analysis/MemDepPrinter.cpp | 7 +- lib/Analysis/MemDerefPrinter.cpp | 12 +- lib/Analysis/MemoryBuiltins.cpp | 137 +- lib/Analysis/MemoryDependenceAnalysis.cpp | 42 +- lib/Analysis/MemoryLocation.cpp | 7 +- lib/Analysis/MemorySSA.cpp | 315 +- lib/Analysis/MemorySSAUpdater.cpp | 239 +- lib/Analysis/ModuleDebugInfoPrinter.cpp | 7 +- lib/Analysis/ModuleSummaryAnalysis.cpp | 276 +- lib/Analysis/MustExecute.cpp | 16 +- lib/Analysis/ObjCARCAliasAnalysis.cpp | 32 +- lib/Analysis/ObjCARCAnalysisUtils.cpp | 7 +- lib/Analysis/ObjCARCInstKind.cpp | 42 +- lib/Analysis/OptimizationRemarkEmitter.cpp | 7 +- lib/Analysis/OrderedBasicBlock.cpp | 31 +- lib/Analysis/OrderedInstructions.cpp | 7 +- lib/Analysis/PHITransAddr.cpp | 7 +- lib/Analysis/PhiValues.cpp | 7 +- lib/Analysis/PostDominators.cpp | 7 +- lib/Analysis/ProfileSummaryInfo.cpp | 26 +- lib/Analysis/PtrUseVisitor.cpp | 15 +- lib/Analysis/RegionInfo.cpp | 7 +- lib/Analysis/RegionPass.cpp | 16 +- lib/Analysis/RegionPrinter.cpp | 7 +- lib/Analysis/ScalarEvolution.cpp | 794 +- lib/Analysis/ScalarEvolutionAliasAnalysis.cpp | 14 +- lib/Analysis/ScalarEvolutionExpander.cpp | 267 +- lib/Analysis/ScalarEvolutionNormalization.cpp | 7 +- lib/Analysis/ScopedNoAliasAA.cpp | 28 +- lib/Analysis/StackSafetyAnalysis.cpp | 11 +- lib/Analysis/StratifiedSets.h | 7 +- lib/Analysis/SyncDependenceAnalysis.cpp | 35 +- lib/Analysis/SyntheticCountsUtils.cpp | 7 +- lib/Analysis/TargetLibraryInfo.cpp | 431 +- lib/Analysis/TargetTransformInfo.cpp | 184 +- lib/Analysis/Trace.cpp | 7 +- lib/Analysis/TypeBasedAliasAnalysis.cpp | 35 +- lib/Analysis/TypeMetadataUtils.cpp | 7 +- lib/Analysis/ValueLattice.cpp | 7 +- lib/Analysis/ValueLatticeUtils.cpp | 7 +- lib/Analysis/ValueTracking.cpp | 1204 ++- lib/Analysis/VectorUtils.cpp | 148 +- lib/AsmParser/LLLexer.cpp | 31 +- lib/AsmParser/LLLexer.h | 7 +- lib/AsmParser/LLParser.cpp | 711 +- lib/AsmParser/LLParser.h | 17 +- lib/AsmParser/LLToken.h | 20 +- lib/AsmParser/Parser.cpp | 7 +- lib/BinaryFormat/AMDGPUMetadataVerifier.cpp | 160 +- lib/BinaryFormat/Dwarf.cpp | 13 +- lib/BinaryFormat/Magic.cpp | 21 +- lib/BinaryFormat/Minidump.cpp | 14 + lib/BinaryFormat/MsgPackDocument.cpp | 245 + lib/BinaryFormat/MsgPackDocumentYAML.cpp | 249 + lib/BinaryFormat/MsgPackReader.cpp | 7 +- lib/BinaryFormat/MsgPackTypes.cpp | 303 - lib/BinaryFormat/MsgPackWriter.cpp | 7 +- lib/BinaryFormat/Wasm.cpp | 29 +- lib/Bitcode/Reader/BitReader.cpp | 7 +- lib/Bitcode/Reader/BitcodeAnalyzer.cpp | 980 ++ lib/Bitcode/Reader/BitcodeReader.cpp | 1261 ++- lib/Bitcode/Reader/BitstreamReader.cpp | 390 - lib/Bitcode/Reader/MetadataLoader.cpp | 269 +- lib/Bitcode/Reader/MetadataLoader.h | 7 +- lib/Bitcode/Reader/ValueList.cpp | 31 +- lib/Bitcode/Reader/ValueList.h | 44 +- lib/Bitcode/Writer/BitWriter.cpp | 7 +- lib/Bitcode/Writer/BitcodeWriter.cpp | 244 +- lib/Bitcode/Writer/BitcodeWriterPass.cpp | 7 +- lib/Bitcode/Writer/ValueEnumerator.cpp | 22 +- lib/Bitcode/Writer/ValueEnumerator.h | 7 +- lib/Bitstream/Reader/BitstreamReader.cpp | 510 ++ lib/CodeGen/AggressiveAntiDepBreaker.cpp | 7 +- lib/CodeGen/AggressiveAntiDepBreaker.h | 7 +- lib/CodeGen/AllocationOrder.cpp | 7 +- lib/CodeGen/AllocationOrder.h | 7 +- lib/CodeGen/Analysis.cpp | 52 +- lib/CodeGen/AntiDepBreaker.h | 7 +- lib/CodeGen/AsmPrinter/ARMException.cpp | 7 +- lib/CodeGen/AsmPrinter/AccelTable.cpp | 46 +- lib/CodeGen/AsmPrinter/AddressPool.cpp | 31 +- lib/CodeGen/AsmPrinter/AddressPool.h | 9 +- lib/CodeGen/AsmPrinter/AsmPrinter.cpp | 279 +- lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp | 31 +- lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp | 104 +- lib/CodeGen/AsmPrinter/ByteStreamer.h | 17 +- lib/CodeGen/AsmPrinter/CodeViewDebug.cpp | 371 +- lib/CodeGen/AsmPrinter/CodeViewDebug.h | 24 +- lib/CodeGen/AsmPrinter/DIE.cpp | 26 +- lib/CodeGen/AsmPrinter/DIEHash.cpp | 10 +- lib/CodeGen/AsmPrinter/DIEHash.h | 7 +- .../AsmPrinter/DbgEntityHistoryCalculator.cpp | 354 +- lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp | 74 +- lib/CodeGen/AsmPrinter/DebugLocEntry.h | 205 +- lib/CodeGen/AsmPrinter/DebugLocStream.cpp | 7 +- lib/CodeGen/AsmPrinter/DebugLocStream.h | 7 +- lib/CodeGen/AsmPrinter/DwarfCFIException.cpp | 7 +- lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp | 184 +- lib/CodeGen/AsmPrinter/DwarfCompileUnit.h | 29 +- lib/CodeGen/AsmPrinter/DwarfDebug.cpp | 668 +- lib/CodeGen/AsmPrinter/DwarfDebug.h | 92 +- lib/CodeGen/AsmPrinter/DwarfException.h | 7 +- lib/CodeGen/AsmPrinter/DwarfExpression.cpp | 120 +- lib/CodeGen/AsmPrinter/DwarfExpression.h | 86 +- lib/CodeGen/AsmPrinter/DwarfFile.cpp | 17 +- lib/CodeGen/AsmPrinter/DwarfFile.h | 10 +- lib/CodeGen/AsmPrinter/DwarfStringPool.cpp | 7 +- lib/CodeGen/AsmPrinter/DwarfStringPool.h | 7 +- lib/CodeGen/AsmPrinter/DwarfUnit.cpp | 197 +- lib/CodeGen/AsmPrinter/DwarfUnit.h | 38 +- lib/CodeGen/AsmPrinter/EHStreamer.cpp | 20 +- lib/CodeGen/AsmPrinter/EHStreamer.h | 7 +- lib/CodeGen/AsmPrinter/ErlangGCPrinter.cpp | 7 +- lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp | 7 +- lib/CodeGen/AsmPrinter/WasmException.cpp | 11 +- lib/CodeGen/AsmPrinter/WasmException.h | 7 +- lib/CodeGen/AsmPrinter/WinCFGuard.cpp | 7 +- lib/CodeGen/AsmPrinter/WinCFGuard.h | 7 +- lib/CodeGen/AsmPrinter/WinException.cpp | 49 +- lib/CodeGen/AsmPrinter/WinException.h | 10 +- lib/CodeGen/AtomicExpandPass.cpp | 70 +- lib/CodeGen/BasicTargetTransformInfo.cpp | 7 +- lib/CodeGen/BranchFolding.cpp | 72 +- lib/CodeGen/BranchFolding.h | 7 +- lib/CodeGen/BranchRelaxation.cpp | 7 +- lib/CodeGen/BreakFalseDeps.cpp | 7 +- lib/CodeGen/BuiltinGCs.cpp | 7 +- lib/CodeGen/CFIInstrInserter.cpp | 7 +- lib/CodeGen/CalcSpillWeights.cpp | 7 +- lib/CodeGen/CallingConvLower.cpp | 7 +- lib/CodeGen/CodeGen.cpp | 10 +- lib/CodeGen/CodeGenPrepare.cpp | 523 +- lib/CodeGen/CriticalAntiDepBreaker.cpp | 7 +- lib/CodeGen/CriticalAntiDepBreaker.h | 7 +- lib/CodeGen/DFAPacketizer.cpp | 7 +- lib/CodeGen/DeadMachineInstructionElim.cpp | 15 +- lib/CodeGen/DetectDeadLanes.cpp | 7 +- lib/CodeGen/DwarfEHPrepare.cpp | 11 +- lib/CodeGen/EarlyIfConversion.cpp | 7 +- lib/CodeGen/EdgeBundles.cpp | 9 +- lib/CodeGen/ExecutionDomainFix.cpp | 16 +- lib/CodeGen/ExpandISelPseudos.cpp | 74 - lib/CodeGen/ExpandMemCmp.cpp | 68 +- lib/CodeGen/ExpandPostRAPseudos.cpp | 7 +- lib/CodeGen/ExpandReductions.cpp | 59 +- lib/CodeGen/FEntryInserter.cpp | 7 +- lib/CodeGen/FaultMaps.cpp | 7 +- lib/CodeGen/FinalizeISel.cpp | 76 + lib/CodeGen/FuncletLayout.cpp | 7 +- lib/CodeGen/GCMetadata.cpp | 7 +- lib/CodeGen/GCMetadataPrinter.cpp | 7 +- lib/CodeGen/GCRootLowering.cpp | 9 +- lib/CodeGen/GCStrategy.cpp | 7 +- lib/CodeGen/GlobalISel/CSEInfo.cpp | 47 +- lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp | 21 +- lib/CodeGen/GlobalISel/CallLowering.cpp | 154 +- lib/CodeGen/GlobalISel/Combiner.cpp | 12 +- lib/CodeGen/GlobalISel/CombinerHelper.cpp | 220 +- lib/CodeGen/GlobalISel/GISelChangeObserver.cpp | 8 +- lib/CodeGen/GlobalISel/GlobalISel.cpp | 7 +- lib/CodeGen/GlobalISel/IRTranslator.cpp | 1284 ++- lib/CodeGen/GlobalISel/InstructionSelect.cpp | 19 +- lib/CodeGen/GlobalISel/InstructionSelector.cpp | 19 +- lib/CodeGen/GlobalISel/LegalityPredicates.cpp | 86 +- lib/CodeGen/GlobalISel/LegalizeMutations.cpp | 54 +- lib/CodeGen/GlobalISel/Legalizer.cpp | 54 +- lib/CodeGen/GlobalISel/LegalizerHelper.cpp | 2936 +++++- lib/CodeGen/GlobalISel/LegalizerInfo.cpp | 186 +- lib/CodeGen/GlobalISel/Localizer.cpp | 233 +- lib/CodeGen/GlobalISel/MachineIRBuilder.cpp | 429 +- lib/CodeGen/GlobalISel/RegBankSelect.cpp | 139 +- lib/CodeGen/GlobalISel/RegisterBank.cpp | 7 +- lib/CodeGen/GlobalISel/RegisterBankInfo.cpp | 115 +- lib/CodeGen/GlobalISel/Utils.cpp | 159 +- lib/CodeGen/GlobalMerge.cpp | 29 +- lib/CodeGen/HardwareLoops.cpp | 463 + lib/CodeGen/IfConversion.cpp | 9 +- lib/CodeGen/ImplicitNullChecks.cpp | 25 +- lib/CodeGen/IndirectBrExpandPass.cpp | 15 +- lib/CodeGen/InlineSpiller.cpp | 52 +- lib/CodeGen/InterferenceCache.cpp | 7 +- lib/CodeGen/InterferenceCache.h | 7 +- lib/CodeGen/InterleavedAccessPass.cpp | 19 +- lib/CodeGen/InterleavedLoadCombinePass.cpp | 10 +- lib/CodeGen/IntrinsicLowering.cpp | 115 +- lib/CodeGen/LLVMTargetMachine.cpp | 16 +- lib/CodeGen/LatencyPriorityQueue.cpp | 7 +- lib/CodeGen/LazyMachineBlockFrequencyInfo.cpp | 7 +- lib/CodeGen/LexicalScopes.cpp | 7 +- lib/CodeGen/LiveDebugValues.cpp | 720 +- lib/CodeGen/LiveDebugVariables.cpp | 181 +- lib/CodeGen/LiveDebugVariables.h | 7 +- lib/CodeGen/LiveInterval.cpp | 64 +- lib/CodeGen/LiveIntervalUnion.cpp | 7 +- lib/CodeGen/LiveIntervals.cpp | 13 +- lib/CodeGen/LivePhysRegs.cpp | 7 +- lib/CodeGen/LiveRangeCalc.cpp | 16 +- lib/CodeGen/LiveRangeCalc.h | 7 +- lib/CodeGen/LiveRangeEdit.cpp | 9 +- lib/CodeGen/LiveRangeShrink.cpp | 7 +- lib/CodeGen/LiveRangeUtils.h | 7 +- lib/CodeGen/LiveRegMatrix.cpp | 7 +- lib/CodeGen/LiveRegUnits.cpp | 23 +- lib/CodeGen/LiveStacks.cpp | 7 +- lib/CodeGen/LiveVariables.cpp | 9 +- lib/CodeGen/LocalStackSlotAllocation.cpp | 23 +- lib/CodeGen/LoopTraversal.cpp | 7 +- lib/CodeGen/LowLevelType.cpp | 7 +- lib/CodeGen/LowerEmuTLS.cpp | 7 +- lib/CodeGen/MIRCanonicalizerPass.cpp | 65 +- lib/CodeGen/MIRParser/MILexer.cpp | 8 +- lib/CodeGen/MIRParser/MILexer.h | 8 +- lib/CodeGen/MIRParser/MIParser.cpp | 574 +- lib/CodeGen/MIRParser/MIParser.h | 125 - lib/CodeGen/MIRParser/MIRParser.cpp | 184 +- lib/CodeGen/MIRPrinter.cpp | 67 +- lib/CodeGen/MIRPrintingPass.cpp | 7 +- lib/CodeGen/MachineBasicBlock.cpp | 17 +- lib/CodeGen/MachineBlockFrequencyInfo.cpp | 7 +- lib/CodeGen/MachineBlockPlacement.cpp | 398 +- lib/CodeGen/MachineBranchProbabilityInfo.cpp | 7 +- lib/CodeGen/MachineCSE.cpp | 181 +- lib/CodeGen/MachineCombiner.cpp | 26 +- lib/CodeGen/MachineCopyPropagation.cpp | 7 +- lib/CodeGen/MachineDominanceFrontier.cpp | 7 +- lib/CodeGen/MachineDominators.cpp | 7 +- lib/CodeGen/MachineFrameInfo.cpp | 18 +- lib/CodeGen/MachineFunction.cpp | 87 +- lib/CodeGen/MachineFunctionPass.cpp | 7 +- lib/CodeGen/MachineFunctionPrinterPass.cpp | 7 +- lib/CodeGen/MachineInstr.cpp | 128 +- lib/CodeGen/MachineInstrBundle.cpp | 7 +- lib/CodeGen/MachineLICM.cpp | 7 +- lib/CodeGen/MachineLoopInfo.cpp | 7 +- lib/CodeGen/MachineModuleInfo.cpp | 30 +- lib/CodeGen/MachineModuleInfoImpls.cpp | 7 +- lib/CodeGen/MachineOperand.cpp | 29 +- lib/CodeGen/MachineOptimizationRemarkEmitter.cpp | 7 +- lib/CodeGen/MachineOutliner.cpp | 42 +- lib/CodeGen/MachinePipeliner.cpp | 534 +- lib/CodeGen/MachinePostDominators.cpp | 7 +- lib/CodeGen/MachineRegionInfo.cpp | 7 +- lib/CodeGen/MachineRegisterInfo.cpp | 20 +- lib/CodeGen/MachineSSAUpdater.cpp | 7 +- lib/CodeGen/MachineScheduler.cpp | 144 +- lib/CodeGen/MachineSink.cpp | 17 +- lib/CodeGen/MachineTraceMetrics.cpp | 7 +- lib/CodeGen/MachineVerifier.cpp | 510 +- lib/CodeGen/MacroFusion.cpp | 19 +- lib/CodeGen/OptimizePHIs.cpp | 14 +- lib/CodeGen/PHIElimination.cpp | 7 +- lib/CodeGen/PHIEliminationUtils.cpp | 7 +- lib/CodeGen/PHIEliminationUtils.h | 7 +- lib/CodeGen/ParallelCG.cpp | 7 +- lib/CodeGen/PatchableFunction.cpp | 7 +- lib/CodeGen/PeepholeOptimizer.cpp | 24 +- lib/CodeGen/PostRAHazardRecognizer.cpp | 7 +- lib/CodeGen/PostRASchedulerList.cpp | 7 +- lib/CodeGen/PreISelIntrinsicLowering.cpp | 13 +- lib/CodeGen/ProcessImplicitDefs.cpp | 7 +- lib/CodeGen/PrologEpilogInserter.cpp | 198 +- lib/CodeGen/PseudoSourceValue.cpp | 7 +- lib/CodeGen/ReachingDefAnalysis.cpp | 7 +- lib/CodeGen/RegAllocBase.cpp | 23 +- lib/CodeGen/RegAllocBase.h | 7 +- lib/CodeGen/RegAllocBasic.cpp | 7 +- lib/CodeGen/RegAllocFast.cpp | 240 +- lib/CodeGen/RegAllocGreedy.cpp | 65 +- lib/CodeGen/RegAllocPBQP.cpp | 7 +- lib/CodeGen/RegUsageInfoCollector.cpp | 90 +- lib/CodeGen/RegUsageInfoPropagate.cpp | 7 +- lib/CodeGen/RegisterClassInfo.cpp | 11 +- lib/CodeGen/RegisterCoalescer.cpp | 140 +- lib/CodeGen/RegisterCoalescer.h | 7 +- lib/CodeGen/RegisterPressure.cpp | 15 +- lib/CodeGen/RegisterScavenging.cpp | 45 +- lib/CodeGen/RegisterUsageInfo.cpp | 7 +- lib/CodeGen/RenameIndependentSubregs.cpp | 7 +- lib/CodeGen/ResetMachineFunctionPass.cpp | 9 +- lib/CodeGen/SafeStack.cpp | 60 +- lib/CodeGen/SafeStackColoring.cpp | 7 +- lib/CodeGen/SafeStackColoring.h | 7 +- lib/CodeGen/SafeStackLayout.cpp | 7 +- lib/CodeGen/SafeStackLayout.h | 7 +- lib/CodeGen/ScalarizeMaskedMemIntrin.cpp | 306 +- lib/CodeGen/ScheduleDAG.cpp | 47 +- lib/CodeGen/ScheduleDAGInstrs.cpp | 66 +- lib/CodeGen/ScheduleDAGPrinter.cpp | 7 +- lib/CodeGen/ScoreboardHazardRecognizer.cpp | 7 +- lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 3287 +++++-- lib/CodeGen/SelectionDAG/FastISel.cpp | 81 +- lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp | 83 +- lib/CodeGen/SelectionDAG/InstrEmitter.cpp | 92 +- lib/CodeGen/SelectionDAG/InstrEmitter.h | 14 +- lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 472 +- lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp | 168 +- lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp | 447 +- lib/CodeGen/SelectionDAG/LegalizeTypes.cpp | 8 +- lib/CodeGen/SelectionDAG/LegalizeTypes.h | 50 +- lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp | 7 +- lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp | 181 +- lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp | 646 +- lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp | 11 +- lib/CodeGen/SelectionDAG/SDNodeDbgValue.h | 10 +- lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp | 10 +- lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp | 94 +- lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp | 107 +- lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h | 7 +- lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp | 7 +- lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 1429 +-- .../SelectionDAG/SelectionDAGAddressAnalysis.cpp | 139 +- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp | 2305 ++--- lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h | 383 +- lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp | 152 +- lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp | 446 +- lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp | 7 +- .../SelectionDAG/SelectionDAGTargetInfo.cpp | 7 +- lib/CodeGen/SelectionDAG/StatepointLowering.cpp | 109 +- lib/CodeGen/SelectionDAG/StatepointLowering.h | 14 +- lib/CodeGen/SelectionDAG/TargetLowering.cpp | 1723 +++- lib/CodeGen/ShadowStackGCLowering.cpp | 13 +- lib/CodeGen/ShrinkWrap.cpp | 16 +- lib/CodeGen/SjLjEHPrepare.cpp | 31 +- lib/CodeGen/SlotIndexes.cpp | 24 +- lib/CodeGen/SpillPlacement.cpp | 7 +- lib/CodeGen/SpillPlacement.h | 7 +- lib/CodeGen/Spiller.h | 7 +- lib/CodeGen/SplitKit.cpp | 16 +- lib/CodeGen/SplitKit.h | 7 +- lib/CodeGen/StackColoring.cpp | 16 +- lib/CodeGen/StackMapLivenessAnalysis.cpp | 7 +- lib/CodeGen/StackMaps.cpp | 7 +- lib/CodeGen/StackProtector.cpp | 70 +- lib/CodeGen/StackSlotColoring.cpp | 11 +- lib/CodeGen/SwiftErrorValueTracking.cpp | 312 + lib/CodeGen/SwitchLoweringUtils.cpp | 489 + lib/CodeGen/TailDuplication.cpp | 7 +- lib/CodeGen/TailDuplicator.cpp | 16 +- lib/CodeGen/TargetFrameLoweringImpl.cpp | 7 +- lib/CodeGen/TargetInstrInfo.cpp | 41 +- lib/CodeGen/TargetLoweringBase.cpp | 137 +- lib/CodeGen/TargetLoweringObjectFileImpl.cpp | 46 +- lib/CodeGen/TargetOptionsImpl.cpp | 7 +- lib/CodeGen/TargetPassConfig.cpp | 106 +- lib/CodeGen/TargetRegisterInfo.cpp | 13 +- lib/CodeGen/TargetSchedule.cpp | 7 +- lib/CodeGen/TargetSubtargetInfo.cpp | 69 +- lib/CodeGen/TwoAddressInstructionPass.cpp | 12 +- lib/CodeGen/UnreachableBlockElim.cpp | 43 +- lib/CodeGen/ValueTypes.cpp | 43 +- lib/CodeGen/VirtRegMap.cpp | 9 +- lib/CodeGen/WasmEHPrepare.cpp | 180 +- lib/CodeGen/WinEHPrepare.cpp | 20 +- lib/CodeGen/XRayInstrumentation.cpp | 9 +- .../CodeView/AppendingTypeTableBuilder.cpp | 16 +- lib/DebugInfo/CodeView/CVSymbolVisitor.cpp | 11 +- lib/DebugInfo/CodeView/CVTypeVisitor.cpp | 26 +- lib/DebugInfo/CodeView/CodeViewError.cpp | 9 +- lib/DebugInfo/CodeView/CodeViewRecordIO.cpp | 173 +- .../CodeView/ContinuationRecordBuilder.cpp | 20 +- .../CodeView/DebugChecksumsSubsection.cpp | 7 +- lib/DebugInfo/CodeView/DebugCrossExSubsection.cpp | 7 +- lib/DebugInfo/CodeView/DebugCrossImpSubsection.cpp | 7 +- .../CodeView/DebugFrameDataSubsection.cpp | 7 +- .../CodeView/DebugInlineeLinesSubsection.cpp | 7 +- lib/DebugInfo/CodeView/DebugLinesSubsection.cpp | 7 +- .../CodeView/DebugStringTableSubsection.cpp | 7 +- lib/DebugInfo/CodeView/DebugSubsection.cpp | 7 +- lib/DebugInfo/CodeView/DebugSubsectionRecord.cpp | 7 +- lib/DebugInfo/CodeView/DebugSubsectionVisitor.cpp | 7 +- .../CodeView/DebugSymbolRVASubsection.cpp | 7 +- lib/DebugInfo/CodeView/DebugSymbolsSubsection.cpp | 9 +- lib/DebugInfo/CodeView/EnumTables.cpp | 28 +- lib/DebugInfo/CodeView/Formatters.cpp | 7 +- lib/DebugInfo/CodeView/GlobalTypeTableBuilder.cpp | 16 +- .../CodeView/LazyRandomTypeCollection.cpp | 7 +- lib/DebugInfo/CodeView/Line.cpp | 7 +- lib/DebugInfo/CodeView/MergingTypeTableBuilder.cpp | 13 +- lib/DebugInfo/CodeView/RecordName.cpp | 7 +- lib/DebugInfo/CodeView/RecordSerialization.cpp | 7 +- lib/DebugInfo/CodeView/SimpleTypeSerializer.cpp | 18 +- lib/DebugInfo/CodeView/StringsAndChecksums.cpp | 7 +- lib/DebugInfo/CodeView/SymbolDumper.cpp | 42 +- lib/DebugInfo/CodeView/SymbolRecordHelpers.cpp | 7 +- lib/DebugInfo/CodeView/SymbolRecordMapping.cpp | 19 +- lib/DebugInfo/CodeView/SymbolSerializer.cpp | 7 +- lib/DebugInfo/CodeView/TypeDumpVisitor.cpp | 11 +- lib/DebugInfo/CodeView/TypeHashing.cpp | 15 +- lib/DebugInfo/CodeView/TypeIndex.cpp | 7 +- lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp | 13 +- lib/DebugInfo/CodeView/TypeRecordHelpers.cpp | 7 +- lib/DebugInfo/CodeView/TypeRecordMapping.cpp | 266 +- lib/DebugInfo/CodeView/TypeStreamMerger.cpp | 7 +- lib/DebugInfo/CodeView/TypeTableCollection.cpp | 13 +- .../DWARF/DWARFAbbreviationDeclaration.cpp | 15 +- lib/DebugInfo/DWARF/DWARFAcceleratorTable.cpp | 21 +- lib/DebugInfo/DWARF/DWARFAddressRange.cpp | 7 +- lib/DebugInfo/DWARF/DWARFCompileUnit.cpp | 7 +- lib/DebugInfo/DWARF/DWARFContext.cpp | 250 +- lib/DebugInfo/DWARF/DWARFDataExtractor.cpp | 23 +- lib/DebugInfo/DWARF/DWARFDebugAbbrev.cpp | 9 +- lib/DebugInfo/DWARF/DWARFDebugAddr.cpp | 34 +- lib/DebugInfo/DWARF/DWARFDebugArangeSet.cpp | 7 +- lib/DebugInfo/DWARF/DWARFDebugAranges.cpp | 26 +- lib/DebugInfo/DWARF/DWARFDebugFrame.cpp | 20 +- lib/DebugInfo/DWARF/DWARFDebugInfoEntry.cpp | 7 +- lib/DebugInfo/DWARF/DWARFDebugLine.cpp | 266 +- lib/DebugInfo/DWARF/DWARFDebugLoc.cpp | 34 +- lib/DebugInfo/DWARF/DWARFDebugMacro.cpp | 7 +- lib/DebugInfo/DWARF/DWARFDebugPubTable.cpp | 7 +- lib/DebugInfo/DWARF/DWARFDebugRangeList.cpp | 9 +- lib/DebugInfo/DWARF/DWARFDebugRnglists.cpp | 17 +- lib/DebugInfo/DWARF/DWARFDie.cpp | 134 +- lib/DebugInfo/DWARF/DWARFExpression.cpp | 93 +- lib/DebugInfo/DWARF/DWARFFormValue.cpp | 84 +- lib/DebugInfo/DWARF/DWARFGdbIndex.cpp | 13 +- lib/DebugInfo/DWARF/DWARFListTable.cpp | 11 +- lib/DebugInfo/DWARF/DWARFTypeUnit.cpp | 7 +- lib/DebugInfo/DWARF/DWARFUnit.cpp | 147 +- lib/DebugInfo/DWARF/DWARFUnitIndex.cpp | 14 +- lib/DebugInfo/DWARF/DWARFVerifier.cpp | 125 +- lib/DebugInfo/GSYM/FunctionInfo.cpp | 22 + lib/DebugInfo/GSYM/InlineInfo.cpp | 59 + lib/DebugInfo/GSYM/Range.cpp | 55 + lib/DebugInfo/MSF/MSFBuilder.cpp | 7 +- lib/DebugInfo/MSF/MSFCommon.cpp | 7 +- lib/DebugInfo/MSF/MSFError.cpp | 9 +- lib/DebugInfo/MSF/MappedBlockStream.cpp | 7 +- lib/DebugInfo/PDB/DIA/DIADataStream.cpp | 7 +- lib/DebugInfo/PDB/DIA/DIAEnumDebugStreams.cpp | 7 +- lib/DebugInfo/PDB/DIA/DIAEnumFrameData.cpp | 7 +- lib/DebugInfo/PDB/DIA/DIAEnumInjectedSources.cpp | 7 +- lib/DebugInfo/PDB/DIA/DIAEnumLineNumbers.cpp | 7 +- lib/DebugInfo/PDB/DIA/DIAEnumSectionContribs.cpp | 7 +- lib/DebugInfo/PDB/DIA/DIAEnumSourceFiles.cpp | 7 +- lib/DebugInfo/PDB/DIA/DIAEnumSymbols.cpp | 7 +- lib/DebugInfo/PDB/DIA/DIAEnumTables.cpp | 7 +- lib/DebugInfo/PDB/DIA/DIAFrameData.cpp | 7 +- lib/DebugInfo/PDB/DIA/DIAInjectedSource.cpp | 11 +- lib/DebugInfo/PDB/DIA/DIALineNumber.cpp | 7 +- lib/DebugInfo/PDB/DIA/DIARawSymbol.cpp | 7 +- lib/DebugInfo/PDB/DIA/DIASectionContrib.cpp | 7 +- lib/DebugInfo/PDB/DIA/DIASession.cpp | 7 +- lib/DebugInfo/PDB/DIA/DIASourceFile.cpp | 7 +- lib/DebugInfo/PDB/DIA/DIATable.cpp | 7 +- lib/DebugInfo/PDB/GenericError.cpp | 9 +- lib/DebugInfo/PDB/IPDBSourceFile.cpp | 7 +- lib/DebugInfo/PDB/Native/DbiModuleDescriptor.cpp | 7 +- .../PDB/Native/DbiModuleDescriptorBuilder.cpp | 13 +- lib/DebugInfo/PDB/Native/DbiModuleList.cpp | 7 +- lib/DebugInfo/PDB/Native/DbiStream.cpp | 108 +- lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp | 7 +- lib/DebugInfo/PDB/Native/EnumTables.cpp | 7 +- lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp | 22 +- lib/DebugInfo/PDB/Native/GlobalsStream.cpp | 7 +- lib/DebugInfo/PDB/Native/Hash.cpp | 7 +- lib/DebugInfo/PDB/Native/HashTable.cpp | 7 +- lib/DebugInfo/PDB/Native/InfoStream.cpp | 7 +- lib/DebugInfo/PDB/Native/InfoStreamBuilder.cpp | 7 +- lib/DebugInfo/PDB/Native/InjectedSourceStream.cpp | 65 + lib/DebugInfo/PDB/Native/ModuleDebugStream.cpp | 23 +- lib/DebugInfo/PDB/Native/NamedStreamMap.cpp | 15 +- lib/DebugInfo/PDB/Native/NativeCompilandSymbol.cpp | 7 +- lib/DebugInfo/PDB/Native/NativeEnumGlobals.cpp | 7 +- .../PDB/Native/NativeEnumInjectedSources.cpp | 120 + lib/DebugInfo/PDB/Native/NativeEnumModules.cpp | 7 +- lib/DebugInfo/PDB/Native/NativeEnumTypes.cpp | 7 +- lib/DebugInfo/PDB/Native/NativeExeSymbol.cpp | 7 +- lib/DebugInfo/PDB/Native/NativeRawSymbol.cpp | 7 +- lib/DebugInfo/PDB/Native/NativeSession.cpp | 20 +- .../PDB/Native/NativeSymbolEnumerator.cpp | 7 +- lib/DebugInfo/PDB/Native/NativeTypeArray.cpp | 7 +- lib/DebugInfo/PDB/Native/NativeTypeBuiltin.cpp | 7 +- lib/DebugInfo/PDB/Native/NativeTypeEnum.cpp | 7 +- lib/DebugInfo/PDB/Native/NativeTypeFunctionSig.cpp | 7 +- lib/DebugInfo/PDB/Native/NativeTypePointer.cpp | 7 +- lib/DebugInfo/PDB/Native/NativeTypeUDT.cpp | 7 +- lib/DebugInfo/PDB/Native/PDBFile.cpp | 95 +- lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp | 12 +- lib/DebugInfo/PDB/Native/PDBStringTable.cpp | 7 +- lib/DebugInfo/PDB/Native/PDBStringTableBuilder.cpp | 139 +- lib/DebugInfo/PDB/Native/PublicsStream.cpp | 7 +- lib/DebugInfo/PDB/Native/RawError.cpp | 2 + lib/DebugInfo/PDB/Native/SymbolStream.cpp | 7 +- lib/DebugInfo/PDB/Native/TpiHashing.cpp | 7 +- lib/DebugInfo/PDB/Native/TpiStream.cpp | 20 +- lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp | 16 +- lib/DebugInfo/PDB/PDB.cpp | 7 +- lib/DebugInfo/PDB/PDBContext.cpp | 31 +- lib/DebugInfo/PDB/PDBExtras.cpp | 47 +- lib/DebugInfo/PDB/PDBInterfaceAnchors.cpp | 7 +- lib/DebugInfo/PDB/PDBSymDumper.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbol.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolAnnotation.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolBlock.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolCompiland.cpp | 17 +- lib/DebugInfo/PDB/PDBSymbolCompilandDetails.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolCompilandEnv.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolCustom.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolData.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolExe.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolFunc.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolFuncDebugEnd.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolFuncDebugStart.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolLabel.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolPublicSymbol.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolThunk.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolTypeArray.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolTypeBaseClass.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolTypeBuiltin.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolTypeCustom.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolTypeDimension.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolTypeEnum.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolTypeFriend.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolTypeFunctionArg.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolTypeFunctionSig.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolTypeManaged.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolTypePointer.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolTypeTypedef.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolTypeUDT.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolTypeVTable.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolTypeVTableShape.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolUnknown.cpp | 7 +- lib/DebugInfo/PDB/PDBSymbolUsingNamespace.cpp | 7 +- lib/DebugInfo/PDB/UDTLayout.cpp | 7 +- lib/DebugInfo/Symbolize/DIPrinter.cpp | 38 +- lib/DebugInfo/Symbolize/SymbolizableObjectFile.cpp | 124 +- lib/DebugInfo/Symbolize/SymbolizableObjectFile.h | 33 +- lib/DebugInfo/Symbolize/Symbolize.cpp | 223 +- lib/Demangle/Demangle.cpp | 36 + lib/Demangle/ItaniumDemangle.cpp | 7 +- lib/Demangle/MicrosoftDemangle.cpp | 397 +- lib/Demangle/MicrosoftDemangleNodes.cpp | 35 +- lib/ExecutionEngine/ExecutionEngine.cpp | 60 +- lib/ExecutionEngine/ExecutionEngineBindings.cpp | 7 +- lib/ExecutionEngine/GDBRegistrationListener.cpp | 7 +- .../IntelJITEvents/IntelJITEventListener.cpp | 21 +- .../IntelJITEvents/IntelJITEventsWrapper.h | 7 +- .../IntelJITEvents/ittnotify_config.h | 7 +- .../IntelJITEvents/ittnotify_types.h | 7 +- lib/ExecutionEngine/IntelJITEvents/jitprofiling.c | 7 +- lib/ExecutionEngine/IntelJITEvents/jitprofiling.h | 7 +- lib/ExecutionEngine/Interpreter/Execution.cpp | 63 +- .../Interpreter/ExternalFunctions.cpp | 7 +- lib/ExecutionEngine/Interpreter/Interpreter.cpp | 7 +- lib/ExecutionEngine/Interpreter/Interpreter.h | 8 +- .../JITLink/BasicGOTAndStubsBuilder.h | 82 + lib/ExecutionEngine/JITLink/EHFrameSupport.cpp | 544 ++ lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h | 72 + lib/ExecutionEngine/JITLink/JITLink.cpp | 172 + lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp | 481 + lib/ExecutionEngine/JITLink/JITLinkGeneric.h | 256 + .../JITLink/JITLinkMemoryManager.cpp | 105 + lib/ExecutionEngine/JITLink/MachO.cpp | 78 + .../JITLink/MachOAtomGraphBuilder.cpp | 411 + .../JITLink/MachOAtomGraphBuilder.h | 138 + lib/ExecutionEngine/JITLink/MachO_x86_64.cpp | 608 ++ lib/ExecutionEngine/MCJIT/MCJIT.cpp | 7 +- lib/ExecutionEngine/MCJIT/MCJIT.h | 7 +- .../OProfileJIT/OProfileJITEventListener.cpp | 7 +- .../OProfileJIT/OProfileWrapper.cpp | 7 +- lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp | 7 +- lib/ExecutionEngine/Orc/CompileUtils.cpp | 86 + lib/ExecutionEngine/Orc/Core.cpp | 1022 +-- lib/ExecutionEngine/Orc/ExecutionUtils.cpp | 32 +- lib/ExecutionEngine/Orc/IRCompileLayer.cpp | 7 +- lib/ExecutionEngine/Orc/IRTransformLayer.cpp | 7 +- lib/ExecutionEngine/Orc/IndirectionUtils.cpp | 15 +- .../Orc/JITTargetMachineBuilder.cpp | 7 +- lib/ExecutionEngine/Orc/LLJIT.cpp | 262 +- lib/ExecutionEngine/Orc/Layer.cpp | 17 +- lib/ExecutionEngine/Orc/LazyReexports.cpp | 20 +- lib/ExecutionEngine/Orc/Legacy.cpp | 10 +- lib/ExecutionEngine/Orc/NullResolver.cpp | 7 +- lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp | 483 + lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp | 7 +- lib/ExecutionEngine/Orc/OrcABISupport.cpp | 17 +- lib/ExecutionEngine/Orc/OrcCBindings.cpp | 7 +- lib/ExecutionEngine/Orc/OrcCBindingsStack.h | 64 +- lib/ExecutionEngine/Orc/OrcError.cpp | 7 +- lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp | 10 +- lib/ExecutionEngine/Orc/OrcMCJITReplacement.h | 32 +- lib/ExecutionEngine/Orc/RPCUtils.cpp | 7 +- .../Orc/RTDyldObjectLinkingLayer.cpp | 66 +- lib/ExecutionEngine/Orc/ThreadSafeModule.cpp | 7 +- .../PerfJITEvents/PerfJITEventListener.cpp | 27 +- lib/ExecutionEngine/RuntimeDyld/JITSymbol.cpp | 7 +- .../RuntimeDyld/RTDyldMemoryManager.cpp | 16 +- lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp | 147 +- .../RuntimeDyld/RuntimeDyldCOFF.cpp | 7 +- lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h | 7 +- .../RuntimeDyld/RuntimeDyldChecker.cpp | 354 +- .../RuntimeDyld/RuntimeDyldCheckerImpl.h | 59 +- lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp | 10 +- lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h | 9 +- lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h | 55 +- .../RuntimeDyld/RuntimeDyldMachO.cpp | 7 +- lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h | 7 +- .../RuntimeDyld/Targets/RuntimeDyldCOFFI386.h | 15 +- .../RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h | 28 +- .../RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h | 23 +- .../RuntimeDyld/Targets/RuntimeDyldELFMips.cpp | 7 +- .../RuntimeDyld/Targets/RuntimeDyldELFMips.h | 7 +- .../RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h | 9 +- .../RuntimeDyld/Targets/RuntimeDyldMachOARM.h | 11 +- .../RuntimeDyld/Targets/RuntimeDyldMachOI386.h | 9 +- .../RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h | 11 +- lib/ExecutionEngine/SectionMemoryManager.cpp | 34 +- lib/ExecutionEngine/TargetSelect.cpp | 7 +- lib/FuzzMutate/FuzzerCLI.cpp | 7 +- lib/FuzzMutate/IRMutator.cpp | 7 +- lib/FuzzMutate/OpDescriptor.cpp | 7 +- lib/FuzzMutate/Operations.cpp | 7 +- lib/FuzzMutate/RandomIRBuilder.cpp | 10 +- lib/IR/AbstractCallSite.cpp | 134 + lib/IR/AsmWriter.cpp | 150 +- lib/IR/AttributeImpl.h | 51 +- lib/IR/Attributes.cpp | 150 +- lib/IR/AutoUpgrade.cpp | 406 +- lib/IR/BasicBlock.cpp | 63 +- lib/IR/Comdat.cpp | 7 +- lib/IR/ConstantFold.cpp | 134 +- lib/IR/ConstantFold.h | 8 +- lib/IR/ConstantRange.cpp | 892 +- lib/IR/Constants.cpp | 61 +- lib/IR/ConstantsContext.h | 7 +- lib/IR/Core.cpp | 158 +- lib/IR/DIBuilder.cpp | 18 +- lib/IR/DataLayout.cpp | 36 +- lib/IR/DebugInfo.cpp | 107 +- lib/IR/DebugInfoMetadata.cpp | 121 +- lib/IR/DebugLoc.cpp | 7 +- lib/IR/DiagnosticHandler.cpp | 7 +- lib/IR/DiagnosticInfo.cpp | 87 +- lib/IR/DiagnosticPrinter.cpp | 7 +- lib/IR/DomTreeUpdater.cpp | 529 -- lib/IR/Dominators.cpp | 7 +- lib/IR/Function.cpp | 218 +- lib/IR/GVMaterializer.cpp | 7 +- lib/IR/Globals.cpp | 35 +- lib/IR/IRBuilder.cpp | 55 +- lib/IR/IRPrintingPasses.cpp | 7 +- lib/IR/InlineAsm.cpp | 7 +- lib/IR/Instruction.cpp | 76 +- lib/IR/Instructions.cpp | 341 +- lib/IR/IntrinsicInst.cpp | 116 +- lib/IR/LLVMContext.cpp | 35 +- lib/IR/LLVMContextImpl.cpp | 7 +- lib/IR/LLVMContextImpl.h | 48 +- lib/IR/LegacyPassManager.cpp | 26 +- lib/IR/MDBuilder.cpp | 57 +- lib/IR/Mangler.cpp | 13 +- lib/IR/Metadata.cpp | 12 +- lib/IR/MetadataImpl.h | 7 +- lib/IR/Module.cpp | 32 +- lib/IR/ModuleSummaryIndex.cpp | 192 +- lib/IR/Operator.cpp | 7 +- lib/IR/OptBisect.cpp | 83 +- lib/IR/Pass.cpp | 33 +- lib/IR/PassInstrumentation.cpp | 7 +- lib/IR/PassManager.cpp | 7 +- lib/IR/PassRegistry.cpp | 7 +- lib/IR/PassTimingInfo.cpp | 28 +- lib/IR/ProfileSummary.cpp | 13 +- lib/IR/RemarkStreamer.cpp | 154 + lib/IR/SafepointIRVerifier.cpp | 18 +- lib/IR/Statepoint.cpp | 37 +- lib/IR/SymbolTableListTraitsImpl.h | 10 +- lib/IR/Type.cpp | 38 +- lib/IR/TypeFinder.cpp | 7 +- lib/IR/Use.cpp | 7 +- lib/IR/User.cpp | 7 +- lib/IR/Value.cpp | 85 +- lib/IR/ValueSymbolTable.cpp | 7 +- lib/IR/Verifier.cpp | 451 +- lib/IRReader/IRReader.cpp | 13 +- lib/LTO/Caching.cpp | 31 +- lib/LTO/LTO.cpp | 193 +- lib/LTO/LTOBackend.cpp | 41 +- lib/LTO/LTOCodeGenerator.cpp | 60 +- lib/LTO/LTOModule.cpp | 38 +- lib/LTO/SummaryBasedOptimizations.cpp | 7 +- lib/LTO/ThinLTOCodeGenerator.cpp | 207 +- lib/LTO/UpdateCompilerUsed.cpp | 7 +- lib/LineEditor/LineEditor.cpp | 7 +- lib/Linker/IRMover.cpp | 74 +- lib/Linker/LinkDiagnosticInfo.h | 7 +- lib/Linker/LinkModules.cpp | 7 +- lib/MC/ConstantPools.cpp | 7 +- lib/MC/ELFObjectWriter.cpp | 59 +- lib/MC/MCAsmBackend.cpp | 12 +- lib/MC/MCAsmInfo.cpp | 11 +- lib/MC/MCAsmInfoCOFF.cpp | 7 +- lib/MC/MCAsmInfoDarwin.cpp | 7 +- lib/MC/MCAsmInfoELF.cpp | 7 +- lib/MC/MCAsmInfoWasm.cpp | 8 +- lib/MC/MCAsmInfoXCOFF.cpp | 18 + lib/MC/MCAsmMacro.cpp | 7 +- lib/MC/MCAsmStreamer.cpp | 61 +- lib/MC/MCAssembler.cpp | 24 +- lib/MC/MCCodeEmitter.cpp | 7 +- lib/MC/MCCodePadder.cpp | 7 +- lib/MC/MCCodeView.cpp | 7 +- lib/MC/MCContext.cpp | 97 +- lib/MC/MCDisassembler/Disassembler.cpp | 35 +- lib/MC/MCDisassembler/Disassembler.h | 41 +- lib/MC/MCDisassembler/MCDisassembler.cpp | 16 +- lib/MC/MCDisassembler/MCExternalSymbolizer.cpp | 7 +- lib/MC/MCDisassembler/MCRelocationInfo.cpp | 7 +- lib/MC/MCDisassembler/MCSymbolizer.cpp | 7 +- lib/MC/MCDwarf.cpp | 108 +- lib/MC/MCELFObjectTargetWriter.cpp | 12 +- lib/MC/MCELFStreamer.cpp | 10 +- lib/MC/MCExpr.cpp | 47 +- lib/MC/MCFragment.cpp | 7 +- lib/MC/MCInst.cpp | 7 +- lib/MC/MCInstPrinter.cpp | 13 +- lib/MC/MCInstrAnalysis.cpp | 7 +- lib/MC/MCInstrDesc.cpp | 7 +- lib/MC/MCLabel.cpp | 7 +- lib/MC/MCLinkerOptimizationHint.cpp | 7 +- lib/MC/MCMachOStreamer.cpp | 11 +- lib/MC/MCMachObjectTargetWriter.cpp | 7 +- lib/MC/MCNullStreamer.cpp | 7 +- lib/MC/MCObjectFileInfo.cpp | 28 +- lib/MC/MCObjectStreamer.cpp | 9 +- lib/MC/MCObjectWriter.cpp | 7 +- lib/MC/MCParser/AsmLexer.cpp | 32 +- lib/MC/MCParser/AsmParser.cpp | 69 +- lib/MC/MCParser/COFFAsmParser.cpp | 7 +- lib/MC/MCParser/DarwinAsmParser.cpp | 9 +- lib/MC/MCParser/ELFAsmParser.cpp | 11 +- lib/MC/MCParser/MCAsmLexer.cpp | 7 +- lib/MC/MCParser/MCAsmParser.cpp | 7 +- lib/MC/MCParser/MCAsmParserExtension.cpp | 7 +- lib/MC/MCParser/MCTargetAsmParser.cpp | 7 +- lib/MC/MCParser/WasmAsmParser.cpp | 174 +- lib/MC/MCRegisterInfo.cpp | 7 +- lib/MC/MCSchedule.cpp | 23 +- lib/MC/MCSection.cpp | 7 +- lib/MC/MCSectionCOFF.cpp | 9 +- lib/MC/MCSectionELF.cpp | 13 +- lib/MC/MCSectionMachO.cpp | 7 +- lib/MC/MCSectionWasm.cpp | 16 +- lib/MC/MCSectionXCOFF.cpp | 33 + lib/MC/MCStreamer.cpp | 32 +- lib/MC/MCSubtargetInfo.cpp | 251 +- lib/MC/MCSymbol.cpp | 7 +- lib/MC/MCSymbolELF.cpp | 17 +- lib/MC/MCTargetOptions.cpp | 18 +- lib/MC/MCValue.cpp | 7 +- lib/MC/MCWasmObjectTargetWriter.cpp | 11 +- lib/MC/MCWasmStreamer.cpp | 26 +- lib/MC/MCWin64EH.cpp | 108 +- lib/MC/MCWinCOFFStreamer.cpp | 11 +- lib/MC/MCWinEH.cpp | 7 +- lib/MC/MCXCOFFObjectTargetWriter.cpp | 16 + lib/MC/MCXCOFFStreamer.cpp | 59 + lib/MC/MachObjectWriter.cpp | 22 +- lib/MC/StringTableBuilder.cpp | 14 +- lib/MC/SubtargetFeature.cpp | 206 +- lib/MC/WasmObjectWriter.cpp | 423 +- lib/MC/WinCOFFObjectWriter.cpp | 9 +- lib/MC/XCOFFObjectWriter.cpp | 94 + lib/MCA/Context.cpp | 14 +- lib/MCA/HWEventListener.cpp | 7 +- lib/MCA/HardwareUnits/HardwareUnit.cpp | 7 +- lib/MCA/HardwareUnits/LSUnit.cpp | 256 +- lib/MCA/HardwareUnits/RegisterFile.cpp | 53 +- lib/MCA/HardwareUnits/ResourceManager.cpp | 98 +- lib/MCA/HardwareUnits/RetireControlUnit.cpp | 7 +- lib/MCA/HardwareUnits/Scheduler.cpp | 166 +- lib/MCA/InstrBuilder.cpp | 37 +- lib/MCA/Instruction.cpp | 117 +- lib/MCA/Pipeline.cpp | 12 +- lib/MCA/Stages/DispatchStage.cpp | 51 +- lib/MCA/Stages/EntryStage.cpp | 11 +- lib/MCA/Stages/ExecuteStage.cpp | 83 +- lib/MCA/Stages/InstructionTables.cpp | 7 +- lib/MCA/Stages/MicroOpQueueStage.cpp | 70 + lib/MCA/Stages/RetireStage.cpp | 7 +- lib/MCA/Stages/Stage.cpp | 7 +- lib/MCA/Support.cpp | 28 +- lib/Object/Archive.cpp | 18 +- lib/Object/ArchiveWriter.cpp | 149 +- lib/Object/Binary.cpp | 12 +- lib/Object/COFFImportFile.cpp | 14 +- lib/Object/COFFModuleDefinition.cpp | 7 +- lib/Object/COFFObjectFile.cpp | 53 +- lib/Object/Decompressor.cpp | 7 +- lib/Object/ELF.cpp | 39 +- lib/Object/ELFObjectFile.cpp | 58 +- lib/Object/Error.cpp | 30 +- lib/Object/IRObjectFile.cpp | 22 +- lib/Object/IRSymtab.cpp | 25 +- lib/Object/MachOObjectFile.cpp | 445 +- lib/Object/MachOUniversal.cpp | 7 +- lib/Object/Minidump.cpp | 137 + lib/Object/ModuleSymbolTable.cpp | 7 +- lib/Object/Object.cpp | 132 +- lib/Object/ObjectFile.cpp | 24 +- lib/Object/RecordStreamer.cpp | 9 +- lib/Object/RecordStreamer.h | 19 +- lib/Object/RelocationResolver.cpp | 550 ++ lib/Object/SymbolSize.cpp | 7 +- lib/Object/SymbolicFile.cpp | 10 +- lib/Object/WasmObjectFile.cpp | 332 +- lib/Object/WindowsMachineFlag.cpp | 44 + lib/Object/WindowsResource.cpp | 203 +- lib/Object/XCOFFObjectFile.cpp | 584 ++ lib/ObjectYAML/COFFYAML.cpp | 16 +- lib/ObjectYAML/CodeViewYAMLDebugSections.cpp | 7 +- lib/ObjectYAML/CodeViewYAMLSymbols.cpp | 17 +- lib/ObjectYAML/CodeViewYAMLTypeHashing.cpp | 7 +- lib/ObjectYAML/CodeViewYAMLTypes.cpp | 11 +- lib/ObjectYAML/DWARFEmitter.cpp | 7 +- lib/ObjectYAML/DWARFVisitor.cpp | 7 +- lib/ObjectYAML/DWARFVisitor.h | 7 +- lib/ObjectYAML/DWARFYAML.cpp | 7 +- lib/ObjectYAML/ELFYAML.cpp | 244 +- lib/ObjectYAML/MachOYAML.cpp | 7 +- lib/ObjectYAML/MinidumpYAML.cpp | 673 ++ lib/ObjectYAML/ObjectYAML.cpp | 22 +- lib/ObjectYAML/WasmYAML.cpp | 80 +- lib/ObjectYAML/XCOFFYAML.cpp | 109 + lib/ObjectYAML/YAML.cpp | 14 +- lib/OptRemarks/OptRemarksParser.cpp | 368 - lib/Option/Arg.cpp | 10 +- lib/Option/ArgList.cpp | 22 +- lib/Option/OptTable.cpp | 81 +- lib/Option/Option.cpp | 120 +- lib/Passes/PassBuilder.cpp | 375 +- lib/Passes/PassPlugin.cpp | 7 +- lib/Passes/PassRegistry.def | 59 +- lib/Passes/StandardInstrumentations.cpp | 7 +- lib/ProfileData/Coverage/CoverageMapping.cpp | 18 +- lib/ProfileData/Coverage/CoverageMappingReader.cpp | 235 +- lib/ProfileData/Coverage/CoverageMappingWriter.cpp | 24 +- lib/ProfileData/GCOV.cpp | 26 +- lib/ProfileData/InstrProf.cpp | 293 +- lib/ProfileData/InstrProfReader.cpp | 47 +- lib/ProfileData/InstrProfWriter.cpp | 107 +- lib/ProfileData/ProfileSummaryBuilder.cpp | 20 +- lib/ProfileData/SampleProf.cpp | 7 +- lib/ProfileData/SampleProfReader.cpp | 11 +- lib/ProfileData/SampleProfWriter.cpp | 12 +- lib/Remarks/Remark.cpp | 132 + lib/Remarks/RemarkFormat.cpp | 30 + lib/Remarks/RemarkParser.cpp | 119 + lib/Remarks/RemarkStringTable.cpp | 48 + lib/Remarks/YAMLRemarkParser.cpp | 327 + lib/Remarks/YAMLRemarkParser.h | 96 + lib/Remarks/YAMLRemarkSerializer.cpp | 167 + lib/Support/AArch64TargetParser.cpp | 17 +- lib/Support/AMDGPUMetadata.cpp | 23 +- lib/Support/APFloat.cpp | 51 +- lib/Support/APInt.cpp | 84 +- lib/Support/APSInt.cpp | 13 +- lib/Support/ARMAttributeParser.cpp | 32 +- lib/Support/ARMBuildAttrs.cpp | 8 +- lib/Support/ARMTargetParser.cpp | 265 +- lib/Support/ARMWinEH.cpp | 7 +- lib/Support/Allocator.cpp | 7 +- lib/Support/Atomic.cpp | 7 +- lib/Support/BinaryStreamError.cpp | 7 +- lib/Support/BinaryStreamReader.cpp | 40 +- lib/Support/BinaryStreamRef.cpp | 7 +- lib/Support/BinaryStreamWriter.cpp | 20 +- lib/Support/BlockFrequency.cpp | 7 +- lib/Support/BranchProbability.cpp | 11 +- lib/Support/BuryPointer.cpp | 7 +- lib/Support/COM.cpp | 7 +- lib/Support/CRC.cpp | 68 + lib/Support/CachePruning.cpp | 18 +- lib/Support/Chrono.cpp | 7 +- lib/Support/CodeGenCoverage.cpp | 7 +- lib/Support/CommandLine.cpp | 445 +- lib/Support/Compression.cpp | 7 +- lib/Support/ConvertUTF.cpp | 7 +- lib/Support/ConvertUTFWrapper.cpp | 7 +- lib/Support/CrashRecoveryContext.cpp | 7 +- lib/Support/DAGDeltaAlgorithm.cpp | 7 +- lib/Support/DJB.cpp | 42 +- lib/Support/DataExtractor.cpp | 62 +- lib/Support/Debug.cpp | 7 +- lib/Support/DeltaAlgorithm.cpp | 7 +- lib/Support/DynamicLibrary.cpp | 7 +- lib/Support/Errno.cpp | 9 +- lib/Support/Error.cpp | 7 +- lib/Support/ErrorHandling.cpp | 29 +- lib/Support/FileCheck.cpp | 1122 ++- lib/Support/FileOutputBuffer.cpp | 81 +- lib/Support/FileUtilities.cpp | 7 +- lib/Support/FoldingSet.cpp | 7 +- lib/Support/FormatVariadic.cpp | 7 +- lib/Support/FormattedStream.cpp | 7 +- lib/Support/GlobPattern.cpp | 7 +- lib/Support/GraphWriter.cpp | 7 +- lib/Support/Hashing.cpp | 7 +- lib/Support/Host.cpp | 102 +- lib/Support/InitLLVM.cpp | 8 +- lib/Support/IntEqClasses.cpp | 7 +- lib/Support/IntervalMap.cpp | 7 +- lib/Support/ItaniumManglingCanonicalizer.cpp | 8 +- lib/Support/JSON.cpp | 221 +- lib/Support/JamCRC.cpp | 7 +- lib/Support/KnownBits.cpp | 50 +- lib/Support/LEB128.cpp | 7 +- lib/Support/LineIterator.cpp | 7 +- lib/Support/LockFileManager.cpp | 7 +- lib/Support/LowLevelType.cpp | 11 +- lib/Support/ManagedStatic.cpp | 7 +- lib/Support/MathExtras.cpp | 7 +- lib/Support/Memory.cpp | 36 +- lib/Support/MemoryBuffer.cpp | 96 +- lib/Support/Mutex.cpp | 7 +- lib/Support/NativeFormatting.cpp | 7 +- lib/Support/Optional.cpp | 14 + lib/Support/Options.cpp | 7 +- lib/Support/Parallel.cpp | 38 +- lib/Support/Path.cpp | 93 +- lib/Support/PluginLoader.cpp | 7 +- lib/Support/PrettyStackTrace.cpp | 92 +- lib/Support/Process.cpp | 7 +- lib/Support/Program.cpp | 7 +- lib/Support/RWMutex.cpp | 7 +- lib/Support/RandomNumberGenerator.cpp | 17 +- lib/Support/Regex.cpp | 7 +- lib/Support/SHA1.cpp | 7 +- lib/Support/ScaledNumber.cpp | 7 +- lib/Support/Signals.cpp | 11 +- lib/Support/Signposts.cpp | 119 + lib/Support/SmallPtrSet.cpp | 7 +- lib/Support/SmallVector.cpp | 7 +- lib/Support/SourceMgr.cpp | 18 +- lib/Support/SpecialCaseList.cpp | 7 +- lib/Support/Statistic.cpp | 10 +- lib/Support/StringExtras.cpp | 7 +- lib/Support/StringMap.cpp | 7 +- lib/Support/StringPool.cpp | 7 +- lib/Support/StringRef.cpp | 7 +- lib/Support/StringSaver.cpp | 7 +- lib/Support/SymbolRemappingReader.cpp | 7 +- lib/Support/SystemUtils.cpp | 7 +- lib/Support/TarWriter.cpp | 7 +- lib/Support/TargetParser.cpp | 61 +- lib/Support/TargetRegistry.cpp | 7 +- lib/Support/ThreadLocal.cpp | 7 +- lib/Support/ThreadPool.cpp | 7 +- lib/Support/Threading.cpp | 7 +- lib/Support/TimeProfiler.cpp | 199 + lib/Support/Timer.cpp | 30 +- lib/Support/ToolOutputFile.cpp | 7 +- lib/Support/TrigramIndex.cpp | 7 +- lib/Support/Triple.cpp | 36 +- lib/Support/Twine.cpp | 7 +- lib/Support/Unicode.cpp | 7 +- lib/Support/Unix/COM.inc | 7 +- lib/Support/Unix/DynamicLibrary.inc | 7 +- lib/Support/Unix/Host.inc | 24 +- lib/Support/Unix/Memory.inc | 81 +- lib/Support/Unix/Mutex.inc | 7 +- lib/Support/Unix/Path.inc | 219 +- lib/Support/Unix/Process.inc | 20 +- lib/Support/Unix/Program.inc | 23 +- lib/Support/Unix/RWMutex.inc | 7 +- lib/Support/Unix/Signals.inc | 81 +- lib/Support/Unix/ThreadLocal.inc | 7 +- lib/Support/Unix/Threading.inc | 52 +- lib/Support/Unix/Unix.h | 9 +- lib/Support/Unix/Watchdog.inc | 7 +- lib/Support/Valgrind.cpp | 7 +- lib/Support/VersionTuple.cpp | 7 +- lib/Support/VirtualFileSystem.cpp | 159 +- lib/Support/Watchdog.cpp | 7 +- lib/Support/Windows/COM.inc | 7 +- lib/Support/Windows/DynamicLibrary.inc | 7 +- lib/Support/Windows/Host.inc | 7 +- lib/Support/Windows/Memory.inc | 96 +- lib/Support/Windows/Mutex.inc | 7 +- lib/Support/Windows/Path.inc | 98 +- lib/Support/Windows/Process.inc | 9 +- lib/Support/Windows/Program.inc | 7 +- lib/Support/Windows/RWMutex.inc | 7 +- lib/Support/Windows/Signals.inc | 11 +- lib/Support/Windows/ThreadLocal.inc | 7 +- lib/Support/Windows/Threading.inc | 23 +- lib/Support/Windows/Watchdog.inc | 7 +- lib/Support/Windows/WindowsSupport.h | 7 +- lib/Support/WithColor.cpp | 7 +- lib/Support/YAMLParser.cpp | 7 +- lib/Support/YAMLTraits.cpp | 58 +- lib/Support/Z3Solver.cpp | 900 ++ lib/Support/circular_raw_ostream.cpp | 7 +- lib/Support/raw_os_ostream.cpp | 7 +- lib/Support/raw_ostream.cpp | 9 +- lib/TableGen/Error.cpp | 7 +- lib/TableGen/JSONBackend.cpp | 7 +- lib/TableGen/Main.cpp | 7 +- lib/TableGen/Record.cpp | 204 +- lib/TableGen/SetTheory.cpp | 7 +- lib/TableGen/StringMatcher.cpp | 7 +- lib/TableGen/TGLexer.cpp | 28 +- lib/TableGen/TGLexer.h | 15 +- lib/TableGen/TGParser.cpp | 280 +- lib/TableGen/TGParser.h | 11 +- lib/TableGen/TableGenBackend.cpp | 7 +- lib/Target/AArch64/AArch64.h | 9 +- lib/Target/AArch64/AArch64.td | 65 +- lib/Target/AArch64/AArch64A53Fix835769.cpp | 7 +- lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp | 7 +- lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp | 7 +- lib/Target/AArch64/AArch64AsmPrinter.cpp | 281 +- lib/Target/AArch64/AArch64BranchTargets.cpp | 7 +- lib/Target/AArch64/AArch64CallLowering.cpp | 205 +- lib/Target/AArch64/AArch64CallLowering.h | 28 +- lib/Target/AArch64/AArch64CallingConvention.cpp | 134 + lib/Target/AArch64/AArch64CallingConvention.h | 156 +- lib/Target/AArch64/AArch64CallingConvention.td | 33 +- .../AArch64/AArch64CleanupLocalDynamicTLSPass.cpp | 7 +- lib/Target/AArch64/AArch64CollectLOH.cpp | 7 +- lib/Target/AArch64/AArch64CompressJumpTables.cpp | 10 +- lib/Target/AArch64/AArch64CondBrTuning.cpp | 7 +- lib/Target/AArch64/AArch64ConditionOptimizer.cpp | 7 +- lib/Target/AArch64/AArch64ConditionalCompares.cpp | 9 +- .../AArch64/AArch64DeadRegisterDefinitionsPass.cpp | 108 +- lib/Target/AArch64/AArch64ExpandImm.cpp | 411 + lib/Target/AArch64/AArch64ExpandImm.h | 35 + lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp | 619 +- lib/Target/AArch64/AArch64FalkorHWPFFix.cpp | 13 +- lib/Target/AArch64/AArch64FastISel.cpp | 34 +- lib/Target/AArch64/AArch64FrameLowering.cpp | 215 +- lib/Target/AArch64/AArch64FrameLowering.h | 17 +- lib/Target/AArch64/AArch64GenRegisterBankInfo.def | 11 +- lib/Target/AArch64/AArch64ISelDAGToDAG.cpp | 140 +- lib/Target/AArch64/AArch64ISelLowering.cpp | 583 +- lib/Target/AArch64/AArch64ISelLowering.h | 42 +- lib/Target/AArch64/AArch64InstrAtomics.td | 7 +- lib/Target/AArch64/AArch64InstrFormats.td | 50 +- lib/Target/AArch64/AArch64InstrInfo.cpp | 472 +- lib/Target/AArch64/AArch64InstrInfo.h | 51 +- lib/Target/AArch64/AArch64InstrInfo.td | 172 +- lib/Target/AArch64/AArch64InstructionSelector.cpp | 2803 +++++- lib/Target/AArch64/AArch64LegalizerInfo.cpp | 388 +- lib/Target/AArch64/AArch64LegalizerInfo.h | 13 +- lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp | 13 +- lib/Target/AArch64/AArch64MCInstLower.cpp | 7 +- lib/Target/AArch64/AArch64MCInstLower.h | 7 +- lib/Target/AArch64/AArch64MachineFunctionInfo.h | 28 +- lib/Target/AArch64/AArch64MacroFusion.cpp | 7 +- lib/Target/AArch64/AArch64MacroFusion.h | 7 +- lib/Target/AArch64/AArch64PBQPRegAlloc.cpp | 7 +- lib/Target/AArch64/AArch64PBQPRegAlloc.h | 7 +- lib/Target/AArch64/AArch64PerfectShuffle.h | 7 +- lib/Target/AArch64/AArch64PfmCounters.td | 7 +- lib/Target/AArch64/AArch64PreLegalizerCombiner.cpp | 11 +- lib/Target/AArch64/AArch64PromoteConstant.cpp | 10 +- .../AArch64/AArch64RedundantCopyElimination.cpp | 11 +- lib/Target/AArch64/AArch64RegisterBankInfo.cpp | 238 +- lib/Target/AArch64/AArch64RegisterBankInfo.h | 20 +- lib/Target/AArch64/AArch64RegisterBanks.td | 7 +- lib/Target/AArch64/AArch64RegisterInfo.cpp | 49 +- lib/Target/AArch64/AArch64RegisterInfo.h | 11 +- lib/Target/AArch64/AArch64RegisterInfo.td | 26 +- lib/Target/AArch64/AArch64SIMDInstrOpt.cpp | 7 +- lib/Target/AArch64/AArch64SVEInstrInfo.td | 426 +- lib/Target/AArch64/AArch64SchedA53.td | 9 +- lib/Target/AArch64/AArch64SchedA57.td | 9 +- lib/Target/AArch64/AArch64SchedA57WriteRes.td | 7 +- lib/Target/AArch64/AArch64SchedCyclone.td | 9 +- lib/Target/AArch64/AArch64SchedExynosM1.td | 9 +- lib/Target/AArch64/AArch64SchedExynosM3.td | 9 +- lib/Target/AArch64/AArch64SchedExynosM4.td | 45 +- lib/Target/AArch64/AArch64SchedFalkor.td | 9 +- lib/Target/AArch64/AArch64SchedFalkorDetails.td | 7 +- lib/Target/AArch64/AArch64SchedKryo.td | 9 +- lib/Target/AArch64/AArch64SchedKryoDetails.td | 7 +- lib/Target/AArch64/AArch64SchedPredExynos.td | 18 +- lib/Target/AArch64/AArch64SchedPredicates.td | 60 +- lib/Target/AArch64/AArch64SchedThunderX.td | 9 +- lib/Target/AArch64/AArch64SchedThunderX2T99.td | 9 +- lib/Target/AArch64/AArch64Schedule.td | 7 +- lib/Target/AArch64/AArch64SelectionDAGInfo.cpp | 95 +- lib/Target/AArch64/AArch64SelectionDAGInfo.h | 11 +- lib/Target/AArch64/AArch64SpeculationHardening.cpp | 182 +- lib/Target/AArch64/AArch64StackTagging.cpp | 345 + lib/Target/AArch64/AArch64StorePairSuppress.cpp | 9 +- lib/Target/AArch64/AArch64Subtarget.cpp | 8 +- lib/Target/AArch64/AArch64Subtarget.h | 40 +- lib/Target/AArch64/AArch64SystemOperands.td | 8 +- lib/Target/AArch64/AArch64TargetMachine.cpp | 37 +- lib/Target/AArch64/AArch64TargetMachine.h | 7 +- lib/Target/AArch64/AArch64TargetObjectFile.cpp | 7 +- lib/Target/AArch64/AArch64TargetObjectFile.h | 7 +- lib/Target/AArch64/AArch64TargetTransformInfo.cpp | 15 +- lib/Target/AArch64/AArch64TargetTransformInfo.h | 11 +- lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp | 102 +- .../AArch64/Disassembler/AArch64Disassembler.cpp | 49 +- .../AArch64/Disassembler/AArch64Disassembler.h | 7 +- .../Disassembler/AArch64ExternalSymbolizer.cpp | 7 +- .../Disassembler/AArch64ExternalSymbolizer.h | 7 +- .../AArch64/InstPrinter/AArch64InstPrinter.cpp | 1582 ---- .../AArch64/InstPrinter/AArch64InstPrinter.h | 223 - .../AArch64/MCTargetDesc/AArch64AddressingModes.h | 7 +- .../AArch64/MCTargetDesc/AArch64AsmBackend.cpp | 54 +- .../MCTargetDesc/AArch64ELFObjectWriter.cpp | 9 +- .../AArch64/MCTargetDesc/AArch64ELFStreamer.cpp | 11 +- .../AArch64/MCTargetDesc/AArch64ELFStreamer.h | 7 +- .../AArch64/MCTargetDesc/AArch64FixupKinds.h | 7 +- .../AArch64/MCTargetDesc/AArch64InstPrinter.cpp | 1587 ++++ .../AArch64/MCTargetDesc/AArch64InstPrinter.h | 222 + .../AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp | 11 +- lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.h | 7 +- .../AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp | 14 +- lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.cpp | 10 +- lib/Target/AArch64/MCTargetDesc/AArch64MCExpr.h | 9 +- .../AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp | 203 +- .../AArch64/MCTargetDesc/AArch64MCTargetDesc.h | 14 +- .../MCTargetDesc/AArch64MachObjectWriter.cpp | 17 +- .../AArch64/MCTargetDesc/AArch64TargetStreamer.cpp | 8 +- .../AArch64/MCTargetDesc/AArch64TargetStreamer.h | 7 +- .../MCTargetDesc/AArch64WinCOFFObjectWriter.cpp | 7 +- .../MCTargetDesc/AArch64WinCOFFStreamer.cpp | 7 +- .../AArch64/MCTargetDesc/AArch64WinCOFFStreamer.h | 7 +- lib/Target/AArch64/SVEInstrFormats.td | 1340 ++- .../AArch64/TargetInfo/AArch64TargetInfo.cpp | 33 +- lib/Target/AArch64/TargetInfo/AArch64TargetInfo.h | 24 + lib/Target/AArch64/Utils/AArch64BaseInfo.cpp | 7 +- lib/Target/AArch64/Utils/AArch64BaseInfo.h | 50 +- lib/Target/AMDGPU/AMDGPU.h | 52 +- lib/Target/AMDGPU/AMDGPU.td | 570 +- lib/Target/AMDGPU/AMDGPUAliasAnalysis.cpp | 41 +- lib/Target/AMDGPU/AMDGPUAliasAnalysis.h | 13 +- lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp | 7 +- lib/Target/AMDGPU/AMDGPUAnnotateKernelFeatures.cpp | 75 +- lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp | 8 +- lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.cpp | 19 +- lib/Target/AMDGPU/AMDGPUArgumentUsageInfo.h | 43 +- lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp | 339 +- lib/Target/AMDGPU/AMDGPUAsmPrinter.h | 17 +- lib/Target/AMDGPU/AMDGPUAtomicOptimizer.cpp | 314 +- lib/Target/AMDGPU/AMDGPUCallLowering.cpp | 362 +- lib/Target/AMDGPU/AMDGPUCallLowering.h | 20 +- lib/Target/AMDGPU/AMDGPUCallingConv.td | 49 +- lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 136 +- lib/Target/AMDGPU/AMDGPUFeatures.td | 18 +- lib/Target/AMDGPU/AMDGPUFixFunctionBitcasts.cpp | 7 +- lib/Target/AMDGPU/AMDGPUFrameLowering.cpp | 7 +- lib/Target/AMDGPU/AMDGPUFrameLowering.h | 7 +- lib/Target/AMDGPU/AMDGPUGISel.td | 55 +- lib/Target/AMDGPU/AMDGPUGenRegisterBankInfo.def | 113 +- lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.cpp | 220 +- lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h | 41 +- lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 802 +- lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 363 +- lib/Target/AMDGPU/AMDGPUISelLowering.h | 73 +- lib/Target/AMDGPU/AMDGPUInline.cpp | 45 +- lib/Target/AMDGPU/AMDGPUInstrInfo.cpp | 7 +- lib/Target/AMDGPU/AMDGPUInstrInfo.h | 7 +- lib/Target/AMDGPU/AMDGPUInstrInfo.td | 46 +- lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp | 1469 ++- lib/Target/AMDGPU/AMDGPUInstructionSelector.h | 55 +- lib/Target/AMDGPU/AMDGPUInstructions.td | 267 +- lib/Target/AMDGPU/AMDGPUIntrinsicInfo.cpp | 103 - lib/Target/AMDGPU/AMDGPUIntrinsicInfo.h | 58 - lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 1357 ++- lib/Target/AMDGPU/AMDGPULegalizerInfo.h | 50 +- lib/Target/AMDGPU/AMDGPULibCalls.cpp | 151 +- lib/Target/AMDGPU/AMDGPULibFunc.cpp | 62 +- lib/Target/AMDGPU/AMDGPULibFunc.h | 11 +- lib/Target/AMDGPU/AMDGPULowerIntrinsics.cpp | 7 +- lib/Target/AMDGPU/AMDGPULowerKernelArguments.cpp | 38 +- lib/Target/AMDGPU/AMDGPULowerKernelAttributes.cpp | 7 +- lib/Target/AMDGPU/AMDGPUMCInstLower.cpp | 48 +- lib/Target/AMDGPU/AMDGPUMachineCFGStructurizer.cpp | 7 +- lib/Target/AMDGPU/AMDGPUMachineFunction.cpp | 21 +- lib/Target/AMDGPU/AMDGPUMachineFunction.h | 7 +- lib/Target/AMDGPU/AMDGPUMachineModuleInfo.cpp | 17 +- lib/Target/AMDGPU/AMDGPUMachineModuleInfo.h | 80 +- lib/Target/AMDGPU/AMDGPUMacroFusion.cpp | 7 +- lib/Target/AMDGPU/AMDGPUMacroFusion.h | 7 +- .../AMDGPU/AMDGPUOpenCLEnqueuedBlockLowering.cpp | 11 +- lib/Target/AMDGPU/AMDGPUPTNote.h | 7 +- lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.cpp | 77 +- lib/Target/AMDGPU/AMDGPUPerfHintAnalysis.h | 17 +- lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 36 +- lib/Target/AMDGPU/AMDGPUPropagateAttributes.cpp | 336 + lib/Target/AMDGPU/AMDGPURegAsmNames.inc.cpp | 353 - lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp | 1782 +++- lib/Target/AMDGPU/AMDGPURegisterBankInfo.h | 52 +- lib/Target/AMDGPU/AMDGPURegisterBanks.td | 9 +- lib/Target/AMDGPU/AMDGPURegisterInfo.cpp | 27 +- lib/Target/AMDGPU/AMDGPURegisterInfo.h | 7 +- lib/Target/AMDGPU/AMDGPURegisterInfo.td | 9 +- lib/Target/AMDGPU/AMDGPURewriteOutArguments.cpp | 7 +- lib/Target/AMDGPU/AMDGPUSearchableTables.td | 60 +- lib/Target/AMDGPU/AMDGPUSubtarget.cpp | 263 +- lib/Target/AMDGPU/AMDGPUSubtarget.h | 311 +- lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 307 +- lib/Target/AMDGPU/AMDGPUTargetMachine.h | 21 +- lib/Target/AMDGPU/AMDGPUTargetObjectFile.cpp | 7 +- lib/Target/AMDGPU/AMDGPUTargetObjectFile.h | 7 +- lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp | 38 +- lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h | 21 +- .../AMDGPU/AMDGPUUnifyDivergentExitNodes.cpp | 18 +- lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp | 7 +- lib/Target/AMDGPU/AMDILCFGStructurizer.cpp | 7 +- lib/Target/AMDGPU/AMDKernelCodeT.h | 15 +- lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp | 2828 ++++-- lib/Target/AMDGPU/BUFInstructions.td | 957 +- lib/Target/AMDGPU/CaymanInstructions.td | 7 +- lib/Target/AMDGPU/DSInstructions.td | 566 +- .../AMDGPU/Disassembler/AMDGPUDisassembler.cpp | 485 +- .../AMDGPU/Disassembler/AMDGPUDisassembler.h | 32 +- lib/Target/AMDGPU/EvergreenInstructions.td | 7 +- lib/Target/AMDGPU/FLATInstructions.td | 527 +- lib/Target/AMDGPU/GCNDPPCombine.cpp | 259 +- lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 826 +- lib/Target/AMDGPU/GCNHazardRecognizer.h | 41 +- lib/Target/AMDGPU/GCNILPSched.cpp | 7 +- lib/Target/AMDGPU/GCNIterativeScheduler.cpp | 7 +- lib/Target/AMDGPU/GCNIterativeScheduler.h | 7 +- lib/Target/AMDGPU/GCNMinRegStrategy.cpp | 7 +- lib/Target/AMDGPU/GCNNSAReassign.cpp | 343 + lib/Target/AMDGPU/GCNProcessors.td | 114 +- lib/Target/AMDGPU/GCNRegBankReassign.cpp | 800 ++ lib/Target/AMDGPU/GCNRegPressure.cpp | 22 +- lib/Target/AMDGPU/GCNRegPressure.h | 61 +- lib/Target/AMDGPU/GCNSchedStrategy.cpp | 35 +- lib/Target/AMDGPU/GCNSchedStrategy.h | 16 +- .../AMDGPU/InstPrinter/AMDGPUInstPrinter.cpp | 1413 --- lib/Target/AMDGPU/InstPrinter/AMDGPUInstPrinter.h | 250 - .../AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp | 65 +- .../AMDGPU/MCTargetDesc/AMDGPUELFObjectWriter.cpp | 21 +- .../AMDGPU/MCTargetDesc/AMDGPUELFStreamer.cpp | 7 +- lib/Target/AMDGPU/MCTargetDesc/AMDGPUELFStreamer.h | 7 +- lib/Target/AMDGPU/MCTargetDesc/AMDGPUFixupKinds.h | 7 +- .../AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp | 1568 ++++ lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h | 268 + lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.cpp | 29 +- lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCAsmInfo.h | 8 +- .../AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp | 7 +- .../AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.h | 20 +- .../AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.cpp | 41 +- .../AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.h | 12 +- .../AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp | 218 +- .../AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h | 40 +- .../AMDGPU/MCTargetDesc/R600MCCodeEmitter.cpp | 14 +- .../AMDGPU/MCTargetDesc/R600MCTargetDesc.cpp | 7 +- lib/Target/AMDGPU/MCTargetDesc/SIMCCodeEmitter.cpp | 84 +- lib/Target/AMDGPU/MIMGInstructions.td | 484 +- lib/Target/AMDGPU/R600.td | 7 +- lib/Target/AMDGPU/R600AsmPrinter.cpp | 7 +- lib/Target/AMDGPU/R600AsmPrinter.h | 7 +- lib/Target/AMDGPU/R600ClauseMergePass.cpp | 7 +- lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp | 7 +- lib/Target/AMDGPU/R600Defines.h | 7 +- lib/Target/AMDGPU/R600EmitClauseMarkers.cpp | 7 +- lib/Target/AMDGPU/R600ExpandSpecialInstrs.cpp | 7 +- lib/Target/AMDGPU/R600FrameLowering.cpp | 7 +- lib/Target/AMDGPU/R600FrameLowering.h | 7 +- lib/Target/AMDGPU/R600ISelLowering.cpp | 37 +- lib/Target/AMDGPU/R600ISelLowering.h | 14 +- lib/Target/AMDGPU/R600InstrFormats.td | 7 +- lib/Target/AMDGPU/R600InstrInfo.cpp | 8 +- lib/Target/AMDGPU/R600InstrInfo.h | 7 +- lib/Target/AMDGPU/R600Instructions.td | 35 +- lib/Target/AMDGPU/R600MachineFunctionInfo.cpp | 7 +- lib/Target/AMDGPU/R600MachineFunctionInfo.h | 7 +- lib/Target/AMDGPU/R600MachineScheduler.cpp | 7 +- lib/Target/AMDGPU/R600MachineScheduler.h | 7 +- .../AMDGPU/R600OpenCLImageTypeLoweringPass.cpp | 7 +- lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp | 22 +- lib/Target/AMDGPU/R600Packetizer.cpp | 11 +- lib/Target/AMDGPU/R600Processors.td | 18 +- lib/Target/AMDGPU/R600RegisterInfo.cpp | 9 +- lib/Target/AMDGPU/R600RegisterInfo.h | 9 +- lib/Target/AMDGPU/R600Schedule.td | 7 +- lib/Target/AMDGPU/R700Instructions.td | 7 +- lib/Target/AMDGPU/SIAddIMGInit.cpp | 7 +- lib/Target/AMDGPU/SIAnnotateControlFlow.cpp | 64 +- lib/Target/AMDGPU/SIDebuggerInsertNops.cpp | 97 - lib/Target/AMDGPU/SIDefines.h | 178 +- lib/Target/AMDGPU/SIFixSGPRCopies.cpp | 83 +- lib/Target/AMDGPU/SIFixVGPRCopies.cpp | 7 +- lib/Target/AMDGPU/SIFixWWMLiveness.cpp | 418 - lib/Target/AMDGPU/SIFixupVectorISel.cpp | 12 +- lib/Target/AMDGPU/SIFoldOperands.cpp | 363 +- lib/Target/AMDGPU/SIFormMemoryClauses.cpp | 22 +- lib/Target/AMDGPU/SIFrameLowering.cpp | 810 +- lib/Target/AMDGPU/SIFrameLowering.h | 28 +- lib/Target/AMDGPU/SIISelLowering.cpp | 1918 +++- lib/Target/AMDGPU/SIISelLowering.h | 49 +- lib/Target/AMDGPU/SIInsertSkips.cpp | 76 +- lib/Target/AMDGPU/SIInsertWaitcnts.cpp | 417 +- lib/Target/AMDGPU/SIInstrFormats.td | 68 +- lib/Target/AMDGPU/SIInstrInfo.cpp | 1415 ++- lib/Target/AMDGPU/SIInstrInfo.h | 125 +- lib/Target/AMDGPU/SIInstrInfo.td | 654 +- lib/Target/AMDGPU/SIInstructions.td | 425 +- lib/Target/AMDGPU/SIIntrinsics.td | 19 - lib/Target/AMDGPU/SILoadStoreOptimizer.cpp | 60 +- lib/Target/AMDGPU/SILowerControlFlow.cpp | 104 +- lib/Target/AMDGPU/SILowerI1Copies.cpp | 107 +- lib/Target/AMDGPU/SILowerSGPRSpills.cpp | 323 + lib/Target/AMDGPU/SIMachineFunctionInfo.cpp | 271 +- lib/Target/AMDGPU/SIMachineFunctionInfo.h | 377 +- lib/Target/AMDGPU/SIMachineScheduler.cpp | 11 +- lib/Target/AMDGPU/SIMachineScheduler.h | 7 +- lib/Target/AMDGPU/SIMemoryLegalizer.cpp | 322 +- lib/Target/AMDGPU/SIModeRegister.cpp | 9 +- lib/Target/AMDGPU/SIOptimizeExecMasking.cpp | 98 +- lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp | 155 +- lib/Target/AMDGPU/SIPeepholeSDWA.cpp | 36 +- lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp | 221 + lib/Target/AMDGPU/SIProgramInfo.h | 21 +- lib/Target/AMDGPU/SIRegisterInfo.cpp | 660 +- lib/Target/AMDGPU/SIRegisterInfo.h | 78 +- lib/Target/AMDGPU/SIRegisterInfo.td | 633 +- lib/Target/AMDGPU/SISchedule.td | 71 +- lib/Target/AMDGPU/SIShrinkInstructions.cpp | 140 +- lib/Target/AMDGPU/SIWholeQuadMode.cpp | 82 +- lib/Target/AMDGPU/SMInstructions.td | 359 +- lib/Target/AMDGPU/SOPInstructions.td | 666 +- lib/Target/AMDGPU/TargetInfo/AMDGPUTargetInfo.cpp | 9 +- lib/Target/AMDGPU/TargetInfo/AMDGPUTargetInfo.h | 29 + lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp | 36 +- lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.h | 14 +- lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 410 +- lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 203 +- lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp | 723 ++ lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.h | 135 + lib/Target/AMDGPU/Utils/AMDKernelCodeTInfo.h | 11 +- lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.cpp | 7 +- lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.h | 7 +- lib/Target/AMDGPU/VIInstrFormats.td | 7 +- lib/Target/AMDGPU/VIInstructions.td | 7 +- lib/Target/AMDGPU/VOP1Instructions.td | 487 +- lib/Target/AMDGPU/VOP2Instructions.td | 889 +- lib/Target/AMDGPU/VOP3Instructions.td | 501 +- lib/Target/AMDGPU/VOP3PInstructions.td | 220 +- lib/Target/AMDGPU/VOPCInstructions.td | 972 +- lib/Target/AMDGPU/VOPInstructions.td | 182 +- lib/Target/ARC/ARC.h | 8 +- lib/Target/ARC/ARC.td | 7 +- lib/Target/ARC/ARCAsmPrinter.cpp | 26 +- lib/Target/ARC/ARCBranchFinalize.cpp | 7 +- lib/Target/ARC/ARCCallingConv.td | 7 +- lib/Target/ARC/ARCExpandPseudos.cpp | 7 +- lib/Target/ARC/ARCFrameLowering.cpp | 59 +- lib/Target/ARC/ARCFrameLowering.h | 7 +- lib/Target/ARC/ARCISelDAGToDAG.cpp | 7 +- lib/Target/ARC/ARCISelLowering.cpp | 7 +- lib/Target/ARC/ARCISelLowering.h | 7 +- lib/Target/ARC/ARCInstrFormats.td | 71 +- lib/Target/ARC/ARCInstrInfo.cpp | 54 +- lib/Target/ARC/ARCInstrInfo.h | 17 +- lib/Target/ARC/ARCInstrInfo.td | 122 +- lib/Target/ARC/ARCMCInstLower.cpp | 7 +- lib/Target/ARC/ARCMCInstLower.h | 7 +- lib/Target/ARC/ARCMachineFunctionInfo.cpp | 7 +- lib/Target/ARC/ARCMachineFunctionInfo.h | 7 +- lib/Target/ARC/ARCOptAddrMode.cpp | 507 ++ lib/Target/ARC/ARCRegisterInfo.cpp | 15 +- lib/Target/ARC/ARCRegisterInfo.h | 9 +- lib/Target/ARC/ARCRegisterInfo.td | 7 +- lib/Target/ARC/ARCSubtarget.cpp | 7 +- lib/Target/ARC/ARCSubtarget.h | 7 +- lib/Target/ARC/ARCTargetMachine.cpp | 13 +- lib/Target/ARC/ARCTargetMachine.h | 7 +- lib/Target/ARC/ARCTargetStreamer.h | 7 +- lib/Target/ARC/ARCTargetTransformInfo.h | 7 +- lib/Target/ARC/Disassembler/ARCDisassembler.cpp | 8 +- lib/Target/ARC/InstPrinter/ARCInstPrinter.cpp | 180 - lib/Target/ARC/InstPrinter/ARCInstPrinter.h | 46 - lib/Target/ARC/MCTargetDesc/ARCInfo.h | 7 +- lib/Target/ARC/MCTargetDesc/ARCInstPrinter.cpp | 179 + lib/Target/ARC/MCTargetDesc/ARCInstPrinter.h | 45 + lib/Target/ARC/MCTargetDesc/ARCMCAsmInfo.cpp | 7 +- lib/Target/ARC/MCTargetDesc/ARCMCAsmInfo.h | 7 +- lib/Target/ARC/MCTargetDesc/ARCMCTargetDesc.cpp | 11 +- lib/Target/ARC/MCTargetDesc/ARCMCTargetDesc.h | 9 +- lib/Target/ARC/TargetInfo/ARCTargetInfo.cpp | 9 +- lib/Target/ARC/TargetInfo/ARCTargetInfo.h | 20 + lib/Target/ARM/A15SDOptimizer.cpp | 7 +- lib/Target/ARM/ARM.h | 18 +- lib/Target/ARM/ARM.td | 185 +- lib/Target/ARM/ARMAsmPrinter.cpp | 153 +- lib/Target/ARM/ARMAsmPrinter.h | 14 +- lib/Target/ARM/ARMBaseInstrInfo.cpp | 412 +- lib/Target/ARM/ARMBaseInstrInfo.h | 72 +- lib/Target/ARM/ARMBaseRegisterInfo.cpp | 51 +- lib/Target/ARM/ARMBaseRegisterInfo.h | 9 +- lib/Target/ARM/ARMBasicBlockInfo.cpp | 146 + lib/Target/ARM/ARMBasicBlockInfo.h | 59 +- lib/Target/ARM/ARMCallLowering.cpp | 176 +- lib/Target/ARM/ARMCallLowering.h | 20 +- lib/Target/ARM/ARMCallingConv.cpp | 284 + lib/Target/ARM/ARMCallingConv.h | 308 +- lib/Target/ARM/ARMCallingConv.td | 52 +- lib/Target/ARM/ARMCodeGenPrepare.cpp | 205 +- lib/Target/ARM/ARMComputeBlockSize.cpp | 81 - lib/Target/ARM/ARMConstantIslandPass.cpp | 246 +- lib/Target/ARM/ARMConstantPoolValue.cpp | 7 +- lib/Target/ARM/ARMConstantPoolValue.h | 7 +- lib/Target/ARM/ARMExpandPseudoInsts.cpp | 28 +- lib/Target/ARM/ARMFastISel.cpp | 53 +- lib/Target/ARM/ARMFeatures.h | 7 +- lib/Target/ARM/ARMFrameLowering.cpp | 117 +- lib/Target/ARM/ARMFrameLowering.h | 7 +- lib/Target/ARM/ARMHazardRecognizer.cpp | 7 +- lib/Target/ARM/ARMHazardRecognizer.h | 7 +- lib/Target/ARM/ARMISelDAGToDAG.cpp | 213 +- lib/Target/ARM/ARMISelLowering.cpp | 1556 +++- lib/Target/ARM/ARMISelLowering.h | 101 +- lib/Target/ARM/ARMInstrFormats.td | 115 +- lib/Target/ARM/ARMInstrInfo.cpp | 9 +- lib/Target/ARM/ARMInstrInfo.h | 7 +- lib/Target/ARM/ARMInstrInfo.td | 380 +- lib/Target/ARM/ARMInstrMVE.td | 4591 ++++++++++ lib/Target/ARM/ARMInstrNEON.td | 1093 ++- lib/Target/ARM/ARMInstrThumb.td | 75 +- lib/Target/ARM/ARMInstrThumb2.td | 487 +- lib/Target/ARM/ARMInstrVFP.td | 367 +- lib/Target/ARM/ARMInstructionSelector.cpp | 268 +- lib/Target/ARM/ARMLegalizerInfo.cpp | 161 +- lib/Target/ARM/ARMLegalizerInfo.h | 7 +- lib/Target/ARM/ARMLoadStoreOptimizer.cpp | 149 +- lib/Target/ARM/ARMLowOverheadLoops.cpp | 384 + lib/Target/ARM/ARMMCInstLower.cpp | 7 +- lib/Target/ARM/ARMMachineFunctionInfo.cpp | 7 +- lib/Target/ARM/ARMMachineFunctionInfo.h | 16 +- lib/Target/ARM/ARMMacroFusion.cpp | 7 +- lib/Target/ARM/ARMMacroFusion.h | 7 +- lib/Target/ARM/ARMOptimizeBarriersPass.cpp | 7 +- lib/Target/ARM/ARMParallelDSP.cpp | 889 +- lib/Target/ARM/ARMPerfectShuffle.h | 7 +- lib/Target/ARM/ARMPredicates.td | 211 + lib/Target/ARM/ARMRegisterBankInfo.cpp | 51 +- lib/Target/ARM/ARMRegisterBankInfo.h | 7 +- lib/Target/ARM/ARMRegisterBanks.td | 7 +- lib/Target/ARM/ARMRegisterInfo.cpp | 7 +- lib/Target/ARM/ARMRegisterInfo.h | 7 +- lib/Target/ARM/ARMRegisterInfo.td | 132 +- lib/Target/ARM/ARMSchedule.td | 9 +- lib/Target/ARM/ARMScheduleA57.td | 13 +- lib/Target/ARM/ARMScheduleA57WriteRes.td | 7 +- lib/Target/ARM/ARMScheduleA8.td | 7 +- lib/Target/ARM/ARMScheduleA9.td | 7 +- lib/Target/ARM/ARMScheduleM3.td | 21 - lib/Target/ARM/ARMScheduleM4.td | 119 + lib/Target/ARM/ARMScheduleR52.td | 7 +- lib/Target/ARM/ARMScheduleSwift.td | 7 +- lib/Target/ARM/ARMScheduleV6.td | 7 +- lib/Target/ARM/ARMSelectionDAGInfo.cpp | 9 +- lib/Target/ARM/ARMSelectionDAGInfo.h | 7 +- lib/Target/ARM/ARMSubtarget.cpp | 73 +- lib/Target/ARM/ARMSubtarget.h | 78 +- lib/Target/ARM/ARMSystemRegister.td | 7 +- lib/Target/ARM/ARMTargetMachine.cpp | 43 +- lib/Target/ARM/ARMTargetMachine.h | 7 +- lib/Target/ARM/ARMTargetObjectFile.cpp | 7 +- lib/Target/ARM/ARMTargetObjectFile.h | 7 +- lib/Target/ARM/ARMTargetTransformInfo.cpp | 275 +- lib/Target/ARM/ARMTargetTransformInfo.h | 23 +- lib/Target/ARM/AsmParser/ARMAsmParser.cpp | 1739 +++- lib/Target/ARM/Disassembler/ARMDisassembler.cpp | 1391 ++- lib/Target/ARM/InstPrinter/ARMInstPrinter.cpp | 1571 ---- lib/Target/ARM/InstPrinter/ARMInstPrinter.h | 243 - lib/Target/ARM/LICENSE.TXT | 47 - lib/Target/ARM/MCTargetDesc/ARMAddressingModes.h | 11 +- lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp | 142 +- lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h | 9 +- lib/Target/ARM/MCTargetDesc/ARMAsmBackendDarwin.h | 7 +- lib/Target/ARM/MCTargetDesc/ARMAsmBackendELF.h | 7 +- lib/Target/ARM/MCTargetDesc/ARMAsmBackendWinCOFF.h | 7 +- lib/Target/ARM/MCTargetDesc/ARMBaseInfo.h | 18 +- lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp | 15 +- lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp | 11 +- lib/Target/ARM/MCTargetDesc/ARMFixupKinds.h | 16 +- lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp | 1678 ++++ lib/Target/ARM/MCTargetDesc/ARMInstPrinter.h | 272 + lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp | 7 +- lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.h | 7 +- lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp | 459 +- lib/Target/ARM/MCTargetDesc/ARMMCExpr.cpp | 7 +- lib/Target/ARM/MCTargetDesc/ARMMCExpr.h | 7 +- lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp | 35 +- lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h | 27 +- .../ARM/MCTargetDesc/ARMMachORelocationInfo.cpp | 7 +- .../ARM/MCTargetDesc/ARMMachObjectWriter.cpp | 7 +- lib/Target/ARM/MCTargetDesc/ARMTargetStreamer.cpp | 62 +- lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp | 7 +- lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.h | 7 +- .../ARM/MCTargetDesc/ARMWinCOFFObjectWriter.cpp | 7 +- lib/Target/ARM/MCTargetDesc/ARMWinCOFFStreamer.cpp | 7 +- lib/Target/ARM/MLxExpansionPass.cpp | 7 +- lib/Target/ARM/TargetInfo/ARMTargetInfo.cpp | 9 +- lib/Target/ARM/TargetInfo/ARMTargetInfo.h | 23 + lib/Target/ARM/Thumb1FrameLowering.cpp | 120 +- lib/Target/ARM/Thumb1FrameLowering.h | 7 +- lib/Target/ARM/Thumb1InstrInfo.cpp | 7 +- lib/Target/ARM/Thumb1InstrInfo.h | 7 +- lib/Target/ARM/Thumb2ITBlockPass.cpp | 221 +- lib/Target/ARM/Thumb2InstrInfo.cpp | 58 +- lib/Target/ARM/Thumb2InstrInfo.h | 13 +- lib/Target/ARM/Thumb2SizeReduction.cpp | 13 +- lib/Target/ARM/ThumbRegisterInfo.cpp | 75 +- lib/Target/ARM/ThumbRegisterInfo.h | 13 +- lib/Target/ARM/Utils/ARMBaseInfo.cpp | 7 +- lib/Target/ARM/Utils/ARMBaseInfo.h | 31 +- lib/Target/AVR/AVR.h | 7 +- lib/Target/AVR/AVR.td | 7 +- lib/Target/AVR/AVRAsmPrinter.cpp | 29 +- lib/Target/AVR/AVRCallingConv.td | 7 +- lib/Target/AVR/AVRExpandPseudoInsts.cpp | 17 +- lib/Target/AVR/AVRFrameLowering.cpp | 12 +- lib/Target/AVR/AVRFrameLowering.h | 7 +- lib/Target/AVR/AVRISelDAGToDAG.cpp | 7 +- lib/Target/AVR/AVRISelLowering.cpp | 55 +- lib/Target/AVR/AVRISelLowering.h | 20 +- lib/Target/AVR/AVRInstrFormats.td | 7 +- lib/Target/AVR/AVRInstrInfo.cpp | 10 +- lib/Target/AVR/AVRInstrInfo.h | 7 +- lib/Target/AVR/AVRInstrInfo.td | 53 +- lib/Target/AVR/AVRMCInstLower.cpp | 7 +- lib/Target/AVR/AVRMCInstLower.h | 7 +- lib/Target/AVR/AVRMachineFunctionInfo.h | 7 +- lib/Target/AVR/AVRRegisterInfo.cpp | 30 +- lib/Target/AVR/AVRRegisterInfo.h | 16 +- lib/Target/AVR/AVRRegisterInfo.td | 11 +- lib/Target/AVR/AVRRelaxMemOperations.cpp | 7 +- lib/Target/AVR/AVRSelectionDAGInfo.h | 7 +- lib/Target/AVR/AVRSubtarget.cpp | 19 +- lib/Target/AVR/AVRSubtarget.h | 12 +- lib/Target/AVR/AVRTargetMachine.cpp | 8 +- lib/Target/AVR/AVRTargetMachine.h | 7 +- lib/Target/AVR/AVRTargetObjectFile.cpp | 7 +- lib/Target/AVR/AVRTargetObjectFile.h | 7 +- lib/Target/AVR/AsmParser/AVRAsmParser.cpp | 24 +- lib/Target/AVR/Disassembler/AVRDisassembler.cpp | 8 +- lib/Target/AVR/InstPrinter/AVRInstPrinter.cpp | 171 - lib/Target/AVR/InstPrinter/AVRInstPrinter.h | 54 - lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp | 7 +- lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h | 7 +- lib/Target/AVR/MCTargetDesc/AVRELFObjectWriter.cpp | 7 +- lib/Target/AVR/MCTargetDesc/AVRELFStreamer.h | 7 +- lib/Target/AVR/MCTargetDesc/AVRFixupKinds.h | 7 +- lib/Target/AVR/MCTargetDesc/AVRInstPrinter.cpp | 170 + lib/Target/AVR/MCTargetDesc/AVRInstPrinter.h | 53 + lib/Target/AVR/MCTargetDesc/AVRMCAsmInfo.cpp | 8 +- lib/Target/AVR/MCTargetDesc/AVRMCAsmInfo.h | 7 +- lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp | 7 +- lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.h | 7 +- lib/Target/AVR/MCTargetDesc/AVRMCELFStreamer.cpp | 7 +- lib/Target/AVR/MCTargetDesc/AVRMCELFStreamer.h | 7 +- lib/Target/AVR/MCTargetDesc/AVRMCExpr.cpp | 7 +- lib/Target/AVR/MCTargetDesc/AVRMCExpr.h | 7 +- lib/Target/AVR/MCTargetDesc/AVRMCTargetDesc.cpp | 10 +- lib/Target/AVR/MCTargetDesc/AVRMCTargetDesc.h | 9 +- lib/Target/AVR/MCTargetDesc/AVRTargetStreamer.cpp | 7 +- lib/Target/AVR/MCTargetDesc/AVRTargetStreamer.h | 7 +- lib/Target/AVR/TargetInfo/AVRTargetInfo.cpp | 9 +- lib/Target/AVR/TargetInfo/AVRTargetInfo.h | 18 + lib/Target/BPF/AsmParser/BPFAsmParser.cpp | 10 +- lib/Target/BPF/BPF.h | 12 +- lib/Target/BPF/BPF.td | 8 +- lib/Target/BPF/BPFAbstractMemberAccess.cpp | 482 + lib/Target/BPF/BPFAsmPrinter.cpp | 42 +- lib/Target/BPF/BPFCORE.h | 24 + lib/Target/BPF/BPFCallingConv.td | 7 +- lib/Target/BPF/BPFFrameLowering.cpp | 7 +- lib/Target/BPF/BPFFrameLowering.h | 7 +- lib/Target/BPF/BPFISelDAGToDAG.cpp | 7 +- lib/Target/BPF/BPFISelLowering.cpp | 64 +- lib/Target/BPF/BPFISelLowering.h | 11 +- lib/Target/BPF/BPFInstrFormats.td | 8 +- lib/Target/BPF/BPFInstrInfo.cpp | 7 +- lib/Target/BPF/BPFInstrInfo.h | 7 +- lib/Target/BPF/BPFInstrInfo.td | 111 +- lib/Target/BPF/BPFMCInstLower.cpp | 7 +- lib/Target/BPF/BPFMCInstLower.h | 7 +- lib/Target/BPF/BPFMIChecking.cpp | 104 +- lib/Target/BPF/BPFMIPeephole.cpp | 7 +- lib/Target/BPF/BPFMISimplifyPatchable.cpp | 163 + lib/Target/BPF/BPFRegisterInfo.cpp | 9 +- lib/Target/BPF/BPFRegisterInfo.h | 9 +- lib/Target/BPF/BPFRegisterInfo.td | 7 +- lib/Target/BPF/BPFSelectionDAGInfo.cpp | 7 +- lib/Target/BPF/BPFSelectionDAGInfo.h | 7 +- lib/Target/BPF/BPFSubtarget.cpp | 13 +- lib/Target/BPF/BPFSubtarget.h | 12 +- lib/Target/BPF/BPFTargetMachine.cpp | 20 +- lib/Target/BPF/BPFTargetMachine.h | 7 +- lib/Target/BPF/BTF.def | 9 +- lib/Target/BPF/BTF.h | 98 +- lib/Target/BPF/BTFDebug.cpp | 727 +- lib/Target/BPF/BTFDebug.h | 120 +- lib/Target/BPF/Disassembler/BPFDisassembler.cpp | 13 +- lib/Target/BPF/InstPrinter/BPFInstPrinter.cpp | 108 - lib/Target/BPF/InstPrinter/BPFInstPrinter.h | 41 - lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp | 19 +- lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp | 39 +- lib/Target/BPF/MCTargetDesc/BPFInstPrinter.cpp | 107 + lib/Target/BPF/MCTargetDesc/BPFInstPrinter.h | 40 + lib/Target/BPF/MCTargetDesc/BPFMCAsmInfo.h | 7 +- lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp | 14 +- lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.cpp | 11 +- lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h | 11 +- lib/Target/BPF/TargetInfo/BPFTargetInfo.cpp | 18 +- lib/Target/BPF/TargetInfo/BPFTargetInfo.h | 22 + lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp | 29 +- lib/Target/Hexagon/BitTracker.cpp | 7 +- lib/Target/Hexagon/BitTracker.h | 7 +- .../Hexagon/Disassembler/HexagonDisassembler.cpp | 10 +- lib/Target/Hexagon/Hexagon.h | 7 +- lib/Target/Hexagon/Hexagon.td | 7 +- lib/Target/Hexagon/HexagonAsmPrinter.cpp | 20 +- lib/Target/Hexagon/HexagonAsmPrinter.h | 14 +- lib/Target/Hexagon/HexagonBitSimplify.cpp | 7 +- lib/Target/Hexagon/HexagonBitTracker.cpp | 7 +- lib/Target/Hexagon/HexagonBitTracker.h | 7 +- lib/Target/Hexagon/HexagonBlockRanges.cpp | 7 +- lib/Target/Hexagon/HexagonBlockRanges.h | 7 +- lib/Target/Hexagon/HexagonBranchRelaxation.cpp | 7 +- lib/Target/Hexagon/HexagonCFGOptimizer.cpp | 7 +- lib/Target/Hexagon/HexagonCallingConv.td | 7 +- lib/Target/Hexagon/HexagonCommonGEP.cpp | 24 +- lib/Target/Hexagon/HexagonConstExtenders.cpp | 7 +- lib/Target/Hexagon/HexagonConstPropagation.cpp | 186 +- lib/Target/Hexagon/HexagonCopyToCombine.cpp | 11 +- lib/Target/Hexagon/HexagonDepArch.h | 7 +- lib/Target/Hexagon/HexagonDepArch.td | 7 +- lib/Target/Hexagon/HexagonDepDecoders.h | 79 - lib/Target/Hexagon/HexagonDepDecoders.inc | 78 + lib/Target/Hexagon/HexagonDepIICHVX.td | 7 +- lib/Target/Hexagon/HexagonDepIICScalar.td | 7 +- lib/Target/Hexagon/HexagonDepITypes.h | 7 +- lib/Target/Hexagon/HexagonDepITypes.td | 7 +- lib/Target/Hexagon/HexagonDepInstrFormats.td | 7 +- lib/Target/Hexagon/HexagonDepInstrInfo.td | 7 +- lib/Target/Hexagon/HexagonDepMapAsm2Intrin.td | 7 +- lib/Target/Hexagon/HexagonDepMappings.td | 7 +- lib/Target/Hexagon/HexagonDepOperands.td | 7 +- lib/Target/Hexagon/HexagonDepTimingClasses.h | 7 +- lib/Target/Hexagon/HexagonEarlyIfConv.cpp | 7 +- lib/Target/Hexagon/HexagonExpandCondsets.cpp | 9 +- lib/Target/Hexagon/HexagonFixupHwLoops.cpp | 7 +- lib/Target/Hexagon/HexagonFrameLowering.cpp | 15 +- lib/Target/Hexagon/HexagonFrameLowering.h | 7 +- lib/Target/Hexagon/HexagonGenExtract.cpp | 9 +- lib/Target/Hexagon/HexagonGenInsert.cpp | 11 +- lib/Target/Hexagon/HexagonGenMux.cpp | 11 +- lib/Target/Hexagon/HexagonGenPredicate.cpp | 73 +- lib/Target/Hexagon/HexagonHardwareLoops.cpp | 7 +- lib/Target/Hexagon/HexagonHazardRecognizer.cpp | 7 +- lib/Target/Hexagon/HexagonHazardRecognizer.h | 7 +- lib/Target/Hexagon/HexagonIICHVX.td | 19 +- lib/Target/Hexagon/HexagonIICScalar.td | 7 +- lib/Target/Hexagon/HexagonISelDAGToDAG.cpp | 12 +- lib/Target/Hexagon/HexagonISelDAGToDAG.h | 7 +- lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp | 7 +- lib/Target/Hexagon/HexagonISelLowering.cpp | 100 +- lib/Target/Hexagon/HexagonISelLowering.h | 15 +- lib/Target/Hexagon/HexagonISelLoweringHVX.cpp | 9 +- lib/Target/Hexagon/HexagonInstrFormats.td | 7 +- lib/Target/Hexagon/HexagonInstrFormatsV5.td | 7 +- lib/Target/Hexagon/HexagonInstrFormatsV60.td | 7 +- lib/Target/Hexagon/HexagonInstrFormatsV65.td | 7 +- lib/Target/Hexagon/HexagonInstrInfo.cpp | 62 +- lib/Target/Hexagon/HexagonInstrInfo.h | 21 +- lib/Target/Hexagon/HexagonIntrinsics.td | 7 +- lib/Target/Hexagon/HexagonIntrinsicsV5.td | 7 +- lib/Target/Hexagon/HexagonIntrinsicsV60.td | 7 +- lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp | 19 +- lib/Target/Hexagon/HexagonMCInstLower.cpp | 7 +- lib/Target/Hexagon/HexagonMachineFunctionInfo.cpp | 7 +- lib/Target/Hexagon/HexagonMachineFunctionInfo.h | 7 +- lib/Target/Hexagon/HexagonMachineScheduler.cpp | 9 +- lib/Target/Hexagon/HexagonMachineScheduler.h | 7 +- lib/Target/Hexagon/HexagonMapAsm2IntrinV62.gen.td | 7 +- lib/Target/Hexagon/HexagonMapAsm2IntrinV65.gen.td | 7 +- lib/Target/Hexagon/HexagonNewValueJump.cpp | 7 +- lib/Target/Hexagon/HexagonOperands.td | 7 +- lib/Target/Hexagon/HexagonOptAddrMode.cpp | 7 +- lib/Target/Hexagon/HexagonOptimizeSZextends.cpp | 7 +- lib/Target/Hexagon/HexagonPatterns.td | 11 +- lib/Target/Hexagon/HexagonPatternsV65.td | 7 +- lib/Target/Hexagon/HexagonPeephole.cpp | 7 +- lib/Target/Hexagon/HexagonPseudo.td | 12 +- lib/Target/Hexagon/HexagonRDFOpt.cpp | 7 +- lib/Target/Hexagon/HexagonRegisterInfo.cpp | 9 +- lib/Target/Hexagon/HexagonRegisterInfo.h | 9 +- lib/Target/Hexagon/HexagonRegisterInfo.td | 7 +- lib/Target/Hexagon/HexagonSchedule.td | 7 +- lib/Target/Hexagon/HexagonScheduleV5.td | 7 +- lib/Target/Hexagon/HexagonScheduleV55.td | 7 +- lib/Target/Hexagon/HexagonScheduleV60.td | 7 +- lib/Target/Hexagon/HexagonScheduleV62.td | 7 +- lib/Target/Hexagon/HexagonScheduleV65.td | 7 +- lib/Target/Hexagon/HexagonScheduleV66.td | 7 +- lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp | 7 +- lib/Target/Hexagon/HexagonSelectionDAGInfo.h | 7 +- .../Hexagon/HexagonSplitConst32AndConst64.cpp | 7 +- lib/Target/Hexagon/HexagonSplitDouble.cpp | 11 +- lib/Target/Hexagon/HexagonStoreWidening.cpp | 15 +- lib/Target/Hexagon/HexagonSubtarget.cpp | 7 +- lib/Target/Hexagon/HexagonSubtarget.h | 7 +- lib/Target/Hexagon/HexagonTargetMachine.cpp | 8 +- lib/Target/Hexagon/HexagonTargetMachine.h | 7 +- lib/Target/Hexagon/HexagonTargetObjectFile.cpp | 14 +- lib/Target/Hexagon/HexagonTargetObjectFile.h | 7 +- lib/Target/Hexagon/HexagonTargetStreamer.h | 7 +- lib/Target/Hexagon/HexagonTargetTransformInfo.cpp | 12 +- lib/Target/Hexagon/HexagonTargetTransformInfo.h | 7 +- lib/Target/Hexagon/HexagonVExtract.cpp | 7 +- lib/Target/Hexagon/HexagonVLIWPacketizer.cpp | 7 +- lib/Target/Hexagon/HexagonVLIWPacketizer.h | 7 +- .../Hexagon/HexagonVectorLoopCarriedReuse.cpp | 222 +- lib/Target/Hexagon/HexagonVectorPrint.cpp | 7 +- .../Hexagon/MCTargetDesc/HexagonAsmBackend.cpp | 8 +- lib/Target/Hexagon/MCTargetDesc/HexagonBaseInfo.h | 7 +- .../MCTargetDesc/HexagonELFObjectWriter.cpp | 9 +- .../Hexagon/MCTargetDesc/HexagonFixupKinds.h | 7 +- .../Hexagon/MCTargetDesc/HexagonInstPrinter.cpp | 8 +- .../Hexagon/MCTargetDesc/HexagonInstPrinter.h | 7 +- .../Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp | 7 +- lib/Target/Hexagon/MCTargetDesc/HexagonMCAsmInfo.h | 7 +- .../Hexagon/MCTargetDesc/HexagonMCChecker.cpp | 8 +- lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.h | 7 +- .../Hexagon/MCTargetDesc/HexagonMCCodeEmitter.cpp | 10 +- .../Hexagon/MCTargetDesc/HexagonMCCodeEmitter.h | 14 +- .../Hexagon/MCTargetDesc/HexagonMCCompound.cpp | 8 +- .../Hexagon/MCTargetDesc/HexagonMCDuplexInfo.cpp | 7 +- .../Hexagon/MCTargetDesc/HexagonMCELFStreamer.cpp | 9 +- .../Hexagon/MCTargetDesc/HexagonMCELFStreamer.h | 10 +- lib/Target/Hexagon/MCTargetDesc/HexagonMCExpr.cpp | 7 +- lib/Target/Hexagon/MCTargetDesc/HexagonMCExpr.h | 7 +- .../Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp | 8 +- .../Hexagon/MCTargetDesc/HexagonMCInstrInfo.h | 7 +- .../Hexagon/MCTargetDesc/HexagonMCShuffler.cpp | 8 +- .../Hexagon/MCTargetDesc/HexagonMCShuffler.h | 7 +- .../Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp | 9 +- .../Hexagon/MCTargetDesc/HexagonMCTargetDesc.h | 8 +- .../Hexagon/MCTargetDesc/HexagonShuffler.cpp | 9 +- lib/Target/Hexagon/MCTargetDesc/HexagonShuffler.h | 9 +- lib/Target/Hexagon/RDFCopy.cpp | 7 +- lib/Target/Hexagon/RDFCopy.h | 7 +- lib/Target/Hexagon/RDFDeadCode.cpp | 7 +- lib/Target/Hexagon/RDFDeadCode.h | 7 +- lib/Target/Hexagon/RDFGraph.cpp | 29 +- lib/Target/Hexagon/RDFGraph.h | 34 +- lib/Target/Hexagon/RDFLiveness.cpp | 8 +- lib/Target/Hexagon/RDFLiveness.h | 9 +- lib/Target/Hexagon/RDFRegisters.cpp | 7 +- lib/Target/Hexagon/RDFRegisters.h | 7 +- .../Hexagon/TargetInfo/HexagonTargetInfo.cpp | 10 +- lib/Target/Hexagon/TargetInfo/HexagonTargetInfo.h | 20 + lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp | 10 +- .../Lanai/Disassembler/LanaiDisassembler.cpp | 13 +- lib/Target/Lanai/Disassembler/LanaiDisassembler.h | 7 +- lib/Target/Lanai/InstPrinter/LanaiInstPrinter.cpp | 305 - lib/Target/Lanai/InstPrinter/LanaiInstPrinter.h | 66 - lib/Target/Lanai/Lanai.h | 15 +- lib/Target/Lanai/Lanai.td | 7 +- lib/Target/Lanai/LanaiAluCode.h | 7 +- lib/Target/Lanai/LanaiAsmPrinter.cpp | 19 +- lib/Target/Lanai/LanaiCallingConv.td | 7 +- lib/Target/Lanai/LanaiDelaySlotFiller.cpp | 7 +- lib/Target/Lanai/LanaiFrameLowering.cpp | 9 +- lib/Target/Lanai/LanaiFrameLowering.h | 8 +- lib/Target/Lanai/LanaiISelDAGToDAG.cpp | 9 +- lib/Target/Lanai/LanaiISelLowering.cpp | 7 +- lib/Target/Lanai/LanaiISelLowering.h | 7 +- lib/Target/Lanai/LanaiInstrFormats.td | 7 +- lib/Target/Lanai/LanaiInstrInfo.cpp | 24 +- lib/Target/Lanai/LanaiInstrInfo.h | 16 +- lib/Target/Lanai/LanaiInstrInfo.td | 7 +- lib/Target/Lanai/LanaiMCInstLower.cpp | 7 +- lib/Target/Lanai/LanaiMCInstLower.h | 7 +- lib/Target/Lanai/LanaiMachineFunctionInfo.cpp | 7 +- lib/Target/Lanai/LanaiMachineFunctionInfo.h | 7 +- lib/Target/Lanai/LanaiMemAluCombiner.cpp | 12 +- lib/Target/Lanai/LanaiRegisterInfo.cpp | 17 +- lib/Target/Lanai/LanaiRegisterInfo.h | 11 +- lib/Target/Lanai/LanaiRegisterInfo.td | 7 +- lib/Target/Lanai/LanaiSchedule.td | 7 +- lib/Target/Lanai/LanaiSelectionDAGInfo.cpp | 7 +- lib/Target/Lanai/LanaiSelectionDAGInfo.h | 7 +- lib/Target/Lanai/LanaiSubtarget.cpp | 7 +- lib/Target/Lanai/LanaiSubtarget.h | 7 +- lib/Target/Lanai/LanaiTargetMachine.cpp | 8 +- lib/Target/Lanai/LanaiTargetMachine.h | 7 +- lib/Target/Lanai/LanaiTargetObjectFile.cpp | 7 +- lib/Target/Lanai/LanaiTargetObjectFile.h | 7 +- lib/Target/Lanai/LanaiTargetTransformInfo.h | 7 +- lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp | 7 +- lib/Target/Lanai/MCTargetDesc/LanaiBaseInfo.h | 7 +- .../Lanai/MCTargetDesc/LanaiELFObjectWriter.cpp | 9 +- lib/Target/Lanai/MCTargetDesc/LanaiFixupKinds.h | 7 +- lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.cpp | 307 + lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.h | 65 + lib/Target/Lanai/MCTargetDesc/LanaiMCAsmInfo.cpp | 7 +- lib/Target/Lanai/MCTargetDesc/LanaiMCAsmInfo.h | 7 +- .../Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp | 9 +- lib/Target/Lanai/MCTargetDesc/LanaiMCExpr.cpp | 7 +- lib/Target/Lanai/MCTargetDesc/LanaiMCExpr.h | 7 +- .../Lanai/MCTargetDesc/LanaiMCTargetDesc.cpp | 10 +- lib/Target/Lanai/MCTargetDesc/LanaiMCTargetDesc.h | 9 +- lib/Target/Lanai/TargetInfo/LanaiTargetInfo.cpp | 13 +- lib/Target/Lanai/TargetInfo/LanaiTargetInfo.h | 20 + lib/Target/MSP430/AsmParser/MSP430AsmParser.cpp | 8 +- .../MSP430/Disassembler/MSP430Disassembler.cpp | 8 +- .../MSP430/InstPrinter/MSP430InstPrinter.cpp | 138 - lib/Target/MSP430/InstPrinter/MSP430InstPrinter.h | 50 - .../MSP430/MCTargetDesc/MSP430AsmBackend.cpp | 7 +- .../MSP430/MCTargetDesc/MSP430ELFObjectWriter.cpp | 7 +- .../MSP430/MCTargetDesc/MSP430ELFStreamer.cpp | 7 +- lib/Target/MSP430/MCTargetDesc/MSP430FixupKinds.h | 7 +- .../MSP430/MCTargetDesc/MSP430InstPrinter.cpp | 137 + lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.h | 49 + lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.cpp | 8 +- lib/Target/MSP430/MCTargetDesc/MSP430MCAsmInfo.h | 7 +- .../MSP430/MCTargetDesc/MSP430MCCodeEmitter.cpp | 7 +- .../MSP430/MCTargetDesc/MSP430MCTargetDesc.cpp | 10 +- .../MSP430/MCTargetDesc/MSP430MCTargetDesc.h | 9 +- lib/Target/MSP430/MSP430.h | 7 +- lib/Target/MSP430/MSP430.td | 7 +- lib/Target/MSP430/MSP430AsmPrinter.cpp | 85 +- lib/Target/MSP430/MSP430BranchSelector.cpp | 7 +- lib/Target/MSP430/MSP430CallingConv.td | 7 +- lib/Target/MSP430/MSP430FrameLowering.cpp | 7 +- lib/Target/MSP430/MSP430FrameLowering.h | 7 +- lib/Target/MSP430/MSP430ISelDAGToDAG.cpp | 7 +- lib/Target/MSP430/MSP430ISelLowering.cpp | 7 +- lib/Target/MSP430/MSP430ISelLowering.h | 7 +- lib/Target/MSP430/MSP430InstrFormats.td | 7 +- lib/Target/MSP430/MSP430InstrInfo.cpp | 10 +- lib/Target/MSP430/MSP430InstrInfo.h | 7 +- lib/Target/MSP430/MSP430InstrInfo.td | 7 +- lib/Target/MSP430/MSP430MCInstLower.cpp | 7 +- lib/Target/MSP430/MSP430MCInstLower.h | 7 +- lib/Target/MSP430/MSP430MachineFunctionInfo.cpp | 7 +- lib/Target/MSP430/MSP430MachineFunctionInfo.h | 7 +- lib/Target/MSP430/MSP430RegisterInfo.cpp | 9 +- lib/Target/MSP430/MSP430RegisterInfo.h | 9 +- lib/Target/MSP430/MSP430RegisterInfo.td | 7 +- lib/Target/MSP430/MSP430Subtarget.cpp | 7 +- lib/Target/MSP430/MSP430Subtarget.h | 7 +- lib/Target/MSP430/MSP430TargetMachine.cpp | 8 +- lib/Target/MSP430/MSP430TargetMachine.h | 7 +- lib/Target/MSP430/TargetInfo/MSP430TargetInfo.cpp | 10 +- lib/Target/MSP430/TargetInfo/MSP430TargetInfo.h | 20 + lib/Target/Mips/AsmParser/MipsAsmParser.cpp | 409 +- lib/Target/Mips/Disassembler/MipsDisassembler.cpp | 17 +- lib/Target/Mips/InstPrinter/MipsInstPrinter.cpp | 288 - lib/Target/Mips/InstPrinter/MipsInstPrinter.h | 113 - .../Mips/MCTargetDesc/MipsABIFlagsSection.cpp | 7 +- lib/Target/Mips/MCTargetDesc/MipsABIFlagsSection.h | 7 +- lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp | 14 +- lib/Target/Mips/MCTargetDesc/MipsABIInfo.h | 7 +- lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp | 11 +- lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h | 7 +- lib/Target/Mips/MCTargetDesc/MipsBaseInfo.h | 12 +- .../Mips/MCTargetDesc/MipsELFObjectWriter.cpp | 9 +- lib/Target/Mips/MCTargetDesc/MipsELFStreamer.cpp | 9 +- lib/Target/Mips/MCTargetDesc/MipsELFStreamer.h | 10 +- lib/Target/Mips/MCTargetDesc/MipsFixupKinds.h | 12 +- lib/Target/Mips/MCTargetDesc/MipsInstPrinter.cpp | 287 + lib/Target/Mips/MCTargetDesc/MipsInstPrinter.h | 112 + lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.cpp | 7 +- lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.h | 7 +- lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp | 15 +- lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.h | 7 +- lib/Target/Mips/MCTargetDesc/MipsMCExpr.cpp | 21 +- lib/Target/Mips/MCTargetDesc/MipsMCExpr.h | 7 +- lib/Target/Mips/MCTargetDesc/MipsMCNaCl.h | 7 +- lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp | 12 +- lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.h | 12 +- .../Mips/MCTargetDesc/MipsNaClELFStreamer.cpp | 11 +- lib/Target/Mips/MCTargetDesc/MipsOptionRecord.cpp | 7 +- .../Mips/MCTargetDesc/MipsTargetStreamer.cpp | 83 +- lib/Target/Mips/MicroMips32r6InstrFormats.td | 7 +- lib/Target/Mips/MicroMips32r6InstrInfo.td | 32 +- lib/Target/Mips/MicroMipsDSPInstrFormats.td | 7 +- lib/Target/Mips/MicroMipsDSPInstrInfo.td | 7 +- lib/Target/Mips/MicroMipsInstrFPU.td | 19 +- lib/Target/Mips/MicroMipsInstrFormats.td | 7 +- lib/Target/Mips/MicroMipsInstrInfo.td | 36 +- lib/Target/Mips/MicroMipsSizeReduction.cpp | 7 +- lib/Target/Mips/Mips.h | 7 +- lib/Target/Mips/Mips.td | 13 +- lib/Target/Mips/Mips16FrameLowering.cpp | 7 +- lib/Target/Mips/Mips16FrameLowering.h | 7 +- lib/Target/Mips/Mips16HardFloat.cpp | 9 +- lib/Target/Mips/Mips16HardFloatInfo.cpp | 7 +- lib/Target/Mips/Mips16HardFloatInfo.h | 7 +- lib/Target/Mips/Mips16ISelDAGToDAG.cpp | 7 +- lib/Target/Mips/Mips16ISelDAGToDAG.h | 7 +- lib/Target/Mips/Mips16ISelLowering.cpp | 17 +- lib/Target/Mips/Mips16ISelLowering.h | 8 +- lib/Target/Mips/Mips16InstrFormats.td | 7 +- lib/Target/Mips/Mips16InstrInfo.cpp | 7 +- lib/Target/Mips/Mips16InstrInfo.h | 7 +- lib/Target/Mips/Mips16InstrInfo.td | 15 +- lib/Target/Mips/Mips16RegisterInfo.cpp | 7 +- lib/Target/Mips/Mips16RegisterInfo.h | 7 +- lib/Target/Mips/Mips32r6InstrFormats.td | 7 +- lib/Target/Mips/Mips32r6InstrInfo.td | 12 +- lib/Target/Mips/Mips64InstrInfo.td | 92 +- lib/Target/Mips/Mips64r6InstrInfo.td | 10 +- lib/Target/Mips/MipsAnalyzeImmediate.cpp | 7 +- lib/Target/Mips/MipsAnalyzeImmediate.h | 7 +- lib/Target/Mips/MipsAsmPrinter.cpp | 63 +- lib/Target/Mips/MipsAsmPrinter.h | 13 +- lib/Target/Mips/MipsBranchExpansion.cpp | 7 +- lib/Target/Mips/MipsCCState.cpp | 7 +- lib/Target/Mips/MipsCCState.h | 7 +- lib/Target/Mips/MipsCallLowering.cpp | 265 +- lib/Target/Mips/MipsCallLowering.h | 31 +- lib/Target/Mips/MipsCallingConv.td | 7 +- lib/Target/Mips/MipsCondMov.td | 29 +- lib/Target/Mips/MipsConstantIslandPass.cpp | 15 +- lib/Target/Mips/MipsDSPInstrFormats.td | 7 +- lib/Target/Mips/MipsDSPInstrInfo.td | 12 +- lib/Target/Mips/MipsDelaySlotFiller.cpp | 45 +- lib/Target/Mips/MipsEVAInstrFormats.td | 7 +- lib/Target/Mips/MipsEVAInstrInfo.td | 7 +- lib/Target/Mips/MipsExpandPseudo.cpp | 7 +- lib/Target/Mips/MipsFastISel.cpp | 55 +- lib/Target/Mips/MipsFrameLowering.cpp | 7 +- lib/Target/Mips/MipsFrameLowering.h | 7 +- lib/Target/Mips/MipsISelDAGToDAG.cpp | 7 +- lib/Target/Mips/MipsISelDAGToDAG.h | 7 +- lib/Target/Mips/MipsISelLowering.cpp | 175 +- lib/Target/Mips/MipsISelLowering.h | 21 +- lib/Target/Mips/MipsInstrFPU.td | 26 +- lib/Target/Mips/MipsInstrFormats.td | 8 +- lib/Target/Mips/MipsInstrInfo.cpp | 23 +- lib/Target/Mips/MipsInstrInfo.h | 7 +- lib/Target/Mips/MipsInstrInfo.td | 114 +- lib/Target/Mips/MipsInstructionSelector.cpp | 447 +- lib/Target/Mips/MipsLegalizerInfo.cpp | 93 +- lib/Target/Mips/MipsLegalizerInfo.h | 7 +- lib/Target/Mips/MipsMCInstLower.cpp | 9 +- lib/Target/Mips/MipsMCInstLower.h | 7 +- lib/Target/Mips/MipsMSAInstrFormats.td | 7 +- lib/Target/Mips/MipsMSAInstrInfo.td | 90 +- lib/Target/Mips/MipsMTInstrFormats.td | 7 +- lib/Target/Mips/MipsMTInstrInfo.td | 7 +- lib/Target/Mips/MipsMachineFunction.cpp | 105 +- lib/Target/Mips/MipsMachineFunction.h | 14 +- lib/Target/Mips/MipsOptimizePICCall.cpp | 7 +- lib/Target/Mips/MipsOptionRecord.h | 7 +- lib/Target/Mips/MipsOs16.cpp | 7 +- lib/Target/Mips/MipsPreLegalizerCombiner.cpp | 18 +- lib/Target/Mips/MipsRegisterBankInfo.cpp | 598 +- lib/Target/Mips/MipsRegisterBankInfo.h | 132 +- lib/Target/Mips/MipsRegisterBanks.td | 9 +- lib/Target/Mips/MipsRegisterInfo.cpp | 40 +- lib/Target/Mips/MipsRegisterInfo.h | 9 +- lib/Target/Mips/MipsRegisterInfo.td | 54 +- lib/Target/Mips/MipsSEFrameLowering.cpp | 7 +- lib/Target/Mips/MipsSEFrameLowering.h | 7 +- lib/Target/Mips/MipsSEISelDAGToDAG.cpp | 113 +- lib/Target/Mips/MipsSEISelDAGToDAG.h | 11 +- lib/Target/Mips/MipsSEISelLowering.cpp | 126 +- lib/Target/Mips/MipsSEISelLowering.h | 15 +- lib/Target/Mips/MipsSEInstrInfo.cpp | 12 +- lib/Target/Mips/MipsSEInstrInfo.h | 7 +- lib/Target/Mips/MipsSERegisterInfo.cpp | 7 +- lib/Target/Mips/MipsSERegisterInfo.h | 7 +- lib/Target/Mips/MipsSchedule.td | 7 +- lib/Target/Mips/MipsScheduleGeneric.td | 934 +- lib/Target/Mips/MipsScheduleP5600.td | 67 +- lib/Target/Mips/MipsSubtarget.cpp | 21 +- lib/Target/Mips/MipsSubtarget.h | 11 +- lib/Target/Mips/MipsTargetMachine.cpp | 17 +- lib/Target/Mips/MipsTargetMachine.h | 13 +- lib/Target/Mips/MipsTargetObjectFile.cpp | 7 +- lib/Target/Mips/MipsTargetObjectFile.h | 7 +- lib/Target/Mips/MipsTargetStreamer.h | 11 +- lib/Target/Mips/TargetInfo/MipsTargetInfo.cpp | 10 +- lib/Target/Mips/TargetInfo/MipsTargetInfo.h | 23 + lib/Target/NVPTX/InstPrinter/NVPTXInstPrinter.cpp | 296 - lib/Target/NVPTX/InstPrinter/NVPTXInstPrinter.h | 52 - lib/Target/NVPTX/MCTargetDesc/NVPTXBaseInfo.h | 7 +- lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp | 309 + lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h | 53 + lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.cpp | 16 +- lib/Target/NVPTX/MCTargetDesc/NVPTXMCAsmInfo.h | 7 +- .../NVPTX/MCTargetDesc/NVPTXMCTargetDesc.cpp | 10 +- lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.h | 10 +- .../NVPTX/MCTargetDesc/NVPTXTargetStreamer.cpp | 26 +- .../NVPTX/MCTargetDesc/NVPTXTargetStreamer.h | 10 +- lib/Target/NVPTX/ManagedStringPool.h | 7 +- lib/Target/NVPTX/NVPTX.h | 20 +- lib/Target/NVPTX/NVPTX.td | 9 +- lib/Target/NVPTX/NVPTXAllocaHoisting.cpp | 7 +- lib/Target/NVPTX/NVPTXAllocaHoisting.h | 7 +- lib/Target/NVPTX/NVPTXAsmPrinter.cpp | 83 +- lib/Target/NVPTX/NVPTXAsmPrinter.h | 18 +- lib/Target/NVPTX/NVPTXAssignValidGlobalNames.cpp | 7 +- lib/Target/NVPTX/NVPTXFrameLowering.cpp | 7 +- lib/Target/NVPTX/NVPTXFrameLowering.h | 7 +- lib/Target/NVPTX/NVPTXGenericToNVVM.cpp | 7 +- lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp | 14 +- lib/Target/NVPTX/NVPTXISelDAGToDAG.h | 8 +- lib/Target/NVPTX/NVPTXISelLowering.cpp | 283 +- lib/Target/NVPTX/NVPTXISelLowering.h | 11 +- lib/Target/NVPTX/NVPTXImageOptimizer.cpp | 7 +- lib/Target/NVPTX/NVPTXInstrFormats.td | 7 +- lib/Target/NVPTX/NVPTXInstrInfo.cpp | 7 +- lib/Target/NVPTX/NVPTXInstrInfo.h | 7 +- lib/Target/NVPTX/NVPTXInstrInfo.td | 23 +- lib/Target/NVPTX/NVPTXIntrinsics.td | 658 +- lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp | 7 +- lib/Target/NVPTX/NVPTXLowerAggrCopies.h | 7 +- lib/Target/NVPTX/NVPTXLowerAlloca.cpp | 8 +- lib/Target/NVPTX/NVPTXLowerArgs.cpp | 11 +- lib/Target/NVPTX/NVPTXMCExpr.cpp | 7 +- lib/Target/NVPTX/NVPTXMCExpr.h | 7 +- lib/Target/NVPTX/NVPTXMachineFunctionInfo.h | 7 +- lib/Target/NVPTX/NVPTXPeephole.cpp | 7 +- lib/Target/NVPTX/NVPTXPrologEpilogPass.cpp | 11 +- lib/Target/NVPTX/NVPTXProxyRegErasure.cpp | 7 +- lib/Target/NVPTX/NVPTXRegisterInfo.cpp | 9 +- lib/Target/NVPTX/NVPTXRegisterInfo.h | 9 +- lib/Target/NVPTX/NVPTXRegisterInfo.td | 7 +- lib/Target/NVPTX/NVPTXReplaceImageHandles.cpp | 8 +- lib/Target/NVPTX/NVPTXSubtarget.cpp | 7 +- lib/Target/NVPTX/NVPTXSubtarget.h | 7 +- lib/Target/NVPTX/NVPTXTargetMachine.cpp | 27 +- lib/Target/NVPTX/NVPTXTargetMachine.h | 7 +- lib/Target/NVPTX/NVPTXTargetObjectFile.h | 7 +- lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp | 8 +- lib/Target/NVPTX/NVPTXTargetTransformInfo.h | 9 +- lib/Target/NVPTX/NVPTXUtilities.cpp | 8 +- lib/Target/NVPTX/NVPTXUtilities.h | 7 +- lib/Target/NVPTX/NVVMIntrRange.cpp | 7 +- lib/Target/NVPTX/NVVMReflect.cpp | 7 +- lib/Target/NVPTX/TargetInfo/NVPTXTargetInfo.cpp | 10 +- lib/Target/NVPTX/TargetInfo/NVPTXTargetInfo.h | 21 + lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp | 15 +- .../PowerPC/Disassembler/PPCDisassembler.cpp | 22 +- lib/Target/PowerPC/InstPrinter/PPCInstPrinter.cpp | 532 -- lib/Target/PowerPC/InstPrinter/PPCInstPrinter.h | 77 - lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp | 117 +- .../PowerPC/MCTargetDesc/PPCELFObjectWriter.cpp | 10 +- lib/Target/PowerPC/MCTargetDesc/PPCFixupKinds.h | 7 +- lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp | 543 ++ lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.h | 76 + lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp | 13 +- lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.h | 17 +- .../PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp | 9 +- lib/Target/PowerPC/MCTargetDesc/PPCMCCodeEmitter.h | 14 +- lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.cpp | 7 +- lib/Target/PowerPC/MCTargetDesc/PPCMCExpr.h | 7 +- .../PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp | 67 +- lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h | 14 +- .../PowerPC/MCTargetDesc/PPCMachObjectWriter.cpp | 7 +- lib/Target/PowerPC/MCTargetDesc/PPCPredicates.cpp | 7 +- lib/Target/PowerPC/MCTargetDesc/PPCPredicates.h | 7 +- .../PowerPC/MCTargetDesc/PPCXCOFFObjectWriter.cpp | 29 + lib/Target/PowerPC/P9InstrResources.td | 371 +- lib/Target/PowerPC/PPC.h | 22 +- lib/Target/PowerPC/PPC.td | 38 +- lib/Target/PowerPC/PPCAsmPrinter.cpp | 223 +- lib/Target/PowerPC/PPCBoolRetToInt.cpp | 7 +- lib/Target/PowerPC/PPCBranchCoalescing.cpp | 11 +- lib/Target/PowerPC/PPCBranchSelector.cpp | 262 +- lib/Target/PowerPC/PPCCCState.cpp | 7 +- lib/Target/PowerPC/PPCCCState.h | 7 +- lib/Target/PowerPC/PPCCTRLoops.cpp | 585 +- lib/Target/PowerPC/PPCCallingConv.cpp | 162 + lib/Target/PowerPC/PPCCallingConv.h | 36 +- lib/Target/PowerPC/PPCCallingConv.td | 50 +- lib/Target/PowerPC/PPCEarlyReturn.cpp | 19 +- lib/Target/PowerPC/PPCExpandISEL.cpp | 7 +- lib/Target/PowerPC/PPCFastISel.cpp | 108 +- lib/Target/PowerPC/PPCFrameLowering.cpp | 211 +- lib/Target/PowerPC/PPCFrameLowering.h | 31 +- lib/Target/PowerPC/PPCHazardRecognizers.cpp | 10 +- lib/Target/PowerPC/PPCHazardRecognizers.h | 7 +- lib/Target/PowerPC/PPCISelDAGToDAG.cpp | 94 +- lib/Target/PowerPC/PPCISelLowering.cpp | 1087 ++- lib/Target/PowerPC/PPCISelLowering.h | 117 +- lib/Target/PowerPC/PPCInstr64Bit.td | 66 +- lib/Target/PowerPC/PPCInstrAltivec.td | 37 +- lib/Target/PowerPC/PPCInstrBuilder.h | 7 +- lib/Target/PowerPC/PPCInstrFormats.td | 21 +- lib/Target/PowerPC/PPCInstrHTM.td | 49 +- lib/Target/PowerPC/PPCInstrInfo.cpp | 388 +- lib/Target/PowerPC/PPCInstrInfo.h | 100 +- lib/Target/PowerPC/PPCInstrInfo.td | 84 +- lib/Target/PowerPC/PPCInstrQPX.td | 7 +- lib/Target/PowerPC/PPCInstrSPE.td | 19 +- lib/Target/PowerPC/PPCInstrVSX.td | 531 +- lib/Target/PowerPC/PPCLoopPreIncPrep.cpp | 15 +- lib/Target/PowerPC/PPCMCInstLower.cpp | 17 +- lib/Target/PowerPC/PPCMIPeephole.cpp | 186 +- lib/Target/PowerPC/PPCMachineFunctionInfo.cpp | 7 +- lib/Target/PowerPC/PPCMachineFunctionInfo.h | 16 +- lib/Target/PowerPC/PPCMachineScheduler.cpp | 83 + lib/Target/PowerPC/PPCMachineScheduler.h | 49 + lib/Target/PowerPC/PPCPerfectShuffle.h | 7 +- lib/Target/PowerPC/PPCPfmCounters.td | 7 +- lib/Target/PowerPC/PPCPreEmitPeephole.cpp | 7 +- lib/Target/PowerPC/PPCQPXLoadSplat.cpp | 11 +- lib/Target/PowerPC/PPCReduceCRLogicals.cpp | 52 +- lib/Target/PowerPC/PPCRegisterInfo.cpp | 217 +- lib/Target/PowerPC/PPCRegisterInfo.h | 18 +- lib/Target/PowerPC/PPCRegisterInfo.td | 9 +- lib/Target/PowerPC/PPCSchedule.td | 8 +- lib/Target/PowerPC/PPCSchedule440.td | 7 +- lib/Target/PowerPC/PPCScheduleA2.td | 7 +- lib/Target/PowerPC/PPCScheduleE500.td | 7 +- lib/Target/PowerPC/PPCScheduleE500mc.td | 7 +- lib/Target/PowerPC/PPCScheduleE5500.td | 7 +- lib/Target/PowerPC/PPCScheduleG3.td | 7 +- lib/Target/PowerPC/PPCScheduleG4.td | 7 +- lib/Target/PowerPC/PPCScheduleG4Plus.td | 7 +- lib/Target/PowerPC/PPCScheduleG5.td | 7 +- lib/Target/PowerPC/PPCScheduleP7.td | 7 +- lib/Target/PowerPC/PPCScheduleP8.td | 7 +- lib/Target/PowerPC/PPCScheduleP9.td | 77 +- lib/Target/PowerPC/PPCSubtarget.cpp | 29 +- lib/Target/PowerPC/PPCSubtarget.h | 28 +- lib/Target/PowerPC/PPCTLSDynamicCall.cpp | 11 +- lib/Target/PowerPC/PPCTOCRegDeps.cpp | 11 +- lib/Target/PowerPC/PPCTargetMachine.cpp | 74 +- lib/Target/PowerPC/PPCTargetMachine.h | 11 +- lib/Target/PowerPC/PPCTargetObjectFile.cpp | 7 +- lib/Target/PowerPC/PPCTargetObjectFile.h | 7 +- lib/Target/PowerPC/PPCTargetStreamer.h | 7 +- lib/Target/PowerPC/PPCTargetTransformInfo.cpp | 449 +- lib/Target/PowerPC/PPCTargetTransformInfo.h | 21 +- lib/Target/PowerPC/PPCVSXCopy.cpp | 11 +- lib/Target/PowerPC/PPCVSXFMAMutate.cpp | 7 +- lib/Target/PowerPC/PPCVSXSwapRemoval.cpp | 12 +- lib/Target/PowerPC/README_P9.txt | 8 +- .../PowerPC/TargetInfo/PowerPCTargetInfo.cpp | 10 +- lib/Target/PowerPC/TargetInfo/PowerPCTargetInfo.h | 22 + lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp | 393 +- .../RISCV/Disassembler/RISCVDisassembler.cpp | 20 +- lib/Target/RISCV/InstPrinter/RISCVInstPrinter.cpp | 115 - lib/Target/RISCV/InstPrinter/RISCVInstPrinter.h | 55 - lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp | 93 +- lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h | 54 +- .../RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp | 70 +- lib/Target/RISCV/MCTargetDesc/RISCVELFStreamer.cpp | 32 +- lib/Target/RISCV/MCTargetDesc/RISCVELFStreamer.h | 7 +- lib/Target/RISCV/MCTargetDesc/RISCVFixupKinds.h | 36 +- lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp | 114 + lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h | 54 + lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.cpp | 8 +- lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.h | 7 +- .../RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp | 150 +- lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.cpp | 120 +- lib/Target/RISCV/MCTargetDesc/RISCVMCExpr.h | 23 +- .../RISCV/MCTargetDesc/RISCVMCTargetDesc.cpp | 18 +- lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.h | 10 +- .../RISCV/MCTargetDesc/RISCVTargetStreamer.cpp | 7 +- .../RISCV/MCTargetDesc/RISCVTargetStreamer.h | 7 +- lib/Target/RISCV/RISCV.h | 7 +- lib/Target/RISCV/RISCV.td | 25 +- lib/Target/RISCV/RISCVAsmPrinter.cpp | 65 +- lib/Target/RISCV/RISCVCallingConv.td | 18 +- lib/Target/RISCV/RISCVExpandPseudoInsts.cpp | 196 +- lib/Target/RISCV/RISCVFrameLowering.cpp | 80 +- lib/Target/RISCV/RISCVFrameLowering.h | 7 +- lib/Target/RISCV/RISCVISelDAGToDAG.cpp | 15 +- lib/Target/RISCV/RISCVISelLowering.cpp | 1185 ++- lib/Target/RISCV/RISCVISelLowering.h | 86 +- lib/Target/RISCV/RISCVInstrFormats.td | 36 +- lib/Target/RISCV/RISCVInstrFormatsC.td | 7 +- lib/Target/RISCV/RISCVInstrInfo.cpp | 36 +- lib/Target/RISCV/RISCVInstrInfo.h | 9 +- lib/Target/RISCV/RISCVInstrInfo.td | 320 +- lib/Target/RISCV/RISCVInstrInfoA.td | 89 +- lib/Target/RISCV/RISCVInstrInfoC.td | 57 +- lib/Target/RISCV/RISCVInstrInfoD.td | 41 +- lib/Target/RISCV/RISCVInstrInfoF.td | 97 +- lib/Target/RISCV/RISCVInstrInfoM.td | 46 +- lib/Target/RISCV/RISCVMCInstLower.cpp | 37 +- lib/Target/RISCV/RISCVMachineFunctionInfo.h | 9 +- lib/Target/RISCV/RISCVMergeBaseOffset.cpp | 7 +- lib/Target/RISCV/RISCVRegisterInfo.cpp | 53 +- lib/Target/RISCV/RISCVRegisterInfo.h | 9 +- lib/Target/RISCV/RISCVRegisterInfo.td | 9 +- lib/Target/RISCV/RISCVSubtarget.cpp | 22 +- lib/Target/RISCV/RISCVSubtarget.h | 21 +- lib/Target/RISCV/RISCVSystemOperands.td | 27 +- lib/Target/RISCV/RISCVTargetMachine.cpp | 21 +- lib/Target/RISCV/RISCVTargetMachine.h | 9 +- lib/Target/RISCV/RISCVTargetObjectFile.cpp | 103 +- lib/Target/RISCV/RISCVTargetObjectFile.h | 31 +- lib/Target/RISCV/RISCVTargetTransformInfo.cpp | 92 + lib/Target/RISCV/RISCVTargetTransformInfo.h | 52 + lib/Target/RISCV/TargetInfo/RISCVTargetInfo.cpp | 14 +- lib/Target/RISCV/TargetInfo/RISCVTargetInfo.h | 21 + lib/Target/RISCV/Utils/RISCVBaseInfo.cpp | 71 + lib/Target/RISCV/Utils/RISCVBaseInfo.h | 44 +- lib/Target/RISCV/Utils/RISCVMatInt.cpp | 32 +- lib/Target/RISCV/Utils/RISCVMatInt.h | 16 +- lib/Target/Sparc/AsmParser/SparcAsmParser.cpp | 11 +- lib/Target/Sparc/DelaySlotFiller.cpp | 7 +- .../Sparc/Disassembler/SparcDisassembler.cpp | 14 +- lib/Target/Sparc/InstPrinter/SparcInstPrinter.cpp | 220 - lib/Target/Sparc/InstPrinter/SparcInstPrinter.h | 57 - lib/Target/Sparc/LeonFeatures.td | 7 +- lib/Target/Sparc/LeonPasses.cpp | 7 +- lib/Target/Sparc/LeonPasses.h | 7 +- lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp | 7 +- .../Sparc/MCTargetDesc/SparcELFObjectWriter.cpp | 7 +- lib/Target/Sparc/MCTargetDesc/SparcFixupKinds.h | 7 +- lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.cpp | 219 + lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.h | 56 + lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.cpp | 7 +- lib/Target/Sparc/MCTargetDesc/SparcMCAsmInfo.h | 7 +- .../Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp | 14 +- lib/Target/Sparc/MCTargetDesc/SparcMCExpr.cpp | 7 +- lib/Target/Sparc/MCTargetDesc/SparcMCExpr.h | 7 +- .../Sparc/MCTargetDesc/SparcMCTargetDesc.cpp | 10 +- lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.h | 11 +- .../Sparc/MCTargetDesc/SparcTargetStreamer.cpp | 9 +- .../Sparc/MCTargetDesc/SparcTargetStreamer.h | 7 +- lib/Target/Sparc/Sparc.h | 7 +- lib/Target/Sparc/Sparc.td | 7 +- lib/Target/Sparc/SparcAsmPrinter.cpp | 23 +- lib/Target/Sparc/SparcCallingConv.td | 7 +- lib/Target/Sparc/SparcFrameLowering.cpp | 7 +- lib/Target/Sparc/SparcFrameLowering.h | 7 +- lib/Target/Sparc/SparcISelDAGToDAG.cpp | 12 +- lib/Target/Sparc/SparcISelLowering.cpp | 10 +- lib/Target/Sparc/SparcISelLowering.h | 7 +- lib/Target/Sparc/SparcInstr64Bit.td | 7 +- lib/Target/Sparc/SparcInstrAliases.td | 7 +- lib/Target/Sparc/SparcInstrFormats.td | 7 +- lib/Target/Sparc/SparcInstrInfo.cpp | 7 +- lib/Target/Sparc/SparcInstrInfo.h | 7 +- lib/Target/Sparc/SparcInstrInfo.td | 7 +- lib/Target/Sparc/SparcInstrVIS.td | 7 +- lib/Target/Sparc/SparcMCInstLower.cpp | 7 +- lib/Target/Sparc/SparcMachineFunctionInfo.cpp | 7 +- lib/Target/Sparc/SparcMachineFunctionInfo.h | 7 +- lib/Target/Sparc/SparcRegisterInfo.cpp | 15 +- lib/Target/Sparc/SparcRegisterInfo.h | 9 +- lib/Target/Sparc/SparcRegisterInfo.td | 7 +- lib/Target/Sparc/SparcSchedule.td | 7 +- lib/Target/Sparc/SparcSubtarget.cpp | 7 +- lib/Target/Sparc/SparcSubtarget.h | 7 +- lib/Target/Sparc/SparcTargetMachine.cpp | 12 +- lib/Target/Sparc/SparcTargetMachine.h | 7 +- lib/Target/Sparc/SparcTargetObjectFile.cpp | 7 +- lib/Target/Sparc/SparcTargetObjectFile.h | 7 +- lib/Target/Sparc/TargetInfo/SparcTargetInfo.cpp | 10 +- lib/Target/Sparc/TargetInfo/SparcTargetInfo.h | 22 + lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp | 35 +- .../SystemZ/Disassembler/SystemZDisassembler.cpp | 8 +- .../SystemZ/InstPrinter/SystemZInstPrinter.cpp | 234 - .../SystemZ/InstPrinter/SystemZInstPrinter.h | 78 - .../SystemZ/MCTargetDesc/SystemZInstPrinter.cpp | 233 + .../SystemZ/MCTargetDesc/SystemZInstPrinter.h | 77 + .../SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp | 7 +- .../SystemZ/MCTargetDesc/SystemZMCAsmInfo.cpp | 7 +- lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmInfo.h | 7 +- .../SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp | 14 +- lib/Target/SystemZ/MCTargetDesc/SystemZMCFixups.h | 7 +- .../SystemZ/MCTargetDesc/SystemZMCObjectWriter.cpp | 11 +- .../SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp | 11 +- .../SystemZ/MCTargetDesc/SystemZMCTargetDesc.h | 9 +- lib/Target/SystemZ/SystemZ.h | 8 +- lib/Target/SystemZ/SystemZ.td | 7 +- lib/Target/SystemZ/SystemZAsmPrinter.cpp | 70 +- lib/Target/SystemZ/SystemZAsmPrinter.h | 13 +- lib/Target/SystemZ/SystemZCallingConv.cpp | 7 +- lib/Target/SystemZ/SystemZCallingConv.h | 7 +- lib/Target/SystemZ/SystemZCallingConv.td | 7 +- lib/Target/SystemZ/SystemZConstantPoolValue.cpp | 7 +- lib/Target/SystemZ/SystemZConstantPoolValue.h | 7 +- lib/Target/SystemZ/SystemZElimCompare.cpp | 16 +- lib/Target/SystemZ/SystemZExpandPseudo.cpp | 7 +- lib/Target/SystemZ/SystemZFeatures.td | 58 +- lib/Target/SystemZ/SystemZFrameLowering.cpp | 7 +- lib/Target/SystemZ/SystemZFrameLowering.h | 7 +- lib/Target/SystemZ/SystemZHazardRecognizer.cpp | 7 +- lib/Target/SystemZ/SystemZHazardRecognizer.h | 7 +- lib/Target/SystemZ/SystemZISelDAGToDAG.cpp | 109 +- lib/Target/SystemZ/SystemZISelLowering.cpp | 816 +- lib/Target/SystemZ/SystemZISelLowering.h | 44 +- lib/Target/SystemZ/SystemZInstrBuilder.h | 7 +- lib/Target/SystemZ/SystemZInstrDFP.td | 99 +- lib/Target/SystemZ/SystemZInstrFP.td | 302 +- lib/Target/SystemZ/SystemZInstrFormats.td | 378 +- lib/Target/SystemZ/SystemZInstrHFP.td | 7 +- lib/Target/SystemZ/SystemZInstrInfo.cpp | 306 +- lib/Target/SystemZ/SystemZInstrInfo.h | 23 +- lib/Target/SystemZ/SystemZInstrInfo.td | 150 +- lib/Target/SystemZ/SystemZInstrSystem.td | 7 +- lib/Target/SystemZ/SystemZInstrVector.td | 555 +- lib/Target/SystemZ/SystemZLDCleanup.cpp | 7 +- lib/Target/SystemZ/SystemZLongBranch.cpp | 7 +- lib/Target/SystemZ/SystemZMCInstLower.cpp | 7 +- lib/Target/SystemZ/SystemZMCInstLower.h | 7 +- lib/Target/SystemZ/SystemZMachineFunctionInfo.cpp | 7 +- lib/Target/SystemZ/SystemZMachineFunctionInfo.h | 7 +- lib/Target/SystemZ/SystemZMachineScheduler.cpp | 7 +- lib/Target/SystemZ/SystemZMachineScheduler.h | 7 +- lib/Target/SystemZ/SystemZOperands.td | 27 +- lib/Target/SystemZ/SystemZOperators.td | 105 +- lib/Target/SystemZ/SystemZPatterns.td | 7 +- lib/Target/SystemZ/SystemZPostRewrite.cpp | 124 + lib/Target/SystemZ/SystemZProcessors.td | 9 +- lib/Target/SystemZ/SystemZRegisterInfo.cpp | 123 +- lib/Target/SystemZ/SystemZRegisterInfo.h | 9 +- lib/Target/SystemZ/SystemZRegisterInfo.td | 14 +- lib/Target/SystemZ/SystemZSchedule.td | 8 +- lib/Target/SystemZ/SystemZScheduleArch13.td | 1695 ++++ lib/Target/SystemZ/SystemZScheduleZ13.td | 18 +- lib/Target/SystemZ/SystemZScheduleZ14.td | 18 +- lib/Target/SystemZ/SystemZScheduleZ196.td | 7 +- lib/Target/SystemZ/SystemZScheduleZEC12.td | 7 +- lib/Target/SystemZ/SystemZSelectionDAGInfo.cpp | 25 +- lib/Target/SystemZ/SystemZSelectionDAGInfo.h | 7 +- lib/Target/SystemZ/SystemZShortenInst.cpp | 62 +- lib/Target/SystemZ/SystemZSubtarget.cpp | 10 +- lib/Target/SystemZ/SystemZSubtarget.h | 37 +- lib/Target/SystemZ/SystemZTDC.cpp | 11 +- lib/Target/SystemZ/SystemZTargetMachine.cpp | 22 +- lib/Target/SystemZ/SystemZTargetMachine.h | 7 +- lib/Target/SystemZ/SystemZTargetTransformInfo.cpp | 39 +- lib/Target/SystemZ/SystemZTargetTransformInfo.h | 7 +- .../SystemZ/TargetInfo/SystemZTargetInfo.cpp | 9 +- lib/Target/SystemZ/TargetInfo/SystemZTargetInfo.h | 20 + lib/Target/Target.cpp | 7 +- lib/Target/TargetIntrinsicInfo.cpp | 7 +- lib/Target/TargetLoweringObjectFile.cpp | 8 +- lib/Target/TargetMachine.cpp | 22 +- lib/Target/TargetMachineC.cpp | 7 +- .../WebAssembly/AsmParser/WebAssemblyAsmParser.cpp | 278 +- .../Disassembler/WebAssemblyDisassembler.cpp | 58 +- .../InstPrinter/WebAssemblyInstPrinter.cpp | 310 - .../InstPrinter/WebAssemblyInstPrinter.h | 66 - .../MCTargetDesc/WebAssemblyAsmBackend.cpp | 20 +- .../MCTargetDesc/WebAssemblyFixupKinds.h | 13 +- .../MCTargetDesc/WebAssemblyInstPrinter.cpp | 296 + .../MCTargetDesc/WebAssemblyInstPrinter.h | 65 + .../MCTargetDesc/WebAssemblyMCAsmInfo.cpp | 9 +- .../MCTargetDesc/WebAssemblyMCAsmInfo.h | 7 +- .../MCTargetDesc/WebAssemblyMCCodeEmitter.cpp | 35 +- .../MCTargetDesc/WebAssemblyMCTargetDesc.cpp | 24 +- .../MCTargetDesc/WebAssemblyMCTargetDesc.h | 302 +- .../MCTargetDesc/WebAssemblyTargetStreamer.cpp | 24 +- .../MCTargetDesc/WebAssemblyTargetStreamer.h | 20 +- .../MCTargetDesc/WebAssemblyWasmObjectWriter.cpp | 109 +- lib/Target/WebAssembly/README.txt | 2 +- .../TargetInfo/WebAssemblyTargetInfo.cpp | 10 +- .../WebAssembly/TargetInfo/WebAssemblyTargetInfo.h | 26 + lib/Target/WebAssembly/WebAssembly.h | 13 +- lib/Target/WebAssembly/WebAssembly.td | 29 +- .../WebAssemblyAddMissingPrototypes.cpp | 89 +- lib/Target/WebAssembly/WebAssemblyArgumentMove.cpp | 11 +- lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp | 186 +- lib/Target/WebAssembly/WebAssemblyAsmPrinter.h | 16 +- lib/Target/WebAssembly/WebAssemblyCFGSort.cpp | 54 +- lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp | 931 +- .../WebAssembly/WebAssemblyCallIndirectFixup.cpp | 37 +- .../WebAssembly/WebAssemblyDebugValueManager.cpp | 7 +- .../WebAssembly/WebAssemblyDebugValueManager.h | 7 +- .../WebAssemblyEHRestoreStackPointer.cpp | 87 - .../WebAssembly/WebAssemblyExceptionInfo.cpp | 21 +- lib/Target/WebAssembly/WebAssemblyExceptionInfo.h | 7 +- .../WebAssembly/WebAssemblyExplicitLocals.cpp | 55 +- lib/Target/WebAssembly/WebAssemblyFastISel.cpp | 183 +- .../WebAssembly/WebAssemblyFixFunctionBitcasts.cpp | 79 +- .../WebAssemblyFixIrreducibleControlFlow.cpp | 616 +- .../WebAssembly/WebAssemblyFrameLowering.cpp | 14 +- lib/Target/WebAssembly/WebAssemblyFrameLowering.h | 7 +- lib/Target/WebAssembly/WebAssemblyISD.def | 14 +- lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp | 168 +- lib/Target/WebAssembly/WebAssemblyISelLowering.cpp | 556 +- lib/Target/WebAssembly/WebAssemblyISelLowering.h | 21 +- lib/Target/WebAssembly/WebAssemblyInstrAtomics.td | 546 +- .../WebAssembly/WebAssemblyInstrBulkMemory.td | 71 + lib/Target/WebAssembly/WebAssemblyInstrCall.td | 202 +- lib/Target/WebAssembly/WebAssemblyInstrControl.td | 93 +- lib/Target/WebAssembly/WebAssemblyInstrConv.td | 7 +- .../WebAssembly/WebAssemblyInstrExceptRef.td | 27 - lib/Target/WebAssembly/WebAssemblyInstrFloat.td | 7 +- lib/Target/WebAssembly/WebAssemblyInstrFormats.td | 10 +- lib/Target/WebAssembly/WebAssemblyInstrInfo.cpp | 62 +- lib/Target/WebAssembly/WebAssemblyInstrInfo.h | 16 +- lib/Target/WebAssembly/WebAssemblyInstrInfo.td | 129 +- lib/Target/WebAssembly/WebAssemblyInstrInteger.td | 14 +- lib/Target/WebAssembly/WebAssemblyInstrMemory.td | 95 +- lib/Target/WebAssembly/WebAssemblyInstrRef.td | 25 + lib/Target/WebAssembly/WebAssemblyInstrSIMD.td | 215 +- .../WebAssembly/WebAssemblyLateEHPrepare.cpp | 467 +- .../WebAssembly/WebAssemblyLowerBrUnless.cpp | 7 +- .../WebAssemblyLowerEmscriptenEHSjLj.cpp | 95 +- .../WebAssembly/WebAssemblyLowerGlobalDtors.cpp | 30 +- lib/Target/WebAssembly/WebAssemblyMCInstLower.cpp | 118 +- lib/Target/WebAssembly/WebAssemblyMCInstLower.h | 12 +- .../WebAssembly/WebAssemblyMachineFunctionInfo.cpp | 40 +- .../WebAssembly/WebAssemblyMachineFunctionInfo.h | 47 +- .../WebAssembly/WebAssemblyMemIntrinsicResults.cpp | 23 +- .../WebAssemblyOptimizeLiveIntervals.cpp | 13 +- .../WebAssembly/WebAssemblyOptimizeReturned.cpp | 17 +- lib/Target/WebAssembly/WebAssemblyPeephole.cpp | 39 +- .../WebAssemblyPrepareForLiveIntervals.cpp | 19 +- lib/Target/WebAssembly/WebAssemblyRegColoring.cpp | 31 +- lib/Target/WebAssembly/WebAssemblyRegNumbering.cpp | 9 +- lib/Target/WebAssembly/WebAssemblyRegStackify.cpp | 173 +- lib/Target/WebAssembly/WebAssemblyRegisterInfo.cpp | 30 +- lib/Target/WebAssembly/WebAssemblyRegisterInfo.h | 9 +- lib/Target/WebAssembly/WebAssemblyRegisterInfo.td | 11 +- .../WebAssembly/WebAssemblyReplacePhysRegs.cpp | 7 +- .../WebAssemblyRuntimeLibcallSignatures.cpp | 143 +- .../WebAssemblyRuntimeLibcallSignatures.h | 11 +- .../WebAssembly/WebAssemblySelectionDAGInfo.cpp | 49 +- .../WebAssembly/WebAssemblySelectionDAGInfo.h | 22 +- .../WebAssembly/WebAssemblySetP2AlignOperands.cpp | 123 +- lib/Target/WebAssembly/WebAssemblySubtarget.cpp | 12 +- lib/Target/WebAssembly/WebAssemblySubtarget.h | 22 +- .../WebAssembly/WebAssemblyTargetMachine.cpp | 250 +- lib/Target/WebAssembly/WebAssemblyTargetMachine.h | 18 +- .../WebAssembly/WebAssemblyTargetObjectFile.cpp | 7 +- .../WebAssembly/WebAssemblyTargetObjectFile.h | 7 +- .../WebAssembly/WebAssemblyTargetTransformInfo.cpp | 9 +- .../WebAssembly/WebAssemblyTargetTransformInfo.h | 7 +- lib/Target/WebAssembly/WebAssemblyUtilities.cpp | 301 +- lib/Target/WebAssembly/WebAssemblyUtilities.h | 27 +- lib/Target/WebAssembly/known_gcc_test_failures.txt | 27 +- lib/Target/X86/AsmParser/X86AsmInstrumentation.cpp | 1089 --- lib/Target/X86/AsmParser/X86AsmInstrumentation.h | 68 - lib/Target/X86/AsmParser/X86AsmParser.cpp | 447 +- lib/Target/X86/AsmParser/X86AsmParserCommon.h | 7 +- lib/Target/X86/AsmParser/X86Operand.h | 58 +- lib/Target/X86/Disassembler/X86Disassembler.cpp | 217 +- .../X86/Disassembler/X86DisassemblerDecoder.cpp | 19 +- .../X86/Disassembler/X86DisassemblerDecoder.h | 14 +- lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp | 202 - lib/Target/X86/InstPrinter/X86ATTInstPrinter.h | 138 - lib/Target/X86/InstPrinter/X86InstComments.cpp | 1310 --- lib/Target/X86/InstPrinter/X86InstComments.h | 27 - .../X86/InstPrinter/X86InstPrinterCommon.cpp | 142 - lib/Target/X86/InstPrinter/X86InstPrinterCommon.h | 38 - lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp | 162 - lib/Target/X86/InstPrinter/X86IntelInstPrinter.h | 157 - lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp | 487 + lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h | 124 + lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp | 82 +- lib/Target/X86/MCTargetDesc/X86BaseInfo.h | 94 +- lib/Target/X86/MCTargetDesc/X86ELFObjectWriter.cpp | 38 +- lib/Target/X86/MCTargetDesc/X86FixupKinds.h | 7 +- lib/Target/X86/MCTargetDesc/X86InstComments.cpp | 1322 +++ lib/Target/X86/MCTargetDesc/X86InstComments.h | 26 + .../X86/MCTargetDesc/X86InstPrinterCommon.cpp | 362 + lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.h | 41 + .../X86/MCTargetDesc/X86IntelInstPrinter.cpp | 445 + lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.h | 144 + lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp | 7 +- lib/Target/X86/MCTargetDesc/X86MCAsmInfo.h | 7 +- lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp | 97 +- lib/Target/X86/MCTargetDesc/X86MCExpr.h | 9 +- lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp | 22 +- lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h | 10 +- .../X86/MCTargetDesc/X86MachObjectWriter.cpp | 7 +- lib/Target/X86/MCTargetDesc/X86TargetStreamer.h | 7 +- .../X86/MCTargetDesc/X86WinCOFFObjectWriter.cpp | 7 +- lib/Target/X86/MCTargetDesc/X86WinCOFFStreamer.cpp | 7 +- .../X86/MCTargetDesc/X86WinCOFFTargetStreamer.cpp | 7 +- lib/Target/X86/ShadowCallStack.cpp | 322 - lib/Target/X86/TargetInfo/X86TargetInfo.cpp | 9 +- lib/Target/X86/TargetInfo/X86TargetInfo.h | 21 + lib/Target/X86/Utils/X86ShuffleDecode.cpp | 14 +- lib/Target/X86/Utils/X86ShuffleDecode.h | 9 +- lib/Target/X86/X86.h | 15 +- lib/Target/X86/X86.td | 1226 ++- lib/Target/X86/X86AsmPrinter.cpp | 274 +- lib/Target/X86/X86AsmPrinter.h | 25 +- lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp | 29 +- lib/Target/X86/X86CallFrameOptimization.cpp | 12 +- lib/Target/X86/X86CallLowering.cpp | 78 +- lib/Target/X86/X86CallLowering.h | 13 +- lib/Target/X86/X86CallingConv.cpp | 162 +- lib/Target/X86/X86CallingConv.h | 104 +- lib/Target/X86/X86CallingConv.td | 28 +- lib/Target/X86/X86CmovConversion.cpp | 35 +- lib/Target/X86/X86CondBrFolding.cpp | 26 +- lib/Target/X86/X86DiscriminateMemOps.cpp | 42 +- lib/Target/X86/X86DomainReassignment.cpp | 12 +- lib/Target/X86/X86EvexToVex.cpp | 21 +- lib/Target/X86/X86ExpandPseudo.cpp | 41 +- lib/Target/X86/X86FastISel.cpp | 264 +- lib/Target/X86/X86FixupBWInsts.cpp | 13 +- lib/Target/X86/X86FixupLEAs.cpp | 393 +- lib/Target/X86/X86FixupSetCC.cpp | 37 +- lib/Target/X86/X86FlagsCopyLowering.cpp | 56 +- lib/Target/X86/X86FloatingPoint.cpp | 28 +- lib/Target/X86/X86FrameLowering.cpp | 80 +- lib/Target/X86/X86FrameLowering.h | 11 +- lib/Target/X86/X86GenRegisterBankInfo.def | 7 +- lib/Target/X86/X86ISelDAGToDAG.cpp | 1590 +++- lib/Target/X86/X86ISelLowering.cpp | 9548 +++++++++++++------- lib/Target/X86/X86ISelLowering.h | 216 +- lib/Target/X86/X86IndirectBranchTracking.cpp | 49 +- lib/Target/X86/X86InsertPrefetch.cpp | 10 +- lib/Target/X86/X86Instr3DNow.td | 11 +- lib/Target/X86/X86InstrAVX512.td | 3488 +++---- lib/Target/X86/X86InstrArithmetic.td | 101 +- lib/Target/X86/X86InstrBuilder.h | 7 +- lib/Target/X86/X86InstrCMovSetCC.td | 176 +- lib/Target/X86/X86InstrCompiler.td | 323 +- lib/Target/X86/X86InstrControl.td | 64 +- lib/Target/X86/X86InstrExtension.td | 11 +- lib/Target/X86/X86InstrFMA.td | 13 +- lib/Target/X86/X86InstrFMA3Info.cpp | 17 +- lib/Target/X86/X86InstrFMA3Info.h | 7 +- lib/Target/X86/X86InstrFPStack.td | 341 +- lib/Target/X86/X86InstrFoldTables.cpp | 186 +- lib/Target/X86/X86InstrFoldTables.h | 7 +- lib/Target/X86/X86InstrFormats.td | 33 +- lib/Target/X86/X86InstrFragmentsSIMD.td | 368 +- lib/Target/X86/X86InstrInfo.cpp | 1116 +-- lib/Target/X86/X86InstrInfo.h | 79 +- lib/Target/X86/X86InstrInfo.td | 439 +- lib/Target/X86/X86InstrMMX.td | 13 +- lib/Target/X86/X86InstrMPX.td | 7 +- lib/Target/X86/X86InstrSGX.td | 7 +- lib/Target/X86/X86InstrSSE.td | 1917 ++-- lib/Target/X86/X86InstrSVM.td | 7 +- lib/Target/X86/X86InstrShiftRotate.td | 98 +- lib/Target/X86/X86InstrSystem.td | 26 +- lib/Target/X86/X86InstrTSX.td | 7 +- lib/Target/X86/X86InstrVMX.td | 7 +- lib/Target/X86/X86InstrVecCompiler.td | 104 +- lib/Target/X86/X86InstrXOP.td | 33 +- lib/Target/X86/X86InstructionSelector.cpp | 92 +- lib/Target/X86/X86InterleavedAccess.cpp | 27 +- lib/Target/X86/X86IntrinsicsInfo.h | 781 +- lib/Target/X86/X86LegalizerInfo.cpp | 30 +- lib/Target/X86/X86LegalizerInfo.h | 7 +- lib/Target/X86/X86MCInstLower.cpp | 274 +- lib/Target/X86/X86MachineFunctionInfo.cpp | 7 +- lib/Target/X86/X86MachineFunctionInfo.h | 7 +- lib/Target/X86/X86MacroFusion.cpp | 164 +- lib/Target/X86/X86MacroFusion.h | 7 +- lib/Target/X86/X86OptimizeLEAs.cpp | 14 +- lib/Target/X86/X86PadShortFunction.cpp | 16 +- lib/Target/X86/X86PfmCounters.td | 7 +- lib/Target/X86/X86RegisterBankInfo.cpp | 24 +- lib/Target/X86/X86RegisterBankInfo.h | 7 +- lib/Target/X86/X86RegisterBanks.td | 7 +- lib/Target/X86/X86RegisterInfo.cpp | 37 +- lib/Target/X86/X86RegisterInfo.h | 23 +- lib/Target/X86/X86RegisterInfo.td | 44 +- lib/Target/X86/X86RetpolineThunks.cpp | 7 +- lib/Target/X86/X86SchedBroadwell.td | 169 +- lib/Target/X86/X86SchedHaswell.td | 195 +- lib/Target/X86/X86SchedPredicates.td | 31 +- lib/Target/X86/X86SchedSandyBridge.td | 96 +- lib/Target/X86/X86SchedSkylakeClient.td | 193 +- lib/Target/X86/X86SchedSkylakeServer.td | 212 +- lib/Target/X86/X86Schedule.td | 14 +- lib/Target/X86/X86ScheduleAtom.td | 12 +- lib/Target/X86/X86ScheduleBdVer2.td | 599 +- lib/Target/X86/X86ScheduleBtVer2.td | 45 +- lib/Target/X86/X86ScheduleSLM.td | 10 +- lib/Target/X86/X86ScheduleZnver1.td | 10 +- lib/Target/X86/X86SelectionDAGInfo.cpp | 222 +- lib/Target/X86/X86SelectionDAGInfo.h | 7 +- lib/Target/X86/X86ShuffleDecodeConstantPool.cpp | 7 +- lib/Target/X86/X86ShuffleDecodeConstantPool.h | 7 +- lib/Target/X86/X86SpeculativeLoadHardening.cpp | 41 +- lib/Target/X86/X86Subtarget.cpp | 22 +- lib/Target/X86/X86Subtarget.h | 47 +- lib/Target/X86/X86TargetMachine.cpp | 33 +- lib/Target/X86/X86TargetMachine.h | 7 +- lib/Target/X86/X86TargetObjectFile.cpp | 7 +- lib/Target/X86/X86TargetObjectFile.h | 7 +- lib/Target/X86/X86TargetTransformInfo.cpp | 529 +- lib/Target/X86/X86TargetTransformInfo.h | 76 +- lib/Target/X86/X86VZeroUpper.cpp | 7 +- lib/Target/X86/X86WinAllocaExpander.cpp | 46 +- lib/Target/X86/X86WinEHState.cpp | 45 +- .../XCore/Disassembler/XCoreDisassembler.cpp | 12 +- lib/Target/XCore/InstPrinter/XCoreInstPrinter.cpp | 90 - lib/Target/XCore/InstPrinter/XCoreInstPrinter.h | 47 - lib/Target/XCore/MCTargetDesc/XCoreInstPrinter.cpp | 89 + lib/Target/XCore/MCTargetDesc/XCoreInstPrinter.h | 46 + lib/Target/XCore/MCTargetDesc/XCoreMCAsmInfo.cpp | 7 +- lib/Target/XCore/MCTargetDesc/XCoreMCAsmInfo.h | 7 +- .../XCore/MCTargetDesc/XCoreMCTargetDesc.cpp | 10 +- lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.h | 9 +- lib/Target/XCore/TargetInfo/XCoreTargetInfo.cpp | 10 +- lib/Target/XCore/TargetInfo/XCoreTargetInfo.h | 20 + lib/Target/XCore/XCore.h | 7 +- lib/Target/XCore/XCore.td | 7 +- lib/Target/XCore/XCoreAsmPrinter.cpp | 31 +- lib/Target/XCore/XCoreCallingConv.td | 7 +- lib/Target/XCore/XCoreFrameLowering.cpp | 7 +- lib/Target/XCore/XCoreFrameLowering.h | 7 +- lib/Target/XCore/XCoreFrameToArgsOffsetElim.cpp | 7 +- lib/Target/XCore/XCoreISelDAGToDAG.cpp | 7 +- lib/Target/XCore/XCoreISelLowering.cpp | 82 +- lib/Target/XCore/XCoreISelLowering.h | 9 +- lib/Target/XCore/XCoreInstrFormats.td | 7 +- lib/Target/XCore/XCoreInstrInfo.cpp | 7 +- lib/Target/XCore/XCoreInstrInfo.h | 7 +- lib/Target/XCore/XCoreInstrInfo.td | 7 +- lib/Target/XCore/XCoreLowerThreadLocal.cpp | 7 +- lib/Target/XCore/XCoreMCInstLower.cpp | 7 +- lib/Target/XCore/XCoreMCInstLower.h | 7 +- lib/Target/XCore/XCoreMachineFunctionInfo.cpp | 7 +- lib/Target/XCore/XCoreMachineFunctionInfo.h | 7 +- lib/Target/XCore/XCoreRegisterInfo.cpp | 11 +- lib/Target/XCore/XCoreRegisterInfo.h | 9 +- lib/Target/XCore/XCoreRegisterInfo.td | 7 +- lib/Target/XCore/XCoreSelectionDAGInfo.cpp | 7 +- lib/Target/XCore/XCoreSelectionDAGInfo.h | 7 +- lib/Target/XCore/XCoreSubtarget.cpp | 7 +- lib/Target/XCore/XCoreSubtarget.h | 7 +- lib/Target/XCore/XCoreTargetMachine.cpp | 8 +- lib/Target/XCore/XCoreTargetMachine.h | 7 +- lib/Target/XCore/XCoreTargetObjectFile.cpp | 7 +- lib/Target/XCore/XCoreTargetObjectFile.h | 7 +- lib/Target/XCore/XCoreTargetStreamer.h | 7 +- lib/Target/XCore/XCoreTargetTransformInfo.h | 7 +- lib/Testing/Support/Annotations.cpp | 95 + lib/Testing/Support/Error.cpp | 7 +- lib/TextAPI/ELF/ELFStub.cpp | 7 +- lib/TextAPI/ELF/TBEHandler.cpp | 7 +- lib/TextAPI/MachO/Architecture.cpp | 77 + lib/TextAPI/MachO/ArchitectureSet.cpp | 69 + lib/TextAPI/MachO/InterfaceFile.cpp | 81 + lib/TextAPI/MachO/PackedVersion.cpp | 113 + lib/TextAPI/MachO/Symbol.cpp | 49 + lib/TextAPI/MachO/TextAPIContext.h | 33 + lib/TextAPI/MachO/TextStub.cpp | 660 ++ lib/TextAPI/MachO/TextStubCommon.cpp | 178 + lib/TextAPI/MachO/TextStubCommon.h | 81 + lib/ToolDrivers/llvm-dlltool/DlltoolDriver.cpp | 10 +- lib/ToolDrivers/llvm-lib/LibDriver.cpp | 156 +- lib/ToolDrivers/llvm-lib/Options.td | 16 +- .../AggressiveInstCombine.cpp | 7 +- .../AggressiveInstCombineInternal.h | 7 +- .../AggressiveInstCombine/TruncInstCombine.cpp | 7 +- lib/Transforms/Coroutines/CoroCleanup.cpp | 9 +- lib/Transforms/Coroutines/CoroEarly.cpp | 11 +- lib/Transforms/Coroutines/CoroElide.cpp | 7 +- lib/Transforms/Coroutines/CoroFrame.cpp | 62 +- lib/Transforms/Coroutines/CoroInstr.h | 7 +- lib/Transforms/Coroutines/CoroInternal.h | 7 +- lib/Transforms/Coroutines/CoroSplit.cpp | 30 +- lib/Transforms/Coroutines/Coroutines.cpp | 15 +- lib/Transforms/IPO/AlwaysInliner.cpp | 41 +- lib/Transforms/IPO/ArgumentPromotion.cpp | 93 +- lib/Transforms/IPO/Attributor.cpp | 1690 ++++ lib/Transforms/IPO/BarrierNoopPass.cpp | 7 +- lib/Transforms/IPO/BlockExtractor.cpp | 122 +- lib/Transforms/IPO/CalledValuePropagation.cpp | 7 +- lib/Transforms/IPO/ConstantMerge.cpp | 29 +- lib/Transforms/IPO/CrossDSOCFI.cpp | 17 +- lib/Transforms/IPO/DeadArgumentElimination.cpp | 9 +- lib/Transforms/IPO/ElimAvailExtern.cpp | 7 +- lib/Transforms/IPO/ExtractGV.cpp | 7 +- lib/Transforms/IPO/ForceFunctionAttrs.cpp | 8 +- lib/Transforms/IPO/FunctionAttrs.cpp | 73 +- lib/Transforms/IPO/FunctionImport.cpp | 57 +- lib/Transforms/IPO/GlobalDCE.cpp | 7 +- lib/Transforms/IPO/GlobalOpt.cpp | 144 +- lib/Transforms/IPO/GlobalSplit.cpp | 7 +- lib/Transforms/IPO/HotColdSplitting.cpp | 424 +- lib/Transforms/IPO/IPConstantPropagation.cpp | 50 +- lib/Transforms/IPO/IPO.cpp | 8 +- lib/Transforms/IPO/InferFunctionAttrs.cpp | 9 +- lib/Transforms/IPO/InlineSimple.cpp | 13 +- lib/Transforms/IPO/Inliner.cpp | 19 +- lib/Transforms/IPO/Internalize.cpp | 30 +- lib/Transforms/IPO/LoopExtractor.cpp | 14 +- lib/Transforms/IPO/LowerTypeTests.cpp | 41 +- lib/Transforms/IPO/MergeFunctions.cpp | 70 +- lib/Transforms/IPO/PartialInlining.cpp | 75 +- lib/Transforms/IPO/PassManagerBuilder.cpp | 206 +- lib/Transforms/IPO/PruneEH.cpp | 18 +- lib/Transforms/IPO/SCCP.cpp | 1 + lib/Transforms/IPO/SampleProfile.cpp | 66 +- lib/Transforms/IPO/StripDeadPrototypes.cpp | 7 +- lib/Transforms/IPO/StripSymbols.cpp | 7 +- lib/Transforms/IPO/SyntheticCountsPropagation.cpp | 7 +- lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp | 50 +- lib/Transforms/IPO/WholeProgramDevirt.cpp | 53 +- lib/Transforms/InstCombine/InstCombineAddSub.cpp | 193 +- lib/Transforms/InstCombine/InstCombineAndOrXor.cpp | 120 +- .../InstCombine/InstCombineAtomicRMW.cpp | 159 + lib/Transforms/InstCombine/InstCombineCalls.cpp | 1158 +-- lib/Transforms/InstCombine/InstCombineCasts.cpp | 90 +- lib/Transforms/InstCombine/InstCombineCompares.cpp | 643 +- lib/Transforms/InstCombine/InstCombineInternal.h | 101 +- .../InstCombine/InstCombineLoadStoreAlloca.cpp | 62 +- .../InstCombine/InstCombineMulDivRem.cpp | 103 +- lib/Transforms/InstCombine/InstCombinePHI.cpp | 15 +- lib/Transforms/InstCombine/InstCombineSelect.cpp | 288 +- lib/Transforms/InstCombine/InstCombineShifts.cpp | 98 +- .../InstCombine/InstCombineSimplifyDemanded.cpp | 112 +- .../InstCombine/InstCombineVectorOps.cpp | 321 +- .../InstCombine/InstructionCombining.cpp | 348 +- .../Instrumentation/AddressSanitizer.cpp | 775 +- lib/Transforms/Instrumentation/BoundsChecking.cpp | 12 +- lib/Transforms/Instrumentation/CFGMST.h | 16 +- lib/Transforms/Instrumentation/CGProfile.cpp | 7 +- .../Instrumentation/ControlHeightReduction.cpp | 70 +- .../Instrumentation/DataFlowSanitizer.cpp | 171 +- .../Instrumentation/EfficiencySanitizer.cpp | 900 -- lib/Transforms/Instrumentation/GCOVProfiling.cpp | 154 +- .../Instrumentation/HWAddressSanitizer.cpp | 594 +- .../Instrumentation/IndirectCallPromotion.cpp | 21 +- lib/Transforms/Instrumentation/InstrOrderFile.cpp | 211 + lib/Transforms/Instrumentation/InstrProfiling.cpp | 213 +- lib/Transforms/Instrumentation/Instrumentation.cpp | 25 +- .../Instrumentation/MaximumSpanningTree.h | 10 +- lib/Transforms/Instrumentation/MemorySanitizer.cpp | 395 +- .../Instrumentation/PGOInstrumentation.cpp | 445 +- lib/Transforms/Instrumentation/PGOMemOPSizeOpt.cpp | 9 +- lib/Transforms/Instrumentation/PoisonChecking.cpp | 357 + .../Instrumentation/SanitizerCoverage.cpp | 199 +- lib/Transforms/Instrumentation/ThreadSanitizer.cpp | 141 +- lib/Transforms/ObjCARC/ARCRuntimeEntryPoints.h | 31 +- lib/Transforms/ObjCARC/BlotMapVector.h | 7 +- lib/Transforms/ObjCARC/DependencyAnalysis.cpp | 7 +- lib/Transforms/ObjCARC/DependencyAnalysis.h | 7 +- lib/Transforms/ObjCARC/ObjCARC.cpp | 7 +- lib/Transforms/ObjCARC/ObjCARC.h | 7 +- lib/Transforms/ObjCARC/ObjCARCAPElim.cpp | 7 +- lib/Transforms/ObjCARC/ObjCARCContract.cpp | 251 +- lib/Transforms/ObjCARC/ObjCARCExpand.cpp | 7 +- lib/Transforms/ObjCARC/ObjCARCOpts.cpp | 80 +- lib/Transforms/ObjCARC/ProvenanceAnalysis.cpp | 7 +- lib/Transforms/ObjCARC/ProvenanceAnalysis.h | 7 +- .../ObjCARC/ProvenanceAnalysisEvaluator.cpp | 7 +- lib/Transforms/ObjCARC/PtrState.cpp | 7 +- lib/Transforms/ObjCARC/PtrState.h | 7 +- lib/Transforms/Scalar/ADCE.cpp | 20 +- lib/Transforms/Scalar/AlignmentFromAssumptions.cpp | 7 +- lib/Transforms/Scalar/BDCE.cpp | 17 +- lib/Transforms/Scalar/CallSiteSplitting.cpp | 12 +- lib/Transforms/Scalar/ConstantHoisting.cpp | 40 +- lib/Transforms/Scalar/ConstantProp.cpp | 7 +- .../Scalar/CorrelatedValuePropagation.cpp | 305 +- lib/Transforms/Scalar/DCE.cpp | 7 +- lib/Transforms/Scalar/DeadStoreElimination.cpp | 102 +- lib/Transforms/Scalar/DivRemPairs.cpp | 7 +- lib/Transforms/Scalar/EarlyCSE.cpp | 239 +- lib/Transforms/Scalar/FlattenCFGPass.cpp | 7 +- lib/Transforms/Scalar/Float2Int.cpp | 29 +- lib/Transforms/Scalar/GVN.cpp | 104 +- lib/Transforms/Scalar/GVNHoist.cpp | 9 +- lib/Transforms/Scalar/GVNSink.cpp | 22 +- lib/Transforms/Scalar/GuardWidening.cpp | 212 +- lib/Transforms/Scalar/IVUsersPrinter.cpp | 7 +- lib/Transforms/Scalar/IndVarSimplify.cpp | 716 +- .../Scalar/InductiveRangeCheckElimination.cpp | 101 +- lib/Transforms/Scalar/InferAddressSpaces.cpp | 53 +- lib/Transforms/Scalar/InstSimplifyPass.cpp | 7 +- lib/Transforms/Scalar/JumpThreading.cpp | 93 +- lib/Transforms/Scalar/LICM.cpp | 466 +- .../Scalar/LoopAccessAnalysisPrinter.cpp | 7 +- lib/Transforms/Scalar/LoopDataPrefetch.cpp | 11 +- lib/Transforms/Scalar/LoopDeletion.cpp | 7 +- lib/Transforms/Scalar/LoopDistribute.cpp | 22 +- lib/Transforms/Scalar/LoopFuse.cpp | 1215 +++ lib/Transforms/Scalar/LoopIdiomRecognize.cpp | 94 +- lib/Transforms/Scalar/LoopInstSimplify.cpp | 9 +- lib/Transforms/Scalar/LoopInterchange.cpp | 130 +- lib/Transforms/Scalar/LoopLoadElimination.cpp | 62 +- lib/Transforms/Scalar/LoopPassManager.cpp | 7 +- lib/Transforms/Scalar/LoopPredication.cpp | 524 +- lib/Transforms/Scalar/LoopRerollPass.cpp | 17 +- lib/Transforms/Scalar/LoopRotation.cpp | 12 +- lib/Transforms/Scalar/LoopSimplifyCFG.cpp | 237 +- lib/Transforms/Scalar/LoopSink.cpp | 14 +- lib/Transforms/Scalar/LoopStrengthReduce.cpp | 344 +- lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp | 10 +- lib/Transforms/Scalar/LoopUnrollPass.cpp | 97 +- lib/Transforms/Scalar/LoopUnswitch.cpp | 26 +- lib/Transforms/Scalar/LoopVersioningLICM.cpp | 17 +- lib/Transforms/Scalar/LowerAtomic.cpp | 17 +- lib/Transforms/Scalar/LowerExpectIntrinsic.cpp | 7 +- lib/Transforms/Scalar/LowerGuardIntrinsic.cpp | 7 +- lib/Transforms/Scalar/LowerWidenableCondition.cpp | 85 + lib/Transforms/Scalar/MakeGuardsExplicit.cpp | 7 +- lib/Transforms/Scalar/MemCpyOptimizer.cpp | 24 +- lib/Transforms/Scalar/MergeICmps.cpp | 728 +- lib/Transforms/Scalar/MergedLoadStoreMotion.cpp | 7 +- lib/Transforms/Scalar/NaryReassociate.cpp | 11 +- lib/Transforms/Scalar/NewGVN.cpp | 61 +- lib/Transforms/Scalar/PartiallyInlineLibCalls.cpp | 7 +- lib/Transforms/Scalar/PlaceSafepoints.cpp | 42 +- lib/Transforms/Scalar/Reassociate.cpp | 103 +- lib/Transforms/Scalar/Reg2Mem.cpp | 7 +- lib/Transforms/Scalar/RewriteStatepointsForGC.cpp | 254 +- lib/Transforms/Scalar/SCCP.cpp | 76 +- lib/Transforms/Scalar/SROA.cpp | 215 +- lib/Transforms/Scalar/Scalar.cpp | 11 +- lib/Transforms/Scalar/Scalarizer.cpp | 70 +- .../Scalar/SeparateConstOffsetFromGEP.cpp | 7 +- lib/Transforms/Scalar/SimpleLoopUnswitch.cpp | 130 +- lib/Transforms/Scalar/SimplifyCFGPass.cpp | 7 +- lib/Transforms/Scalar/Sink.cpp | 7 +- lib/Transforms/Scalar/SpeculateAroundPHIs.cpp | 15 +- lib/Transforms/Scalar/SpeculativeExecution.cpp | 8 +- .../Scalar/StraightLineStrengthReduce.cpp | 15 +- lib/Transforms/Scalar/StructurizeCFG.cpp | 47 +- lib/Transforms/Scalar/TailRecursionElimination.cpp | 13 +- lib/Transforms/Scalar/WarnMissedTransforms.cpp | 9 +- lib/Transforms/Utils/ASanStackFrameLayout.cpp | 9 +- lib/Transforms/Utils/AddDiscriminators.cpp | 11 +- lib/Transforms/Utils/BasicBlockUtils.cpp | 141 +- lib/Transforms/Utils/BreakCriticalEdges.cpp | 34 +- lib/Transforms/Utils/BuildLibCalls.cpp | 339 +- lib/Transforms/Utils/BypassSlowDivision.cpp | 7 +- lib/Transforms/Utils/CallPromotionUtils.cpp | 21 +- lib/Transforms/Utils/CanonicalizeAliases.cpp | 7 +- lib/Transforms/Utils/CloneFunction.cpp | 37 +- lib/Transforms/Utils/CloneModule.cpp | 7 +- lib/Transforms/Utils/CodeExtractor.cpp | 496 +- lib/Transforms/Utils/CtorUtils.cpp | 7 +- lib/Transforms/Utils/DemoteRegToStack.cpp | 16 +- lib/Transforms/Utils/EntryExitInstrumenter.cpp | 11 +- lib/Transforms/Utils/EscapeEnumerator.cpp | 13 +- lib/Transforms/Utils/Evaluator.cpp | 113 +- lib/Transforms/Utils/FlattenCFG.cpp | 7 +- lib/Transforms/Utils/FunctionComparator.cpp | 53 +- lib/Transforms/Utils/FunctionImportUtils.cpp | 44 +- lib/Transforms/Utils/GlobalStatus.cpp | 7 +- lib/Transforms/Utils/GuardUtils.cpp | 7 +- .../Utils/ImportedFunctionsInliningStatistics.cpp | 7 +- lib/Transforms/Utils/InlineFunction.cpp | 172 +- lib/Transforms/Utils/InstructionNamer.cpp | 7 +- lib/Transforms/Utils/IntegerDivision.cpp | 7 +- lib/Transforms/Utils/LCSSA.cpp | 47 +- lib/Transforms/Utils/LibCallsShrinkWrap.cpp | 7 +- lib/Transforms/Utils/Local.cpp | 387 +- lib/Transforms/Utils/LoopRotationUtils.cpp | 26 +- lib/Transforms/Utils/LoopSimplify.cpp | 126 +- lib/Transforms/Utils/LoopUnroll.cpp | 424 +- lib/Transforms/Utils/LoopUnrollAndJam.cpp | 17 +- lib/Transforms/Utils/LoopUnrollPeel.cpp | 210 +- lib/Transforms/Utils/LoopUnrollRuntime.cpp | 61 +- lib/Transforms/Utils/LoopUtils.cpp | 106 +- lib/Transforms/Utils/LoopVersioning.cpp | 12 +- lib/Transforms/Utils/LowerInvoke.cpp | 10 +- lib/Transforms/Utils/LowerMemIntrinsics.cpp | 29 +- lib/Transforms/Utils/LowerSwitch.cpp | 218 +- lib/Transforms/Utils/Mem2Reg.cpp | 7 +- lib/Transforms/Utils/MetaRenamer.cpp | 7 +- lib/Transforms/Utils/ModuleUtils.cpp | 88 +- lib/Transforms/Utils/NameAnonGlobals.cpp | 7 +- lib/Transforms/Utils/PredicateInfo.cpp | 18 +- lib/Transforms/Utils/PromoteMemoryToRegister.cpp | 66 +- lib/Transforms/Utils/SSAUpdater.cpp | 10 +- lib/Transforms/Utils/SSAUpdaterBulk.cpp | 7 +- lib/Transforms/Utils/SanitizerStats.cpp | 15 +- lib/Transforms/Utils/SimplifyCFG.cpp | 203 +- lib/Transforms/Utils/SimplifyIndVar.cpp | 200 +- lib/Transforms/Utils/SimplifyLibCalls.cpp | 510 +- lib/Transforms/Utils/SizeOpts.cpp | 37 + lib/Transforms/Utils/SplitModule.cpp | 7 +- lib/Transforms/Utils/StripGCRelocates.cpp | 7 +- .../Utils/StripNonLineTableDebugInfo.cpp | 7 +- lib/Transforms/Utils/SymbolRewriter.cpp | 7 +- lib/Transforms/Utils/UnifyFunctionExitNodes.cpp | 7 +- lib/Transforms/Utils/Utils.cpp | 10 +- lib/Transforms/Utils/VNCoercion.cpp | 66 +- lib/Transforms/Utils/ValueMapper.cpp | 22 +- lib/Transforms/Vectorize/LoadStoreVectorizer.cpp | 21 +- .../Vectorize/LoopVectorizationLegality.cpp | 347 +- .../Vectorize/LoopVectorizationPlanner.h | 23 +- lib/Transforms/Vectorize/LoopVectorize.cpp | 476 +- lib/Transforms/Vectorize/SLPVectorizer.cpp | 1362 ++- lib/Transforms/Vectorize/VPRecipeBuilder.h | 14 +- lib/Transforms/Vectorize/VPlan.cpp | 23 +- lib/Transforms/Vectorize/VPlan.h | 60 +- lib/Transforms/Vectorize/VPlanDominatorTree.h | 7 +- lib/Transforms/Vectorize/VPlanHCFGBuilder.cpp | 11 +- lib/Transforms/Vectorize/VPlanHCFGBuilder.h | 7 +- lib/Transforms/Vectorize/VPlanHCFGTransforms.cpp | 7 +- lib/Transforms/Vectorize/VPlanHCFGTransforms.h | 7 +- lib/Transforms/Vectorize/VPlanLoopInfo.h | 7 +- lib/Transforms/Vectorize/VPlanPredicator.cpp | 248 + lib/Transforms/Vectorize/VPlanPredicator.h | 74 + lib/Transforms/Vectorize/VPlanSLP.cpp | 7 +- lib/Transforms/Vectorize/VPlanValue.h | 7 +- lib/Transforms/Vectorize/VPlanVerifier.cpp | 7 +- lib/Transforms/Vectorize/VPlanVerifier.h | 7 +- lib/Transforms/Vectorize/Vectorize.cpp | 7 +- lib/WindowsManifest/WindowsManifestMerger.cpp | 7 +- lib/XRay/BlockIndexer.cpp | 7 +- lib/XRay/BlockPrinter.cpp | 7 +- lib/XRay/BlockVerifier.cpp | 7 +- lib/XRay/FDRRecordProducer.cpp | 7 +- lib/XRay/FDRRecords.cpp | 7 +- lib/XRay/FDRTraceExpander.cpp | 7 +- lib/XRay/FDRTraceWriter.cpp | 7 +- lib/XRay/FileHeaderReader.cpp | 7 +- lib/XRay/InstrumentationMap.cpp | 26 +- lib/XRay/LogBuilderConsumer.cpp | 7 +- lib/XRay/Profile.cpp | 18 +- lib/XRay/RecordInitializer.cpp | 7 +- lib/XRay/RecordPrinter.cpp | 7 +- lib/XRay/Trace.cpp | 26 +- tools/bugpoint/BugDriver.cpp | 7 +- tools/bugpoint/BugDriver.h | 7 +- tools/bugpoint/CrashDebugger.cpp | 7 +- tools/bugpoint/ExecutionDriver.cpp | 7 +- tools/bugpoint/ExtractFunction.cpp | 7 +- tools/bugpoint/FindBugs.cpp | 7 +- tools/bugpoint/ListReducer.h | 7 +- tools/bugpoint/Miscompilation.cpp | 27 +- tools/bugpoint/OptimizerDriver.cpp | 7 +- tools/bugpoint/ToolRunner.cpp | 7 +- tools/bugpoint/ToolRunner.h | 7 +- tools/bugpoint/bugpoint.cpp | 7 +- tools/llc/llc.cpp | 62 +- tools/lli/RemoteJITUtils.h | 9 +- tools/lli/lli.cpp | 97 +- tools/llvm-ar/llvm-ar.cpp | 168 +- tools/llvm-as/llvm-as.cpp | 29 +- tools/llvm-bcanalyzer/llvm-bcanalyzer.cpp | 984 +- tools/llvm-cov/CodeCoverage.cpp | 20 +- tools/llvm-cov/CoverageExporter.h | 7 +- tools/llvm-cov/CoverageExporterJson.cpp | 69 +- tools/llvm-cov/CoverageExporterJson.h | 7 +- tools/llvm-cov/CoverageExporterLcov.cpp | 9 +- tools/llvm-cov/CoverageExporterLcov.h | 7 +- tools/llvm-cov/CoverageFilters.cpp | 7 +- tools/llvm-cov/CoverageFilters.h | 7 +- tools/llvm-cov/CoverageReport.cpp | 7 +- tools/llvm-cov/CoverageReport.h | 7 +- tools/llvm-cov/CoverageSummaryInfo.cpp | 7 +- tools/llvm-cov/CoverageSummaryInfo.h | 7 +- tools/llvm-cov/CoverageViewOptions.h | 9 +- tools/llvm-cov/RenderingSupport.h | 7 +- tools/llvm-cov/SourceCoverageView.cpp | 11 +- tools/llvm-cov/SourceCoverageView.h | 7 +- tools/llvm-cov/SourceCoverageViewHTML.cpp | 7 +- tools/llvm-cov/SourceCoverageViewHTML.h | 7 +- tools/llvm-cov/SourceCoverageViewText.cpp | 7 +- tools/llvm-cov/SourceCoverageViewText.h | 7 +- tools/llvm-cov/TestingSupport.cpp | 20 +- tools/llvm-cov/gcov.cpp | 15 +- tools/llvm-cov/llvm-cov.cpp | 7 +- tools/llvm-cxxdump/Error.cpp | 7 +- tools/llvm-cxxdump/Error.h | 7 +- tools/llvm-cxxdump/llvm-cxxdump.cpp | 24 +- tools/llvm-cxxdump/llvm-cxxdump.h | 7 +- tools/llvm-cxxfilt/llvm-cxxfilt.cpp | 93 +- tools/llvm-cxxmap/llvm-cxxmap.cpp | 7 +- tools/llvm-diff/DiffConsumer.cpp | 9 +- tools/llvm-diff/DiffConsumer.h | 7 +- tools/llvm-diff/DiffLog.cpp | 7 +- tools/llvm-diff/DiffLog.h | 7 +- tools/llvm-diff/DifferenceEngine.cpp | 9 +- tools/llvm-diff/DifferenceEngine.h | 7 +- tools/llvm-diff/llvm-diff.cpp | 7 +- tools/llvm-dis/llvm-dis.cpp | 7 +- tools/llvm-dwarfdump/Statistics.cpp | 178 +- tools/llvm-dwarfdump/llvm-dwarfdump.cpp | 111 +- tools/llvm-extract/llvm-extract.cpp | 131 +- tools/llvm-link/llvm-link.cpp | 7 +- tools/llvm-lto/llvm-lto.cpp | 118 +- tools/llvm-lto2/llvm-lto2.cpp | 55 +- tools/llvm-mc/Disassembler.cpp | 7 +- tools/llvm-mc/Disassembler.h | 7 +- tools/llvm-mc/llvm-mc.cpp | 21 +- tools/llvm-mca/CodeRegion.cpp | 114 +- tools/llvm-mca/CodeRegion.h | 40 +- tools/llvm-mca/CodeRegionGenerator.cpp | 19 +- tools/llvm-mca/CodeRegionGenerator.h | 7 +- tools/llvm-mca/PipelinePrinter.cpp | 7 +- tools/llvm-mca/PipelinePrinter.h | 7 +- tools/llvm-mca/Views/BottleneckAnalysis.cpp | 624 ++ tools/llvm-mca/Views/BottleneckAnalysis.h | 341 + tools/llvm-mca/Views/DispatchStatistics.cpp | 7 +- tools/llvm-mca/Views/DispatchStatistics.h | 7 +- tools/llvm-mca/Views/InstructionInfoView.cpp | 10 +- tools/llvm-mca/Views/InstructionInfoView.h | 7 +- tools/llvm-mca/Views/RegisterFileStatistics.cpp | 7 +- tools/llvm-mca/Views/RegisterFileStatistics.h | 7 +- tools/llvm-mca/Views/ResourcePressureView.cpp | 7 +- tools/llvm-mca/Views/ResourcePressureView.h | 7 +- .../llvm-mca/Views/RetireControlUnitStatistics.cpp | 7 +- tools/llvm-mca/Views/RetireControlUnitStatistics.h | 7 +- tools/llvm-mca/Views/SchedulerStatistics.cpp | 37 +- tools/llvm-mca/Views/SchedulerStatistics.h | 11 +- tools/llvm-mca/Views/SummaryView.cpp | 25 +- tools/llvm-mca/Views/SummaryView.h | 13 +- tools/llvm-mca/Views/TimelineView.cpp | 7 +- tools/llvm-mca/Views/TimelineView.h | 7 +- tools/llvm-mca/Views/View.cpp | 7 +- tools/llvm-mca/Views/View.h | 7 +- tools/llvm-mca/llvm-mca.cpp | 56 +- tools/llvm-modextract/llvm-modextract.cpp | 7 +- tools/llvm-nm/llvm-nm.cpp | 517 +- tools/llvm-objcopy/Buffer.cpp | 50 +- tools/llvm-objcopy/Buffer.h | 16 +- tools/llvm-objcopy/COFF/COFFObjcopy.cpp | 158 +- tools/llvm-objcopy/COFF/COFFObjcopy.h | 12 +- tools/llvm-objcopy/COFF/Object.cpp | 91 +- tools/llvm-objcopy/COFF/Object.h | 79 +- tools/llvm-objcopy/COFF/Reader.cpp | 112 +- tools/llvm-objcopy/COFF/Reader.h | 9 +- tools/llvm-objcopy/COFF/Writer.cpp | 167 +- tools/llvm-objcopy/COFF/Writer.h | 10 +- tools/llvm-objcopy/CopyConfig.cpp | 661 +- tools/llvm-objcopy/CopyConfig.h | 130 +- tools/llvm-objcopy/ELF/ELFObjcopy.cpp | 684 +- tools/llvm-objcopy/ELF/ELFObjcopy.h | 18 +- tools/llvm-objcopy/ELF/Object.cpp | 1198 ++- tools/llvm-objcopy/ELF/Object.h | 314 +- tools/llvm-objcopy/MachO/MachOObjcopy.cpp | 68 + tools/llvm-objcopy/MachO/MachOObjcopy.h | 31 + tools/llvm-objcopy/MachO/MachOReader.cpp | 241 + tools/llvm-objcopy/MachO/MachOReader.h | 48 + tools/llvm-objcopy/MachO/MachOWriter.cpp | 590 ++ tools/llvm-objcopy/MachO/MachOWriter.h | 64 + tools/llvm-objcopy/MachO/Object.cpp | 15 + tools/llvm-objcopy/MachO/Object.h | 232 + tools/llvm-objcopy/ObjcopyOpts.td | 166 +- tools/llvm-objcopy/StripOpts.td | 61 +- tools/llvm-objcopy/llvm-objcopy.cpp | 237 +- tools/llvm-objcopy/llvm-objcopy.h | 8 +- tools/llvm-objdump/COFFDump.cpp | 82 +- tools/llvm-objdump/ELFDump.cpp | 252 +- tools/llvm-objdump/MachODump.cpp | 978 +- tools/llvm-objdump/WasmDump.cpp | 40 +- tools/llvm-objdump/llvm-objdump.cpp | 2215 ++--- tools/llvm-objdump/llvm-objdump.h | 153 +- tools/llvm-pdbutil/BytesOutputStyle.cpp | 11 +- tools/llvm-pdbutil/BytesOutputStyle.h | 7 +- tools/llvm-pdbutil/DumpOutputStyle.cpp | 192 +- tools/llvm-pdbutil/DumpOutputStyle.h | 16 +- tools/llvm-pdbutil/ExplainOutputStyle.cpp | 7 +- tools/llvm-pdbutil/ExplainOutputStyle.h | 7 +- tools/llvm-pdbutil/FormatUtil.cpp | 7 +- tools/llvm-pdbutil/FormatUtil.h | 7 +- tools/llvm-pdbutil/InputFile.cpp | 16 +- tools/llvm-pdbutil/InputFile.h | 7 +- tools/llvm-pdbutil/LinePrinter.cpp | 10 +- tools/llvm-pdbutil/LinePrinter.h | 10 +- tools/llvm-pdbutil/MinimalSymbolDumper.cpp | 159 +- tools/llvm-pdbutil/MinimalSymbolDumper.h | 7 +- tools/llvm-pdbutil/MinimalTypeDumper.cpp | 29 +- tools/llvm-pdbutil/MinimalTypeDumper.h | 14 +- tools/llvm-pdbutil/OutputStyle.h | 7 +- tools/llvm-pdbutil/PdbYaml.cpp | 10 +- tools/llvm-pdbutil/PdbYaml.h | 7 +- tools/llvm-pdbutil/PrettyBuiltinDumper.cpp | 7 +- tools/llvm-pdbutil/PrettyBuiltinDumper.h | 7 +- tools/llvm-pdbutil/PrettyClassDefinitionDumper.cpp | 7 +- tools/llvm-pdbutil/PrettyClassDefinitionDumper.h | 7 +- .../PrettyClassLayoutGraphicalDumper.cpp | 7 +- .../PrettyClassLayoutGraphicalDumper.h | 7 +- tools/llvm-pdbutil/PrettyCompilandDumper.cpp | 7 +- tools/llvm-pdbutil/PrettyCompilandDumper.h | 7 +- tools/llvm-pdbutil/PrettyEnumDumper.cpp | 7 +- tools/llvm-pdbutil/PrettyEnumDumper.h | 7 +- tools/llvm-pdbutil/PrettyExternalSymbolDumper.cpp | 7 +- tools/llvm-pdbutil/PrettyExternalSymbolDumper.h | 7 +- tools/llvm-pdbutil/PrettyFunctionDumper.cpp | 14 +- tools/llvm-pdbutil/PrettyFunctionDumper.h | 7 +- tools/llvm-pdbutil/PrettyTypeDumper.cpp | 7 +- tools/llvm-pdbutil/PrettyTypeDumper.h | 7 +- tools/llvm-pdbutil/PrettyTypedefDumper.cpp | 7 +- tools/llvm-pdbutil/PrettyTypedefDumper.h | 7 +- tools/llvm-pdbutil/PrettyVariableDumper.cpp | 7 +- tools/llvm-pdbutil/PrettyVariableDumper.h | 7 +- tools/llvm-pdbutil/StreamUtil.cpp | 7 +- tools/llvm-pdbutil/StreamUtil.h | 7 +- tools/llvm-pdbutil/TypeReferenceTracker.cpp | 160 + tools/llvm-pdbutil/TypeReferenceTracker.h | 69 + tools/llvm-pdbutil/YAMLOutputStyle.cpp | 12 +- tools/llvm-pdbutil/YAMLOutputStyle.h | 7 +- tools/llvm-pdbutil/llvm-pdbutil.cpp | 43 +- tools/llvm-pdbutil/llvm-pdbutil.h | 9 +- tools/llvm-profdata/llvm-profdata.cpp | 153 +- tools/llvm-readobj/ARMEHABIPrinter.h | 17 +- tools/llvm-readobj/ARMWinEHPrinter.cpp | 21 +- tools/llvm-readobj/ARMWinEHPrinter.h | 9 +- tools/llvm-readobj/COFFDumper.cpp | 187 +- tools/llvm-readobj/COFFImportDumper.cpp | 9 +- tools/llvm-readobj/DwarfCFIEHPrinter.h | 7 +- tools/llvm-readobj/ELFDumper.cpp | 2259 +++-- tools/llvm-readobj/Error.cpp | 7 +- tools/llvm-readobj/Error.h | 7 +- tools/llvm-readobj/MachODumper.cpp | 43 +- tools/llvm-readobj/ObjDumper.cpp | 218 +- tools/llvm-readobj/ObjDumper.h | 56 +- tools/llvm-readobj/StackMapPrinter.h | 19 +- tools/llvm-readobj/WasmDumper.cpp | 62 +- tools/llvm-readobj/Win64EHDumper.cpp | 7 +- tools/llvm-readobj/Win64EHDumper.h | 7 +- tools/llvm-readobj/WindowsResourceDumper.cpp | 7 +- tools/llvm-readobj/WindowsResourceDumper.h | 7 +- tools/llvm-readobj/XCOFFDumper.cpp | 190 + tools/llvm-readobj/llvm-readobj.cpp | 272 +- tools/llvm-readobj/llvm-readobj.h | 14 +- tools/llvm-rtdyld/llvm-rtdyld.cpp | 361 +- tools/llvm-stress/llvm-stress.cpp | 9 +- tools/llvm-symbolizer/llvm-symbolizer.cpp | 161 +- tools/llvm-xray/func-id-helper.cpp | 21 +- tools/llvm-xray/func-id-helper.h | 7 +- tools/llvm-xray/llvm-xray.cpp | 7 +- tools/llvm-xray/trie-node.h | 7 +- tools/llvm-xray/xray-account.cpp | 11 +- tools/llvm-xray/xray-account.h | 19 +- tools/llvm-xray/xray-color-helper.cpp | 7 +- tools/llvm-xray/xray-color-helper.h | 7 +- tools/llvm-xray/xray-converter.cpp | 120 +- tools/llvm-xray/xray-converter.h | 7 +- tools/llvm-xray/xray-extract.cpp | 11 +- tools/llvm-xray/xray-fdr-dump.cpp | 18 +- tools/llvm-xray/xray-graph-diff.cpp | 7 +- tools/llvm-xray/xray-graph-diff.h | 7 +- tools/llvm-xray/xray-graph.cpp | 11 +- tools/llvm-xray/xray-graph.h | 10 +- tools/llvm-xray/xray-registry.cpp | 7 +- tools/llvm-xray/xray-registry.h | 7 +- tools/llvm-xray/xray-stacks.cpp | 17 +- tools/opt/AnalysisWrappers.cpp | 7 +- tools/opt/BreakpointPrinter.cpp | 11 +- tools/opt/BreakpointPrinter.h | 7 +- tools/opt/Debugify.cpp | 7 +- tools/opt/Debugify.h | 7 +- tools/opt/GraphPrinters.cpp | 7 +- tools/opt/NewPMDriver.cpp | 58 +- tools/opt/NewPMDriver.h | 14 +- tools/opt/PassPrinters.cpp | 7 +- tools/opt/PassPrinters.h | 7 +- tools/opt/PrintSCC.cpp | 7 +- tools/opt/opt.cpp | 151 +- utils/TableGen/AsmMatcherEmitter.cpp | 246 +- utils/TableGen/AsmWriterEmitter.cpp | 26 +- utils/TableGen/AsmWriterInst.cpp | 40 +- utils/TableGen/AsmWriterInst.h | 7 +- utils/TableGen/Attributes.cpp | 7 +- utils/TableGen/CTagsEmitter.cpp | 7 +- utils/TableGen/CallingConvEmitter.cpp | 51 +- utils/TableGen/CodeEmitterGen.cpp | 106 +- utils/TableGen/CodeGenDAGPatterns.cpp | 110 +- utils/TableGen/CodeGenDAGPatterns.h | 25 +- utils/TableGen/CodeGenHwModes.cpp | 7 +- utils/TableGen/CodeGenHwModes.h | 7 +- utils/TableGen/CodeGenInstruction.cpp | 79 +- utils/TableGen/CodeGenInstruction.h | 8 +- utils/TableGen/CodeGenIntrinsics.h | 20 +- utils/TableGen/CodeGenMapTable.cpp | 7 +- utils/TableGen/CodeGenRegisters.cpp | 15 +- utils/TableGen/CodeGenRegisters.h | 7 +- utils/TableGen/CodeGenSchedule.cpp | 52 +- utils/TableGen/CodeGenSchedule.h | 7 +- utils/TableGen/CodeGenTarget.cpp | 100 +- utils/TableGen/CodeGenTarget.h | 7 +- utils/TableGen/DAGISelEmitter.cpp | 7 +- utils/TableGen/DAGISelMatcher.cpp | 31 +- utils/TableGen/DAGISelMatcher.h | 66 +- utils/TableGen/DAGISelMatcherEmitter.cpp | 27 +- utils/TableGen/DAGISelMatcherGen.cpp | 60 +- utils/TableGen/DAGISelMatcherOpt.cpp | 15 +- utils/TableGen/DFAPacketizerEmitter.cpp | 7 +- utils/TableGen/DisassemblerEmitter.cpp | 7 +- utils/TableGen/ExegesisEmitter.cpp | 7 +- utils/TableGen/FastISelEmitter.cpp | 7 +- utils/TableGen/FixedLenDecoderEmitter.cpp | 147 +- utils/TableGen/GlobalISelEmitter.cpp | 157 +- utils/TableGen/InfoByHwMode.cpp | 14 +- utils/TableGen/InfoByHwMode.h | 13 +- utils/TableGen/InstrDocsEmitter.cpp | 7 +- utils/TableGen/InstrInfoEmitter.cpp | 89 +- utils/TableGen/IntrinsicEmitter.cpp | 120 +- utils/TableGen/OptParserEmitter.cpp | 7 +- utils/TableGen/PredicateExpander.cpp | 7 +- utils/TableGen/PredicateExpander.h | 7 +- utils/TableGen/PseudoLoweringEmitter.cpp | 7 +- utils/TableGen/RISCVCompressInstEmitter.cpp | 59 +- utils/TableGen/RegisterBankEmitter.cpp | 7 +- utils/TableGen/RegisterInfoEmitter.cpp | 7 +- utils/TableGen/SDNodeProperties.cpp | 13 +- utils/TableGen/SDNodeProperties.h | 7 +- utils/TableGen/SearchableTableEmitter.cpp | 35 +- utils/TableGen/SequenceToOffsetTable.h | 7 +- utils/TableGen/SubtargetEmitter.cpp | 140 +- utils/TableGen/SubtargetFeatureInfo.cpp | 27 +- utils/TableGen/SubtargetFeatureInfo.h | 16 +- utils/TableGen/TableGen.cpp | 18 +- utils/TableGen/TableGenBackends.h | 7 +- utils/TableGen/Types.cpp | 7 +- utils/TableGen/Types.h | 7 +- utils/TableGen/WebAssemblyDisassemblerEmitter.cpp | 38 +- utils/TableGen/WebAssemblyDisassemblerEmitter.h | 7 +- utils/TableGen/X86DisassemblerShared.h | 7 +- utils/TableGen/X86DisassemblerTables.cpp | 81 +- utils/TableGen/X86DisassemblerTables.h | 7 +- utils/TableGen/X86EVEX2VEXTablesEmitter.cpp | 64 +- utils/TableGen/X86FoldTablesEmitter.cpp | 71 +- utils/TableGen/X86ModRMFilters.cpp | 7 +- utils/TableGen/X86ModRMFilters.h | 7 +- utils/TableGen/X86RecognizableInstr.cpp | 128 +- utils/TableGen/X86RecognizableInstr.h | 24 +- 4643 files changed, 231107 insertions(+), 109687 deletions(-) delete mode 100644 include/llvm-c/OptRemarks.h create mode 100644 include/llvm-c/Remarks.h create mode 100644 include/llvm/ADT/fallible_iterator.h create mode 100644 include/llvm/Analysis/DomTreeUpdater.h create mode 100644 include/llvm/Analysis/VecFuncs.def create mode 100644 include/llvm/BinaryFormat/Minidump.h create mode 100644 include/llvm/BinaryFormat/MinidumpConstants.def create mode 100644 include/llvm/BinaryFormat/MsgPackDocument.h delete mode 100644 include/llvm/BinaryFormat/MsgPackTypes.h create mode 100644 include/llvm/BinaryFormat/XCOFF.h delete mode 100644 include/llvm/Bitcode/BitCodes.h create mode 100644 include/llvm/Bitcode/BitcodeAnalyzer.h delete mode 100644 include/llvm/Bitcode/BitstreamReader.h delete mode 100644 include/llvm/Bitcode/BitstreamWriter.h create mode 100644 include/llvm/Bitstream/BitCodes.h create mode 100644 include/llvm/Bitstream/BitstreamReader.h create mode 100644 include/llvm/Bitstream/BitstreamWriter.h create mode 100644 include/llvm/CodeGen/CSEConfigBase.h create mode 100644 include/llvm/CodeGen/MIRParser/MIParser.h create mode 100644 include/llvm/CodeGen/Register.h create mode 100644 include/llvm/CodeGen/SwiftErrorValueTracking.h create mode 100644 include/llvm/CodeGen/SwitchLoweringUtils.h create mode 100644 include/llvm/DebugInfo/GSYM/FileEntry.h create mode 100644 include/llvm/DebugInfo/GSYM/FunctionInfo.h create mode 100644 include/llvm/DebugInfo/GSYM/InlineInfo.h create mode 100644 include/llvm/DebugInfo/GSYM/LineEntry.h create mode 100644 include/llvm/DebugInfo/GSYM/Range.h create mode 100644 include/llvm/DebugInfo/GSYM/StringTable.h create mode 100644 include/llvm/DebugInfo/PDB/Native/InjectedSourceStream.h create mode 100644 include/llvm/DebugInfo/PDB/Native/NativeEnumInjectedSources.h delete mode 100644 include/llvm/Demangle/Compiler.h create mode 100644 include/llvm/Demangle/DemangleConfig.h create mode 100644 include/llvm/Demangle/README.txt create mode 100644 include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h create mode 100644 include/llvm/ExecutionEngine/JITLink/JITLink.h create mode 100644 include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h create mode 100644 include/llvm/ExecutionEngine/JITLink/MachO.h create mode 100644 include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h create mode 100644 include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h create mode 100644 include/llvm/ExecutionEngine/OrcV1Deprecation.h delete mode 100644 include/llvm/IR/DomTreeUpdater.h create mode 100644 include/llvm/IR/RemarkStreamer.h create mode 100644 include/llvm/MC/MCAsmInfoXCOFF.h create mode 100644 include/llvm/MC/MCSectionXCOFF.h create mode 100644 include/llvm/MC/MCSymbolXCOFF.h create mode 100644 include/llvm/MC/MCXCOFFObjectWriter.h create mode 100644 include/llvm/MC/MCXCOFFStreamer.h create mode 100644 include/llvm/MCA/Stages/MicroOpQueueStage.h create mode 100644 include/llvm/Object/Minidump.h delete mode 100644 include/llvm/Object/RelocVisitor.h create mode 100644 include/llvm/Object/RelocationResolver.h create mode 100644 include/llvm/Object/WindowsMachineFlag.h create mode 100644 include/llvm/Object/XCOFFObjectFile.h create mode 100644 include/llvm/ObjectYAML/MinidumpYAML.h create mode 100644 include/llvm/ObjectYAML/XCOFFYAML.h create mode 100644 include/llvm/Remarks/Remark.h create mode 100644 include/llvm/Remarks/RemarkFormat.h create mode 100644 include/llvm/Remarks/RemarkParser.h create mode 100644 include/llvm/Remarks/RemarkSerializer.h create mode 100644 include/llvm/Remarks/RemarkStringTable.h create mode 100644 include/llvm/Support/CRC.h create mode 100644 include/llvm/Support/GenericIteratedDominanceFrontier.h create mode 100644 include/llvm/Support/SMTAPI.h create mode 100644 include/llvm/Support/ScalableSize.h create mode 100644 include/llvm/Support/Signposts.h create mode 100644 include/llvm/Support/TimeProfiler.h create mode 100644 include/llvm/Testing/Support/Annotations.h create mode 100644 include/llvm/TextAPI/MachO/Architecture.def create mode 100644 include/llvm/TextAPI/MachO/Architecture.h create mode 100644 include/llvm/TextAPI/MachO/ArchitectureSet.h create mode 100644 include/llvm/TextAPI/MachO/InterfaceFile.h create mode 100644 include/llvm/TextAPI/MachO/PackedVersion.h create mode 100644 include/llvm/TextAPI/MachO/Symbol.h create mode 100644 include/llvm/TextAPI/MachO/TextAPIReader.h create mode 100644 include/llvm/TextAPI/MachO/TextAPIWriter.h create mode 100644 include/llvm/Transforms/IPO/Attributor.h create mode 100644 include/llvm/Transforms/Instrumentation/AddressSanitizer.h create mode 100644 include/llvm/Transforms/Instrumentation/HWAddressSanitizer.h create mode 100644 include/llvm/Transforms/Instrumentation/InstrOrderFile.h create mode 100644 include/llvm/Transforms/Instrumentation/PoisonChecking.h create mode 100644 include/llvm/Transforms/Scalar/LoopFuse.h create mode 100644 include/llvm/Transforms/Scalar/LowerWidenableCondition.h create mode 100644 include/llvm/Transforms/Scalar/MergeICmps.h create mode 100644 include/llvm/Transforms/Utils/SizeOpts.h create mode 100644 lib/Analysis/DomTreeUpdater.cpp delete mode 100644 lib/Analysis/IteratedDominanceFrontier.cpp create mode 100644 lib/BinaryFormat/Minidump.cpp create mode 100644 lib/BinaryFormat/MsgPackDocument.cpp create mode 100644 lib/BinaryFormat/MsgPackDocumentYAML.cpp delete mode 100644 lib/BinaryFormat/MsgPackTypes.cpp create mode 100644 lib/Bitcode/Reader/BitcodeAnalyzer.cpp delete mode 100644 lib/Bitcode/Reader/BitstreamReader.cpp create mode 100644 lib/Bitstream/Reader/BitstreamReader.cpp delete mode 100644 lib/CodeGen/ExpandISelPseudos.cpp create mode 100644 lib/CodeGen/FinalizeISel.cpp create mode 100644 lib/CodeGen/HardwareLoops.cpp delete mode 100644 lib/CodeGen/MIRParser/MIParser.h create mode 100644 lib/CodeGen/SwiftErrorValueTracking.cpp create mode 100644 lib/CodeGen/SwitchLoweringUtils.cpp create mode 100644 lib/DebugInfo/GSYM/FunctionInfo.cpp create mode 100644 lib/DebugInfo/GSYM/InlineInfo.cpp create mode 100644 lib/DebugInfo/GSYM/Range.cpp create mode 100644 lib/DebugInfo/PDB/Native/InjectedSourceStream.cpp create mode 100644 lib/DebugInfo/PDB/Native/NativeEnumInjectedSources.cpp create mode 100644 lib/Demangle/Demangle.cpp create mode 100644 lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h create mode 100644 lib/ExecutionEngine/JITLink/EHFrameSupport.cpp create mode 100644 lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h create mode 100644 lib/ExecutionEngine/JITLink/JITLink.cpp create mode 100644 lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp create mode 100644 lib/ExecutionEngine/JITLink/JITLinkGeneric.h create mode 100644 lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp create mode 100644 lib/ExecutionEngine/JITLink/MachO.cpp create mode 100644 lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp create mode 100644 lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h create mode 100644 lib/ExecutionEngine/JITLink/MachO_x86_64.cpp create mode 100644 lib/ExecutionEngine/Orc/CompileUtils.cpp create mode 100644 lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp create mode 100644 lib/IR/AbstractCallSite.cpp delete mode 100644 lib/IR/DomTreeUpdater.cpp create mode 100644 lib/IR/RemarkStreamer.cpp create mode 100644 lib/MC/MCAsmInfoXCOFF.cpp create mode 100644 lib/MC/MCSectionXCOFF.cpp create mode 100644 lib/MC/MCXCOFFObjectTargetWriter.cpp create mode 100644 lib/MC/MCXCOFFStreamer.cpp create mode 100644 lib/MC/XCOFFObjectWriter.cpp create mode 100644 lib/MCA/Stages/MicroOpQueueStage.cpp create mode 100644 lib/Object/Minidump.cpp create mode 100644 lib/Object/RelocationResolver.cpp create mode 100644 lib/Object/WindowsMachineFlag.cpp create mode 100644 lib/Object/XCOFFObjectFile.cpp create mode 100644 lib/ObjectYAML/MinidumpYAML.cpp create mode 100644 lib/ObjectYAML/XCOFFYAML.cpp delete mode 100644 lib/OptRemarks/OptRemarksParser.cpp create mode 100644 lib/Remarks/Remark.cpp create mode 100644 lib/Remarks/RemarkFormat.cpp create mode 100644 lib/Remarks/RemarkParser.cpp create mode 100644 lib/Remarks/RemarkStringTable.cpp create mode 100644 lib/Remarks/YAMLRemarkParser.cpp create mode 100644 lib/Remarks/YAMLRemarkParser.h create mode 100644 lib/Remarks/YAMLRemarkSerializer.cpp create mode 100644 lib/Support/CRC.cpp create mode 100644 lib/Support/Optional.cpp create mode 100644 lib/Support/Signposts.cpp create mode 100644 lib/Support/TimeProfiler.cpp create mode 100644 lib/Support/Z3Solver.cpp create mode 100644 lib/Target/AArch64/AArch64CallingConvention.cpp create mode 100644 lib/Target/AArch64/AArch64ExpandImm.cpp create mode 100644 lib/Target/AArch64/AArch64ExpandImm.h create mode 100644 lib/Target/AArch64/AArch64StackTagging.cpp delete mode 100644 lib/Target/AArch64/InstPrinter/AArch64InstPrinter.cpp delete mode 100644 lib/Target/AArch64/InstPrinter/AArch64InstPrinter.h create mode 100644 lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.cpp create mode 100644 lib/Target/AArch64/MCTargetDesc/AArch64InstPrinter.h create mode 100644 lib/Target/AArch64/TargetInfo/AArch64TargetInfo.h delete mode 100644 lib/Target/AMDGPU/AMDGPUIntrinsicInfo.cpp delete mode 100644 lib/Target/AMDGPU/AMDGPUIntrinsicInfo.h create mode 100644 lib/Target/AMDGPU/AMDGPUPropagateAttributes.cpp delete mode 100644 lib/Target/AMDGPU/AMDGPURegAsmNames.inc.cpp create mode 100644 lib/Target/AMDGPU/GCNNSAReassign.cpp create mode 100644 lib/Target/AMDGPU/GCNRegBankReassign.cpp delete mode 100644 lib/Target/AMDGPU/InstPrinter/AMDGPUInstPrinter.cpp delete mode 100644 lib/Target/AMDGPU/InstPrinter/AMDGPUInstPrinter.h create mode 100644 lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp create mode 100644 lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.h delete mode 100644 lib/Target/AMDGPU/SIDebuggerInsertNops.cpp delete mode 100644 lib/Target/AMDGPU/SIFixWWMLiveness.cpp delete mode 100644 lib/Target/AMDGPU/SIIntrinsics.td create mode 100644 lib/Target/AMDGPU/SILowerSGPRSpills.cpp create mode 100644 lib/Target/AMDGPU/SIPreAllocateWWMRegs.cpp create mode 100644 lib/Target/AMDGPU/TargetInfo/AMDGPUTargetInfo.h create mode 100644 lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.cpp create mode 100644 lib/Target/AMDGPU/Utils/AMDGPUPALMetadata.h create mode 100644 lib/Target/ARC/ARCOptAddrMode.cpp delete mode 100644 lib/Target/ARC/InstPrinter/ARCInstPrinter.cpp delete mode 100644 lib/Target/ARC/InstPrinter/ARCInstPrinter.h create mode 100644 lib/Target/ARC/MCTargetDesc/ARCInstPrinter.cpp create mode 100644 lib/Target/ARC/MCTargetDesc/ARCInstPrinter.h create mode 100644 lib/Target/ARC/TargetInfo/ARCTargetInfo.h create mode 100644 lib/Target/ARM/ARMBasicBlockInfo.cpp create mode 100644 lib/Target/ARM/ARMCallingConv.cpp delete mode 100644 lib/Target/ARM/ARMComputeBlockSize.cpp create mode 100644 lib/Target/ARM/ARMInstrMVE.td create mode 100644 lib/Target/ARM/ARMLowOverheadLoops.cpp create mode 100644 lib/Target/ARM/ARMPredicates.td delete mode 100644 lib/Target/ARM/ARMScheduleM3.td create mode 100644 lib/Target/ARM/ARMScheduleM4.td delete mode 100644 lib/Target/ARM/InstPrinter/ARMInstPrinter.cpp delete mode 100644 lib/Target/ARM/InstPrinter/ARMInstPrinter.h delete mode 100755 lib/Target/ARM/LICENSE.TXT create mode 100644 lib/Target/ARM/MCTargetDesc/ARMInstPrinter.cpp create mode 100644 lib/Target/ARM/MCTargetDesc/ARMInstPrinter.h create mode 100644 lib/Target/ARM/TargetInfo/ARMTargetInfo.h delete mode 100644 lib/Target/AVR/InstPrinter/AVRInstPrinter.cpp delete mode 100644 lib/Target/AVR/InstPrinter/AVRInstPrinter.h create mode 100644 lib/Target/AVR/MCTargetDesc/AVRInstPrinter.cpp create mode 100644 lib/Target/AVR/MCTargetDesc/AVRInstPrinter.h create mode 100644 lib/Target/AVR/TargetInfo/AVRTargetInfo.h create mode 100644 lib/Target/BPF/BPFAbstractMemberAccess.cpp create mode 100644 lib/Target/BPF/BPFCORE.h create mode 100644 lib/Target/BPF/BPFMISimplifyPatchable.cpp delete mode 100644 lib/Target/BPF/InstPrinter/BPFInstPrinter.cpp delete mode 100644 lib/Target/BPF/InstPrinter/BPFInstPrinter.h create mode 100644 lib/Target/BPF/MCTargetDesc/BPFInstPrinter.cpp create mode 100644 lib/Target/BPF/MCTargetDesc/BPFInstPrinter.h create mode 100644 lib/Target/BPF/TargetInfo/BPFTargetInfo.h delete mode 100644 lib/Target/Hexagon/HexagonDepDecoders.h create mode 100644 lib/Target/Hexagon/HexagonDepDecoders.inc create mode 100644 lib/Target/Hexagon/TargetInfo/HexagonTargetInfo.h delete mode 100644 lib/Target/Lanai/InstPrinter/LanaiInstPrinter.cpp delete mode 100644 lib/Target/Lanai/InstPrinter/LanaiInstPrinter.h create mode 100644 lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.cpp create mode 100644 lib/Target/Lanai/MCTargetDesc/LanaiInstPrinter.h create mode 100644 lib/Target/Lanai/TargetInfo/LanaiTargetInfo.h delete mode 100644 lib/Target/MSP430/InstPrinter/MSP430InstPrinter.cpp delete mode 100644 lib/Target/MSP430/InstPrinter/MSP430InstPrinter.h create mode 100644 lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.cpp create mode 100644 lib/Target/MSP430/MCTargetDesc/MSP430InstPrinter.h create mode 100644 lib/Target/MSP430/TargetInfo/MSP430TargetInfo.h delete mode 100644 lib/Target/Mips/InstPrinter/MipsInstPrinter.cpp delete mode 100644 lib/Target/Mips/InstPrinter/MipsInstPrinter.h create mode 100644 lib/Target/Mips/MCTargetDesc/MipsInstPrinter.cpp create mode 100644 lib/Target/Mips/MCTargetDesc/MipsInstPrinter.h create mode 100644 lib/Target/Mips/TargetInfo/MipsTargetInfo.h delete mode 100644 lib/Target/NVPTX/InstPrinter/NVPTXInstPrinter.cpp delete mode 100644 lib/Target/NVPTX/InstPrinter/NVPTXInstPrinter.h create mode 100644 lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.cpp create mode 100644 lib/Target/NVPTX/MCTargetDesc/NVPTXInstPrinter.h create mode 100644 lib/Target/NVPTX/TargetInfo/NVPTXTargetInfo.h delete mode 100644 lib/Target/PowerPC/InstPrinter/PPCInstPrinter.cpp delete mode 100644 lib/Target/PowerPC/InstPrinter/PPCInstPrinter.h create mode 100644 lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp create mode 100644 lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.h create mode 100644 lib/Target/PowerPC/MCTargetDesc/PPCXCOFFObjectWriter.cpp create mode 100644 lib/Target/PowerPC/PPCCallingConv.cpp create mode 100644 lib/Target/PowerPC/PPCMachineScheduler.cpp create mode 100644 lib/Target/PowerPC/PPCMachineScheduler.h create mode 100644 lib/Target/PowerPC/TargetInfo/PowerPCTargetInfo.h delete mode 100644 lib/Target/RISCV/InstPrinter/RISCVInstPrinter.cpp delete mode 100644 lib/Target/RISCV/InstPrinter/RISCVInstPrinter.h create mode 100644 lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.cpp create mode 100644 lib/Target/RISCV/MCTargetDesc/RISCVInstPrinter.h create mode 100644 lib/Target/RISCV/RISCVTargetTransformInfo.cpp create mode 100644 lib/Target/RISCV/RISCVTargetTransformInfo.h create mode 100644 lib/Target/RISCV/TargetInfo/RISCVTargetInfo.h delete mode 100644 lib/Target/Sparc/InstPrinter/SparcInstPrinter.cpp delete mode 100644 lib/Target/Sparc/InstPrinter/SparcInstPrinter.h create mode 100644 lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.cpp create mode 100644 lib/Target/Sparc/MCTargetDesc/SparcInstPrinter.h create mode 100644 lib/Target/Sparc/TargetInfo/SparcTargetInfo.h delete mode 100644 lib/Target/SystemZ/InstPrinter/SystemZInstPrinter.cpp delete mode 100644 lib/Target/SystemZ/InstPrinter/SystemZInstPrinter.h create mode 100644 lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.cpp create mode 100644 lib/Target/SystemZ/MCTargetDesc/SystemZInstPrinter.h create mode 100644 lib/Target/SystemZ/SystemZPostRewrite.cpp create mode 100644 lib/Target/SystemZ/SystemZScheduleArch13.td create mode 100644 lib/Target/SystemZ/TargetInfo/SystemZTargetInfo.h delete mode 100644 lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.cpp delete mode 100644 lib/Target/WebAssembly/InstPrinter/WebAssemblyInstPrinter.h create mode 100644 lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.cpp create mode 100644 lib/Target/WebAssembly/MCTargetDesc/WebAssemblyInstPrinter.h create mode 100644 lib/Target/WebAssembly/TargetInfo/WebAssemblyTargetInfo.h delete mode 100644 lib/Target/WebAssembly/WebAssemblyEHRestoreStackPointer.cpp create mode 100644 lib/Target/WebAssembly/WebAssemblyInstrBulkMemory.td delete mode 100644 lib/Target/WebAssembly/WebAssemblyInstrExceptRef.td create mode 100644 lib/Target/WebAssembly/WebAssemblyInstrRef.td delete mode 100644 lib/Target/X86/AsmParser/X86AsmInstrumentation.cpp delete mode 100644 lib/Target/X86/AsmParser/X86AsmInstrumentation.h delete mode 100644 lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp delete mode 100644 lib/Target/X86/InstPrinter/X86ATTInstPrinter.h delete mode 100644 lib/Target/X86/InstPrinter/X86InstComments.cpp delete mode 100644 lib/Target/X86/InstPrinter/X86InstComments.h delete mode 100644 lib/Target/X86/InstPrinter/X86InstPrinterCommon.cpp delete mode 100644 lib/Target/X86/InstPrinter/X86InstPrinterCommon.h delete mode 100644 lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp delete mode 100644 lib/Target/X86/InstPrinter/X86IntelInstPrinter.h create mode 100644 lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.cpp create mode 100644 lib/Target/X86/MCTargetDesc/X86ATTInstPrinter.h create mode 100644 lib/Target/X86/MCTargetDesc/X86InstComments.cpp create mode 100644 lib/Target/X86/MCTargetDesc/X86InstComments.h create mode 100644 lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.cpp create mode 100644 lib/Target/X86/MCTargetDesc/X86InstPrinterCommon.h create mode 100644 lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.cpp create mode 100644 lib/Target/X86/MCTargetDesc/X86IntelInstPrinter.h delete mode 100644 lib/Target/X86/ShadowCallStack.cpp create mode 100644 lib/Target/X86/TargetInfo/X86TargetInfo.h delete mode 100644 lib/Target/XCore/InstPrinter/XCoreInstPrinter.cpp delete mode 100644 lib/Target/XCore/InstPrinter/XCoreInstPrinter.h create mode 100644 lib/Target/XCore/MCTargetDesc/XCoreInstPrinter.cpp create mode 100644 lib/Target/XCore/MCTargetDesc/XCoreInstPrinter.h create mode 100644 lib/Target/XCore/TargetInfo/XCoreTargetInfo.h create mode 100644 lib/Testing/Support/Annotations.cpp create mode 100644 lib/TextAPI/MachO/Architecture.cpp create mode 100644 lib/TextAPI/MachO/ArchitectureSet.cpp create mode 100644 lib/TextAPI/MachO/InterfaceFile.cpp create mode 100644 lib/TextAPI/MachO/PackedVersion.cpp create mode 100644 lib/TextAPI/MachO/Symbol.cpp create mode 100644 lib/TextAPI/MachO/TextAPIContext.h create mode 100644 lib/TextAPI/MachO/TextStub.cpp create mode 100644 lib/TextAPI/MachO/TextStubCommon.cpp create mode 100644 lib/TextAPI/MachO/TextStubCommon.h create mode 100644 lib/Transforms/IPO/Attributor.cpp create mode 100644 lib/Transforms/InstCombine/InstCombineAtomicRMW.cpp delete mode 100644 lib/Transforms/Instrumentation/EfficiencySanitizer.cpp create mode 100644 lib/Transforms/Instrumentation/InstrOrderFile.cpp create mode 100644 lib/Transforms/Instrumentation/PoisonChecking.cpp create mode 100644 lib/Transforms/Scalar/LoopFuse.cpp create mode 100644 lib/Transforms/Scalar/LowerWidenableCondition.cpp create mode 100644 lib/Transforms/Utils/SizeOpts.cpp create mode 100644 lib/Transforms/Vectorize/VPlanPredicator.cpp create mode 100644 lib/Transforms/Vectorize/VPlanPredicator.h create mode 100644 tools/llvm-mca/Views/BottleneckAnalysis.cpp create mode 100644 tools/llvm-mca/Views/BottleneckAnalysis.h create mode 100644 tools/llvm-objcopy/MachO/MachOObjcopy.cpp create mode 100644 tools/llvm-objcopy/MachO/MachOObjcopy.h create mode 100644 tools/llvm-objcopy/MachO/MachOReader.cpp create mode 100644 tools/llvm-objcopy/MachO/MachOReader.h create mode 100644 tools/llvm-objcopy/MachO/MachOWriter.cpp create mode 100644 tools/llvm-objcopy/MachO/MachOWriter.h create mode 100644 tools/llvm-objcopy/MachO/Object.cpp create mode 100644 tools/llvm-objcopy/MachO/Object.h create mode 100644 tools/llvm-pdbutil/TypeReferenceTracker.cpp create mode 100644 tools/llvm-pdbutil/TypeReferenceTracker.h create mode 100644 tools/llvm-readobj/XCOFFDumper.cpp diff --git a/LICENSE.TXT b/LICENSE.TXT index e4d67d16fea1..fa6ac5400070 100644 --- a/LICENSE.TXT +++ b/LICENSE.TXT @@ -1,5 +1,240 @@ ============================================================================== -LLVM Release License +The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: +============================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + +============================================================================== +Software from third parties included in the LLVM Project: +============================================================================== +The LLVM Project contains third party software which is under different license +terms. All such code will be identified clearly using at least one of two +mechanisms: +1) It will be in a separate directory tree with its own `LICENSE.txt` or + `LICENSE` file at the top containing the specific license and restrictions + which apply to that software, or +2) It will contain specific license and restriction terms at the top of every + file. + +============================================================================== +Legacy LLVM License (https://llvm.org/docs/DeveloperPolicy.html#legacy): ============================================================================== University of Illinois/NCSA Open Source License @@ -42,27 +277,3 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE SOFTWARE. -============================================================================== -Copyrights and Licenses for Third Party Software Distributed with LLVM: -============================================================================== -The LLVM software contains code written by third parties. Such software will -have its own individual LICENSE.TXT file in the directory in which it appears. -This file will describe the copyrights, license, and restrictions which apply -to that code. - -The disclaimer of warranty in the University of Illinois Open Source License -applies to all code in the LLVM Distribution, and nothing in any of the -other licenses gives permission to use the names of the LLVM Team or the -University of Illinois to endorse or promote products derived from this -Software. - -The following pieces of software have additional or alternate copyrights, -licenses, and/or restrictions: - -Program Directory -------- --------- -Google Test llvm/utils/unittest/googletest -OpenBSD regex llvm/lib/Support/{reg*, COPYRIGHT.regex} -pyyaml tests llvm/test/YAMLParser/{*.data, LICENSE.TXT} -ARM contributions llvm/lib/Target/ARM/LICENSE.TXT -md5 contributions llvm/lib/Support/MD5.cpp llvm/include/llvm/Support/MD5.h diff --git a/include/llvm-c/Analysis.h b/include/llvm-c/Analysis.h index 36dcb89e0e08..cb9e8ece3c53 100644 --- a/include/llvm-c/Analysis.h +++ b/include/llvm-c/Analysis.h @@ -1,9 +1,9 @@ /*===-- llvm-c/Analysis.h - Analysis Library C Interface --------*- C++ -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| diff --git a/include/llvm-c/BitReader.h b/include/llvm-c/BitReader.h index d1fc302767ba..b307ee979f8a 100644 --- a/include/llvm-c/BitReader.h +++ b/include/llvm-c/BitReader.h @@ -1,9 +1,9 @@ /*===-- llvm-c/BitReader.h - BitReader Library C Interface ------*- C++ -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| diff --git a/include/llvm-c/BitWriter.h b/include/llvm-c/BitWriter.h index 797d03179ab3..187051555b9a 100644 --- a/include/llvm-c/BitWriter.h +++ b/include/llvm-c/BitWriter.h @@ -1,9 +1,9 @@ /*===-- llvm-c/BitWriter.h - BitWriter Library C Interface ------*- C++ -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| diff --git a/include/llvm-c/Comdat.h b/include/llvm-c/Comdat.h index 499996d68a53..81fee3fc9a6b 100644 --- a/include/llvm-c/Comdat.h +++ b/include/llvm-c/Comdat.h @@ -1,9 +1,9 @@ /*===-- llvm-c/Comdat.h - Module Comdat C Interface -------------*- C++ -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| diff --git a/include/llvm-c/Core.h b/include/llvm-c/Core.h index 06de058bdc58..cac2f297056d 100644 --- a/include/llvm-c/Core.h +++ b/include/llvm-c/Core.h @@ -1,9 +1,9 @@ /*===-- llvm-c/Core.h - Core Library C Interface ------------------*- C -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| @@ -65,6 +65,7 @@ typedef enum { LLVMInvoke = 5, /* removed 6 due to API changes */ LLVMUnreachable = 7, + LLVMCallBr = 67, /* Standard Unary Operators */ LLVMFNeg = 66, @@ -2401,6 +2402,13 @@ LLVMValueRef LLVMGetPersonalityFn(LLVMValueRef Fn); */ void LLVMSetPersonalityFn(LLVMValueRef Fn, LLVMValueRef PersonalityFn); +/** + * Obtain the intrinsic ID number which matches the given function name. + * + * @see llvm::Function::lookupIntrinsicID() + */ +unsigned LLVMLookupIntrinsicID(const char *Name, size_t NameLen); + /** * Obtain the ID number from a function instance. * @@ -2612,52 +2620,138 @@ void LLVMSetParamAlignment(LLVMValueRef Arg, unsigned Align); */ /** - * @} + * @defgroup LLVMCCoreValueGlobalIFunc IFuncs + * + * Functions in this group relate to indirect functions. + * + * Functions in this group expect LLVMValueRef instances that correspond + * to llvm::GlobalIFunc instances. + * + * @{ */ /** - * @} + * Add a global indirect function to a module under a specified name. + * + * @see llvm::GlobalIFunc::create() */ +LLVMValueRef LLVMAddGlobalIFunc(LLVMModuleRef M, + const char *Name, size_t NameLen, + LLVMTypeRef Ty, unsigned AddrSpace, + LLVMValueRef Resolver); /** - * @} + * Obtain a GlobalIFunc value from a Module by its name. + * + * The returned value corresponds to a llvm::GlobalIFunc value. + * + * @see llvm::Module::getNamedIFunc() */ +LLVMValueRef LLVMGetNamedGlobalIFunc(LLVMModuleRef M, + const char *Name, size_t NameLen); /** - * @defgroup LLVMCCoreValueMetadata Metadata + * Obtain an iterator to the first GlobalIFunc in a Module. * - * @{ + * @see llvm::Module::ifunc_begin() */ +LLVMValueRef LLVMGetFirstGlobalIFunc(LLVMModuleRef M); /** - * Obtain a MDString value from a context. + * Obtain an iterator to the last GlobalIFunc in a Module. * - * The returned instance corresponds to the llvm::MDString class. + * @see llvm::Module::ifunc_end() + */ +LLVMValueRef LLVMGetLastGlobalIFunc(LLVMModuleRef M); + +/** + * Advance a GlobalIFunc iterator to the next GlobalIFunc. * - * The instance is specified by string data of a specified length. The - * string content is copied, so the backing memory can be freed after - * this function returns. + * Returns NULL if the iterator was already at the end and there are no more + * global aliases. */ -LLVMValueRef LLVMMDStringInContext(LLVMContextRef C, const char *Str, - unsigned SLen); +LLVMValueRef LLVMGetNextGlobalIFunc(LLVMValueRef IFunc); /** - * Obtain a MDString value from the global context. + * Decrement a GlobalIFunc iterator to the previous GlobalIFunc. + * + * Returns NULL if the iterator was already at the beginning and there are + * no previous global aliases. */ -LLVMValueRef LLVMMDString(const char *Str, unsigned SLen); +LLVMValueRef LLVMGetPreviousGlobalIFunc(LLVMValueRef IFunc); + +/** + * Retrieves the resolver function associated with this indirect function, or + * NULL if it doesn't not exist. + * + * @see llvm::GlobalIFunc::getResolver() + */ +LLVMValueRef LLVMGetGlobalIFuncResolver(LLVMValueRef IFunc); /** - * Obtain a MDNode value from a context. + * Sets the resolver function associated with this indirect function. * - * The returned value corresponds to the llvm::MDNode class. + * @see llvm::GlobalIFunc::setResolver() */ -LLVMValueRef LLVMMDNodeInContext(LLVMContextRef C, LLVMValueRef *Vals, - unsigned Count); +void LLVMSetGlobalIFuncResolver(LLVMValueRef IFunc, LLVMValueRef Resolver); /** - * Obtain a MDNode value from the global context. + * Remove a global indirect function from its parent module and delete it. + * + * @see llvm::GlobalIFunc::eraseFromParent() */ -LLVMValueRef LLVMMDNode(LLVMValueRef *Vals, unsigned Count); +void LLVMEraseGlobalIFunc(LLVMValueRef IFunc); + +/** + * Remove a global indirect function from its parent module. + * + * This unlinks the global indirect function from its containing module but + * keeps it alive. + * + * @see llvm::GlobalIFunc::removeFromParent() + */ +void LLVMRemoveGlobalIFunc(LLVMValueRef IFunc); + +/** + * @} + */ + +/** + * @} + */ + +/** + * @} + */ + +/** + * @} + */ + +/** + * @defgroup LLVMCCoreValueMetadata Metadata + * + * @{ + */ + +/** + * Create an MDString value from a given string value. + * + * The MDString value does not take ownership of the given string, it remains + * the responsibility of the caller to free it. + * + * @see llvm::MDString::get() + */ +LLVMMetadataRef LLVMMDStringInContext2(LLVMContextRef C, const char *Str, + size_t SLen); + +/** + * Create an MDNode value with the given array of operands. + * + * @see llvm::MDNode::get() + */ +LLVMMetadataRef LLVMMDNodeInContext2(LLVMContextRef C, LLVMMetadataRef *MDs, + size_t Count); /** * Obtain a Metadata as a Value. @@ -2699,6 +2793,17 @@ unsigned LLVMGetMDNodeNumOperands(LLVMValueRef V); */ void LLVMGetMDNodeOperands(LLVMValueRef V, LLVMValueRef *Dest); +/** Deprecated: Use LLVMMDStringInContext2 instead. */ +LLVMValueRef LLVMMDStringInContext(LLVMContextRef C, const char *Str, + unsigned SLen); +/** Deprecated: Use LLVMMDStringInContext2 instead. */ +LLVMValueRef LLVMMDString(const char *Str, unsigned SLen); +/** Deprecated: Use LLVMMDNodeInContext2 instead. */ +LLVMValueRef LLVMMDNodeInContext(LLVMContextRef C, LLVMValueRef *Vals, + unsigned Count); +/** Deprecated: Use LLVMMDNodeInContext2 instead. */ +LLVMValueRef LLVMMDNode(LLVMValueRef *Vals, unsigned Count); + /** * @} */ @@ -2811,6 +2916,24 @@ LLVMBasicBlockRef LLVMGetPreviousBasicBlock(LLVMBasicBlockRef BB); */ LLVMBasicBlockRef LLVMGetEntryBasicBlock(LLVMValueRef Fn); +/** + * Insert the given basic block after the insertion point of the given builder. + * + * The insertion point must be valid. + * + * @see llvm::Function::BasicBlockListType::insertAfter() + */ +void LLVMInsertExistingBasicBlockAfterInsertBlock(LLVMBuilderRef Builder, + LLVMBasicBlockRef BB); + +/** + * Append the given basic block to the basic block list of the given function. + * + * @see llvm::Function::BasicBlockListType::push_back() + */ +void LLVMAppendExistingBasicBlock(LLVMValueRef Fn, + LLVMBasicBlockRef BB); + /** * Create a new basic block without inserting it into a function. * @@ -3387,9 +3510,59 @@ void LLVMInsertIntoBuilderWithName(LLVMBuilderRef Builder, LLVMValueRef Instr, void LLVMDisposeBuilder(LLVMBuilderRef Builder); /* Metadata */ + +/** + * Get location information used by debugging information. + * + * @see llvm::IRBuilder::getCurrentDebugLocation() + */ +LLVMMetadataRef LLVMGetCurrentDebugLocation2(LLVMBuilderRef Builder); + +/** + * Set location information used by debugging information. + * + * To clear the location metadata of the given instruction, pass NULL to \p Loc. + * + * @see llvm::IRBuilder::SetCurrentDebugLocation() + */ +void LLVMSetCurrentDebugLocation2(LLVMBuilderRef Builder, LLVMMetadataRef Loc); + +/** + * Attempts to set the debug location for the given instruction using the + * current debug location for the given builder. If the builder has no current + * debug location, this function is a no-op. + * + * @see llvm::IRBuilder::SetInstDebugLocation() + */ +void LLVMSetInstDebugLocation(LLVMBuilderRef Builder, LLVMValueRef Inst); + +/** + * Get the dafult floating-point math metadata for a given builder. + * + * @see llvm::IRBuilder::getDefaultFPMathTag() + */ +LLVMMetadataRef LLVMBuilderGetDefaultFPMathTag(LLVMBuilderRef Builder); + +/** + * Set the default floating-point math metadata for the given builder. + * + * To clear the metadata, pass NULL to \p FPMathTag. + * + * @see llvm::IRBuilder::setDefaultFPMathTag() + */ +void LLVMBuilderSetDefaultFPMathTag(LLVMBuilderRef Builder, + LLVMMetadataRef FPMathTag); + +/** + * Deprecated: Passing the NULL location will crash. + * Use LLVMGetCurrentDebugLocation2 instead. + */ void LLVMSetCurrentDebugLocation(LLVMBuilderRef Builder, LLVMValueRef L); +/** + * Deprecated: Returning the NULL location will crash. + * Use LLVMGetCurrentDebugLocation2 instead. + */ LLVMValueRef LLVMGetCurrentDebugLocation(LLVMBuilderRef Builder); -void LLVMSetInstDebugLocation(LLVMBuilderRef Builder, LLVMValueRef Inst); /* Terminators */ LLVMValueRef LLVMBuildRetVoid(LLVMBuilderRef); diff --git a/include/llvm-c/DataTypes.h b/include/llvm-c/DataTypes.h index 7081c83ffc2b..893b22b49ffc 100644 --- a/include/llvm-c/DataTypes.h +++ b/include/llvm-c/DataTypes.h @@ -1,9 +1,9 @@ /*===-- include/llvm-c/DataTypes.h - Define fixed size types ------*- C -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| diff --git a/include/llvm-c/DebugInfo.h b/include/llvm-c/DebugInfo.h index 87a72034b0e8..33c8110a863c 100644 --- a/include/llvm-c/DebugInfo.h +++ b/include/llvm-c/DebugInfo.h @@ -1,9 +1,8 @@ //===------------ DebugInfo.h - LLVM C API Debug Info API -----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -51,13 +50,12 @@ typedef enum { LLVMDIFlagIntroducedVirtual = 1 << 18, LLVMDIFlagBitField = 1 << 19, LLVMDIFlagNoReturn = 1 << 20, - LLVMDIFlagMainSubprogram = 1 << 21, LLVMDIFlagTypePassByValue = 1 << 22, LLVMDIFlagTypePassByReference = 1 << 23, LLVMDIFlagEnumClass = 1 << 24, LLVMDIFlagFixedEnum = LLVMDIFlagEnumClass, // Deprecated. LLVMDIFlagThunk = 1 << 25, - LLVMDIFlagTrivial = 1 << 26, + LLVMDIFlagNonTrivial = 1 << 26, LLVMDIFlagBigEndian = 1 << 27, LLVMDIFlagLittleEndian = 1 << 28, LLVMDIFlagIndirectVirtualBase = (1 << 2) | (1 << 5), @@ -161,7 +159,8 @@ enum { LLVMDIObjCPropertyMetadataKind, LLVMDIImportedEntityMetadataKind, LLVMDIMacroMetadataKind, - LLVMDIMacroFileMetadataKind + LLVMDIMacroFileMetadataKind, + LLVMDICommonBlockMetadataKind }; typedef unsigned LLVMMetadataKind; @@ -452,6 +451,49 @@ unsigned LLVMDILocationGetColumn(LLVMMetadataRef Location); */ LLVMMetadataRef LLVMDILocationGetScope(LLVMMetadataRef Location); +/** + * Get the "inline at" location associated with this debug location. + * \param Location The debug location. + * + * @see DILocation::getInlinedAt() + */ +LLVMMetadataRef LLVMDILocationGetInlinedAt(LLVMMetadataRef Location); + +/** + * Get the metadata of the file associated with a given scope. + * \param Scope The scope object. + * + * @see DIScope::getFile() + */ +LLVMMetadataRef LLVMDIScopeGetFile(LLVMMetadataRef Scope); + +/** + * Get the directory of a given file. + * \param File The file object. + * \param Len The length of the returned string. + * + * @see DIFile::getDirectory() + */ +const char *LLVMDIFileGetDirectory(LLVMMetadataRef File, unsigned *Len); + +/** + * Get the name of a given file. + * \param File The file object. + * \param Len The length of the returned string. + * + * @see DIFile::getFilename() + */ +const char *LLVMDIFileGetFilename(LLVMMetadataRef File, unsigned *Len); + +/** + * Get the source of a given file. + * \param File The file object. + * \param Len The length of the returned string. + * + * @see DIFile::getSource() + */ +const char *LLVMDIFileGetSource(LLVMMetadataRef File, unsigned *Len); + /** * Create a type array. * \param Builder The DIBuilder. @@ -479,6 +521,19 @@ LLVMDIBuilderCreateSubroutineType(LLVMDIBuilderRef Builder, unsigned NumParameterTypes, LLVMDIFlags Flags); +/** + * Create debugging information entry for an enumerator. + * @param Builder The DIBuilder. + * @param Name Enumerator name. + * @param NameLen Length of enumerator name. + * @param Value Enumerator value. + * @param IsUnsigned True if the value is unsigned. + */ +LLVMMetadataRef LLVMDIBuilderCreateEnumerator(LLVMDIBuilderRef Builder, + const char *Name, size_t NameLen, + int64_t Value, + LLVMBool IsUnsigned); + /** * Create debugging information entry for an enumeration. * \param Builder The DIBuilder. @@ -1017,6 +1072,48 @@ LLVMMetadataRef LLVMDIBuilderCreateGlobalVariableExpression( size_t NameLen, const char *Linkage, size_t LinkLen, LLVMMetadataRef File, unsigned LineNo, LLVMMetadataRef Ty, LLVMBool LocalToUnit, LLVMMetadataRef Expr, LLVMMetadataRef Decl, uint32_t AlignInBits); + +/** + * Retrieves the \c DIVariable associated with this global variable expression. + * \param GVE The global variable expression. + * + * @see llvm::DIGlobalVariableExpression::getVariable() + */ +LLVMMetadataRef LLVMDIGlobalVariableExpressionGetVariable(LLVMMetadataRef GVE); + +/** + * Retrieves the \c DIExpression associated with this global variable expression. + * \param GVE The global variable expression. + * + * @see llvm::DIGlobalVariableExpression::getExpression() + */ +LLVMMetadataRef LLVMDIGlobalVariableExpressionGetExpression( + LLVMMetadataRef GVE); + +/** + * Get the metadata of the file associated with a given variable. + * \param Var The variable object. + * + * @see DIVariable::getFile() + */ +LLVMMetadataRef LLVMDIVariableGetFile(LLVMMetadataRef Var); + +/** + * Get the metadata of the scope associated with a given variable. + * \param Var The variable object. + * + * @see DIVariable::getScope() + */ +LLVMMetadataRef LLVMDIVariableGetScope(LLVMMetadataRef Var); + +/** + * Get the source line where this \c DIVariable is declared. + * \param Var The DIVariable. + * + * @see DIVariable::getLine() + */ +unsigned LLVMDIVariableGetLine(LLVMMetadataRef Var); + /** * Create a new temporary \c MDNode. Suitable for use in constructing cyclic * \c MDNode structures. A temporary \c MDNode is not uniqued, may be RAUW'd, @@ -1180,6 +1277,30 @@ LLVMMetadataRef LLVMGetSubprogram(LLVMValueRef Func); */ void LLVMSetSubprogram(LLVMValueRef Func, LLVMMetadataRef SP); +/** + * Get the line associated with a given subprogram. + * \param Subprogram The subprogram object. + * + * @see DISubprogram::getLine() + */ +unsigned LLVMDISubprogramGetLine(LLVMMetadataRef Subprogram); + +/** + * Get the debug location for the given instruction. + * + * @see llvm::Instruction::getDebugLoc() + */ +LLVMMetadataRef LLVMInstructionGetDebugLoc(LLVMValueRef Inst); + +/** + * Set the debug location for the given instruction. + * + * To clear the location metadata of the given instruction, pass NULL to \p Loc. + * + * @see llvm::Instruction::setDebugLoc() + */ +void LLVMInstructionSetDebugLoc(LLVMValueRef Inst, LLVMMetadataRef Loc); + /** * Obtain the enumerated type of a Metadata instance. * diff --git a/include/llvm-c/Disassembler.h b/include/llvm-c/Disassembler.h index 5e80b95848cf..3adcc3c47a3f 100644 --- a/include/llvm-c/Disassembler.h +++ b/include/llvm-c/Disassembler.h @@ -1,9 +1,9 @@ /*===-- llvm-c/Disassembler.h - Disassembler Public C Interface ---*- C -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| diff --git a/include/llvm-c/DisassemblerTypes.h b/include/llvm-c/DisassemblerTypes.h index e8754ac77055..389e5ee454a8 100644 --- a/include/llvm-c/DisassemblerTypes.h +++ b/include/llvm-c/DisassemblerTypes.h @@ -1,9 +1,9 @@ /*===-- llvm-c/DisassemblerTypedefs.h -----------------------------*- C -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*/ diff --git a/include/llvm-c/Error.h b/include/llvm-c/Error.h index 71e84661222b..52943063c697 100644 --- a/include/llvm-c/Error.h +++ b/include/llvm-c/Error.h @@ -1,9 +1,9 @@ /*===------- llvm-c/Error.h - llvm::Error class C Interface -------*- C -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| @@ -60,7 +60,7 @@ void LLVMDisposeErrorMessage(char *ErrMsg); /** * Returns the type id for llvm StringError. */ -LLVMErrorTypeId LLVMGetStringErrorTypeId(); +LLVMErrorTypeId LLVMGetStringErrorTypeId(void); #ifdef __cplusplus } diff --git a/include/llvm-c/ErrorHandling.h b/include/llvm-c/ErrorHandling.h index 2059b3aeb158..4927349d8983 100644 --- a/include/llvm-c/ErrorHandling.h +++ b/include/llvm-c/ErrorHandling.h @@ -1,9 +1,9 @@ /*===-- llvm-c/ErrorHandling.h - Error Handling C Interface -------*- C -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| diff --git a/include/llvm-c/ExecutionEngine.h b/include/llvm-c/ExecutionEngine.h index e8ebef9ab15d..ef714cd06384 100644 --- a/include/llvm-c/ExecutionEngine.h +++ b/include/llvm-c/ExecutionEngine.h @@ -1,9 +1,9 @@ /*===-- llvm-c/ExecutionEngine.h - ExecutionEngine Lib C Iface --*- C++ -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| diff --git a/include/llvm-c/IRReader.h b/include/llvm-c/IRReader.h index 5b58d9921fb0..4d0b696e9583 100644 --- a/include/llvm-c/IRReader.h +++ b/include/llvm-c/IRReader.h @@ -1,9 +1,9 @@ /*===-- llvm-c/IRReader.h - IR Reader C Interface -----------------*- C -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| diff --git a/include/llvm-c/Initialization.h b/include/llvm-c/Initialization.h index e45eafb139f2..36c41dbd8d31 100644 --- a/include/llvm-c/Initialization.h +++ b/include/llvm-c/Initialization.h @@ -1,9 +1,9 @@ /*===-- llvm-c/Initialization.h - Initialization C Interface ------*- C -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| diff --git a/include/llvm-c/LinkTimeOptimizer.h b/include/llvm-c/LinkTimeOptimizer.h index 8bcf59969ccb..19b4f5cf7491 100644 --- a/include/llvm-c/LinkTimeOptimizer.h +++ b/include/llvm-c/LinkTimeOptimizer.h @@ -1,9 +1,8 @@ //===-- llvm/LinkTimeOptimizer.h - LTO Public C Interface -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm-c/Linker.h b/include/llvm-c/Linker.h index d02c37f94c86..908513041661 100644 --- a/include/llvm-c/Linker.h +++ b/include/llvm-c/Linker.h @@ -1,9 +1,9 @@ /*===-- llvm-c/Linker.h - Module Linker C Interface -------------*- C++ -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| diff --git a/include/llvm-c/Object.h b/include/llvm-c/Object.h index a2980e89fe3d..1e9b703a68ff 100644 --- a/include/llvm-c/Object.h +++ b/include/llvm-c/Object.h @@ -1,9 +1,9 @@ /*===-- llvm-c/Object.h - Object Lib C Iface --------------------*- C++ -*-===*/ /* */ -/* The LLVM Compiler Infrastructure */ -/* */ -/* This file is distributed under the University of Illinois Open Source */ -/* License. See LICENSE.TXT for details. */ +/* Part of the LLVM Project, under the Apache License v2.0 with LLVM */ +/* Exceptions. */ +/* See https://llvm.org/LICENSE.txt for license information. */ +/* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ /* */ /*===----------------------------------------------------------------------===*/ /* */ @@ -34,29 +34,140 @@ extern "C" { */ // Opaque type wrappers -typedef struct LLVMOpaqueObjectFile *LLVMObjectFileRef; typedef struct LLVMOpaqueSectionIterator *LLVMSectionIteratorRef; typedef struct LLVMOpaqueSymbolIterator *LLVMSymbolIteratorRef; typedef struct LLVMOpaqueRelocationIterator *LLVMRelocationIteratorRef; -// ObjectFile creation -LLVMObjectFileRef LLVMCreateObjectFile(LLVMMemoryBufferRef MemBuf); -void LLVMDisposeObjectFile(LLVMObjectFileRef ObjectFile); +typedef enum { + LLVMBinaryTypeArchive, /**< Archive file. */ + LLVMBinaryTypeMachOUniversalBinary, /**< Mach-O Universal Binary file. */ + LLVMBinaryTypeCOFFImportFile, /**< COFF Import file. */ + LLVMBinaryTypeIR, /**< LLVM IR. */ + LLVMBinaryTypeWinRes, /**< Windows resource (.res) file. */ + LLVMBinaryTypeCOFF, /**< COFF Object file. */ + LLVMBinaryTypeELF32L, /**< ELF 32-bit, little endian. */ + LLVMBinaryTypeELF32B, /**< ELF 32-bit, big endian. */ + LLVMBinaryTypeELF64L, /**< ELF 64-bit, little endian. */ + LLVMBinaryTypeELF64B, /**< ELF 64-bit, big endian. */ + LLVMBinaryTypeMachO32L, /**< MachO 32-bit, little endian. */ + LLVMBinaryTypeMachO32B, /**< MachO 32-bit, big endian. */ + LLVMBinaryTypeMachO64L, /**< MachO 64-bit, little endian. */ + LLVMBinaryTypeMachO64B, /**< MachO 64-bit, big endian. */ + LLVMBinaryTypeWasm, /**< Web Assembly. */ +} LLVMBinaryType; + +/** + * Create a binary file from the given memory buffer. + * + * The exact type of the binary file will be inferred automatically, and the + * appropriate implementation selected. The context may be NULL except if + * the resulting file is an LLVM IR file. + * + * The memory buffer is not consumed by this function. It is the responsibilty + * of the caller to free it with \c LLVMDisposeMemoryBuffer. + * + * If NULL is returned, the \p ErrorMessage parameter is populated with the + * error's description. It is then the caller's responsibility to free this + * message by calling \c LLVMDisposeMessage. + * + * @see llvm::object::createBinary + */ +LLVMBinaryRef LLVMCreateBinary(LLVMMemoryBufferRef MemBuf, + LLVMContextRef Context, + char **ErrorMessage); + +/** + * Dispose of a binary file. + * + * The binary file does not own its backing buffer. It is the responsibilty + * of the caller to free it with \c LLVMDisposeMemoryBuffer. + */ +void LLVMDisposeBinary(LLVMBinaryRef BR); + +/** + * Retrieves a copy of the memory buffer associated with this object file. + * + * The returned buffer is merely a shallow copy and does not own the actual + * backing buffer of the binary. Nevertheless, it is the responsibility of the + * caller to free it with \c LLVMDisposeMemoryBuffer. + * + * @see llvm::object::getMemoryBufferRef + */ +LLVMMemoryBufferRef LLVMBinaryCopyMemoryBuffer(LLVMBinaryRef BR); + +/** + * Retrieve the specific type of a binary. + * + * @see llvm::object::Binary::getType + */ +LLVMBinaryType LLVMBinaryGetType(LLVMBinaryRef BR); + +/* + * For a Mach-O universal binary file, retrieves the object file corresponding + * to the given architecture if it is present as a slice. + * + * If NULL is returned, the \p ErrorMessage parameter is populated with the + * error's description. It is then the caller's responsibility to free this + * message by calling \c LLVMDisposeMessage. + * + * It is the responsiblity of the caller to free the returned object file by + * calling \c LLVMDisposeBinary. + */ +LLVMBinaryRef LLVMMachOUniversalBinaryCopyObjectForArch(LLVMBinaryRef BR, + const char *Arch, + size_t ArchLen, + char **ErrorMessage); + +/** + * Retrieve a copy of the section iterator for this object file. + * + * If there are no sections, the result is NULL. + * + * The returned iterator is merely a shallow copy. Nevertheless, it is + * the responsibility of the caller to free it with + * \c LLVMDisposeSectionIterator. + * + * @see llvm::object::sections() + */ +LLVMSectionIteratorRef LLVMObjectFileCopySectionIterator(LLVMBinaryRef BR); + +/** + * Returns whether the given section iterator is at the end. + * + * @see llvm::object::section_end + */ +LLVMBool LLVMObjectFileIsSectionIteratorAtEnd(LLVMBinaryRef BR, + LLVMSectionIteratorRef SI); + +/** + * Retrieve a copy of the symbol iterator for this object file. + * + * If there are no symbols, the result is NULL. + * + * The returned iterator is merely a shallow copy. Nevertheless, it is + * the responsibility of the caller to free it with + * \c LLVMDisposeSymbolIterator. + * + * @see llvm::object::symbols() + */ +LLVMSymbolIteratorRef LLVMObjectFileCopySymbolIterator(LLVMBinaryRef BR); + +/** + * Returns whether the given symbol iterator is at the end. + * + * @see llvm::object::symbol_end + */ +LLVMBool LLVMObjectFileIsSymbolIteratorAtEnd(LLVMBinaryRef BR, + LLVMSymbolIteratorRef SI); -// ObjectFile Section iterators -LLVMSectionIteratorRef LLVMGetSections(LLVMObjectFileRef ObjectFile); void LLVMDisposeSectionIterator(LLVMSectionIteratorRef SI); -LLVMBool LLVMIsSectionIteratorAtEnd(LLVMObjectFileRef ObjectFile, - LLVMSectionIteratorRef SI); + void LLVMMoveToNextSection(LLVMSectionIteratorRef SI); void LLVMMoveToContainingSection(LLVMSectionIteratorRef Sect, LLVMSymbolIteratorRef Sym); // ObjectFile Symbol iterators -LLVMSymbolIteratorRef LLVMGetSymbols(LLVMObjectFileRef ObjectFile); void LLVMDisposeSymbolIterator(LLVMSymbolIteratorRef SI); -LLVMBool LLVMIsSymbolIteratorAtEnd(LLVMObjectFileRef ObjectFile, - LLVMSymbolIteratorRef SI); void LLVMMoveToNextSymbol(LLVMSymbolIteratorRef SI); // SectionRef accessors @@ -89,6 +200,28 @@ uint64_t LLVMGetRelocationType(LLVMRelocationIteratorRef RI); const char *LLVMGetRelocationTypeName(LLVMRelocationIteratorRef RI); const char *LLVMGetRelocationValueString(LLVMRelocationIteratorRef RI); +/** Deprecated: Use LLVMBinaryRef instead. */ +typedef struct LLVMOpaqueObjectFile *LLVMObjectFileRef; + +/** Deprecated: Use LLVMCreateBinary instead. */ +LLVMObjectFileRef LLVMCreateObjectFile(LLVMMemoryBufferRef MemBuf); + +/** Deprecated: Use LLVMDisposeBinary instead. */ +void LLVMDisposeObjectFile(LLVMObjectFileRef ObjectFile); + +/** Deprecated: Use LLVMObjectFileCopySectionIterator instead. */ +LLVMSectionIteratorRef LLVMGetSections(LLVMObjectFileRef ObjectFile); + +/** Deprecated: Use LLVMObjectFileIsSectionIteratorAtEnd instead. */ +LLVMBool LLVMIsSectionIteratorAtEnd(LLVMObjectFileRef ObjectFile, + LLVMSectionIteratorRef SI); + +/** Deprecated: Use LLVMObjectFileCopySymbolIterator instead. */ +LLVMSymbolIteratorRef LLVMGetSymbols(LLVMObjectFileRef ObjectFile); + +/** Deprecated: Use LLVMObjectFileIsSymbolIteratorAtEnd instead. */ +LLVMBool LLVMIsSymbolIteratorAtEnd(LLVMObjectFileRef ObjectFile, + LLVMSymbolIteratorRef SI); /** * @} */ diff --git a/include/llvm-c/OptRemarks.h b/include/llvm-c/OptRemarks.h deleted file mode 100644 index 6a90394e711c..000000000000 --- a/include/llvm-c/OptRemarks.h +++ /dev/null @@ -1,204 +0,0 @@ -/*===-- llvm-c/OptRemarks.h - OptRemarks Public C Interface -------*- C -*-===*\ -|* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| -|* *| -|*===----------------------------------------------------------------------===*| -|* *| -|* This header provides a public interface to an opt-remark library. *| -|* LLVM provides an implementation of this interface. *| -|* *| -\*===----------------------------------------------------------------------===*/ - -#ifndef LLVM_C_OPT_REMARKS_H -#define LLVM_C_OPT_REMARKS_H - -#include "llvm-c/Core.h" -#include "llvm-c/Types.h" -#ifdef __cplusplus -#include -extern "C" { -#else -#include -#endif /* !defined(__cplusplus) */ - -/** - * @defgroup LLVMCOPTREMARKS OptRemarks - * @ingroup LLVMC - * - * @{ - */ - -#define OPT_REMARKS_API_VERSION 0 - -/** - * String containing a buffer and a length. The buffer is not guaranteed to be - * zero-terminated. - * - * \since OPT_REMARKS_API_VERSION=0 - */ -typedef struct { - const char *Str; - uint32_t Len; -} LLVMOptRemarkStringRef; - -/** - * DebugLoc containing File, Line and Column. - * - * \since OPT_REMARKS_API_VERSION=0 - */ -typedef struct { - // File: - LLVMOptRemarkStringRef SourceFile; - // Line: - uint32_t SourceLineNumber; - // Column: - uint32_t SourceColumnNumber; -} LLVMOptRemarkDebugLoc; - -/** - * Element of the "Args" list. The key might give more information about what - * are the semantics of the value, e.g. "Callee" will tell you that the value - * is a symbol that names a function. - * - * \since OPT_REMARKS_API_VERSION=0 - */ -typedef struct { - // e.g. "Callee" - LLVMOptRemarkStringRef Key; - // e.g. "malloc" - LLVMOptRemarkStringRef Value; - - // "DebugLoc": Optional - LLVMOptRemarkDebugLoc DebugLoc; -} LLVMOptRemarkArg; - -/** - * One remark entry. - * - * \since OPT_REMARKS_API_VERSION=0 - */ -typedef struct { - // e.g. !Missed, !Passed - LLVMOptRemarkStringRef RemarkType; - // "Pass": Required - LLVMOptRemarkStringRef PassName; - // "Name": Required - LLVMOptRemarkStringRef RemarkName; - // "Function": Required - LLVMOptRemarkStringRef FunctionName; - - // "DebugLoc": Optional - LLVMOptRemarkDebugLoc DebugLoc; - // "Hotness": Optional - uint32_t Hotness; - // "Args": Optional. It is an array of `num_args` elements. - uint32_t NumArgs; - LLVMOptRemarkArg *Args; -} LLVMOptRemarkEntry; - -typedef struct LLVMOptRemarkOpaqueParser *LLVMOptRemarkParserRef; - -/** - * Creates a remark parser that can be used to read and parse the buffer located - * in \p Buf of size \p Size. - * - * \p Buf cannot be NULL. - * - * This function should be paired with LLVMOptRemarkParserDispose() to avoid - * leaking resources. - * - * \since OPT_REMARKS_API_VERSION=0 - */ -extern LLVMOptRemarkParserRef LLVMOptRemarkParserCreate(const void *Buf, - uint64_t Size); - -/** - * Returns the next remark in the file. - * - * The value pointed to by the return value is invalidated by the next call to - * LLVMOptRemarkParserGetNext(). - * - * If the parser reaches the end of the buffer, the return value will be NULL. - * - * In the case of an error, the return value will be NULL, and: - * - * 1) LLVMOptRemarkParserHasError() will return `1`. - * - * 2) LLVMOptRemarkParserGetErrorMessage() will return a descriptive error - * message. - * - * An error may occur if: - * - * 1) An argument is invalid. - * - * 2) There is a YAML parsing error. This type of error aborts parsing - * immediately and returns `1`. It can occur on malformed YAML. - * - * 3) Remark parsing error. If this type of error occurs, the parser won't call - * the handler and will continue to the next one. It can occur on malformed - * remarks, like missing or extra fields in the file. - * - * Here is a quick example of the usage: - * - * ``` - * LLVMOptRemarkParserRef Parser = LLVMOptRemarkParserCreate(Buf, Size); - * LLVMOptRemarkEntry *Remark = NULL; - * while ((Remark == LLVMOptRemarkParserGetNext(Parser))) { - * // use Remark - * } - * bool HasError = LLVMOptRemarkParserHasError(Parser); - * LLVMOptRemarkParserDispose(Parser); - * ``` - * - * \since OPT_REMARKS_API_VERSION=0 - */ -extern LLVMOptRemarkEntry * -LLVMOptRemarkParserGetNext(LLVMOptRemarkParserRef Parser); - -/** - * Returns `1` if the parser encountered an error while parsing the buffer. - * - * \since OPT_REMARKS_API_VERSION=0 - */ -extern LLVMBool LLVMOptRemarkParserHasError(LLVMOptRemarkParserRef Parser); - -/** - * Returns a null-terminated string containing an error message. - * - * In case of no error, the result is `NULL`. - * - * The memory of the string is bound to the lifetime of \p Parser. If - * LLVMOptRemarkParserDispose() is called, the memory of the string will be - * released. - * - * \since OPT_REMARKS_API_VERSION=0 - */ -extern const char * -LLVMOptRemarkParserGetErrorMessage(LLVMOptRemarkParserRef Parser); - -/** - * Releases all the resources used by \p Parser. - * - * \since OPT_REMARKS_API_VERSION=0 - */ -extern void LLVMOptRemarkParserDispose(LLVMOptRemarkParserRef Parser); - -/** - * Returns the version of the opt-remarks dylib. - * - * \since OPT_REMARKS_API_VERSION=0 - */ -extern uint32_t LLVMOptRemarkVersion(void); - -/** - * @} // endgoup LLVMCOPTREMARKS - */ - -#ifdef __cplusplus -} -#endif /* !defined(__cplusplus) */ - -#endif /* LLVM_C_OPT_REMARKS_H */ diff --git a/include/llvm-c/OrcBindings.h b/include/llvm-c/OrcBindings.h index 570db87fee94..9e92371b5a3a 100644 --- a/include/llvm-c/OrcBindings.h +++ b/include/llvm-c/OrcBindings.h @@ -1,9 +1,9 @@ /*===----------- llvm-c/OrcBindings.h - Orc Lib C Iface ---------*- C++ -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| diff --git a/include/llvm-c/Remarks.h b/include/llvm-c/Remarks.h new file mode 100644 index 000000000000..88eb5120c57c --- /dev/null +++ b/include/llvm-c/Remarks.h @@ -0,0 +1,329 @@ +/*===-- llvm-c/Remarks.h - Remarks Public C Interface -------------*- C -*-===*\ +|* *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| +|* *| +|*===----------------------------------------------------------------------===*| +|* *| +|* This header provides a public interface to a remark diagnostics library. *| +|* LLVM provides an implementation of this interface. *| +|* *| +\*===----------------------------------------------------------------------===*/ + +#ifndef LLVM_C_REMARKS_H +#define LLVM_C_REMARKS_H + +#include "llvm-c/Types.h" +#ifdef __cplusplus +#include +extern "C" { +#else +#include +#endif /* !defined(__cplusplus) */ + +/** + * @defgroup LLVMCREMARKS Remarks + * @ingroup LLVMC + * + * @{ + */ + +#define REMARKS_API_VERSION 0 + +/** + * The type of the emitted remark. + */ +enum LLVMRemarkType { + LLVMRemarkTypeUnknown, + LLVMRemarkTypePassed, + LLVMRemarkTypeMissed, + LLVMRemarkTypeAnalysis, + LLVMRemarkTypeAnalysisFPCommute, + LLVMRemarkTypeAnalysisAliasing, + LLVMRemarkTypeFailure +}; + +/** + * String containing a buffer and a length. The buffer is not guaranteed to be + * zero-terminated. + * + * \since REMARKS_API_VERSION=0 + */ +typedef struct LLVMRemarkOpaqueString *LLVMRemarkStringRef; + +/** + * Returns the buffer holding the string. + * + * \since REMARKS_API_VERSION=0 + */ +extern const char *LLVMRemarkStringGetData(LLVMRemarkStringRef String); + +/** + * Returns the size of the string. + * + * \since REMARKS_API_VERSION=0 + */ +extern uint32_t LLVMRemarkStringGetLen(LLVMRemarkStringRef String); + +/** + * DebugLoc containing File, Line and Column. + * + * \since REMARKS_API_VERSION=0 + */ +typedef struct LLVMRemarkOpaqueDebugLoc *LLVMRemarkDebugLocRef; + +/** + * Return the path to the source file for a debug location. + * + * \since REMARKS_API_VERSION=0 + */ +extern LLVMRemarkStringRef +LLVMRemarkDebugLocGetSourceFilePath(LLVMRemarkDebugLocRef DL); + +/** + * Return the line in the source file for a debug location. + * + * \since REMARKS_API_VERSION=0 + */ +extern uint32_t LLVMRemarkDebugLocGetSourceLine(LLVMRemarkDebugLocRef DL); + +/** + * Return the column in the source file for a debug location. + * + * \since REMARKS_API_VERSION=0 + */ +extern uint32_t LLVMRemarkDebugLocGetSourceColumn(LLVMRemarkDebugLocRef DL); + +/** + * Element of the "Args" list. The key might give more information about what + * the semantics of the value are, e.g. "Callee" will tell you that the value + * is a symbol that names a function. + * + * \since REMARKS_API_VERSION=0 + */ +typedef struct LLVMRemarkOpaqueArg *LLVMRemarkArgRef; + +/** + * Returns the key of an argument. The key defines what the value is, and the + * same key can appear multiple times in the list of arguments. + * + * \since REMARKS_API_VERSION=0 + */ +extern LLVMRemarkStringRef LLVMRemarkArgGetKey(LLVMRemarkArgRef Arg); + +/** + * Returns the value of an argument. This is a string that can contain newlines. + * + * \since REMARKS_API_VERSION=0 + */ +extern LLVMRemarkStringRef LLVMRemarkArgGetValue(LLVMRemarkArgRef Arg); + +/** + * Returns the debug location that is attached to the value of this argument. + * + * If there is no debug location, the return value will be `NULL`. + * + * \since REMARKS_API_VERSION=0 + */ +extern LLVMRemarkDebugLocRef LLVMRemarkArgGetDebugLoc(LLVMRemarkArgRef Arg); + +/** + * A remark emitted by the compiler. + * + * \since REMARKS_API_VERSION=0 + */ +typedef struct LLVMRemarkOpaqueEntry *LLVMRemarkEntryRef; + +/** + * Free the resources used by the remark entry. + * + * \since REMARKS_API_VERSION=0 + */ +extern void LLVMRemarkEntryDispose(LLVMRemarkEntryRef Remark); + +/** + * The type of the remark. For example, it can allow users to only keep the + * missed optimizations from the compiler. + * + * \since REMARKS_API_VERSION=0 + */ +extern enum LLVMRemarkType LLVMRemarkEntryGetType(LLVMRemarkEntryRef Remark); + +/** + * Get the name of the pass that emitted this remark. + * + * \since REMARKS_API_VERSION=0 + */ +extern LLVMRemarkStringRef +LLVMRemarkEntryGetPassName(LLVMRemarkEntryRef Remark); + +/** + * Get an identifier of the remark. + * + * \since REMARKS_API_VERSION=0 + */ +extern LLVMRemarkStringRef +LLVMRemarkEntryGetRemarkName(LLVMRemarkEntryRef Remark); + +/** + * Get the name of the function being processed when the remark was emitted. + * + * \since REMARKS_API_VERSION=0 + */ +extern LLVMRemarkStringRef +LLVMRemarkEntryGetFunctionName(LLVMRemarkEntryRef Remark); + +/** + * Returns the debug location that is attached to this remark. + * + * If there is no debug location, the return value will be `NULL`. + * + * \since REMARKS_API_VERSION=0 + */ +extern LLVMRemarkDebugLocRef +LLVMRemarkEntryGetDebugLoc(LLVMRemarkEntryRef Remark); + +/** + * Return the hotness of the remark. + * + * A hotness of `0` means this value is not set. + * + * \since REMARKS_API_VERSION=0 + */ +extern uint64_t LLVMRemarkEntryGetHotness(LLVMRemarkEntryRef Remark); + +/** + * The number of arguments the remark holds. + * + * \since REMARKS_API_VERSION=0 + */ +extern uint32_t LLVMRemarkEntryGetNumArgs(LLVMRemarkEntryRef Remark); + +/** + * Get a new iterator to iterate over a remark's argument. + * + * If there are no arguments in \p Remark, the return value will be `NULL`. + * + * The lifetime of the returned value is bound to the lifetime of \p Remark. + * + * \since REMARKS_API_VERSION=0 + */ +extern LLVMRemarkArgRef LLVMRemarkEntryGetFirstArg(LLVMRemarkEntryRef Remark); + +/** + * Get the next argument in \p Remark from the position of \p It. + * + * Returns `NULL` if there are no more arguments available. + * + * The lifetime of the returned value is bound to the lifetime of \p Remark. + * + * \since REMARKS_API_VERSION=0 + */ +extern LLVMRemarkArgRef LLVMRemarkEntryGetNextArg(LLVMRemarkArgRef It, + LLVMRemarkEntryRef Remark); + +typedef struct LLVMRemarkOpaqueParser *LLVMRemarkParserRef; + +/** + * Creates a remark parser that can be used to parse the buffer located in \p + * Buf of size \p Size bytes. + * + * \p Buf cannot be `NULL`. + * + * This function should be paired with LLVMRemarkParserDispose() to avoid + * leaking resources. + * + * \since REMARKS_API_VERSION=0 + */ +extern LLVMRemarkParserRef LLVMRemarkParserCreateYAML(const void *Buf, + uint64_t Size); + +/** + * Returns the next remark in the file. + * + * The value pointed to by the return value needs to be disposed using a call to + * LLVMRemarkEntryDispose(). + * + * All the entries in the returned value that are of LLVMRemarkStringRef type + * will become invalidated once a call to LLVMRemarkParserDispose is made. + * + * If the parser reaches the end of the buffer, the return value will be `NULL`. + * + * In the case of an error, the return value will be `NULL`, and: + * + * 1) LLVMRemarkParserHasError() will return `1`. + * + * 2) LLVMRemarkParserGetErrorMessage() will return a descriptive error + * message. + * + * An error may occur if: + * + * 1) An argument is invalid. + * + * 2) There is a parsing error. This can occur on things like malformed YAML. + * + * 3) There is a Remark semantic error. This can occur on well-formed files with + * missing or extra fields. + * + * Here is a quick example of the usage: + * + * ``` + * LLVMRemarkParserRef Parser = LLVMRemarkParserCreateYAML(Buf, Size); + * LLVMRemarkEntryRef Remark = NULL; + * while ((Remark = LLVMRemarkParserGetNext(Parser))) { + * // use Remark + * LLVMRemarkEntryDispose(Remark); // Release memory. + * } + * bool HasError = LLVMRemarkParserHasError(Parser); + * LLVMRemarkParserDispose(Parser); + * ``` + * + * \since REMARKS_API_VERSION=0 + */ +extern LLVMRemarkEntryRef LLVMRemarkParserGetNext(LLVMRemarkParserRef Parser); + +/** + * Returns `1` if the parser encountered an error while parsing the buffer. + * + * \since REMARKS_API_VERSION=0 + */ +extern LLVMBool LLVMRemarkParserHasError(LLVMRemarkParserRef Parser); + +/** + * Returns a null-terminated string containing an error message. + * + * In case of no error, the result is `NULL`. + * + * The memory of the string is bound to the lifetime of \p Parser. If + * LLVMRemarkParserDispose() is called, the memory of the string will be + * released. + * + * \since REMARKS_API_VERSION=0 + */ +extern const char *LLVMRemarkParserGetErrorMessage(LLVMRemarkParserRef Parser); + +/** + * Releases all the resources used by \p Parser. + * + * \since REMARKS_API_VERSION=0 + */ +extern void LLVMRemarkParserDispose(LLVMRemarkParserRef Parser); + +/** + * Returns the version of the remarks library. + * + * \since REMARKS_API_VERSION=0 + */ +extern uint32_t LLVMRemarkVersion(void); + +/** + * @} // endgoup LLVMCREMARKS + */ + +#ifdef __cplusplus +} +#endif /* !defined(__cplusplus) */ + +#endif /* LLVM_C_REMARKS_H */ diff --git a/include/llvm-c/Support.h b/include/llvm-c/Support.h index 37d5d72ff5dc..097f784246c5 100644 --- a/include/llvm-c/Support.h +++ b/include/llvm-c/Support.h @@ -1,9 +1,9 @@ /*===-- llvm-c/Support.h - Support C Interface --------------------*- C -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| diff --git a/include/llvm-c/Target.h b/include/llvm-c/Target.h index 03004ba5eec0..4ef641eaf232 100644 --- a/include/llvm-c/Target.h +++ b/include/llvm-c/Target.h @@ -1,9 +1,9 @@ /*===-- llvm-c/Target.h - Target Lib C Iface --------------------*- C++ -*-===*/ /* */ -/* The LLVM Compiler Infrastructure */ -/* */ -/* This file is distributed under the University of Illinois Open Source */ -/* License. See LICENSE.TXT for details. */ +/* Part of the LLVM Project, under the Apache License v2.0 with LLVM */ +/* Exceptions. */ +/* See https://llvm.org/LICENSE.txt for license information. */ +/* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception */ /* */ /*===----------------------------------------------------------------------===*/ /* */ @@ -22,10 +22,6 @@ #include "llvm-c/Types.h" #include "llvm/Config/llvm-config.h" -#if defined(_MSC_VER) && !defined(inline) -#define inline __inline -#endif - #ifdef __cplusplus extern "C" { #endif diff --git a/include/llvm-c/TargetMachine.h b/include/llvm-c/TargetMachine.h index c06e9edc9aaf..28d7c096871e 100644 --- a/include/llvm-c/TargetMachine.h +++ b/include/llvm-c/TargetMachine.h @@ -1,9 +1,9 @@ /*===-- llvm-c/TargetMachine.h - Target Machine Library C Interface - C++ -*-=*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| diff --git a/include/llvm-c/Transforms/AggressiveInstCombine.h b/include/llvm-c/Transforms/AggressiveInstCombine.h index 8756a22e917a..c0b0141c3da1 100644 --- a/include/llvm-c/Transforms/AggressiveInstCombine.h +++ b/include/llvm-c/Transforms/AggressiveInstCombine.h @@ -1,9 +1,9 @@ /*===-- AggressiveInstCombine.h ---------------------------------*- C++ -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| diff --git a/include/llvm-c/Transforms/Coroutines.h b/include/llvm-c/Transforms/Coroutines.h index 827e30fb2d7c..227e7cf0a360 100644 --- a/include/llvm-c/Transforms/Coroutines.h +++ b/include/llvm-c/Transforms/Coroutines.h @@ -1,9 +1,9 @@ /*===-- Coroutines.h - Coroutines Library C Interface -----------*- C++ -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| diff --git a/include/llvm-c/Transforms/IPO.h b/include/llvm-c/Transforms/IPO.h index 7705b1864dc3..7a82ed464141 100644 --- a/include/llvm-c/Transforms/IPO.h +++ b/include/llvm-c/Transforms/IPO.h @@ -1,9 +1,9 @@ /*===-- IPO.h - Interprocedural Transformations C Interface -----*- C++ -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| diff --git a/include/llvm-c/Transforms/InstCombine.h b/include/llvm-c/Transforms/InstCombine.h index e1c1572d53dc..166f278d9a69 100644 --- a/include/llvm-c/Transforms/InstCombine.h +++ b/include/llvm-c/Transforms/InstCombine.h @@ -1,9 +1,9 @@ /*===-- Scalar.h - Scalar Transformation Library C Interface ----*- C++ -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| diff --git a/include/llvm-c/Transforms/PassManagerBuilder.h b/include/llvm-c/Transforms/PassManagerBuilder.h index 69786b341ab4..d164c00d49c5 100644 --- a/include/llvm-c/Transforms/PassManagerBuilder.h +++ b/include/llvm-c/Transforms/PassManagerBuilder.h @@ -1,9 +1,9 @@ /*===-- llvm-c/Transform/PassManagerBuilder.h - PMB C Interface ---*- C -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| diff --git a/include/llvm-c/Transforms/Scalar.h b/include/llvm-c/Transforms/Scalar.h index 3c3bb4eb9b82..031cf98b2df2 100644 --- a/include/llvm-c/Transforms/Scalar.h +++ b/include/llvm-c/Transforms/Scalar.h @@ -1,9 +1,9 @@ /*===-- Scalar.h - Scalar Transformation Library C Interface ----*- C++ -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| diff --git a/include/llvm-c/Transforms/Utils.h b/include/llvm-c/Transforms/Utils.h index f171f7fbbe3e..63594abfa460 100644 --- a/include/llvm-c/Transforms/Utils.h +++ b/include/llvm-c/Transforms/Utils.h @@ -1,9 +1,9 @@ /*===-- Utils.h - Transformation Utils Library C Interface ------*- C++ -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| @@ -38,6 +38,9 @@ void LLVMAddLowerSwitchPass(LLVMPassManagerRef PM); /** See llvm::createPromoteMemoryToRegisterPass function. */ void LLVMAddPromoteMemoryToRegisterPass(LLVMPassManagerRef PM); +/** See llvm::createAddDiscriminatorsPass function. */ +void LLVMAddAddDiscriminatorsPass(LLVMPassManagerRef PM); + /** * @} */ diff --git a/include/llvm-c/Transforms/Vectorize.h b/include/llvm-c/Transforms/Vectorize.h index e3f9961acfb1..e383481fe4f4 100644 --- a/include/llvm-c/Transforms/Vectorize.h +++ b/include/llvm-c/Transforms/Vectorize.h @@ -1,10 +1,10 @@ /*===---------------------------Vectorize.h --------------------- -*- C -*-===*\ |*===----------- Vectorization Transformation Library C Interface ---------===*| |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| diff --git a/include/llvm-c/Types.h b/include/llvm-c/Types.h index ce1acf3e0421..612c7d3eff32 100644 --- a/include/llvm-c/Types.h +++ b/include/llvm-c/Types.h @@ -1,9 +1,9 @@ /*===-- llvm-c/Support.h - C Interface Types declarations ---------*- C -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| @@ -163,6 +163,11 @@ typedef struct LLVMOpaqueModuleFlagEntry LLVMModuleFlagEntry; */ typedef struct LLVMOpaqueJITEventListener *LLVMJITEventListenerRef; +/** + * @see llvm::object::Binary + */ +typedef struct LLVMOpaqueBinary *LLVMBinaryRef; + /** * @} */ diff --git a/include/llvm-c/lto.h b/include/llvm-c/lto.h index 090cd34af4e9..2467722b1954 100644 --- a/include/llvm-c/lto.h +++ b/include/llvm-c/lto.h @@ -1,9 +1,9 @@ /*===-- llvm-c/lto.h - LTO Public C Interface ---------------------*- C -*-===*\ |* *| -|* The LLVM Compiler Infrastructure *| -|* *| -|* This file is distributed under the University of Illinois Open Source *| -|* License. See LICENSE.TXT for details. *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| |* *| |*===----------------------------------------------------------------------===*| |* *| @@ -44,7 +44,7 @@ typedef bool lto_bool_t; * @{ */ -#define LTO_API_VERSION 23 +#define LTO_API_VERSION 24 /** * \since prior to LTO_API_VERSION=3 @@ -846,7 +846,47 @@ thinlto_codegen_set_cache_size_megabytes(thinlto_code_gen_t cg, extern void thinlto_codegen_set_cache_size_files(thinlto_code_gen_t cg, unsigned max_size_files); - +/** Opaque reference to an LTO input file */ +typedef struct LLVMOpaqueLTOInput *lto_input_t; + +/** + * Creates an LTO input file from a buffer. The path + * argument is used for diagnotics as this function + * otherwise does not know which file the given buffer + * is associated with. + * + * \since LTO_API_VERSION=24 + */ +extern lto_input_t lto_input_create(const void *buffer, + size_t buffer_size, + const char *path); + +/** + * Frees all memory internally allocated by the LTO input file. + * Upon return the lto_module_t is no longer valid. + * + * \since LTO_API_VERSION=24 + */ +extern void lto_input_dispose(lto_input_t input); + +/** + * Returns the number of dependent library specifiers + * for the given LTO input file. + * + * \since LTO_API_VERSION=24 + */ +extern unsigned lto_input_get_num_dependent_libraries(lto_input_t input); + +/** + * Returns the ith dependent library specifier + * for the given LTO input file. The returned + * string is not null-terminated. + * + * \since LTO_API_VERSION=24 + */ +extern const char * lto_input_get_dependent_library(lto_input_t input, + size_t index, + size_t *size); /** * @} // endgroup LLVMCTLTO_CACHING diff --git a/include/llvm/ADT/APFloat.h b/include/llvm/ADT/APFloat.h index c6fa5ad674f6..a9648d35cf5d 100644 --- a/include/llvm/ADT/APFloat.h +++ b/include/llvm/ADT/APFloat.h @@ -1,9 +1,8 @@ //===- llvm/ADT/APFloat.h - Arbitrary Precision Floating Point ---*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -148,6 +147,17 @@ struct APFloatBase { /// \name Floating Point Semantics. /// @{ + enum Semantics { + S_IEEEhalf, + S_IEEEsingle, + S_IEEEdouble, + S_x87DoubleExtended, + S_IEEEquad, + S_PPCDoubleDouble + }; + + static const llvm::fltSemantics &EnumToSemantics(Semantics S); + static Semantics SemanticsToEnum(const llvm::fltSemantics &Sem); static const fltSemantics &IEEEhalf() LLVM_READNONE; static const fltSemantics &IEEEsingle() LLVM_READNONE; diff --git a/include/llvm/ADT/APInt.h b/include/llvm/ADT/APInt.h index 6e106ff8bf5d..2381b75e08b1 100644 --- a/include/llvm/ADT/APInt.h +++ b/include/llvm/ADT/APInt.h @@ -1,9 +1,8 @@ //===-- llvm/ADT/APInt.h - For Arbitrary Precision Integer -----*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -2213,6 +2212,15 @@ Optional SolveQuadraticEquationWrap(APInt A, APInt B, APInt C, // See friend declaration above. This additional declaration is required in // order to compile LLVM with IBM xlC compiler. hash_code hash_value(const APInt &Arg); -} // End of llvm namespace + +/// StoreIntToMemory - Fills the StoreBytes bytes of memory starting from Dst +/// with the integer held in IntVal. +void StoreIntToMemory(const APInt &IntVal, uint8_t *Dst, unsigned StoreBytes); + +/// LoadIntFromMemory - Loads the integer stored in the LoadBytes bytes starting +/// from Src into IntVal, which is assumed to be wide enough and to hold zero. +void LoadIntFromMemory(APInt &IntVal, uint8_t *Src, unsigned LoadBytes); + +} // namespace llvm #endif diff --git a/include/llvm/ADT/APSInt.h b/include/llvm/ADT/APSInt.h index 7ee2c4c62fce..0f991826c457 100644 --- a/include/llvm/ADT/APSInt.h +++ b/include/llvm/ADT/APSInt.h @@ -1,9 +1,8 @@ //===-- llvm/ADT/APSInt.h - Arbitrary Precision Signed Int -----*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -43,6 +42,24 @@ public: /// \param Str the string to be interpreted. explicit APSInt(StringRef Str); + /// Determine sign of this APSInt. + /// + /// \returns true if this APSInt is negative, false otherwise + bool isNegative() const { return isSigned() && APInt::isNegative(); } + + /// Determine if this APSInt Value is non-negative (>= 0) + /// + /// \returns true if this APSInt is non-negative, false otherwise + bool isNonNegative() const { return !isNegative(); } + + /// Determine if this APSInt Value is positive. + /// + /// This tests if the value of this APSInt is positive (> 0). Note + /// that 0 is not a positive value. + /// + /// \returns true if this APSInt is positive. + bool isStrictlyPositive() const { return isNonNegative() && !isNullValue(); } + APSInt &operator=(APInt RHS) { // Retain our current sign. APInt::operator=(std::move(RHS)); diff --git a/include/llvm/ADT/AllocatorList.h b/include/llvm/ADT/AllocatorList.h index 178c6742a87b..405a2e4264df 100644 --- a/include/llvm/ADT/AllocatorList.h +++ b/include/llvm/ADT/AllocatorList.h @@ -1,9 +1,8 @@ //===- llvm/ADT/AllocatorList.h - Custom allocator list ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/ADT/Any.h b/include/llvm/ADT/Any.h index 7faa4c963d3d..5dcd6e73c54f 100644 --- a/include/llvm/ADT/Any.h +++ b/include/llvm/ADT/Any.h @@ -1,9 +1,8 @@ //===- Any.h - Generic type erased holder of any type -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/ArrayRef.h b/include/llvm/ADT/ArrayRef.h index 9cb25b09c6cb..773c88f7c9f9 100644 --- a/include/llvm/ADT/ArrayRef.h +++ b/include/llvm/ADT/ArrayRef.h @@ -1,9 +1,8 @@ //===- ArrayRef.h - Array Reference Wrapper ---------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -431,7 +430,7 @@ namespace llvm { std::copy(Data.begin(), Data.end(), this->begin()); } - OwningArrayRef(OwningArrayRef &&Other) { *this = Other; } + OwningArrayRef(OwningArrayRef &&Other) { *this = std::move(Other); } OwningArrayRef &operator=(OwningArrayRef &&Other) { delete[] this->data(); @@ -526,12 +525,6 @@ namespace llvm { /// @} - // ArrayRefs can be treated like a POD type. - template struct isPodLike; - template struct isPodLike> { - static const bool value = true; - }; - template hash_code hash_value(ArrayRef S) { return hash_combine_range(S.begin(), S.end()); } diff --git a/include/llvm/ADT/BitVector.h b/include/llvm/ADT/BitVector.h index 9ab1da7c6913..fabf5d9cd348 100644 --- a/include/llvm/ADT/BitVector.h +++ b/include/llvm/ADT/BitVector.h @@ -1,9 +1,8 @@ //===- llvm/ADT/BitVector.h - Bit vectors -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/BitmaskEnum.h b/include/llvm/ADT/BitmaskEnum.h index 18c6ba5a3eb8..1a18bc721b21 100644 --- a/include/llvm/ADT/BitmaskEnum.h +++ b/include/llvm/ADT/BitmaskEnum.h @@ -1,9 +1,8 @@ //===-- llvm/ADT/BitmaskEnum.h ----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/ADT/BreadthFirstIterator.h b/include/llvm/ADT/BreadthFirstIterator.h index 6bc63c283b09..e97d76680db8 100644 --- a/include/llvm/ADT/BreadthFirstIterator.h +++ b/include/llvm/ADT/BreadthFirstIterator.h @@ -1,9 +1,8 @@ //===- llvm/ADT/BreadthFirstIterator.h - Breadth First iterator -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -125,7 +124,7 @@ public: const NodeRef &operator*() const { return VisitQueue.front()->first; } - // This is a nonstandard operator-> that dereferenfces the pointer an extra + // This is a nonstandard operator-> that dereferences the pointer an extra // time so that you can actually call methods on the node, because the // contained type is a pointer. NodeRef operator->() const { return **this; } diff --git a/include/llvm/ADT/CachedHashString.h b/include/llvm/ADT/CachedHashString.h index d8f0e7afdd49..80144fb87e0e 100644 --- a/include/llvm/ADT/CachedHashString.h +++ b/include/llvm/ADT/CachedHashString.h @@ -1,9 +1,8 @@ //===- llvm/ADT/CachedHashString.h - Prehashed string/StringRef -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/DAGDeltaAlgorithm.h b/include/llvm/ADT/DAGDeltaAlgorithm.h index 41fdd43efb8a..d4cdc3c86048 100644 --- a/include/llvm/ADT/DAGDeltaAlgorithm.h +++ b/include/llvm/ADT/DAGDeltaAlgorithm.h @@ -1,9 +1,8 @@ //===- DAGDeltaAlgorithm.h - A DAG Minimization Algorithm ------*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception //===----------------------------------------------------------------------===// #ifndef LLVM_ADT_DAGDELTAALGORITHM_H diff --git a/include/llvm/ADT/DeltaAlgorithm.h b/include/llvm/ADT/DeltaAlgorithm.h index 6becb2a60104..114b95499530 100644 --- a/include/llvm/ADT/DeltaAlgorithm.h +++ b/include/llvm/ADT/DeltaAlgorithm.h @@ -1,9 +1,8 @@ //===- DeltaAlgorithm.h - A Set Minimization Algorithm ---------*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception //===----------------------------------------------------------------------===// #ifndef LLVM_ADT_DELTAALGORITHM_H diff --git a/include/llvm/ADT/DenseMap.h b/include/llvm/ADT/DenseMap.h index 1f50502fff92..a05cf8130d3c 100644 --- a/include/llvm/ADT/DenseMap.h +++ b/include/llvm/ADT/DenseMap.h @@ -1,9 +1,8 @@ //===- llvm/ADT/DenseMap.h - Dense probed hash table ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -64,7 +63,7 @@ struct DenseMapPair : public std::pair { template DenseMapPair(AltPairT &&AltPair, typename std::enable_if>::value>::type * = 0) + AltPairT, std::pair>::value>::type * = nullptr) : std::pair(std::forward(AltPair)) {} KeyT &getFirst() { return std::pair::first; } @@ -146,7 +145,8 @@ public: } const KeyT EmptyKey = getEmptyKey(), TombstoneKey = getTombstoneKey(); - if (isPodLike::value && isPodLike::value) { + if (is_trivially_copyable::value && + is_trivially_copyable::value) { // Use a simpler loop when these are trivial types. for (BucketT *P = getBuckets(), *E = getBucketsEnd(); P != E; ++P) P->getFirst() = EmptyKey; @@ -422,7 +422,8 @@ protected: setNumEntries(other.getNumEntries()); setNumTombstones(other.getNumTombstones()); - if (isPodLike::value && isPodLike::value) + if (is_trivially_copyable::value && + is_trivially_copyable::value) memcpy(reinterpret_cast(getBuckets()), other.getBuckets(), getNumBuckets() * sizeof(BucketT)); else diff --git a/include/llvm/ADT/DenseMapInfo.h b/include/llvm/ADT/DenseMapInfo.h index 5d12b424fb37..5ef6f3ad1b04 100644 --- a/include/llvm/ADT/DenseMapInfo.h +++ b/include/llvm/ADT/DenseMapInfo.h @@ -1,9 +1,8 @@ //===- llvm/ADT/DenseMapInfo.h - Type traits for DenseMap -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,6 +17,7 @@ #include "llvm/ADT/Hashing.h" #include "llvm/ADT/StringRef.h" #include "llvm/Support/PointerLikeTypeTraits.h" +#include "llvm/Support/ScalableSize.h" #include #include #include @@ -269,6 +269,21 @@ template <> struct DenseMapInfo { static bool isEqual(hash_code LHS, hash_code RHS) { return LHS == RHS; } }; +template <> struct DenseMapInfo { + static inline ElementCount getEmptyKey() { return {~0U, true}; } + static inline ElementCount getTombstoneKey() { return {~0U - 1, false}; } + static unsigned getHashValue(const ElementCount& EltCnt) { + if (EltCnt.Scalable) + return (EltCnt.Min * 37U) - 1U; + + return EltCnt.Min * 37U; + } + + static bool isEqual(const ElementCount& LHS, const ElementCount& RHS) { + return LHS == RHS; + } +}; + } // end namespace llvm #endif // LLVM_ADT_DENSEMAPINFO_H diff --git a/include/llvm/ADT/DenseSet.h b/include/llvm/ADT/DenseSet.h index e85a38587e41..9afb715ae1db 100644 --- a/include/llvm/ADT/DenseSet.h +++ b/include/llvm/ADT/DenseSet.h @@ -1,9 +1,8 @@ //===- llvm/ADT/DenseSet.h - Dense probed hash table ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -131,7 +130,7 @@ public: class ConstIterator { typename MapTy::const_iterator I; - friend class DenseSet; + friend class DenseSetImpl; friend class Iterator; public: diff --git a/include/llvm/ADT/DepthFirstIterator.h b/include/llvm/ADT/DepthFirstIterator.h index 1f3766d3c9de..11967f5eefcc 100644 --- a/include/llvm/ADT/DepthFirstIterator.h +++ b/include/llvm/ADT/DepthFirstIterator.h @@ -1,9 +1,8 @@ //===- llvm/ADT/DepthFirstIterator.h - Depth First iterator -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/EpochTracker.h b/include/llvm/ADT/EpochTracker.h index 49ef192364e8..a782b4756898 100644 --- a/include/llvm/ADT/EpochTracker.h +++ b/include/llvm/ADT/EpochTracker.h @@ -1,9 +1,8 @@ //===- llvm/ADT/EpochTracker.h - ADT epoch tracking --------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/EquivalenceClasses.h b/include/llvm/ADT/EquivalenceClasses.h index e3f48433c69f..2cb7108c0794 100644 --- a/include/llvm/ADT/EquivalenceClasses.h +++ b/include/llvm/ADT/EquivalenceClasses.h @@ -1,9 +1,8 @@ //===- llvm/ADT/EquivalenceClasses.h - Generic Equiv. Classes ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/FoldingSet.h b/include/llvm/ADT/FoldingSet.h index e363e69d032a..d5837e51bcfc 100644 --- a/include/llvm/ADT/FoldingSet.h +++ b/include/llvm/ADT/FoldingSet.h @@ -1,9 +1,8 @@ //===- llvm/ADT/FoldingSet.h - Uniquing Hash Set ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/FunctionExtras.h b/include/llvm/ADT/FunctionExtras.h index 2b75dc6ac219..121aa527a5da 100644 --- a/include/llvm/ADT/FunctionExtras.h +++ b/include/llvm/ADT/FunctionExtras.h @@ -1,9 +1,8 @@ //===- FunctionExtras.h - Function type erasure utilities -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/ADT/GraphTraits.h b/include/llvm/ADT/GraphTraits.h index d39b50fdc488..3ce91225d80d 100644 --- a/include/llvm/ADT/GraphTraits.h +++ b/include/llvm/ADT/GraphTraits.h @@ -1,9 +1,8 @@ //===- llvm/ADT/GraphTraits.h - Graph traits template -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/Hashing.h b/include/llvm/ADT/Hashing.h index 9175c545b7c9..008188bfa210 100644 --- a/include/llvm/ADT/Hashing.h +++ b/include/llvm/ADT/Hashing.h @@ -1,9 +1,8 @@ //===-- llvm/ADT/Hashing.h - Utilities for hashing --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -192,7 +191,7 @@ inline uint64_t hash_1to3_bytes(const char *s, size_t len, uint64_t seed) { uint8_t b = s[len >> 1]; uint8_t c = s[len - 1]; uint32_t y = static_cast(a) + (static_cast(b) << 8); - uint32_t z = len + (static_cast(c) << 2); + uint32_t z = static_cast(len) + (static_cast(c) << 2); return shift_mix(y * k2 ^ z * k3 ^ seed) * k2; } diff --git a/include/llvm/ADT/ImmutableList.h b/include/llvm/ADT/ImmutableList.h index 0541dc2566ed..c9ee494734e7 100644 --- a/include/llvm/ADT/ImmutableList.h +++ b/include/llvm/ADT/ImmutableList.h @@ -1,9 +1,8 @@ //==--- ImmutableList.h - Immutable (functional) list interface --*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -242,10 +241,6 @@ template struct DenseMapInfo> { } }; -template struct isPodLike; -template -struct isPodLike> { static const bool value = true; }; - } // end namespace llvm #endif // LLVM_ADT_IMMUTABLELIST_H diff --git a/include/llvm/ADT/ImmutableMap.h b/include/llvm/ADT/ImmutableMap.h index cbc27ff17ccf..86fd7fefaec3 100644 --- a/include/llvm/ADT/ImmutableMap.h +++ b/include/llvm/ADT/ImmutableMap.h @@ -1,9 +1,8 @@ //===--- ImmutableMap.h - Immutable (functional) map interface --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/ImmutableSet.h b/include/llvm/ADT/ImmutableSet.h index b1d5f4ac42e4..587105431533 100644 --- a/include/llvm/ADT/ImmutableSet.h +++ b/include/llvm/ADT/ImmutableSet.h @@ -1,9 +1,8 @@ //===--- ImmutableSet.h - Immutable (functional) set interface --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/IndexedMap.h b/include/llvm/ADT/IndexedMap.h index 2ee80d2cde63..b44f16b91d76 100644 --- a/include/llvm/ADT/IndexedMap.h +++ b/include/llvm/ADT/IndexedMap.h @@ -1,9 +1,8 @@ //===- llvm/ADT/IndexedMap.h - An index map implementation ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/IntEqClasses.h b/include/llvm/ADT/IntEqClasses.h index 0baee2f11a79..08f46a3079ef 100644 --- a/include/llvm/ADT/IntEqClasses.h +++ b/include/llvm/ADT/IntEqClasses.h @@ -1,9 +1,8 @@ //===-- llvm/ADT/IntEqClasses.h - Equiv. Classes of Integers ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/IntervalMap.h b/include/llvm/ADT/IntervalMap.h index 2af61049e5af..12828c4cfdab 100644 --- a/include/llvm/ADT/IntervalMap.h +++ b/include/llvm/ADT/IntervalMap.h @@ -1,9 +1,8 @@ //===- llvm/ADT/IntervalMap.h - A sorted interval map -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/IntrusiveRefCntPtr.h b/include/llvm/ADT/IntrusiveRefCntPtr.h index 430ef86afbd9..6d97fe15db8b 100644 --- a/include/llvm/ADT/IntrusiveRefCntPtr.h +++ b/include/llvm/ADT/IntrusiveRefCntPtr.h @@ -1,9 +1,8 @@ //==- llvm/ADT/IntrusiveRefCntPtr.h - Smart Refcounting Pointer --*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/MapVector.h b/include/llvm/ADT/MapVector.h index 47b4987f210a..1de1124f4ea2 100644 --- a/include/llvm/ADT/MapVector.h +++ b/include/llvm/ADT/MapVector.h @@ -1,9 +1,8 @@ //===- llvm/ADT/MapVector.h - Map w/ deterministic value order --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/None.h b/include/llvm/ADT/None.h index 4b6bc1e005b5..004ca0ac50ac 100644 --- a/include/llvm/ADT/None.h +++ b/include/llvm/ADT/None.h @@ -1,9 +1,8 @@ //===-- None.h - Simple null value for implicit construction ------*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/Optional.h b/include/llvm/ADT/Optional.h index 76937d632ae1..b45a74002e10 100644 --- a/include/llvm/ADT/Optional.h +++ b/include/llvm/ADT/Optional.h @@ -1,9 +1,8 @@ //===- Optional.h - Simple variant for passing optional values --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -17,94 +16,197 @@ #define LLVM_ADT_OPTIONAL_H #include "llvm/ADT/None.h" -#include "llvm/Support/AlignOf.h" #include "llvm/Support/Compiler.h" #include "llvm/Support/type_traits.h" -#include #include +#include #include #include namespace llvm { +class raw_ostream; + namespace optional_detail { + +struct in_place_t {}; + /// Storage for any type. -template ::value> struct OptionalStorage { - AlignedCharArrayUnion storage; - bool hasVal = false; +template ::value> +class OptionalStorage { + union { + char empty; + T value; + }; + bool hasVal; - OptionalStorage() = default; +public: + ~OptionalStorage() { reset(); } - OptionalStorage(const T &y) : hasVal(true) { new (storage.buffer) T(y); } - OptionalStorage(const OptionalStorage &O) : hasVal(O.hasVal) { - if (hasVal) - new (storage.buffer) T(*O.getPointer()); + OptionalStorage() noexcept : empty(), hasVal(false) {} + + OptionalStorage(OptionalStorage const &other) : OptionalStorage() { + if (other.hasValue()) { + emplace(other.value); + } } - OptionalStorage(T &&y) : hasVal(true) { - new (storage.buffer) T(std::forward(y)); + OptionalStorage(OptionalStorage &&other) : OptionalStorage() { + if (other.hasValue()) { + emplace(std::move(other.value)); + } } - OptionalStorage(OptionalStorage &&O) : hasVal(O.hasVal) { - if (O.hasVal) { - new (storage.buffer) T(std::move(*O.getPointer())); + + template + explicit OptionalStorage(in_place_t, Args &&... args) + : value(std::forward(args)...), hasVal(true) {} + + void reset() noexcept { + if (hasVal) { + value.~T(); + hasVal = false; } } - OptionalStorage &operator=(T &&y) { - if (hasVal) - *getPointer() = std::move(y); - else { - new (storage.buffer) T(std::move(y)); + bool hasValue() const noexcept { return hasVal; } + + T &getValue() LLVM_LVALUE_FUNCTION noexcept { + assert(hasVal); + return value; + } + T const &getValue() const LLVM_LVALUE_FUNCTION noexcept { + assert(hasVal); + return value; + } +#if LLVM_HAS_RVALUE_REFERENCE_THIS + T &&getValue() && noexcept { + assert(hasVal); + return std::move(value); + } +#endif + + template void emplace(Args &&... args) { + reset(); + ::new ((void *)std::addressof(value)) T(std::forward(args)...); + hasVal = true; + } + + OptionalStorage &operator=(T const &y) { + if (hasValue()) { + value = y; + } else { + ::new ((void *)std::addressof(value)) T(y); hasVal = true; } return *this; } - OptionalStorage &operator=(OptionalStorage &&O) { - if (!O.hasVal) - reset(); - else { - *this = std::move(*O.getPointer()); + OptionalStorage &operator=(T &&y) { + if (hasValue()) { + value = std::move(y); + } else { + ::new ((void *)std::addressof(value)) T(std::move(y)); + hasVal = true; } return *this; } - // FIXME: these assignments (& the equivalent const T&/const Optional& ctors) - // could be made more efficient by passing by value, possibly unifying them - // with the rvalue versions above - but this could place a different set of - // requirements (notably: the existence of a default ctor) when implemented - // in that way. Careful SFINAE to avoid such pitfalls would be required. - OptionalStorage &operator=(const T &y) { - if (hasVal) - *getPointer() = y; - else { - new (storage.buffer) T(y); - hasVal = true; + OptionalStorage &operator=(OptionalStorage const &other) { + if (other.hasValue()) { + if (hasValue()) { + value = other.value; + } else { + ::new ((void *)std::addressof(value)) T(other.value); + hasVal = true; + } + } else { + reset(); } return *this; } - OptionalStorage &operator=(const OptionalStorage &O) { - if (!O.hasVal) + + OptionalStorage &operator=(OptionalStorage &&other) { + if (other.hasValue()) { + if (hasValue()) { + value = std::move(other.value); + } else { + ::new ((void *)std::addressof(value)) T(std::move(other.value)); + hasVal = true; + } + } else { reset(); - else - *this = *O.getPointer(); + } return *this; } +}; - ~OptionalStorage() { reset(); } +template class OptionalStorage { + union { + char empty; + T value; + }; + bool hasVal = false; + +public: + ~OptionalStorage() = default; + + OptionalStorage() noexcept : empty{} {} + + OptionalStorage(OptionalStorage const &other) = default; + OptionalStorage(OptionalStorage &&other) = default; + + OptionalStorage &operator=(OptionalStorage const &other) = default; + OptionalStorage &operator=(OptionalStorage &&other) = default; + + template + explicit OptionalStorage(in_place_t, Args &&... args) + : value(std::forward(args)...), hasVal(true) {} - void reset() { + void reset() noexcept { if (hasVal) { - (*getPointer()).~T(); + value.~T(); hasVal = false; } } - T *getPointer() { + bool hasValue() const noexcept { return hasVal; } + + T &getValue() LLVM_LVALUE_FUNCTION noexcept { assert(hasVal); - return reinterpret_cast(storage.buffer); + return value; } - const T *getPointer() const { + T const &getValue() const LLVM_LVALUE_FUNCTION noexcept { assert(hasVal); - return reinterpret_cast(storage.buffer); + return value; + } +#if LLVM_HAS_RVALUE_REFERENCE_THIS + T &&getValue() && noexcept { + assert(hasVal); + return std::move(value); + } +#endif + + template void emplace(Args &&... args) { + reset(); + ::new ((void *)std::addressof(value)) T(std::forward(args)...); + hasVal = true; + } + + OptionalStorage &operator=(T const &y) { + if (hasValue()) { + value = y; + } else { + ::new ((void *)std::addressof(value)) T(y); + hasVal = true; + } + return *this; + } + OptionalStorage &operator=(T &&y) { + if (hasValue()) { + value = std::move(y); + } else { + ::new ((void *)std::addressof(value)) T(std::move(y)); + hasVal = true; + } + return *this; } }; @@ -119,10 +221,10 @@ public: constexpr Optional() {} constexpr Optional(NoneType) {} - Optional(const T &y) : Storage(y) {} + Optional(const T &y) : Storage(optional_detail::in_place_t{}, y) {} Optional(const Optional &O) = default; - Optional(T &&y) : Storage(std::forward(y)) {} + Optional(T &&y) : Storage(optional_detail::in_place_t{}, std::move(y)) {} Optional(Optional &&O) = default; Optional &operator=(T &&y) { @@ -133,9 +235,7 @@ public: /// Create a new object by constructing it in place with the given arguments. template void emplace(ArgTypes &&... Args) { - reset(); - Storage.hasVal = true; - new (getPointer()) T(std::forward(Args)...); + Storage.emplace(std::forward(Args)...); } static inline Optional create(const T *y) { @@ -150,23 +250,17 @@ public: void reset() { Storage.reset(); } - const T *getPointer() const { - assert(Storage.hasVal); - return reinterpret_cast(Storage.storage.buffer); - } - T *getPointer() { - assert(Storage.hasVal); - return reinterpret_cast(Storage.storage.buffer); - } - const T &getValue() const LLVM_LVALUE_FUNCTION { return *getPointer(); } - T &getValue() LLVM_LVALUE_FUNCTION { return *getPointer(); } + const T *getPointer() const { return &Storage.getValue(); } + T *getPointer() { return &Storage.getValue(); } + const T &getValue() const LLVM_LVALUE_FUNCTION { return Storage.getValue(); } + T &getValue() LLVM_LVALUE_FUNCTION { return Storage.getValue(); } - explicit operator bool() const { return Storage.hasVal; } - bool hasValue() const { return Storage.hasVal; } + explicit operator bool() const { return hasValue(); } + bool hasValue() const { return Storage.hasValue(); } const T *operator->() const { return getPointer(); } T *operator->() { return getPointer(); } - const T &operator*() const LLVM_LVALUE_FUNCTION { return *getPointer(); } - T &operator*() LLVM_LVALUE_FUNCTION { return *getPointer(); } + const T &operator*() const LLVM_LVALUE_FUNCTION { return getValue(); } + T &operator*() LLVM_LVALUE_FUNCTION { return getValue(); } template constexpr T getValueOr(U &&value) const LLVM_LVALUE_FUNCTION { @@ -174,8 +268,8 @@ public: } #if LLVM_HAS_RVALUE_REFERENCE_THIS - T &&getValue() && { return std::move(*getPointer()); } - T &&operator*() && { return std::move(*getPointer()); } + T &&getValue() && { return std::move(Storage.getValue()); } + T &&operator*() && { return std::move(Storage.getValue()); } template T getValueOr(U &&value) && { @@ -184,11 +278,6 @@ public: #endif }; -template struct isPodLike> { - // An Optional is pod-like if T is. - static const bool value = isPodLike::value; -}; - template bool operator==(const Optional &X, const Optional &Y) { if (X && Y) @@ -323,6 +412,18 @@ template bool operator>=(const T &X, const Optional &Y) { return !(X < Y); } +raw_ostream &operator<<(raw_ostream &OS, NoneType); + +template () + << std::declval())> +raw_ostream &operator<<(raw_ostream &OS, const Optional &O) { + if (O) + OS << *O; + else + OS << None; + return OS; +} + } // end namespace llvm #endif // LLVM_ADT_OPTIONAL_H diff --git a/include/llvm/ADT/PackedVector.h b/include/llvm/ADT/PackedVector.h index 3d53c49536d0..ae7f8cc85743 100644 --- a/include/llvm/ADT/PackedVector.h +++ b/include/llvm/ADT/PackedVector.h @@ -1,9 +1,8 @@ //===- llvm/ADT/PackedVector.h - Packed values vector -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/PointerEmbeddedInt.h b/include/llvm/ADT/PointerEmbeddedInt.h index ab4e1048a5bc..3eb6edb03430 100644 --- a/include/llvm/ADT/PointerEmbeddedInt.h +++ b/include/llvm/ADT/PointerEmbeddedInt.h @@ -1,9 +1,8 @@ //===- llvm/ADT/PointerEmbeddedInt.h ----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/ADT/PointerIntPair.h b/include/llvm/ADT/PointerIntPair.h index 6d1b53a90ad2..24a2bb67a36e 100644 --- a/include/llvm/ADT/PointerIntPair.h +++ b/include/llvm/ADT/PointerIntPair.h @@ -1,9 +1,8 @@ //===- llvm/ADT/PointerIntPair.h - Pair for pointer and int -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,6 +14,7 @@ #define LLVM_ADT_POINTERINTPAIR_H #include "llvm/Support/PointerLikeTypeTraits.h" +#include "llvm/Support/type_traits.h" #include #include #include @@ -126,6 +126,19 @@ public: } }; +// Specialize is_trivially_copyable to avoid limitation of llvm::is_trivially_copyable +// when compiled with gcc 4.9. +template +struct is_trivially_copyable> : std::true_type { +#ifdef HAVE_STD_IS_TRIVIALLY_COPYABLE + static_assert(std::is_trivially_copyable>::value, + "inconsistent behavior between llvm:: and std:: implementation of is_trivially_copyable"); +#endif +}; + + template struct PointerIntPairInfo { static_assert(PtrTraits::NumLowBitsAvailable < @@ -176,12 +189,6 @@ struct PointerIntPairInfo { } }; -template struct isPodLike; -template -struct isPodLike> { - static const bool value = true; -}; - // Provide specialization of DenseMapInfo for PointerIntPair. template struct DenseMapInfo> { diff --git a/include/llvm/ADT/PointerSumType.h b/include/llvm/ADT/PointerSumType.h index a19e45a46218..d467f83f58ac 100644 --- a/include/llvm/ADT/PointerSumType.h +++ b/include/llvm/ADT/PointerSumType.h @@ -1,9 +1,8 @@ //===- llvm/ADT/PointerSumType.h --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/ADT/PointerUnion.h b/include/llvm/ADT/PointerUnion.h index 315e58336cba..2bcdf546c6e4 100644 --- a/include/llvm/ADT/PointerUnion.h +++ b/include/llvm/ADT/PointerUnion.h @@ -1,9 +1,8 @@ //===- llvm/ADT/PointerUnion.h - Discriminated Union of 2 Ptrs --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -54,22 +53,98 @@ struct PointerUnionTypeSelectorReturn< typename PointerUnionTypeSelector::Return; }; -/// Provide PointerLikeTypeTraits for void* that is used by PointerUnion -/// for the two template arguments. -template class PointerUnionUIntTraits { -public: - static inline void *getAsVoidPointer(void *P) { return P; } - static inline void *getFromVoidPointer(void *P) { return P; } +namespace pointer_union_detail { + constexpr int constexprMin(int a, int b) { return a < b ? a : b; } + /// Determine the number of bits required to store integers with values < n. + /// This is ceil(log2(n)). + constexpr int bitsRequired(unsigned n) { + return n > 1 ? 1 + bitsRequired((n + 1) / 2) : 0; + } + + // FIXME: In C++14, replace this with + // std::min({PointerLikeTypeTraits::NumLowBitsAvailable...}) + template constexpr int lowBitsAvailable() { + return PointerLikeTypeTraits::NumLowBitsAvailable; + } + template + constexpr int lowBitsAvailable() { + return constexprMin(lowBitsAvailable(), lowBitsAvailable()); + } - enum { - PT1BitsAv = (int)(PointerLikeTypeTraits::NumLowBitsAvailable), - PT2BitsAv = (int)(PointerLikeTypeTraits::NumLowBitsAvailable), - NumLowBitsAvailable = PT1BitsAv < PT2BitsAv ? PT1BitsAv : PT2BitsAv + /// Find the index of a type in a list of types. TypeIndex::Index + /// is the index of T in Us, or sizeof...(Us) if T does not appear in the + /// list. + template struct TypeIndex; + template struct TypeIndex { + static constexpr int Index = 0; }; -}; + template + struct TypeIndex { + static constexpr int Index = 1 + TypeIndex::Index; + }; + template struct TypeIndex { + static constexpr int Index = 0; + }; + + /// Find the first type in a list of types. + template struct GetFirstType { + using type = T; + }; + + /// Provide PointerLikeTypeTraits for void* that is used by PointerUnion + /// for the template arguments. + template class PointerUnionUIntTraits { + public: + static inline void *getAsVoidPointer(void *P) { return P; } + static inline void *getFromVoidPointer(void *P) { return P; } + static constexpr int NumLowBitsAvailable = lowBitsAvailable(); + }; + + /// Implement assigment in terms of construction. + template struct AssignableFrom { + Derived &operator=(T t) { + return static_cast(*this) = Derived(t); + } + }; + + template + class PointerUnionMembers; -/// A discriminated union of two pointer types, with the discriminator in the -/// low bit of the pointer. + template + class PointerUnionMembers { + protected: + ValTy Val; + PointerUnionMembers() = default; + PointerUnionMembers(ValTy Val) : Val(Val) {} + + friend struct PointerLikeTypeTraits; + }; + + template + class PointerUnionMembers + : public PointerUnionMembers { + using Base = PointerUnionMembers; + public: + using Base::Base; + PointerUnionMembers() = default; + PointerUnionMembers(Type V) + : Base(ValTy(const_cast( + PointerLikeTypeTraits::getAsVoidPointer(V)), + I)) {} + + using Base::operator=; + Derived &operator=(Type V) { + this->Val = ValTy( + const_cast(PointerLikeTypeTraits::getAsVoidPointer(V)), + I); + return static_cast(*this); + }; + }; +} + +/// A discriminated union of two or more pointer types, with the discriminator +/// in the low bit of the pointer. /// /// This implementation is extremely efficient in space due to leveraging the /// low bits of the pointer, while exposing a natural and type-safe API. @@ -84,49 +159,44 @@ public: /// P = (float*)0; /// Y = P.get(); // ok. /// X = P.get(); // runtime assertion failure. -template class PointerUnion { -public: - using ValTy = - PointerIntPair>; - -private: - ValTy Val; - - struct IsPT1 { - static const int Num = 0; - }; - struct IsPT2 { - static const int Num = 1; - }; - template struct UNION_DOESNT_CONTAIN_TYPE {}; +template +class PointerUnion + : public pointer_union_detail::PointerUnionMembers< + PointerUnion, + PointerIntPair< + void *, pointer_union_detail::bitsRequired(sizeof...(PTs)), int, + pointer_union_detail::PointerUnionUIntTraits>, + 0, PTs...> { + // The first type is special in some ways, but we don't want PointerUnion to + // be a 'template ' because it's much more + // convenient to have a name for the whole pack. So split off the first type + // here. + using First = typename pointer_union_detail::GetFirstType::type; + using Base = typename PointerUnion::PointerUnionMembers; public: PointerUnion() = default; - PointerUnion(PT1 V) - : Val(const_cast( - PointerLikeTypeTraits::getAsVoidPointer(V))) {} - PointerUnion(PT2 V) - : Val(const_cast(PointerLikeTypeTraits::getAsVoidPointer(V)), - 1) {} + + PointerUnion(std::nullptr_t) : PointerUnion() {} + using Base::Base; /// Test if the pointer held in the union is null, regardless of /// which type it is. bool isNull() const { // Convert from the void* to one of the pointer types, to make sure that // we recursively strip off low bits if we have a nested PointerUnion. - return !PointerLikeTypeTraits::getFromVoidPointer(Val.getPointer()); + return !PointerLikeTypeTraits::getFromVoidPointer( + this->Val.getPointer()); } explicit operator bool() const { return !isNull(); } /// Test if the Union currently holds the type matching T. template int is() const { - using Ty = typename ::llvm::PointerUnionTypeSelector< - PT1, T, IsPT1, - ::llvm::PointerUnionTypeSelector>>::Return; - int TyNo = Ty::Num; - return static_cast(Val.getInt()) == TyNo; + constexpr int Index = pointer_union_detail::TypeIndex::Index; + static_assert(Index < sizeof...(PTs), + "PointerUnion::is given type not in the union"); + return this->Val.getInt() == Index; } /// Returns the value of the specified pointer type. @@ -134,7 +204,7 @@ public: /// If the specified pointer type is incorrect, assert. template T get() const { assert(is() && "Invalid accessor called"); - return PointerLikeTypeTraits::getFromVoidPointer(Val.getPointer()); + return PointerLikeTypeTraits::getFromVoidPointer(this->Val.getPointer()); } /// Returns the current pointer if it is of the specified pointer type, @@ -147,342 +217,100 @@ public: /// If the union is set to the first pointer type get an address pointing to /// it. - PT1 const *getAddrOfPtr1() const { + First const *getAddrOfPtr1() const { return const_cast(this)->getAddrOfPtr1(); } /// If the union is set to the first pointer type get an address pointing to /// it. - PT1 *getAddrOfPtr1() { - assert(is() && "Val is not the first pointer"); + First *getAddrOfPtr1() { + assert(is() && "Val is not the first pointer"); assert( - get() == Val.getPointer() && + get() == this->Val.getPointer() && "Can't get the address because PointerLikeTypeTraits changes the ptr"); - return const_cast( - reinterpret_cast(Val.getAddrOfPointer())); + return const_cast( + reinterpret_cast(this->Val.getAddrOfPointer())); } /// Assignment from nullptr which just clears the union. const PointerUnion &operator=(std::nullptr_t) { - Val.initWithPointer(nullptr); + this->Val.initWithPointer(nullptr); return *this; } - /// Assignment operators - Allow assigning into this union from either - /// pointer type, setting the discriminator to remember what it came from. - const PointerUnion &operator=(const PT1 &RHS) { - Val.initWithPointer( - const_cast(PointerLikeTypeTraits::getAsVoidPointer(RHS))); - return *this; - } - const PointerUnion &operator=(const PT2 &RHS) { - Val.setPointerAndInt( - const_cast(PointerLikeTypeTraits::getAsVoidPointer(RHS)), - 1); - return *this; - } + /// Assignment from elements of the union. + using Base::operator=; - void *getOpaqueValue() const { return Val.getOpaqueValue(); } + void *getOpaqueValue() const { return this->Val.getOpaqueValue(); } static inline PointerUnion getFromOpaqueValue(void *VP) { PointerUnion V; - V.Val = ValTy::getFromOpaqueValue(VP); + V.Val = decltype(V.Val)::getFromOpaqueValue(VP); return V; } }; -template -bool operator==(PointerUnion lhs, PointerUnion rhs) { +template +bool operator==(PointerUnion lhs, PointerUnion rhs) { return lhs.getOpaqueValue() == rhs.getOpaqueValue(); } -template -bool operator!=(PointerUnion lhs, PointerUnion rhs) { +template +bool operator!=(PointerUnion lhs, PointerUnion rhs) { return lhs.getOpaqueValue() != rhs.getOpaqueValue(); } -template -bool operator<(PointerUnion lhs, PointerUnion rhs) { +template +bool operator<(PointerUnion lhs, PointerUnion rhs) { return lhs.getOpaqueValue() < rhs.getOpaqueValue(); } // Teach SmallPtrSet that PointerUnion is "basically a pointer", that has // # low bits available = min(PT1bits,PT2bits)-1. -template -struct PointerLikeTypeTraits> { - static inline void *getAsVoidPointer(const PointerUnion &P) { +template +struct PointerLikeTypeTraits> { + static inline void *getAsVoidPointer(const PointerUnion &P) { return P.getOpaqueValue(); } - static inline PointerUnion getFromVoidPointer(void *P) { - return PointerUnion::getFromOpaqueValue(P); + static inline PointerUnion getFromVoidPointer(void *P) { + return PointerUnion::getFromOpaqueValue(P); } - // The number of bits available are the min of the two pointer types. - enum { - NumLowBitsAvailable = PointerLikeTypeTraits< - typename PointerUnion::ValTy>::NumLowBitsAvailable - }; + // The number of bits available are the min of the pointer types minus the + // bits needed for the discriminator. + static constexpr int NumLowBitsAvailable = PointerLikeTypeTraits::Val)>::NumLowBitsAvailable; }; /// A pointer union of three pointer types. See documentation for PointerUnion /// for usage. -template class PointerUnion3 { -public: - using InnerUnion = PointerUnion; - using ValTy = PointerUnion; - -private: - ValTy Val; - - struct IsInnerUnion { - ValTy Val; - - IsInnerUnion(ValTy val) : Val(val) {} - - template int is() const { - return Val.template is() && - Val.template get().template is(); - } - - template T get() const { - return Val.template get().template get(); - } - }; - - struct IsPT3 { - ValTy Val; - - IsPT3(ValTy val) : Val(val) {} - - template int is() const { return Val.template is(); } - template T get() const { return Val.template get(); } - }; - -public: - PointerUnion3() = default; - PointerUnion3(PT1 V) { Val = InnerUnion(V); } - PointerUnion3(PT2 V) { Val = InnerUnion(V); } - PointerUnion3(PT3 V) { Val = V; } - - /// Test if the pointer held in the union is null, regardless of - /// which type it is. - bool isNull() const { return Val.isNull(); } - explicit operator bool() const { return !isNull(); } - - /// Test if the Union currently holds the type matching T. - template int is() const { - // If T is PT1/PT2 choose IsInnerUnion otherwise choose IsPT3. - using Ty = typename ::llvm::PointerUnionTypeSelector< - PT1, T, IsInnerUnion, - ::llvm::PointerUnionTypeSelector>::Return; - return Ty(Val).template is(); - } - - /// Returns the value of the specified pointer type. - /// - /// If the specified pointer type is incorrect, assert. - template T get() const { - assert(is() && "Invalid accessor called"); - // If T is PT1/PT2 choose IsInnerUnion otherwise choose IsPT3. - using Ty = typename ::llvm::PointerUnionTypeSelector< - PT1, T, IsInnerUnion, - ::llvm::PointerUnionTypeSelector>::Return; - return Ty(Val).template get(); - } - - /// Returns the current pointer if it is of the specified pointer type, - /// otherwises returns null. - template T dyn_cast() const { - if (is()) - return get(); - return T(); - } - - /// Assignment from nullptr which just clears the union. - const PointerUnion3 &operator=(std::nullptr_t) { - Val = nullptr; - return *this; - } - - /// Assignment operators - Allow assigning into this union from either - /// pointer type, setting the discriminator to remember what it came from. - const PointerUnion3 &operator=(const PT1 &RHS) { - Val = InnerUnion(RHS); - return *this; - } - const PointerUnion3 &operator=(const PT2 &RHS) { - Val = InnerUnion(RHS); - return *this; - } - const PointerUnion3 &operator=(const PT3 &RHS) { - Val = RHS; - return *this; - } - - void *getOpaqueValue() const { return Val.getOpaqueValue(); } - static inline PointerUnion3 getFromOpaqueValue(void *VP) { - PointerUnion3 V; - V.Val = ValTy::getFromOpaqueValue(VP); - return V; - } -}; - -// Teach SmallPtrSet that PointerUnion3 is "basically a pointer", that has -// # low bits available = min(PT1bits,PT2bits,PT2bits)-2. template -struct PointerLikeTypeTraits> { - static inline void *getAsVoidPointer(const PointerUnion3 &P) { - return P.getOpaqueValue(); - } - - static inline PointerUnion3 getFromVoidPointer(void *P) { - return PointerUnion3::getFromOpaqueValue(P); - } - - // The number of bits available are the min of the two pointer types. - enum { - NumLowBitsAvailable = PointerLikeTypeTraits< - typename PointerUnion3::ValTy>::NumLowBitsAvailable - }; -}; - -template -bool operator<(PointerUnion3 lhs, - PointerUnion3 rhs) { - return lhs.getOpaqueValue() < rhs.getOpaqueValue(); -} +using PointerUnion3 = PointerUnion; /// A pointer union of four pointer types. See documentation for PointerUnion /// for usage. template -class PointerUnion4 { -public: - using InnerUnion1 = PointerUnion; - using InnerUnion2 = PointerUnion; - using ValTy = PointerUnion; - -private: - ValTy Val; - -public: - PointerUnion4() = default; - PointerUnion4(PT1 V) { Val = InnerUnion1(V); } - PointerUnion4(PT2 V) { Val = InnerUnion1(V); } - PointerUnion4(PT3 V) { Val = InnerUnion2(V); } - PointerUnion4(PT4 V) { Val = InnerUnion2(V); } - - /// Test if the pointer held in the union is null, regardless of - /// which type it is. - bool isNull() const { return Val.isNull(); } - explicit operator bool() const { return !isNull(); } - - /// Test if the Union currently holds the type matching T. - template int is() const { - // If T is PT1/PT2 choose InnerUnion1 otherwise choose InnerUnion2. - using Ty = typename ::llvm::PointerUnionTypeSelector< - PT1, T, InnerUnion1, - ::llvm::PointerUnionTypeSelector>::Return; - return Val.template is() && Val.template get().template is(); - } - - /// Returns the value of the specified pointer type. - /// - /// If the specified pointer type is incorrect, assert. - template T get() const { - assert(is() && "Invalid accessor called"); - // If T is PT1/PT2 choose InnerUnion1 otherwise choose InnerUnion2. - using Ty = typename ::llvm::PointerUnionTypeSelector< - PT1, T, InnerUnion1, - ::llvm::PointerUnionTypeSelector>::Return; - return Val.template get().template get(); - } - - /// Returns the current pointer if it is of the specified pointer type, - /// otherwises returns null. - template T dyn_cast() const { - if (is()) - return get(); - return T(); - } - - /// Assignment from nullptr which just clears the union. - const PointerUnion4 &operator=(std::nullptr_t) { - Val = nullptr; - return *this; - } - - /// Assignment operators - Allow assigning into this union from either - /// pointer type, setting the discriminator to remember what it came from. - const PointerUnion4 &operator=(const PT1 &RHS) { - Val = InnerUnion1(RHS); - return *this; - } - const PointerUnion4 &operator=(const PT2 &RHS) { - Val = InnerUnion1(RHS); - return *this; - } - const PointerUnion4 &operator=(const PT3 &RHS) { - Val = InnerUnion2(RHS); - return *this; - } - const PointerUnion4 &operator=(const PT4 &RHS) { - Val = InnerUnion2(RHS); - return *this; - } - - void *getOpaqueValue() const { return Val.getOpaqueValue(); } - static inline PointerUnion4 getFromOpaqueValue(void *VP) { - PointerUnion4 V; - V.Val = ValTy::getFromOpaqueValue(VP); - return V; - } -}; - -// Teach SmallPtrSet that PointerUnion4 is "basically a pointer", that has -// # low bits available = min(PT1bits,PT2bits,PT2bits)-2. -template -struct PointerLikeTypeTraits> { - static inline void * - getAsVoidPointer(const PointerUnion4 &P) { - return P.getOpaqueValue(); - } - - static inline PointerUnion4 getFromVoidPointer(void *P) { - return PointerUnion4::getFromOpaqueValue(P); - } - - // The number of bits available are the min of the two pointer types. - enum { - NumLowBitsAvailable = PointerLikeTypeTraits< - typename PointerUnion4::ValTy>::NumLowBitsAvailable - }; -}; +using PointerUnion4 = PointerUnion; // Teach DenseMap how to use PointerUnions as keys. -template struct DenseMapInfo> { - using Pair = PointerUnion; - using FirstInfo = DenseMapInfo; - using SecondInfo = DenseMapInfo; +template struct DenseMapInfo> { + using Union = PointerUnion; + using FirstInfo = + DenseMapInfo::type>; - static inline Pair getEmptyKey() { return Pair(FirstInfo::getEmptyKey()); } + static inline Union getEmptyKey() { return Union(FirstInfo::getEmptyKey()); } - static inline Pair getTombstoneKey() { - return Pair(FirstInfo::getTombstoneKey()); + static inline Union getTombstoneKey() { + return Union(FirstInfo::getTombstoneKey()); } - static unsigned getHashValue(const Pair &PairVal) { - intptr_t key = (intptr_t)PairVal.getOpaqueValue(); + static unsigned getHashValue(const Union &UnionVal) { + intptr_t key = (intptr_t)UnionVal.getOpaqueValue(); return DenseMapInfo::getHashValue(key); } - static bool isEqual(const Pair &LHS, const Pair &RHS) { - return LHS.template is() == RHS.template is() && - (LHS.template is() ? FirstInfo::isEqual(LHS.template get(), - RHS.template get()) - : SecondInfo::isEqual(LHS.template get(), - RHS.template get())); + static bool isEqual(const Union &LHS, const Union &RHS) { + return LHS == RHS; } }; diff --git a/include/llvm/ADT/PostOrderIterator.h b/include/llvm/ADT/PostOrderIterator.h index d77b12228cb1..2fe7447a8e77 100644 --- a/include/llvm/ADT/PostOrderIterator.h +++ b/include/llvm/ADT/PostOrderIterator.h @@ -1,9 +1,8 @@ //===- llvm/ADT/PostOrderIterator.h - PostOrder iterator --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/PriorityQueue.h b/include/llvm/ADT/PriorityQueue.h index 8ba871e25304..cf79ee10ba7f 100644 --- a/include/llvm/ADT/PriorityQueue.h +++ b/include/llvm/ADT/PriorityQueue.h @@ -1,9 +1,8 @@ //===- llvm/ADT/PriorityQueue.h - Priority queues ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/PriorityWorklist.h b/include/llvm/ADT/PriorityWorklist.h index aa531f3337d9..96d22c87557e 100644 --- a/include/llvm/ADT/PriorityWorklist.h +++ b/include/llvm/ADT/PriorityWorklist.h @@ -1,9 +1,8 @@ //===- PriorityWorklist.h - Worklist with insertion priority ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/include/llvm/ADT/SCCIterator.h b/include/llvm/ADT/SCCIterator.h index ab1dc4613be0..eb1a5d0938cf 100644 --- a/include/llvm/ADT/SCCIterator.h +++ b/include/llvm/ADT/SCCIterator.h @@ -1,9 +1,8 @@ //===- ADT/SCCIterator.h - Strongly Connected Comp. Iter. -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/ADT/STLExtras.h b/include/llvm/ADT/STLExtras.h index f66ca7c08a73..81dce0168c79 100644 --- a/include/llvm/ADT/STLExtras.h +++ b/include/llvm/ADT/STLExtras.h @@ -1,9 +1,8 @@ //===- llvm/ADT/STLExtras.h - Useful STL related functions ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -241,6 +240,13 @@ inline mapped_iterator map_iterator(ItTy I, FuncTy F) { return mapped_iterator(std::move(I), std::move(F)); } +template +auto map_range(ContainerTy &&C, FuncTy F) + -> decltype(make_range(map_iterator(C.begin(), F), + map_iterator(C.end(), F))) { + return make_range(map_iterator(C.begin(), F), map_iterator(C.end(), F)); +} + /// Helper to determine if type T has a member called rbegin(). template class has_rbegin_impl { using yes = char[1]; @@ -1278,29 +1284,52 @@ auto partition(R &&Range, UnaryPredicate P) -> decltype(adl_begin(Range)) { /// Provide wrappers to std::lower_bound which take ranges instead of having to /// pass begin/end explicitly. -template -auto lower_bound(R &&Range, ForwardIt I) -> decltype(adl_begin(Range)) { - return std::lower_bound(adl_begin(Range), adl_end(Range), I); +template +auto lower_bound(R &&Range, T &&Value) -> decltype(adl_begin(Range)) { + return std::lower_bound(adl_begin(Range), adl_end(Range), + std::forward(Value)); } -template -auto lower_bound(R &&Range, ForwardIt I, Compare C) +template +auto lower_bound(R &&Range, T &&Value, Compare C) -> decltype(adl_begin(Range)) { - return std::lower_bound(adl_begin(Range), adl_end(Range), I, C); + return std::lower_bound(adl_begin(Range), adl_end(Range), + std::forward(Value), C); } /// Provide wrappers to std::upper_bound which take ranges instead of having to /// pass begin/end explicitly. -template -auto upper_bound(R &&Range, ForwardIt I) -> decltype(adl_begin(Range)) { - return std::upper_bound(adl_begin(Range), adl_end(Range), I); +template +auto upper_bound(R &&Range, T &&Value) -> decltype(adl_begin(Range)) { + return std::upper_bound(adl_begin(Range), adl_end(Range), + std::forward(Value)); } -template -auto upper_bound(R &&Range, ForwardIt I, Compare C) +template +auto upper_bound(R &&Range, T &&Value, Compare C) -> decltype(adl_begin(Range)) { - return std::upper_bound(adl_begin(Range), adl_end(Range), I, C); + return std::upper_bound(adl_begin(Range), adl_end(Range), + std::forward(Value), C); +} + +template +void stable_sort(R &&Range) { + std::stable_sort(adl_begin(Range), adl_end(Range)); +} + +template +void stable_sort(R &&Range, Compare C) { + std::stable_sort(adl_begin(Range), adl_end(Range), C); +} + +/// Binary search for the first iterator in a range where a predicate is false. +/// Requires that C is always true below some limit, and always false above it. +template ()))> +auto partition_point(R &&Range, Predicate P) -> decltype(adl_begin(Range)) { + return std::partition_point(adl_begin(Range), adl_end(Range), P); } + /// Wrapper function around std::equal to detect if all elements /// in a container are same. template @@ -1331,6 +1360,33 @@ void erase_if(Container &C, UnaryPredicate P) { C.erase(remove_if(C, P), C.end()); } +/// Given a sequence container Cont, replace the range [ContIt, ContEnd) with +/// the range [ValIt, ValEnd) (which is not from the same container). +template +void replace(Container &Cont, typename Container::iterator ContIt, + typename Container::iterator ContEnd, RandomAccessIterator ValIt, + RandomAccessIterator ValEnd) { + while (true) { + if (ValIt == ValEnd) { + Cont.erase(ContIt, ContEnd); + return; + } else if (ContIt == ContEnd) { + Cont.insert(ContIt, ValIt, ValEnd); + return; + } + *ContIt++ = *ValIt++; + } +} + +/// Given a sequence container Cont, replace the range [ContIt, ContEnd) with +/// the range R. +template> +void replace(Container &Cont, typename Container::iterator ContIt, + typename Container::iterator ContEnd, Range R) { + replace(Cont, ContIt, ContEnd, R.begin(), R.end()); +} + //===----------------------------------------------------------------------===// // Extra additions to //===----------------------------------------------------------------------===// @@ -1418,6 +1474,9 @@ namespace detail { template class enumerator_iter; template struct result_pair { + using value_reference = + typename std::iterator_traits>::reference; + friend class enumerator_iter; result_pair() = default; @@ -1431,8 +1490,8 @@ template struct result_pair { } std::size_t index() const { return Index; } - const ValueOfRange &value() const { return *Iter; } - ValueOfRange &value() { return *Iter; } + const value_reference value() const { return *Iter; } + value_reference value() { return *Iter; } private: std::size_t Index = std::numeric_limits::max(); @@ -1577,6 +1636,19 @@ bool hasNItemsOrMore( return true; } +/// Returns a raw pointer that represents the same address as the argument. +/// +/// The late bound return should be removed once we move to C++14 to better +/// align with the C++20 declaration. Also, this implementation can be removed +/// once we move to C++20 where it's defined as std::to_addres() +/// +/// The std::pointer_traits<>::to_address(p) variations of these overloads has +/// not been implemented. +template auto to_address(const Ptr &P) -> decltype(P.operator->()) { + return P.operator->(); +} +template constexpr T *to_address(T *P) { return P; } + } // end namespace llvm #endif // LLVM_ADT_STLEXTRAS_H diff --git a/include/llvm/ADT/ScopeExit.h b/include/llvm/ADT/ScopeExit.h index bd13755fa999..712d91237739 100644 --- a/include/llvm/ADT/ScopeExit.h +++ b/include/llvm/ADT/ScopeExit.h @@ -1,9 +1,8 @@ //===- llvm/ADT/ScopeExit.h - Execute code at scope exit --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/ScopedHashTable.h b/include/llvm/ADT/ScopedHashTable.h index 22b0c1bdaf4d..40c49ebc0be1 100644 --- a/include/llvm/ADT/ScopedHashTable.h +++ b/include/llvm/ADT/ScopedHashTable.h @@ -1,9 +1,8 @@ //===- ScopedHashTable.h - A simple scoped hash table -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/Sequence.h b/include/llvm/ADT/Sequence.h index 3d4a897bf9a9..8c505f2010dd 100644 --- a/include/llvm/ADT/Sequence.h +++ b/include/llvm/ADT/Sequence.h @@ -1,9 +1,8 @@ //===- Sequence.h - Utility for producing sequences of values ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/ADT/SetOperations.h b/include/llvm/ADT/SetOperations.h index 7c9f2fbe066e..037256a860b2 100644 --- a/include/llvm/ADT/SetOperations.h +++ b/include/llvm/ADT/SetOperations.h @@ -1,9 +1,8 @@ //===-- llvm/ADT/SetOperations.h - Generic Set Operations -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/SetVector.h b/include/llvm/ADT/SetVector.h index 3d6781041320..d0a0d28d1c81 100644 --- a/include/llvm/ADT/SetVector.h +++ b/include/llvm/ADT/SetVector.h @@ -1,9 +1,8 @@ //===- llvm/ADT/SetVector.h - Set with insert order iteration ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/SmallBitVector.h b/include/llvm/ADT/SmallBitVector.h index 0a73dbd60671..742450e6a951 100644 --- a/include/llvm/ADT/SmallBitVector.h +++ b/include/llvm/ADT/SmallBitVector.h @@ -1,9 +1,8 @@ //===- llvm/ADT/SmallBitVector.h - 'Normally small' bit vectors -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/SmallPtrSet.h b/include/llvm/ADT/SmallPtrSet.h index db08e40257ba..913518230d2d 100644 --- a/include/llvm/ADT/SmallPtrSet.h +++ b/include/llvm/ADT/SmallPtrSet.h @@ -1,9 +1,8 @@ //===- llvm/ADT/SmallPtrSet.h - 'Normally small' pointer set ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/SmallSet.h b/include/llvm/ADT/SmallSet.h index 5d84627714bc..6b128c2e2992 100644 --- a/include/llvm/ADT/SmallSet.h +++ b/include/llvm/ADT/SmallSet.h @@ -1,9 +1,8 @@ //===- llvm/ADT/SmallSet.h - 'Normally small' sets --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/SmallString.h b/include/llvm/ADT/SmallString.h index ff46e85ccb09..898be80d0324 100644 --- a/include/llvm/ADT/SmallString.h +++ b/include/llvm/ADT/SmallString.h @@ -1,9 +1,8 @@ //===- llvm/ADT/SmallString.h - 'Normally small' strings --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/SmallVector.h b/include/llvm/ADT/SmallVector.h index 0636abbb1fbf..17586904d212 100644 --- a/include/llvm/ADT/SmallVector.h +++ b/include/llvm/ADT/SmallVector.h @@ -1,9 +1,8 @@ //===- llvm/ADT/SmallVector.h - 'Normally small' vectors --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -42,8 +41,8 @@ protected: unsigned Size = 0, Capacity; SmallVectorBase() = delete; - SmallVectorBase(void *FirstEl, size_t Capacity) - : BeginX(FirstEl), Capacity(Capacity) {} + SmallVectorBase(void *FirstEl, size_t TotalCapacity) + : BeginX(FirstEl), Capacity(TotalCapacity) {} /// This is an implementation of the grow() method which only works /// on POD-like data types and is out of line to reduce code duplication. @@ -64,9 +63,9 @@ public: /// of the buffer when they know that more elements are available, and only /// update the size later. This avoids the cost of value initializing elements /// which will only be overwritten. - void set_size(size_t Size) { - assert(Size <= capacity()); - this->Size = Size; + void set_size(size_t N) { + assert(N <= capacity()); + Size = N; } }; @@ -125,13 +124,9 @@ public: using const_pointer = const T *; // forward iterator creation methods. - LLVM_ATTRIBUTE_ALWAYS_INLINE iterator begin() { return (iterator)this->BeginX; } - LLVM_ATTRIBUTE_ALWAYS_INLINE const_iterator begin() const { return (const_iterator)this->BeginX; } - LLVM_ATTRIBUTE_ALWAYS_INLINE iterator end() { return begin() + size(); } - LLVM_ATTRIBUTE_ALWAYS_INLINE const_iterator end() const { return begin() + size(); } // reverse iterator creation methods. @@ -150,12 +145,10 @@ public: /// Return a pointer to the vector's buffer, even if empty(). const_pointer data() const { return const_pointer(begin()); } - LLVM_ATTRIBUTE_ALWAYS_INLINE reference operator[](size_type idx) { assert(idx < size()); return begin()[idx]; } - LLVM_ATTRIBUTE_ALWAYS_INLINE const_reference operator[](size_type idx) const { assert(idx < size()); return begin()[idx]; @@ -180,9 +173,9 @@ public: } }; -/// SmallVectorTemplateBase - This is where we put method +/// SmallVectorTemplateBase - This is where we put method /// implementations that are designed to work with non-POD-like T's. -template ::value> +template ::value> class SmallVectorTemplateBase : public SmallVectorTemplateCommon { protected: SmallVectorTemplateBase(size_t Size) : SmallVectorTemplateCommon(Size) {} @@ -236,8 +229,8 @@ public: }; // Define this out-of-line to dissuade the C++ compiler from inlining it. -template -void SmallVectorTemplateBase::grow(size_t MinSize) { +template +void SmallVectorTemplateBase::grow(size_t MinSize) { if (MinSize > UINT32_MAX) report_bad_alloc_error("SmallVector capacity overflow during allocation"); @@ -260,9 +253,8 @@ void SmallVectorTemplateBase::grow(size_t MinSize) { this->Capacity = NewCapacity; } - -/// SmallVectorTemplateBase - This is where we put method -/// implementations that are designed to work with POD-like T's. +/// SmallVectorTemplateBase - This is where we put +/// method implementations that are designed to work with POD-like T's. template class SmallVectorTemplateBase : public SmallVectorTemplateCommon { protected: @@ -326,12 +318,13 @@ class SmallVectorImpl : public SmallVectorTemplateBase { public: using iterator = typename SuperClass::iterator; using const_iterator = typename SuperClass::const_iterator; + using reference = typename SuperClass::reference; using size_type = typename SuperClass::size_type; protected: // Default ctor - Initialize to empty. explicit SmallVectorImpl(unsigned N) - : SmallVectorTemplateBase::value>(N) {} + : SmallVectorTemplateBase(N) {} public: SmallVectorImpl(const SmallVectorImpl &) = delete; @@ -393,22 +386,18 @@ public: std::input_iterator_tag>::value>::type> void append(in_iter in_start, in_iter in_end) { size_type NumInputs = std::distance(in_start, in_end); - // Grow allocated space if needed. if (NumInputs > this->capacity() - this->size()) this->grow(this->size()+NumInputs); - // Copy the new elements over. this->uninitialized_copy(in_start, in_end, this->end()); this->set_size(this->size() + NumInputs); } - /// Add the specified range to the end of the SmallVector. + /// Append \p NumInputs copies of \p Elt to the end. void append(size_type NumInputs, const T &Elt) { - // Grow allocated space if needed. if (NumInputs > this->capacity() - this->size()) this->grow(this->size()+NumInputs); - // Copy the new elements over. std::uninitialized_fill_n(this->end(), NumInputs, Elt); this->set_size(this->size() + NumInputs); } @@ -649,11 +638,12 @@ public: insert(I, IL.begin(), IL.end()); } - template void emplace_back(ArgTypes &&... Args) { + template reference emplace_back(ArgTypes &&... Args) { if (LLVM_UNLIKELY(this->size() >= this->capacity())) this->grow(); ::new ((void *)this->end()) T(std::forward(Args)...); this->set_size(this->size() + 1); + return this->back(); } SmallVectorImpl &operator=(const SmallVectorImpl &RHS); diff --git a/include/llvm/ADT/SparseBitVector.h b/include/llvm/ADT/SparseBitVector.h index 84e73bcbace8..12850e14f4ed 100644 --- a/include/llvm/ADT/SparseBitVector.h +++ b/include/llvm/ADT/SparseBitVector.h @@ -1,9 +1,8 @@ //===- llvm/ADT/SparseBitVector.h - Efficient Sparse BitVector --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/SparseMultiSet.h b/include/llvm/ADT/SparseMultiSet.h index 3c8637621510..d9d3ff459267 100644 --- a/include/llvm/ADT/SparseMultiSet.h +++ b/include/llvm/ADT/SparseMultiSet.h @@ -1,9 +1,8 @@ //===- llvm/ADT/SparseMultiSet.h - Sparse multiset --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/SparseSet.h b/include/llvm/ADT/SparseSet.h index 74cc6dab8c74..a6eb9b942e80 100644 --- a/include/llvm/ADT/SparseSet.h +++ b/include/llvm/ADT/SparseSet.h @@ -1,9 +1,8 @@ //===- llvm/ADT/SparseSet.h - Sparse set ------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/Statistic.h b/include/llvm/ADT/Statistic.h index 90c2eefceb6c..2ac59da596ef 100644 --- a/include/llvm/ADT/Statistic.h +++ b/include/llvm/ADT/Statistic.h @@ -1,9 +1,8 @@ //===-- llvm/ADT/Statistic.h - Easy way to expose stats ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/StringExtras.h b/include/llvm/ADT/StringExtras.h index 60a03633a8a6..16ac90bd6c89 100644 --- a/include/llvm/ADT/StringExtras.h +++ b/include/llvm/ADT/StringExtras.h @@ -1,9 +1,8 @@ //===- llvm/ADT/StringExtras.h - Useful string functions --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/StringMap.h b/include/llvm/ADT/StringMap.h index a9f83d3f5091..8a586fc26709 100644 --- a/include/llvm/ADT/StringMap.h +++ b/include/llvm/ADT/StringMap.h @@ -1,9 +1,8 @@ //===- StringMap.h - String Hash table map interface ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -360,6 +359,11 @@ public: return find(Key) == end() ? 0 : 1; } + template + size_type count(const StringMapEntry &MapEntry) const { + return count(MapEntry.getKey()); + } + /// insert - Insert the specified key/value pair into the map. If the key /// already exists in the map, return false and ignore the request, otherwise /// insert it and return true. diff --git a/include/llvm/ADT/StringRef.h b/include/llvm/ADT/StringRef.h index a5ba5b59b5a3..4661b1e68b2f 100644 --- a/include/llvm/ADT/StringRef.h +++ b/include/llvm/ADT/StringRef.h @@ -1,9 +1,8 @@ //===- StringRef.h - Constant String Reference Wrapper ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -63,7 +62,6 @@ namespace llvm { // Workaround memcmp issue with null pointers (undefined behavior) // by providing a specialized version - LLVM_ATTRIBUTE_ALWAYS_INLINE static int compareMemory(const char *Lhs, const char *Rhs, size_t Length) { if (Length == 0) { return 0; } return ::memcmp(Lhs,Rhs,Length); @@ -81,17 +79,14 @@ namespace llvm { StringRef(std::nullptr_t) = delete; /// Construct a string ref from a cstring. - LLVM_ATTRIBUTE_ALWAYS_INLINE /*implicit*/ StringRef(const char *Str) : Data(Str), Length(Str ? ::strlen(Str) : 0) {} /// Construct a string ref from a pointer and length. - LLVM_ATTRIBUTE_ALWAYS_INLINE /*implicit*/ constexpr StringRef(const char *data, size_t length) : Data(data), Length(length) {} /// Construct a string ref from an std::string. - LLVM_ATTRIBUTE_ALWAYS_INLINE /*implicit*/ StringRef(const std::string &Str) : Data(Str.data()), Length(Str.length()) {} @@ -124,17 +119,14 @@ namespace llvm { /// data - Get a pointer to the start of the string (which may not be null /// terminated). LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE const char *data() const { return Data; } /// empty - Check if the string is empty. LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE bool empty() const { return Length == 0; } /// size - Get the string size. LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE size_t size() const { return Length; } /// front - Get the first character in the string. @@ -165,7 +157,6 @@ namespace llvm { /// equals - Check for string equality, this is more efficient than /// compare() when the relative ordering of inequal strings isn't needed. LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE bool equals(StringRef RHS) const { return (Length == RHS.Length && compareMemory(Data, RHS.Data, RHS.Length) == 0); @@ -180,7 +171,6 @@ namespace llvm { /// compare - Compare two strings; the result is -1, 0, or 1 if this string /// is lexicographically less than, equal to, or greater than the \p RHS. LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE int compare(StringRef RHS) const { // Check the prefix for a mismatch. if (int Res = compareMemory(Data, RHS.Data, std::min(Length, RHS.Length))) @@ -263,7 +253,6 @@ namespace llvm { /// Check if this string starts with the given \p Prefix. LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE bool startswith(StringRef Prefix) const { return Length >= Prefix.Length && compareMemory(Data, Prefix.Data, Prefix.Length) == 0; @@ -275,7 +264,6 @@ namespace llvm { /// Check if this string ends with the given \p Suffix. LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE bool endswith(StringRef Suffix) const { return Length >= Suffix.Length && compareMemory(end() - Suffix.Length, Suffix.Data, Suffix.Length) == 0; @@ -294,7 +282,6 @@ namespace llvm { /// \returns The index of the first occurrence of \p C, or npos if not /// found. LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE size_t find(char C, size_t From = 0) const { size_t FindBegin = std::min(From, Length); if (FindBegin < Length) { // Avoid calling memchr with nullptr. @@ -317,7 +304,6 @@ namespace llvm { /// \returns The index of the first character satisfying \p F starting from /// \p From, or npos if not found. LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE size_t find_if(function_ref F, size_t From = 0) const { StringRef S = drop_front(From); while (!S.empty()) { @@ -333,7 +319,6 @@ namespace llvm { /// \returns The index of the first character not satisfying \p F starting /// from \p From, or npos if not found. LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE size_t find_if_not(function_ref F, size_t From = 0) const { return find_if([F](char c) { return !F(c); }, From); } @@ -444,19 +429,16 @@ namespace llvm { /// Return true if the given string is a substring of *this, and false /// otherwise. LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE bool contains(StringRef Other) const { return find(Other) != npos; } /// Return true if the given character is contained in *this, and false /// otherwise. LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE bool contains(char C) const { return find_first_of(C) != npos; } /// Return true if the given string is a substring of *this, and false /// otherwise. LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE bool contains_lower(StringRef Other) const { return find_lower(Other) != npos; } @@ -464,7 +446,6 @@ namespace llvm { /// Return true if the given character is contained in *this, and false /// otherwise. LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE bool contains_lower(char C) const { return find_lower(C) != npos; } /// @} @@ -594,7 +575,6 @@ namespace llvm { /// exceeds the number of characters remaining in the string, the string /// suffix (starting with \p Start) will be returned. LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef substr(size_t Start, size_t N = npos) const { Start = std::min(Start, Length); return StringRef(Data + Start, std::min(N, Length - Start)); @@ -604,7 +584,6 @@ namespace llvm { /// elements remaining. If \p N is greater than the length of the /// string, the entire string is returned. LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef take_front(size_t N = 1) const { if (N >= size()) return *this; @@ -615,7 +594,6 @@ namespace llvm { /// elements remaining. If \p N is greater than the length of the /// string, the entire string is returned. LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef take_back(size_t N = 1) const { if (N >= size()) return *this; @@ -625,7 +603,6 @@ namespace llvm { /// Return the longest prefix of 'this' such that every character /// in the prefix satisfies the given predicate. LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef take_while(function_ref F) const { return substr(0, find_if_not(F)); } @@ -633,7 +610,6 @@ namespace llvm { /// Return the longest prefix of 'this' such that no character in /// the prefix satisfies the given predicate. LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef take_until(function_ref F) const { return substr(0, find_if(F)); } @@ -641,7 +617,6 @@ namespace llvm { /// Return a StringRef equal to 'this' but with the first \p N elements /// dropped. LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef drop_front(size_t N = 1) const { assert(size() >= N && "Dropping more elements than exist"); return substr(N); @@ -650,7 +625,6 @@ namespace llvm { /// Return a StringRef equal to 'this' but with the last \p N elements /// dropped. LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef drop_back(size_t N = 1) const { assert(size() >= N && "Dropping more elements than exist"); return substr(0, size()-N); @@ -659,7 +633,6 @@ namespace llvm { /// Return a StringRef equal to 'this', but with all characters satisfying /// the given predicate dropped from the beginning of the string. LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef drop_while(function_ref F) const { return substr(find_if_not(F)); } @@ -667,14 +640,12 @@ namespace llvm { /// Return a StringRef equal to 'this', but with all characters not /// satisfying the given predicate dropped from the beginning of the string. LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef drop_until(function_ref F) const { return substr(find_if(F)); } /// Returns true if this StringRef has the given prefix and removes that /// prefix. - LLVM_ATTRIBUTE_ALWAYS_INLINE bool consume_front(StringRef Prefix) { if (!startswith(Prefix)) return false; @@ -685,7 +656,6 @@ namespace llvm { /// Returns true if this StringRef has the given suffix and removes that /// suffix. - LLVM_ATTRIBUTE_ALWAYS_INLINE bool consume_back(StringRef Suffix) { if (!endswith(Suffix)) return false; @@ -706,7 +676,6 @@ namespace llvm { /// will be returned. If this is less than \p Start, an empty string will /// be returned. LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE StringRef slice(size_t Start, size_t End) const { Start = std::min(Start, Length); End = std::min(std::max(Start, End), Length); @@ -894,12 +863,10 @@ namespace llvm { /// @name StringRef Comparison Operators /// @{ - LLVM_ATTRIBUTE_ALWAYS_INLINE inline bool operator==(StringRef LHS, StringRef RHS) { return LHS.equals(RHS); } - LLVM_ATTRIBUTE_ALWAYS_INLINE inline bool operator!=(StringRef LHS, StringRef RHS) { return !(LHS == RHS); } inline bool operator<(StringRef LHS, StringRef RHS) { @@ -928,10 +895,6 @@ namespace llvm { LLVM_NODISCARD hash_code hash_value(StringRef S); - // StringRefs can be treated like a POD type. - template struct isPodLike; - template <> struct isPodLike { static const bool value = true; }; - } // end namespace llvm #endif // LLVM_ADT_STRINGREF_H diff --git a/include/llvm/ADT/StringSet.h b/include/llvm/ADT/StringSet.h index 9af44c07df79..af3a44a7b32c 100644 --- a/include/llvm/ADT/StringSet.h +++ b/include/llvm/ADT/StringSet.h @@ -1,9 +1,8 @@ //===- StringSet.h - The LLVM Compiler Driver -------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open -// Source License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -34,6 +33,7 @@ namespace llvm { for (StringRef X : S) insert(X); } + explicit StringSet(AllocatorTy A) : base(A) {} std::pair insert(StringRef Key) { assert(!Key.empty()); @@ -45,6 +45,12 @@ namespace llvm { for (auto It = Begin; It != End; ++It) base::insert(std::make_pair(*It, '\0')); } + + template + std::pair + insert(const StringMapEntry &MapEntry) { + return insert(MapEntry.getKey()); + } }; } // end namespace llvm diff --git a/include/llvm/ADT/StringSwitch.h b/include/llvm/ADT/StringSwitch.h index b7860b98ce5d..fea911f6928b 100644 --- a/include/llvm/ADT/StringSwitch.h +++ b/include/llvm/ADT/StringSwitch.h @@ -1,9 +1,8 @@ //===--- StringSwitch.h - Switch-on-literal-string Construct --------------===/ // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception //===----------------------------------------------------------------------===/ // // This file implements the StringSwitch template, which mimics a switch() @@ -49,7 +48,6 @@ class StringSwitch { Optional Result; public: - LLVM_ATTRIBUTE_ALWAYS_INLINE explicit StringSwitch(StringRef S) : Str(S), Result() { } @@ -66,7 +64,6 @@ public: ~StringSwitch() = default; // Case-sensitive case matchers - LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch &Case(StringLiteral S, T Value) { if (!Result && Str == S) { Result = std::move(Value); @@ -74,7 +71,6 @@ public: return *this; } - LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch& EndsWith(StringLiteral S, T Value) { if (!Result && Str.endswith(S)) { Result = std::move(Value); @@ -82,7 +78,6 @@ public: return *this; } - LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch& StartsWith(StringLiteral S, T Value) { if (!Result && Str.startswith(S)) { Result = std::move(Value); @@ -90,51 +85,43 @@ public: return *this; } - LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch &Cases(StringLiteral S0, StringLiteral S1, T Value) { return Case(S0, Value).Case(S1, Value); } - LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2, T Value) { return Case(S0, Value).Cases(S1, S2, Value); } - LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2, StringLiteral S3, T Value) { return Case(S0, Value).Cases(S1, S2, S3, Value); } - LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2, StringLiteral S3, StringLiteral S4, T Value) { return Case(S0, Value).Cases(S1, S2, S3, S4, Value); } - LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2, StringLiteral S3, StringLiteral S4, StringLiteral S5, T Value) { return Case(S0, Value).Cases(S1, S2, S3, S4, S5, Value); } - LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2, StringLiteral S3, StringLiteral S4, StringLiteral S5, StringLiteral S6, T Value) { return Case(S0, Value).Cases(S1, S2, S3, S4, S5, S6, Value); } - LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2, StringLiteral S3, StringLiteral S4, StringLiteral S5, StringLiteral S6, StringLiteral S7, T Value) { return Case(S0, Value).Cases(S1, S2, S3, S4, S5, S6, S7, Value); } - LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2, StringLiteral S3, StringLiteral S4, StringLiteral S5, StringLiteral S6, StringLiteral S7, StringLiteral S8, @@ -142,7 +129,6 @@ public: return Case(S0, Value).Cases(S1, S2, S3, S4, S5, S6, S7, S8, Value); } - LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch &Cases(StringLiteral S0, StringLiteral S1, StringLiteral S2, StringLiteral S3, StringLiteral S4, StringLiteral S5, StringLiteral S6, StringLiteral S7, StringLiteral S8, @@ -151,7 +137,6 @@ public: } // Case-insensitive case matchers. - LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch &CaseLower(StringLiteral S, T Value) { if (!Result && Str.equals_lower(S)) Result = std::move(Value); @@ -159,7 +144,6 @@ public: return *this; } - LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch &EndsWithLower(StringLiteral S, T Value) { if (!Result && Str.endswith_lower(S)) Result = Value; @@ -167,7 +151,6 @@ public: return *this; } - LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch &StartsWithLower(StringLiteral S, T Value) { if (!Result && Str.startswith_lower(S)) Result = std::move(Value); @@ -175,31 +158,26 @@ public: return *this; } - LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch &CasesLower(StringLiteral S0, StringLiteral S1, T Value) { return CaseLower(S0, Value).CaseLower(S1, Value); } - LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch &CasesLower(StringLiteral S0, StringLiteral S1, StringLiteral S2, T Value) { return CaseLower(S0, Value).CasesLower(S1, S2, Value); } - LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch &CasesLower(StringLiteral S0, StringLiteral S1, StringLiteral S2, StringLiteral S3, T Value) { return CaseLower(S0, Value).CasesLower(S1, S2, S3, Value); } - LLVM_ATTRIBUTE_ALWAYS_INLINE StringSwitch &CasesLower(StringLiteral S0, StringLiteral S1, StringLiteral S2, StringLiteral S3, StringLiteral S4, T Value) { return CaseLower(S0, Value).CasesLower(S1, S2, S3, S4, Value); } LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE R Default(T Value) { if (Result) return std::move(*Result); @@ -207,7 +185,6 @@ public: } LLVM_NODISCARD - LLVM_ATTRIBUTE_ALWAYS_INLINE operator R() { assert(Result && "Fell off the end of a string-switch"); return std::move(*Result); diff --git a/include/llvm/ADT/TinyPtrVector.h b/include/llvm/ADT/TinyPtrVector.h index 1b8e9aa658c3..ac82451a9b21 100644 --- a/include/llvm/ADT/TinyPtrVector.h +++ b/include/llvm/ADT/TinyPtrVector.h @@ -1,9 +1,8 @@ //===- llvm/ADT/TinyPtrVector.h - 'Normally tiny' vectors -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/ADT/Triple.h b/include/llvm/ADT/Triple.h index e06a68e27317..edeb31efab80 100644 --- a/include/llvm/ADT/Triple.h +++ b/include/llvm/ADT/Triple.h @@ -1,9 +1,8 @@ //===-- llvm/ADT/Triple.h - Target triple helper class ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -50,6 +49,7 @@ public: armeb, // ARM (big endian): armeb aarch64, // AArch64 (little endian): aarch64 aarch64_be, // AArch64 (big endian): aarch64_be + aarch64_32, // AArch64 (little endian) ILP32: aarch64_32 arc, // ARC: Synopsys ARC avr, // AVR: Atmel AVR microcontroller bpfel, // eBPF or extended BPF or 64-bit BPF (little endian) @@ -109,6 +109,7 @@ public: ARMSubArch_v8r, ARMSubArch_v8m_baseline, ARMSubArch_v8m_mainline, + ARMSubArch_v8_1m_mainline, ARMSubArch_v7, ARMSubArch_v7em, ARMSubArch_v7m, @@ -187,7 +188,8 @@ public: HermitCore, // HermitCore Unikernel/Multikernel Hurd, // GNU/Hurd WASI, // Experimental WebAssembly OS - LastOSType = WASI + Emscripten, + LastOSType = Emscripten }; enum EnvironmentType { UnknownEnvironment, @@ -201,6 +203,8 @@ public: CODE16, EABI, EABIHF, + ELFv1, + ELFv2, Android, Musl, MuslEABI, @@ -210,8 +214,9 @@ public: Itanium, Cygnus, CoreCLR, - Simulator, // Simulator variants of other systems, e.g., Apple's iOS - LastEnvironmentType = Simulator + Simulator, // Simulator variants of other systems, e.g., Apple's iOS + MacABI, // Mac Catalyst variant of Apple's iOS deployment target. + LastEnvironmentType = MacABI }; enum ObjectFormatType { UnknownObjectFormat, @@ -220,6 +225,7 @@ public: ELF, MachO, Wasm, + XCOFF, }; private: @@ -415,7 +421,7 @@ public: if (LHS[1] != Minor) return LHS[1] < Minor; if (LHS[2] != Micro) - return LHS[1] < Micro; + return LHS[2] < Micro; return false; } @@ -480,6 +486,10 @@ public: return getEnvironment() == Triple::Simulator; } + bool isMacCatalystEnvironment() const { + return getEnvironment() == Triple::MacABI; + } + bool isOSNetBSD() const { return getOS() == Triple::NetBSD; } @@ -524,32 +534,36 @@ public: return getOS() == Triple::Haiku; } - /// Checks if the environment could be MSVC. - bool isWindowsMSVCEnvironment() const { - return getOS() == Triple::Win32 && - (getEnvironment() == Triple::UnknownEnvironment || - getEnvironment() == Triple::MSVC); + /// Tests whether the OS is Windows. + bool isOSWindows() const { + return getOS() == Triple::Win32; } /// Checks if the environment is MSVC. bool isKnownWindowsMSVCEnvironment() const { - return getOS() == Triple::Win32 && getEnvironment() == Triple::MSVC; + return isOSWindows() && getEnvironment() == Triple::MSVC; + } + + /// Checks if the environment could be MSVC. + bool isWindowsMSVCEnvironment() const { + return isKnownWindowsMSVCEnvironment() || + (isOSWindows() && getEnvironment() == Triple::UnknownEnvironment); } bool isWindowsCoreCLREnvironment() const { - return getOS() == Triple::Win32 && getEnvironment() == Triple::CoreCLR; + return isOSWindows() && getEnvironment() == Triple::CoreCLR; } bool isWindowsItaniumEnvironment() const { - return getOS() == Triple::Win32 && getEnvironment() == Triple::Itanium; + return isOSWindows() && getEnvironment() == Triple::Itanium; } bool isWindowsCygwinEnvironment() const { - return getOS() == Triple::Win32 && getEnvironment() == Triple::Cygnus; + return isOSWindows() && getEnvironment() == Triple::Cygnus; } bool isWindowsGNUEnvironment() const { - return getOS() == Triple::Win32 && getEnvironment() == Triple::GNU; + return isOSWindows() && getEnvironment() == Triple::GNU; } /// Tests for either Cygwin or MinGW OS @@ -563,11 +577,6 @@ public: isWindowsItaniumEnvironment(); } - /// Tests whether the OS is Windows. - bool isOSWindows() const { - return getOS() == Triple::Win32; - } - /// Tests whether the OS is NaCl (Native Client) bool isOSNaCl() const { return getOS() == Triple::NaCl; @@ -593,6 +602,11 @@ public: return getOS() == Triple::WASI; } + /// Tests whether the OS is Emscripten. + bool isOSEmscripten() const { + return getOS() == Triple::Emscripten; + } + /// Tests whether the OS uses glibc. bool isOSGlibc() const { return (getOS() == Triple::Linux || getOS() == Triple::KFreeBSD || @@ -600,6 +614,11 @@ public: !isAndroid(); } + /// Tests whether the OS is AIX. + bool isOSAIX() const { + return getOS() == Triple::AIX; + } + /// Tests whether the OS uses the ELF binary format. bool isOSBinFormatELF() const { return getObjectFormat() == Triple::ELF; @@ -620,6 +639,11 @@ public: return getObjectFormat() == Triple::Wasm; } + /// Tests whether the OS uses the XCOFF binary format. + bool isOSBinFormatXCOFF() const { + return getObjectFormat() == Triple::XCOFF; + } + /// Tests whether the target is the PS4 CPU bool isPS4CPU() const { return getArch() == Triple::x86_64 && @@ -656,6 +680,11 @@ public: getEnvironment() == Triple::MuslEABIHF; } + /// Tests whether the target is SPIR (32- or 64-bit). + bool isSPIR() const { + return getArch() == Triple::spir || getArch() == Triple::spir64; + } + /// Tests whether the target is NVPTX (32- or 64-bit). bool isNVPTX() const { return getArch() == Triple::nvptx || getArch() == Triple::nvptx64; @@ -691,6 +720,16 @@ public: return isMIPS32() || isMIPS64(); } + /// Tests whether the target is 64-bit PowerPC (little and big endian). + bool isPPC64() const { + return getArch() == Triple::ppc64 || getArch() == Triple::ppc64le; + } + + /// Tests whether the target is RISC-V (32- and 64-bit). + bool isRISCV() const { + return getArch() == Triple::riscv32 || getArch() == Triple::riscv64; + } + /// Tests whether the target supports comdat bool supportsCOMDAT() const { return !isOSBinFormatMachO(); diff --git a/include/llvm/ADT/Twine.h b/include/llvm/ADT/Twine.h index b60fd0981398..4140c22aad3d 100644 --- a/include/llvm/ADT/Twine.h +++ b/include/llvm/ADT/Twine.h @@ -1,9 +1,8 @@ //===- Twine.h - Fast Temporary String Concatenation ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -274,6 +273,9 @@ namespace llvm { assert(isValid() && "Invalid twine!"); } + /// Delete the implicit conversion from nullptr as Twine(const char *) + /// cannot take nullptr. + /*implicit*/ Twine(std::nullptr_t) = delete; /// Construct from an std::string. /*implicit*/ Twine(const std::string &Str) : LHSKind(StdStringKind) { diff --git a/include/llvm/ADT/UniqueVector.h b/include/llvm/ADT/UniqueVector.h index c86bedd07687..bfea988f1702 100644 --- a/include/llvm/ADT/UniqueVector.h +++ b/include/llvm/ADT/UniqueVector.h @@ -1,9 +1,8 @@ //===- llvm/ADT/UniqueVector.h ----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/ADT/VariadicFunction.h b/include/llvm/ADT/VariadicFunction.h index 9028abe4c72c..5aefb05ecdda 100644 --- a/include/llvm/ADT/VariadicFunction.h +++ b/include/llvm/ADT/VariadicFunction.h @@ -1,9 +1,8 @@ -//===--- VariadicFunctions.h - Variadic Functions ---------------*- C++ -*-===// +//===- VariadicFunction.h - Variadic Functions ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/bit.h b/include/llvm/ADT/bit.h index a4aba7b6a9ee..a790d5ed2d21 100644 --- a/include/llvm/ADT/bit.h +++ b/include/llvm/ADT/bit.h @@ -1,9 +1,8 @@ //===-- llvm/ADT/bit.h - C++20 ----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -41,11 +40,11 @@ template ::type , typename = typename std::enable_if<__is_trivially_copyable(From)>::type #else - // This case is GCC 4.x. clang with libc++ or libstdc++ never get here. Unlike - // llvm/Support/type_traits.h's isPodLike we don't want to provide a - // good-enough answer here: developers in that configuration will hit - // compilation failures on the bots instead of locally. That's acceptable - // because it's very few developers, and only until we move past C++11. +// This case is GCC 4.x. clang with libc++ or libstdc++ never get here. Unlike +// llvm/Support/type_traits.h's is_trivially_copyable we don't want to +// provide a good-enough answer here: developers in that configuration will hit +// compilation failures on the bots instead of locally. That's acceptable +// because it's very few developers, and only until we move past C++11. #endif > inline To bit_cast(const From &from) noexcept { diff --git a/include/llvm/ADT/edit_distance.h b/include/llvm/ADT/edit_distance.h index b2e8ec5c3f6d..4f5134008692 100644 --- a/include/llvm/ADT/edit_distance.h +++ b/include/llvm/ADT/edit_distance.h @@ -1,9 +1,8 @@ //===-- llvm/ADT/edit_distance.h - Array edit distance function --- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/fallible_iterator.h b/include/llvm/ADT/fallible_iterator.h new file mode 100644 index 000000000000..6501ad2233cd --- /dev/null +++ b/include/llvm/ADT/fallible_iterator.h @@ -0,0 +1,243 @@ +//===--- fallible_iterator.h - Wrapper for fallible iterators ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ADT_FALLIBLE_ITERATOR_H +#define LLVM_ADT_FALLIBLE_ITERATOR_H + +#include "llvm/ADT/PointerIntPair.h" +#include "llvm/ADT/iterator_range.h" +#include "llvm/Support/Error.h" + +#include + +namespace llvm { + +/// A wrapper class for fallible iterators. +/// +/// The fallible_iterator template wraps an underlying iterator-like class +/// whose increment and decrement operations are replaced with fallible versions +/// like: +/// +/// @code{.cpp} +/// Error inc(); +/// Error dec(); +/// @endcode +/// +/// It produces an interface that is (mostly) compatible with a traditional +/// c++ iterator, including ++ and -- operators that do not fail. +/// +/// Instances of the wrapper are constructed with an instance of the +/// underlying iterator and (for non-end iterators) a reference to an Error +/// instance. If the underlying increment/decrement operations fail, the Error +/// is returned via this reference, and the resulting iterator value set to an +/// end-of-range sentinel value. This enables the following loop idiom: +/// +/// @code{.cpp} +/// class Archive { // E.g. Potentially malformed on-disk archive +/// public: +/// fallible_iterator children_begin(Error &Err); +/// fallible_iterator children_end(); +/// iterator_range> +/// children(Error &Err) { +/// return make_range(children_begin(Err), children_end()); +/// //... +/// }; +/// +/// void walk(Archive &A) { +/// Error Err = Error::success(); +/// for (auto &C : A.children(Err)) { +/// // Loop body only entered when increment succeeds. +/// } +/// if (Err) { +/// // handle error. +/// } +/// } +/// @endcode +/// +/// The wrapper marks the referenced Error as unchecked after each increment +/// and/or decrement operation, and clears the unchecked flag when a non-end +/// value is compared against end (since, by the increment invariant, not being +/// an end value proves that there was no error, and is equivalent to checking +/// that the Error is success). This allows early exits from the loop body +/// without requiring redundant error checks. +template class fallible_iterator { +private: + template + using enable_if_struct_deref_supported = std::enable_if< + !std::is_void().operator->())>::value, + decltype(std::declval().operator->())>; + +public: + /// Construct a fallible iterator that *cannot* be used as an end-of-range + /// value. + /// + /// A value created by this method can be dereferenced, incremented, + /// decremented and compared, providing the underlying type supports it. + /// + /// The error that is passed in will be initially marked as checked, so if the + /// iterator is not used at all the Error need not be checked. + static fallible_iterator itr(Underlying I, Error &Err) { + (void)!!Err; + return fallible_iterator(std::move(I), &Err); + } + + /// Construct a fallible iteratro that can be used as an end-of-range value. + /// + /// A value created by this method can be dereferenced (if the underlying + /// value points at a valid value) and compared, but not incremented or + /// decremented. + static fallible_iterator end(Underlying I) { + return fallible_iterator(std::move(I), nullptr); + } + + /// Forward dereference to the underlying iterator. + auto operator*() -> decltype(*std::declval()) { return *I; } + + /// Forward const dereference to the underlying iterator. + auto operator*() const -> decltype(*std::declval()) { + return *I; + } + + /// Forward structure dereference to the underlying iterator (if the + /// underlying iterator supports it). + template + typename enable_if_struct_deref_supported::type operator->() { + return I.operator->(); + } + + /// Forward const structure dereference to the underlying iterator (if the + /// underlying iterator supports it). + template + typename enable_if_struct_deref_supported::type operator->() const { + return I.operator->(); + } + + /// Increment the fallible iterator. + /// + /// If the underlying 'inc' operation fails, this will set the Error value + /// and update this iterator value to point to end-of-range. + /// + /// The Error value is marked as needing checking, regardless of whether the + /// 'inc' operation succeeds or fails. + fallible_iterator &operator++() { + assert(getErrPtr() && "Cannot increment end iterator"); + if (auto Err = I.inc()) + handleError(std::move(Err)); + else + resetCheckedFlag(); + return *this; + } + + /// Decrement the fallible iterator. + /// + /// If the underlying 'dec' operation fails, this will set the Error value + /// and update this iterator value to point to end-of-range. + /// + /// The Error value is marked as needing checking, regardless of whether the + /// 'dec' operation succeeds or fails. + fallible_iterator &operator--() { + assert(getErrPtr() && "Cannot decrement end iterator"); + if (auto Err = I.dec()) + handleError(std::move(Err)); + else + resetCheckedFlag(); + return *this; + } + + /// Compare fallible iterators for equality. + /// + /// Returns true if both LHS and RHS are end-of-range values, or if both are + /// non-end-of-range values whose underlying iterator values compare equal. + /// + /// If this is a comparison between an end-of-range iterator and a + /// non-end-of-range iterator, then the Error (referenced by the + /// non-end-of-range value) is marked as checked: Since all + /// increment/decrement operations result in an end-of-range value, comparing + /// false against end-of-range is equivalent to checking that the Error value + /// is success. This flag management enables early returns from loop bodies + /// without redundant Error checks. + friend bool operator==(const fallible_iterator &LHS, + const fallible_iterator &RHS) { + // If both iterators are in the end state they compare + // equal, regardless of whether either is valid. + if (LHS.isEnd() && RHS.isEnd()) + return true; + + assert(LHS.isValid() && RHS.isValid() && + "Invalid iterators can only be compared against end"); + + bool Equal = LHS.I == RHS.I; + + // If the iterators differ and this is a comparison against end then mark + // the Error as checked. + if (!Equal) { + if (LHS.isEnd()) + (void)!!*RHS.getErrPtr(); + else + (void)!!*LHS.getErrPtr(); + } + + return Equal; + } + + /// Compare fallible iterators for inequality. + /// + /// See notes for operator==. + friend bool operator!=(const fallible_iterator &LHS, + const fallible_iterator &RHS) { + return !(LHS == RHS); + } + +private: + fallible_iterator(Underlying I, Error *Err) + : I(std::move(I)), ErrState(Err, false) {} + + Error *getErrPtr() const { return ErrState.getPointer(); } + + bool isEnd() const { return getErrPtr() == nullptr; } + + bool isValid() const { return !ErrState.getInt(); } + + void handleError(Error Err) { + *getErrPtr() = std::move(Err); + ErrState.setPointer(nullptr); + ErrState.setInt(true); + } + + void resetCheckedFlag() { + *getErrPtr() = Error::success(); + } + + Underlying I; + mutable PointerIntPair ErrState; +}; + +/// Convenience wrapper to make a fallible_iterator value from an instance +/// of an underlying iterator and an Error reference. +template +fallible_iterator make_fallible_itr(Underlying I, Error &Err) { + return fallible_iterator::itr(std::move(I), Err); +} + +/// Convenience wrapper to make a fallible_iterator end value from an instance +/// of an underlying iterator. +template +fallible_iterator make_fallible_end(Underlying E) { + return fallible_iterator::end(std::move(E)); +} + +template +iterator_range> +make_fallible_range(Underlying I, Underlying E, Error &Err) { + return make_range(make_fallible_itr(std::move(I), Err), + make_fallible_end(std::move(E))); +} + +} // end namespace llvm + +#endif // LLVM_ADT_FALLIBLE_ITERATOR_H diff --git a/include/llvm/ADT/ilist.h b/include/llvm/ADT/ilist.h index 00bb6d528175..06c7abff965f 100644 --- a/include/llvm/ADT/ilist.h +++ b/include/llvm/ADT/ilist.h @@ -1,9 +1,8 @@ //==-- llvm/ADT/ilist.h - Intrusive Linked List Template ---------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -66,9 +65,8 @@ template struct ilist_callback_traits { void addNodeToList(NodeTy *) {} void removeNodeFromList(NodeTy *) {} - /// Callback before transferring nodes to this list. - /// - /// \pre \c this!=&OldList + /// Callback before transferring nodes to this list. The nodes may already be + /// in this same list. template void transferNodesFromList(ilist_callback_traits &OldList, Iterator /*first*/, Iterator /*last*/) { @@ -287,8 +285,8 @@ private: if (position == last) return; - if (this != &L2) // Notify traits we moved the nodes... - this->transferNodesFromList(L2, first, last); + // Notify traits we moved the nodes... + this->transferNodesFromList(L2, first, last); base_list_type::splice(position, L2, first, last); } diff --git a/include/llvm/ADT/ilist_base.h b/include/llvm/ADT/ilist_base.h index 3d818a48d41d..b8c098b951ad 100644 --- a/include/llvm/ADT/ilist_base.h +++ b/include/llvm/ADT/ilist_base.h @@ -1,9 +1,8 @@ //===- llvm/ADT/ilist_base.h - Intrusive List Base --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/ADT/ilist_iterator.h b/include/llvm/ADT/ilist_iterator.h index 671e644e0154..cbe5cefa96d1 100644 --- a/include/llvm/ADT/ilist_iterator.h +++ b/include/llvm/ADT/ilist_iterator.h @@ -1,9 +1,8 @@ //===- llvm/ADT/ilist_iterator.h - Intrusive List Iterator ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/ADT/ilist_node.h b/include/llvm/ADT/ilist_node.h index dd0e6b4ec2b9..e040d9630a1e 100644 --- a/include/llvm/ADT/ilist_node.h +++ b/include/llvm/ADT/ilist_node.h @@ -1,9 +1,8 @@ //===- llvm/ADT/ilist_node.h - Intrusive Linked List Helper -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ADT/ilist_node_base.h b/include/llvm/ADT/ilist_node_base.h index e5062ac4eaad..f6c518e6eed7 100644 --- a/include/llvm/ADT/ilist_node_base.h +++ b/include/llvm/ADT/ilist_node_base.h @@ -1,9 +1,8 @@ //===- llvm/ADT/ilist_node_base.h - Intrusive List Node Base -----*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/ADT/ilist_node_options.h b/include/llvm/ADT/ilist_node_options.h index 7ff4005f6757..9b95cdbe08c4 100644 --- a/include/llvm/ADT/ilist_node_options.h +++ b/include/llvm/ADT/ilist_node_options.h @@ -1,9 +1,8 @@ //===- llvm/ADT/ilist_node_options.h - ilist_node Options -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/ADT/iterator.h b/include/llvm/ADT/iterator.h index 40e490cf7864..467fd4c00ec5 100644 --- a/include/llvm/ADT/iterator.h +++ b/include/llvm/ADT/iterator.h @@ -1,9 +1,8 @@ //===- iterator.h - Utilities for using and defining iterators --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/ADT/iterator_range.h b/include/llvm/ADT/iterator_range.h index 2ba12866ecf3..774c7c4e3366 100644 --- a/include/llvm/ADT/iterator_range.h +++ b/include/llvm/ADT/iterator_range.h @@ -1,9 +1,8 @@ //===- iterator_range.h - A range adaptor for iterators ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/ADT/simple_ilist.h b/include/llvm/ADT/simple_ilist.h index 4c7598a1acb4..9257b47b9cf8 100644 --- a/include/llvm/ADT/simple_ilist.h +++ b/include/llvm/ADT/simple_ilist.h @@ -1,9 +1,8 @@ //===- llvm/ADT/simple_ilist.h - Simple Intrusive List ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Analysis/AliasAnalysis.h b/include/llvm/Analysis/AliasAnalysis.h index e2a2ac0622e8..948341554f23 100644 --- a/include/llvm/Analysis/AliasAnalysis.h +++ b/include/llvm/Analysis/AliasAnalysis.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/AliasAnalysis.h - Alias Analysis Interface -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -38,6 +37,7 @@ #ifndef LLVM_ANALYSIS_ALIASANALYSIS_H #define LLVM_ANALYSIS_ALIASANALYSIS_H +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/None.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" @@ -286,6 +286,28 @@ createModRefInfo(const FunctionModRefBehavior FMRB) { return ModRefInfo(FMRB & static_cast(ModRefInfo::ModRef)); } +/// This class stores info we want to provide to or retain within an alias +/// query. By default, the root query is stateless and starts with a freshly +/// constructed info object. Specific alias analyses can use this query info to +/// store per-query state that is important for recursive or nested queries to +/// avoid recomputing. To enable preserving this state across multiple queries +/// where safe (due to the IR not changing), use a `BatchAAResults` wrapper. +/// The information stored in an `AAQueryInfo` is currently limitted to the +/// caches used by BasicAA, but can further be extended to fit other AA needs. +class AAQueryInfo { +public: + using LocPair = std::pair; + using AliasCacheT = SmallDenseMap; + AliasCacheT AliasCache; + + using IsCapturedCacheT = SmallDenseMap; + IsCapturedCacheT IsCapturedCache; + + AAQueryInfo() : AliasCache(), IsCapturedCache() {} +}; + +class BatchAAResults; + class AAResults { public: // Make these results default constructable and movable. We have to spell @@ -600,32 +622,8 @@ public: /// helpers above. ModRefInfo getModRefInfo(const Instruction *I, const Optional &OptLoc) { - if (OptLoc == None) { - if (const auto *Call = dyn_cast(I)) { - return createModRefInfo(getModRefBehavior(Call)); - } - } - - const MemoryLocation &Loc = OptLoc.getValueOr(MemoryLocation()); - - switch (I->getOpcode()) { - case Instruction::VAArg: return getModRefInfo((const VAArgInst*)I, Loc); - case Instruction::Load: return getModRefInfo((const LoadInst*)I, Loc); - case Instruction::Store: return getModRefInfo((const StoreInst*)I, Loc); - case Instruction::Fence: return getModRefInfo((const FenceInst*)I, Loc); - case Instruction::AtomicCmpXchg: - return getModRefInfo((const AtomicCmpXchgInst*)I, Loc); - case Instruction::AtomicRMW: - return getModRefInfo((const AtomicRMWInst*)I, Loc); - case Instruction::Call: return getModRefInfo((const CallInst*)I, Loc); - case Instruction::Invoke: return getModRefInfo((const InvokeInst*)I,Loc); - case Instruction::CatchPad: - return getModRefInfo((const CatchPadInst *)I, Loc); - case Instruction::CatchRet: - return getModRefInfo((const CatchReturnInst *)I, Loc); - default: - return ModRefInfo::NoModRef; - } + AAQueryInfo AAQIP; + return getModRefInfo(I, OptLoc, AAQIP); } /// A convenience wrapper for constructing the memory location. @@ -692,6 +690,69 @@ public: } private: + AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB, + AAQueryInfo &AAQI); + bool pointsToConstantMemory(const MemoryLocation &Loc, AAQueryInfo &AAQI, + bool OrLocal = false); + ModRefInfo getModRefInfo(Instruction *I, const CallBase *Call2, + AAQueryInfo &AAQIP); + ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc, + AAQueryInfo &AAQI); + ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2, + AAQueryInfo &AAQI); + ModRefInfo getModRefInfo(const VAArgInst *V, const MemoryLocation &Loc, + AAQueryInfo &AAQI); + ModRefInfo getModRefInfo(const LoadInst *L, const MemoryLocation &Loc, + AAQueryInfo &AAQI); + ModRefInfo getModRefInfo(const StoreInst *S, const MemoryLocation &Loc, + AAQueryInfo &AAQI); + ModRefInfo getModRefInfo(const FenceInst *S, const MemoryLocation &Loc, + AAQueryInfo &AAQI); + ModRefInfo getModRefInfo(const AtomicCmpXchgInst *CX, + const MemoryLocation &Loc, AAQueryInfo &AAQI); + ModRefInfo getModRefInfo(const AtomicRMWInst *RMW, const MemoryLocation &Loc, + AAQueryInfo &AAQI); + ModRefInfo getModRefInfo(const CatchPadInst *I, const MemoryLocation &Loc, + AAQueryInfo &AAQI); + ModRefInfo getModRefInfo(const CatchReturnInst *I, const MemoryLocation &Loc, + AAQueryInfo &AAQI); + ModRefInfo getModRefInfo(const Instruction *I, + const Optional &OptLoc, + AAQueryInfo &AAQIP) { + if (OptLoc == None) { + if (const auto *Call = dyn_cast(I)) { + return createModRefInfo(getModRefBehavior(Call)); + } + } + + const MemoryLocation &Loc = OptLoc.getValueOr(MemoryLocation()); + + switch (I->getOpcode()) { + case Instruction::VAArg: + return getModRefInfo((const VAArgInst *)I, Loc, AAQIP); + case Instruction::Load: + return getModRefInfo((const LoadInst *)I, Loc, AAQIP); + case Instruction::Store: + return getModRefInfo((const StoreInst *)I, Loc, AAQIP); + case Instruction::Fence: + return getModRefInfo((const FenceInst *)I, Loc, AAQIP); + case Instruction::AtomicCmpXchg: + return getModRefInfo((const AtomicCmpXchgInst *)I, Loc, AAQIP); + case Instruction::AtomicRMW: + return getModRefInfo((const AtomicRMWInst *)I, Loc, AAQIP); + case Instruction::Call: + return getModRefInfo((const CallInst *)I, Loc, AAQIP); + case Instruction::Invoke: + return getModRefInfo((const InvokeInst *)I, Loc, AAQIP); + case Instruction::CatchPad: + return getModRefInfo((const CatchPadInst *)I, Loc, AAQIP); + case Instruction::CatchRet: + return getModRefInfo((const CatchReturnInst *)I, Loc, AAQIP); + default: + return ModRefInfo::NoModRef; + } + } + class Concept; template class Model; @@ -703,6 +764,47 @@ private: std::vector> AAs; std::vector AADeps; + + friend class BatchAAResults; +}; + +/// This class is a wrapper over an AAResults, and it is intended to be used +/// only when there are no IR changes inbetween queries. BatchAAResults is +/// reusing the same `AAQueryInfo` to preserve the state across queries, +/// esentially making AA work in "batch mode". The internal state cannot be +/// cleared, so to go "out-of-batch-mode", the user must either use AAResults, +/// or create a new BatchAAResults. +class BatchAAResults { + AAResults &AA; + AAQueryInfo AAQI; + +public: + BatchAAResults(AAResults &AAR) : AA(AAR), AAQI() {} + AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB) { + return AA.alias(LocA, LocB, AAQI); + } + bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal = false) { + return AA.pointsToConstantMemory(Loc, AAQI, OrLocal); + } + ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc) { + return AA.getModRefInfo(Call, Loc, AAQI); + } + ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2) { + return AA.getModRefInfo(Call1, Call2, AAQI); + } + ModRefInfo getModRefInfo(const Instruction *I, + const Optional &OptLoc) { + return AA.getModRefInfo(I, OptLoc, AAQI); + } + ModRefInfo getModRefInfo(Instruction *I, const CallBase *Call2) { + return AA.getModRefInfo(I, Call2, AAQI); + } + ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) { + return AA.getArgModRefInfo(Call, ArgIdx); + } + FunctionModRefBehavior getModRefBehavior(const CallBase *Call) { + return AA.getModRefBehavior(Call); + } }; /// Temporary typedef for legacy code that uses a generic \c AliasAnalysis @@ -735,12 +837,12 @@ public: /// each other. This is the interface that must be implemented by specific /// alias analysis implementations. virtual AliasResult alias(const MemoryLocation &LocA, - const MemoryLocation &LocB) = 0; + const MemoryLocation &LocB, AAQueryInfo &AAQI) = 0; /// Checks whether the given location points to constant memory, or if /// \p OrLocal is true whether it points to a local alloca. virtual bool pointsToConstantMemory(const MemoryLocation &Loc, - bool OrLocal) = 0; + AAQueryInfo &AAQI, bool OrLocal) = 0; /// @} //===--------------------------------------------------------------------===// @@ -764,13 +866,14 @@ public: /// getModRefInfo (for call sites) - Return information about whether /// a particular call site modifies or reads the specified memory location. virtual ModRefInfo getModRefInfo(const CallBase *Call, - const MemoryLocation &Loc) = 0; + const MemoryLocation &Loc, + AAQueryInfo &AAQI) = 0; /// Return information about whether two call sites may refer to the same set /// of memory locations. See the AA documentation for details: /// http://llvm.org/docs/AliasAnalysis.html#ModRefInfo - virtual ModRefInfo getModRefInfo(const CallBase *Call1, - const CallBase *Call2) = 0; + virtual ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2, + AAQueryInfo &AAQI) = 0; /// @} }; @@ -792,14 +895,14 @@ public: void setAAResults(AAResults *NewAAR) override { Result.setAAResults(NewAAR); } - AliasResult alias(const MemoryLocation &LocA, - const MemoryLocation &LocB) override { - return Result.alias(LocA, LocB); + AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB, + AAQueryInfo &AAQI) override { + return Result.alias(LocA, LocB, AAQI); } - bool pointsToConstantMemory(const MemoryLocation &Loc, + bool pointsToConstantMemory(const MemoryLocation &Loc, AAQueryInfo &AAQI, bool OrLocal) override { - return Result.pointsToConstantMemory(Loc, OrLocal); + return Result.pointsToConstantMemory(Loc, AAQI, OrLocal); } ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) override { @@ -814,14 +917,14 @@ public: return Result.getModRefBehavior(F); } - ModRefInfo getModRefInfo(const CallBase *Call, - const MemoryLocation &Loc) override { - return Result.getModRefInfo(Call, Loc); + ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc, + AAQueryInfo &AAQI) override { + return Result.getModRefInfo(Call, Loc, AAQI); } - ModRefInfo getModRefInfo(const CallBase *Call1, - const CallBase *Call2) override { - return Result.getModRefInfo(Call1, Call2); + ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2, + AAQueryInfo &AAQI) override { + return Result.getModRefInfo(Call1, Call2, AAQI); } }; @@ -867,13 +970,16 @@ protected: AAResultsProxy(AAResults *AAR, DerivedT &CurrentResult) : AAR(AAR), CurrentResult(CurrentResult) {} - AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB) { - return AAR ? AAR->alias(LocA, LocB) : CurrentResult.alias(LocA, LocB); + AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB, + AAQueryInfo &AAQI) { + return AAR ? AAR->alias(LocA, LocB, AAQI) + : CurrentResult.alias(LocA, LocB, AAQI); } - bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal) { - return AAR ? AAR->pointsToConstantMemory(Loc, OrLocal) - : CurrentResult.pointsToConstantMemory(Loc, OrLocal); + bool pointsToConstantMemory(const MemoryLocation &Loc, AAQueryInfo &AAQI, + bool OrLocal) { + return AAR ? AAR->pointsToConstantMemory(Loc, AAQI, OrLocal) + : CurrentResult.pointsToConstantMemory(Loc, AAQI, OrLocal); } ModRefInfo getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) { @@ -890,14 +996,16 @@ protected: return AAR ? AAR->getModRefBehavior(F) : CurrentResult.getModRefBehavior(F); } - ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc) { - return AAR ? AAR->getModRefInfo(Call, Loc) - : CurrentResult.getModRefInfo(Call, Loc); + ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc, + AAQueryInfo &AAQI) { + return AAR ? AAR->getModRefInfo(Call, Loc, AAQI) + : CurrentResult.getModRefInfo(Call, Loc, AAQI); } - ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2) { - return AAR ? AAR->getModRefInfo(Call1, Call2) - : CurrentResult.getModRefInfo(Call1, Call2); + ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2, + AAQueryInfo &AAQI) { + return AAR ? AAR->getModRefInfo(Call1, Call2, AAQI) + : CurrentResult.getModRefInfo(Call1, Call2, AAQI); } }; @@ -921,11 +1029,13 @@ protected: AAResultsProxy getBestAAResults() { return AAResultsProxy(AAR, derived()); } public: - AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB) { + AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB, + AAQueryInfo &AAQI) { return MayAlias; } - bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal) { + bool pointsToConstantMemory(const MemoryLocation &Loc, AAQueryInfo &AAQI, + bool OrLocal) { return false; } @@ -941,11 +1051,13 @@ public: return FMRB_UnknownModRefBehavior; } - ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc) { + ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc, + AAQueryInfo &AAQI) { return ModRefInfo::ModRef; } - ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2) { + ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2, + AAQueryInfo &AAQI) { return ModRefInfo::ModRef; } }; @@ -984,6 +1096,11 @@ bool isIdentifiedFunctionLocal(const Value *V); /// This manager effectively wraps the AnalysisManager for registering alias /// analyses. When you register your alias analysis with this manager, it will /// ensure the analysis itself is registered with its AnalysisManager. +/// +/// The result of this analysis is only invalidated if one of the particular +/// aggregated AA results end up being invalidated. This removes the need to +/// explicitly preserve the results of `AAManager`. Note that analyses should no +/// longer be registered once the `AAManager` is run. class AAManager : public AnalysisInfoMixin { public: using Result = AAResults; diff --git a/include/llvm/Analysis/AliasAnalysisEvaluator.h b/include/llvm/Analysis/AliasAnalysisEvaluator.h index 0941814a56c3..972eceaa3ba9 100644 --- a/include/llvm/Analysis/AliasAnalysisEvaluator.h +++ b/include/llvm/Analysis/AliasAnalysisEvaluator.h @@ -1,9 +1,8 @@ //===- AliasAnalysisEvaluator.h - Alias Analysis Accuracy Evaluator -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/Analysis/AliasSetTracker.h b/include/llvm/Analysis/AliasSetTracker.h index 7ed5cd5c4734..34a509b7f4bb 100644 --- a/include/llvm/Analysis/AliasSetTracker.h +++ b/include/llvm/Analysis/AliasSetTracker.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/AliasSetTracker.h - Build Alias Sets -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -37,6 +36,8 @@ namespace llvm { class AliasSetTracker; class BasicBlock; class LoadInst; +class Loop; +class MemorySSA; class AnyMemSetInst; class AnyMemTransferInst; class raw_ostream; @@ -294,7 +295,8 @@ private: void removeFromTracker(AliasSetTracker &AST); void addPointer(AliasSetTracker &AST, PointerRec &Entry, LocationSize Size, - const AAMDNodes &AAInfo, bool KnownMustAlias = false); + const AAMDNodes &AAInfo, bool KnownMustAlias = false, + bool SkipSizeUpdate = false); void addUnknownInst(Instruction *I, AliasAnalysis &AA); void removeUnknownInst(AliasSetTracker &AST, Instruction *I) { @@ -310,10 +312,10 @@ private: } public: - /// Return true if the specified pointer "may" (or must) alias one of the - /// members in the set. - bool aliasesPointer(const Value *Ptr, LocationSize Size, - const AAMDNodes &AAInfo, AliasAnalysis &AA) const; + /// If the specified pointer "may" (or must) alias one of the members in the + /// set return the appropriate AliasResult. Otherwise return NoAlias. + AliasResult aliasesPointer(const Value *Ptr, LocationSize Size, + const AAMDNodes &AAInfo, AliasAnalysis &AA) const; bool aliasesUnknownInst(const Instruction *Inst, AliasAnalysis &AA) const; }; @@ -341,6 +343,8 @@ class AliasSetTracker { struct ASTCallbackVHDenseMapInfo : public DenseMapInfo {}; AliasAnalysis &AA; + MemorySSA *MSSA; + Loop *L; ilist AliasSets; using PointerMapType = DenseMap VarIndices; }; - /// Track alias queries to guard against recursion. - using LocPair = std::pair; - using AliasCacheTy = SmallDenseMap; - AliasCacheTy AliasCache; - /// Tracks phi nodes we have visited. /// /// When interpret "Value" pointer equality as value equality we need to make @@ -200,22 +198,24 @@ private: AliasResult aliasGEP(const GEPOperator *V1, LocationSize V1Size, const AAMDNodes &V1AAInfo, const Value *V2, LocationSize V2Size, const AAMDNodes &V2AAInfo, - const Value *UnderlyingV1, const Value *UnderlyingV2); + const Value *UnderlyingV1, const Value *UnderlyingV2, + AAQueryInfo &AAQI); AliasResult aliasPHI(const PHINode *PN, LocationSize PNSize, const AAMDNodes &PNAAInfo, const Value *V2, LocationSize V2Size, const AAMDNodes &V2AAInfo, - const Value *UnderV2); + const Value *UnderV2, AAQueryInfo &AAQI); AliasResult aliasSelect(const SelectInst *SI, LocationSize SISize, const AAMDNodes &SIAAInfo, const Value *V2, LocationSize V2Size, const AAMDNodes &V2AAInfo, - const Value *UnderV2); + const Value *UnderV2, AAQueryInfo &AAQI); AliasResult aliasCheck(const Value *V1, LocationSize V1Size, AAMDNodes V1AATag, const Value *V2, LocationSize V2Size, AAMDNodes V2AATag, - const Value *O1 = nullptr, const Value *O2 = nullptr); + AAQueryInfo &AAQI, const Value *O1 = nullptr, + const Value *O2 = nullptr); }; /// Analysis pass providing a never-invalidated alias analysis result. diff --git a/include/llvm/Analysis/BlockFrequencyInfo.h b/include/llvm/Analysis/BlockFrequencyInfo.h index 0b2618735697..8bcfd7ff8f58 100644 --- a/include/llvm/Analysis/BlockFrequencyInfo.h +++ b/include/llvm/Analysis/BlockFrequencyInfo.h @@ -1,9 +1,8 @@ //===- BlockFrequencyInfo.h - Block Frequency Analysis ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -68,7 +67,8 @@ public: /// Returns the estimated profile count of \p BB. /// This computes the relative block frequency of \p BB and multiplies it by /// the enclosing function's count (if available) and returns the value. - Optional getBlockProfileCount(const BasicBlock *BB) const; + Optional getBlockProfileCount(const BasicBlock *BB, + bool AllowSynthetic = false) const; /// Returns the estimated profile count of \p Freq. /// This uses the frequency \p Freq and multiplies it by diff --git a/include/llvm/Analysis/BlockFrequencyInfoImpl.h b/include/llvm/Analysis/BlockFrequencyInfoImpl.h index 25b2efd33c98..bfe4fb14a2b8 100644 --- a/include/llvm/Analysis/BlockFrequencyInfoImpl.h +++ b/include/llvm/Analysis/BlockFrequencyInfoImpl.h @@ -1,9 +1,8 @@ //==- BlockFrequencyInfoImpl.h - Block Frequency Implementation --*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -160,10 +159,6 @@ inline raw_ostream &operator<<(raw_ostream &OS, BlockMass X) { } // end namespace bfi_detail -template <> struct isPodLike { - static const bool value = true; -}; - /// Base class for BlockFrequencyInfoImpl /// /// BlockFrequencyInfoImplBase has supporting data structures and some @@ -187,9 +182,9 @@ public: struct BlockNode { using IndexType = uint32_t; - IndexType Index = std::numeric_limits::max(); + IndexType Index; - BlockNode() = default; + BlockNode() : Index(std::numeric_limits::max()) {} BlockNode(IndexType Index) : Index(Index) {} bool operator==(const BlockNode &X) const { return Index == X.Index; } @@ -525,9 +520,11 @@ public: BlockFrequency getBlockFreq(const BlockNode &Node) const; Optional getBlockProfileCount(const Function &F, - const BlockNode &Node) const; + const BlockNode &Node, + bool AllowSynthetic = false) const; Optional getProfileCountFromFreq(const Function &F, - uint64_t Freq) const; + uint64_t Freq, + bool AllowSynthetic = false) const; bool isIrrLoopHeader(const BlockNode &Node); void setBlockFreq(const BlockNode &Node, uint64_t Freq); @@ -973,13 +970,17 @@ public: } Optional getBlockProfileCount(const Function &F, - const BlockT *BB) const { - return BlockFrequencyInfoImplBase::getBlockProfileCount(F, getNode(BB)); + const BlockT *BB, + bool AllowSynthetic = false) const { + return BlockFrequencyInfoImplBase::getBlockProfileCount(F, getNode(BB), + AllowSynthetic); } Optional getProfileCountFromFreq(const Function &F, - uint64_t Freq) const { - return BlockFrequencyInfoImplBase::getProfileCountFromFreq(F, Freq); + uint64_t Freq, + bool AllowSynthetic = false) const { + return BlockFrequencyInfoImplBase::getProfileCountFromFreq(F, Freq, + AllowSynthetic); } bool isIrrLoopHeader(const BlockT *BB) { diff --git a/include/llvm/Analysis/BranchProbabilityInfo.h b/include/llvm/Analysis/BranchProbabilityInfo.h index 45277db46090..97cb730d16c7 100644 --- a/include/llvm/Analysis/BranchProbabilityInfo.h +++ b/include/llvm/Analysis/BranchProbabilityInfo.h @@ -1,9 +1,8 @@ //===- BranchProbabilityInfo.h - Branch Probability Analysis ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/CFG.h b/include/llvm/Analysis/CFG.h index caae0b6e2a8f..bb55e76ac86a 100644 --- a/include/llvm/Analysis/CFG.h +++ b/include/llvm/Analysis/CFG.h @@ -1,9 +1,8 @@ //===-- Analysis/CFG.h - BasicBlock Analyses --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -48,8 +47,8 @@ unsigned GetSuccessorNumber(const BasicBlock *BB, const BasicBlock *Succ); bool isCriticalEdge(const Instruction *TI, unsigned SuccNum, bool AllowIdenticalEdges = false); -/// Determine whether instruction 'To' is reachable from 'From', -/// returning true if uncertain. +/// Determine whether instruction 'To' is reachable from 'From', without passing +/// through any blocks in ExclusionSet, returning true if uncertain. /// /// Determine whether there is a path from From to To within a single function. /// Returns false only if we can prove that once 'From' has been executed then @@ -63,9 +62,10 @@ bool isCriticalEdge(const Instruction *TI, unsigned SuccNum, /// we find a block that dominates the block containing 'To'. DT is most useful /// on branchy code but not loops, and LI is most useful on code with loops but /// does not help on branchy code outside loops. -bool isPotentiallyReachable(const Instruction *From, const Instruction *To, - const DominatorTree *DT = nullptr, - const LoopInfo *LI = nullptr); +bool isPotentiallyReachable( + const Instruction *From, const Instruction *To, + const SmallPtrSetImpl *ExclusionSet = nullptr, + const DominatorTree *DT = nullptr, const LoopInfo *LI = nullptr); /// Determine whether block 'To' is reachable from 'From', returning /// true if uncertain. @@ -89,6 +89,20 @@ bool isPotentiallyReachableFromMany(SmallVectorImpl &Worklist, const DominatorTree *DT = nullptr, const LoopInfo *LI = nullptr); +/// Determine whether there is at least one path from a block in +/// 'Worklist' to 'StopBB' without passing through any blocks in +/// 'ExclusionSet', returning true if uncertain. +/// +/// Determine whether there is a path from at least one block in Worklist to +/// StopBB within a single function without passing through any of the blocks +/// in 'ExclusionSet'. Returns false only if we can prove that once any block +/// in 'Worklist' has been reached then 'StopBB' can not be executed. +/// Conservatively returns true. +bool isPotentiallyReachableFromMany( + SmallVectorImpl &Worklist, BasicBlock *StopBB, + const SmallPtrSetImpl *ExclusionSet, + const DominatorTree *DT = nullptr, const LoopInfo *LI = nullptr); + /// Return true if the control flow in \p RPOTraversal is irreducible. /// /// This is a generic implementation to detect CFG irreducibility based on loop diff --git a/include/llvm/Analysis/CFGPrinter.h b/include/llvm/Analysis/CFGPrinter.h index 5996dd90bcfd..aaefc11653dd 100644 --- a/include/llvm/Analysis/CFGPrinter.h +++ b/include/llvm/Analysis/CFGPrinter.h @@ -1,9 +1,8 @@ //===-- CFGPrinter.h - CFG printer external interface -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/CFLAliasAnalysisUtils.h b/include/llvm/Analysis/CFLAliasAnalysisUtils.h index 981a8ddc2289..02f999a5b913 100644 --- a/include/llvm/Analysis/CFLAliasAnalysisUtils.h +++ b/include/llvm/Analysis/CFLAliasAnalysisUtils.h @@ -1,9 +1,8 @@ //=- CFLAliasAnalysisUtils.h - Utilities for CFL Alias Analysis ----*- C++-*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // \file diff --git a/include/llvm/Analysis/CFLAndersAliasAnalysis.h b/include/llvm/Analysis/CFLAndersAliasAnalysis.h index 8ae72553ab94..7c8b42b1d8d2 100644 --- a/include/llvm/Analysis/CFLAndersAliasAnalysis.h +++ b/include/llvm/Analysis/CFLAndersAliasAnalysis.h @@ -1,9 +1,8 @@ //==- CFLAndersAliasAnalysis.h - Unification-based Alias Analysis -*- C++-*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -61,7 +60,8 @@ public: const cflaa::AliasSummary *getAliasSummary(const Function &); AliasResult query(const MemoryLocation &, const MemoryLocation &); - AliasResult alias(const MemoryLocation &, const MemoryLocation &); + AliasResult alias(const MemoryLocation &, const MemoryLocation &, + AAQueryInfo &); private: /// Ensures that the given function is available in the cache. diff --git a/include/llvm/Analysis/CFLSteensAliasAnalysis.h b/include/llvm/Analysis/CFLSteensAliasAnalysis.h index 09e366f11e18..cc7a47cd9a5f 100644 --- a/include/llvm/Analysis/CFLSteensAliasAnalysis.h +++ b/include/llvm/Analysis/CFLSteensAliasAnalysis.h @@ -1,9 +1,8 @@ //==- CFLSteensAliasAnalysis.h - Unification-based Alias Analysis -*- C++-*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -70,7 +69,8 @@ public: AliasResult query(const MemoryLocation &LocA, const MemoryLocation &LocB); - AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB) { + AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB, + AAQueryInfo &AAQI) { if (LocA.Ptr == LocB.Ptr) return MustAlias; @@ -80,11 +80,11 @@ public: // ConstantExpr, but every query needs to have at least one Value tied to a // Function, and neither GlobalValues nor ConstantExprs are. if (isa(LocA.Ptr) && isa(LocB.Ptr)) - return AAResultBase::alias(LocA, LocB); + return AAResultBase::alias(LocA, LocB, AAQI); AliasResult QueryResult = query(LocA, LocB); if (QueryResult == MayAlias) - return AAResultBase::alias(LocA, LocB); + return AAResultBase::alias(LocA, LocB, AAQI); return QueryResult; } diff --git a/include/llvm/Analysis/CGSCCPassManager.h b/include/llvm/Analysis/CGSCCPassManager.h index 61b99f6c3e6b..8af5fb86995a 100644 --- a/include/llvm/Analysis/CGSCCPassManager.h +++ b/include/llvm/Analysis/CGSCCPassManager.h @@ -1,9 +1,8 @@ //===- CGSCCPassManager.h - Call graph pass management ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -292,6 +291,21 @@ struct CGSCCUpdateResult { /// post-order walk. LazyCallGraph::SCC *UpdatedC; + /// Preserved analyses across SCCs. + /// + /// We specifically want to allow CGSCC passes to mutate ancestor IR + /// (changing both the CG structure and the function IR itself). However, + /// this means we need to take special care to correctly mark what analyses + /// are preserved *across* SCCs. We have to track this out-of-band here + /// because within the main `PassManeger` infrastructure we need to mark + /// everything within an SCC as preserved in order to avoid repeatedly + /// invalidating the same analyses as we unnest pass managers and adaptors. + /// So we track the cross-SCC version of the preserved analyses here from any + /// code that does direct invalidation of SCC analyses, and then use it + /// whenever we move forward in the post-order walk of SCCs before running + /// passes over the new SCC. + PreservedAnalyses CrossSCCPA; + /// A hacky area where the inliner can retain history about inlining /// decisions that mutated the call graph's SCC structure in order to avoid /// infinite inlining. See the comments in the inliner's CG update logic. @@ -339,175 +353,7 @@ public: } /// Runs the CGSCC pass across every SCC in the module. - PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM) { - // Setup the CGSCC analysis manager from its proxy. - CGSCCAnalysisManager &CGAM = - AM.getResult(M).getManager(); - - // Get the call graph for this module. - LazyCallGraph &CG = AM.getResult(M); - - // We keep worklists to allow us to push more work onto the pass manager as - // the passes are run. - SmallPriorityWorklist RCWorklist; - SmallPriorityWorklist CWorklist; - - // Keep sets for invalidated SCCs and RefSCCs that should be skipped when - // iterating off the worklists. - SmallPtrSet InvalidRefSCCSet; - SmallPtrSet InvalidSCCSet; - - SmallDenseSet, 4> - InlinedInternalEdges; - - CGSCCUpdateResult UR = {RCWorklist, CWorklist, InvalidRefSCCSet, - InvalidSCCSet, nullptr, nullptr, - InlinedInternalEdges}; - - // Request PassInstrumentation from analysis manager, will use it to run - // instrumenting callbacks for the passes later. - PassInstrumentation PI = AM.getResult(M); - - PreservedAnalyses PA = PreservedAnalyses::all(); - CG.buildRefSCCs(); - for (auto RCI = CG.postorder_ref_scc_begin(), - RCE = CG.postorder_ref_scc_end(); - RCI != RCE;) { - assert(RCWorklist.empty() && - "Should always start with an empty RefSCC worklist"); - // The postorder_ref_sccs range we are walking is lazily constructed, so - // we only push the first one onto the worklist. The worklist allows us - // to capture *new* RefSCCs created during transformations. - // - // We really want to form RefSCCs lazily because that makes them cheaper - // to update as the program is simplified and allows us to have greater - // cache locality as forming a RefSCC touches all the parts of all the - // functions within that RefSCC. - // - // We also eagerly increment the iterator to the next position because - // the CGSCC passes below may delete the current RefSCC. - RCWorklist.insert(&*RCI++); - - do { - LazyCallGraph::RefSCC *RC = RCWorklist.pop_back_val(); - if (InvalidRefSCCSet.count(RC)) { - LLVM_DEBUG(dbgs() << "Skipping an invalid RefSCC...\n"); - continue; - } - - assert(CWorklist.empty() && - "Should always start with an empty SCC worklist"); - - LLVM_DEBUG(dbgs() << "Running an SCC pass across the RefSCC: " << *RC - << "\n"); - - // Push the initial SCCs in reverse post-order as we'll pop off the - // back and so see this in post-order. - for (LazyCallGraph::SCC &C : llvm::reverse(*RC)) - CWorklist.insert(&C); - - do { - LazyCallGraph::SCC *C = CWorklist.pop_back_val(); - // Due to call graph mutations, we may have invalid SCCs or SCCs from - // other RefSCCs in the worklist. The invalid ones are dead and the - // other RefSCCs should be queued above, so we just need to skip both - // scenarios here. - if (InvalidSCCSet.count(C)) { - LLVM_DEBUG(dbgs() << "Skipping an invalid SCC...\n"); - continue; - } - if (&C->getOuterRefSCC() != RC) { - LLVM_DEBUG(dbgs() - << "Skipping an SCC that is now part of some other " - "RefSCC...\n"); - continue; - } - - do { - // Check that we didn't miss any update scenario. - assert(!InvalidSCCSet.count(C) && "Processing an invalid SCC!"); - assert(C->begin() != C->end() && "Cannot have an empty SCC!"); - assert(&C->getOuterRefSCC() == RC && - "Processing an SCC in a different RefSCC!"); - - UR.UpdatedRC = nullptr; - UR.UpdatedC = nullptr; - - // Check the PassInstrumentation's BeforePass callbacks before - // running the pass, skip its execution completely if asked to - // (callback returns false). - if (!PI.runBeforePass(Pass, *C)) - continue; - - PreservedAnalyses PassPA = Pass.run(*C, CGAM, CG, UR); - - if (UR.InvalidatedSCCs.count(C)) - PI.runAfterPassInvalidated(Pass); - else - PI.runAfterPass(Pass, *C); - - // Update the SCC and RefSCC if necessary. - C = UR.UpdatedC ? UR.UpdatedC : C; - RC = UR.UpdatedRC ? UR.UpdatedRC : RC; - - // If the CGSCC pass wasn't able to provide a valid updated SCC, - // the current SCC may simply need to be skipped if invalid. - if (UR.InvalidatedSCCs.count(C)) { - LLVM_DEBUG(dbgs() - << "Skipping invalidated root or island SCC!\n"); - break; - } - // Check that we didn't miss any update scenario. - assert(C->begin() != C->end() && "Cannot have an empty SCC!"); - - // We handle invalidating the CGSCC analysis manager's information - // for the (potentially updated) SCC here. Note that any other SCCs - // whose structure has changed should have been invalidated by - // whatever was updating the call graph. This SCC gets invalidated - // late as it contains the nodes that were actively being - // processed. - CGAM.invalidate(*C, PassPA); - - // Then intersect the preserved set so that invalidation of module - // analyses will eventually occur when the module pass completes. - PA.intersect(std::move(PassPA)); - - // The pass may have restructured the call graph and refined the - // current SCC and/or RefSCC. We need to update our current SCC and - // RefSCC pointers to follow these. Also, when the current SCC is - // refined, re-run the SCC pass over the newly refined SCC in order - // to observe the most precise SCC model available. This inherently - // cannot cycle excessively as it only happens when we split SCCs - // apart, at most converging on a DAG of single nodes. - // FIXME: If we ever start having RefSCC passes, we'll want to - // iterate there too. - if (UR.UpdatedC) - LLVM_DEBUG(dbgs() - << "Re-running SCC passes after a refinement of the " - "current SCC: " - << *UR.UpdatedC << "\n"); - - // Note that both `C` and `RC` may at this point refer to deleted, - // invalid SCC and RefSCCs respectively. But we will short circuit - // the processing when we check them in the loop above. - } while (UR.UpdatedC); - } while (!CWorklist.empty()); - - // We only need to keep internal inlined edge information within - // a RefSCC, clear it to save on space and let the next time we visit - // any of these functions have a fresh start. - InlinedInternalEdges.clear(); - } while (!RCWorklist.empty()); - } - - // By definition we preserve the call garph, all SCC analyses, and the - // analysis proxies by handling them above and in any nested pass managers. - PA.preserveSet>(); - PA.preserve(); - PA.preserve(); - PA.preserve(); - return PA; - } + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); private: CGSCCPassT Pass; @@ -873,6 +719,210 @@ DevirtSCCRepeatedPass createDevirtSCCRepeatedPass(PassT Pass, return DevirtSCCRepeatedPass(std::move(Pass), MaxIterations); } +// Out-of-line implementation details for templates below this point. + +template +PreservedAnalyses +ModuleToPostOrderCGSCCPassAdaptor::run(Module &M, + ModuleAnalysisManager &AM) { + // Setup the CGSCC analysis manager from its proxy. + CGSCCAnalysisManager &CGAM = + AM.getResult(M).getManager(); + + // Get the call graph for this module. + LazyCallGraph &CG = AM.getResult(M); + + // We keep worklists to allow us to push more work onto the pass manager as + // the passes are run. + SmallPriorityWorklist RCWorklist; + SmallPriorityWorklist CWorklist; + + // Keep sets for invalidated SCCs and RefSCCs that should be skipped when + // iterating off the worklists. + SmallPtrSet InvalidRefSCCSet; + SmallPtrSet InvalidSCCSet; + + SmallDenseSet, 4> + InlinedInternalEdges; + + CGSCCUpdateResult UR = { + RCWorklist, CWorklist, InvalidRefSCCSet, InvalidSCCSet, + nullptr, nullptr, PreservedAnalyses::all(), InlinedInternalEdges}; + + // Request PassInstrumentation from analysis manager, will use it to run + // instrumenting callbacks for the passes later. + PassInstrumentation PI = AM.getResult(M); + + PreservedAnalyses PA = PreservedAnalyses::all(); + CG.buildRefSCCs(); + for (auto RCI = CG.postorder_ref_scc_begin(), + RCE = CG.postorder_ref_scc_end(); + RCI != RCE;) { + assert(RCWorklist.empty() && + "Should always start with an empty RefSCC worklist"); + // The postorder_ref_sccs range we are walking is lazily constructed, so + // we only push the first one onto the worklist. The worklist allows us + // to capture *new* RefSCCs created during transformations. + // + // We really want to form RefSCCs lazily because that makes them cheaper + // to update as the program is simplified and allows us to have greater + // cache locality as forming a RefSCC touches all the parts of all the + // functions within that RefSCC. + // + // We also eagerly increment the iterator to the next position because + // the CGSCC passes below may delete the current RefSCC. + RCWorklist.insert(&*RCI++); + + do { + LazyCallGraph::RefSCC *RC = RCWorklist.pop_back_val(); + if (InvalidRefSCCSet.count(RC)) { + LLVM_DEBUG(dbgs() << "Skipping an invalid RefSCC...\n"); + continue; + } + + assert(CWorklist.empty() && + "Should always start with an empty SCC worklist"); + + LLVM_DEBUG(dbgs() << "Running an SCC pass across the RefSCC: " << *RC + << "\n"); + + // Push the initial SCCs in reverse post-order as we'll pop off the + // back and so see this in post-order. + for (LazyCallGraph::SCC &C : llvm::reverse(*RC)) + CWorklist.insert(&C); + + do { + LazyCallGraph::SCC *C = CWorklist.pop_back_val(); + // Due to call graph mutations, we may have invalid SCCs or SCCs from + // other RefSCCs in the worklist. The invalid ones are dead and the + // other RefSCCs should be queued above, so we just need to skip both + // scenarios here. + if (InvalidSCCSet.count(C)) { + LLVM_DEBUG(dbgs() << "Skipping an invalid SCC...\n"); + continue; + } + if (&C->getOuterRefSCC() != RC) { + LLVM_DEBUG(dbgs() << "Skipping an SCC that is now part of some other " + "RefSCC...\n"); + continue; + } + + // Ensure we can proxy analysis updates from from the CGSCC analysis + // manager into the Function analysis manager by getting a proxy here. + // FIXME: This seems like a bit of a hack. We should find a cleaner + // or more costructive way to ensure this happens. + (void)CGAM.getResult(*C, CG); + + // Each time we visit a new SCC pulled off the worklist, + // a transformation of a child SCC may have also modified this parent + // and invalidated analyses. So we invalidate using the update record's + // cross-SCC preserved set. This preserved set is intersected by any + // CGSCC pass that handles invalidation (primarily pass managers) prior + // to marking its SCC as preserved. That lets us track everything that + // might need invalidation across SCCs without excessive invalidations + // on a single SCC. + // + // This essentially allows SCC passes to freely invalidate analyses + // of any ancestor SCC. If this becomes detrimental to successfully + // caching analyses, we could force each SCC pass to manually + // invalidate the analyses for any SCCs other than themselves which + // are mutated. However, that seems to lose the robustness of the + // pass-manager driven invalidation scheme. + // + // FIXME: This is redundant in one case -- the top of the worklist may + // *also* be the same SCC we just ran over (and invalidated for). In + // that case, we'll end up doing a redundant invalidation here as + // a consequence. + CGAM.invalidate(*C, UR.CrossSCCPA); + + do { + // Check that we didn't miss any update scenario. + assert(!InvalidSCCSet.count(C) && "Processing an invalid SCC!"); + assert(C->begin() != C->end() && "Cannot have an empty SCC!"); + assert(&C->getOuterRefSCC() == RC && + "Processing an SCC in a different RefSCC!"); + + UR.UpdatedRC = nullptr; + UR.UpdatedC = nullptr; + + // Check the PassInstrumentation's BeforePass callbacks before + // running the pass, skip its execution completely if asked to + // (callback returns false). + if (!PI.runBeforePass(Pass, *C)) + continue; + + PreservedAnalyses PassPA = Pass.run(*C, CGAM, CG, UR); + + if (UR.InvalidatedSCCs.count(C)) + PI.runAfterPassInvalidated(Pass); + else + PI.runAfterPass(Pass, *C); + + // Update the SCC and RefSCC if necessary. + C = UR.UpdatedC ? UR.UpdatedC : C; + RC = UR.UpdatedRC ? UR.UpdatedRC : RC; + + // If the CGSCC pass wasn't able to provide a valid updated SCC, + // the current SCC may simply need to be skipped if invalid. + if (UR.InvalidatedSCCs.count(C)) { + LLVM_DEBUG(dbgs() << "Skipping invalidated root or island SCC!\n"); + break; + } + // Check that we didn't miss any update scenario. + assert(C->begin() != C->end() && "Cannot have an empty SCC!"); + + // We handle invalidating the CGSCC analysis manager's information + // for the (potentially updated) SCC here. Note that any other SCCs + // whose structure has changed should have been invalidated by + // whatever was updating the call graph. This SCC gets invalidated + // late as it contains the nodes that were actively being + // processed. + CGAM.invalidate(*C, PassPA); + + // Then intersect the preserved set so that invalidation of module + // analyses will eventually occur when the module pass completes. + // Also intersect with the cross-SCC preserved set to capture any + // cross-SCC invalidation. + UR.CrossSCCPA.intersect(PassPA); + PA.intersect(std::move(PassPA)); + + // The pass may have restructured the call graph and refined the + // current SCC and/or RefSCC. We need to update our current SCC and + // RefSCC pointers to follow these. Also, when the current SCC is + // refined, re-run the SCC pass over the newly refined SCC in order + // to observe the most precise SCC model available. This inherently + // cannot cycle excessively as it only happens when we split SCCs + // apart, at most converging on a DAG of single nodes. + // FIXME: If we ever start having RefSCC passes, we'll want to + // iterate there too. + if (UR.UpdatedC) + LLVM_DEBUG(dbgs() + << "Re-running SCC passes after a refinement of the " + "current SCC: " + << *UR.UpdatedC << "\n"); + + // Note that both `C` and `RC` may at this point refer to deleted, + // invalid SCC and RefSCCs respectively. But we will short circuit + // the processing when we check them in the loop above. + } while (UR.UpdatedC); + } while (!CWorklist.empty()); + + // We only need to keep internal inlined edge information within + // a RefSCC, clear it to save on space and let the next time we visit + // any of these functions have a fresh start. + InlinedInternalEdges.clear(); + } while (!RCWorklist.empty()); + } + + // By definition we preserve the call garph, all SCC analyses, and the + // analysis proxies by handling them above and in any nested pass managers. + PA.preserveSet>(); + PA.preserve(); + PA.preserve(); + PA.preserve(); + return PA; +} + // Clear out the debug logging macro. #undef DEBUG_TYPE diff --git a/include/llvm/Analysis/CallGraph.h b/include/llvm/Analysis/CallGraph.h index f109cf2fac4d..7a10183c4d91 100644 --- a/include/llvm/Analysis/CallGraph.h +++ b/include/llvm/Analysis/CallGraph.h @@ -1,9 +1,8 @@ //===- CallGraph.h - Build a Module's call graph ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -48,8 +47,8 @@ #include "llvm/ADT/GraphTraits.h" #include "llvm/ADT/STLExtras.h" -#include "llvm/IR/CallSite.h" #include "llvm/IR/Function.h" +#include "llvm/IR/InstrTypes.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/PassManager.h" #include "llvm/IR/ValueHandle.h" @@ -230,11 +229,11 @@ public: } /// Adds a function to the list of functions called by this one. - void addCalledFunction(CallSite CS, CallGraphNode *M) { - assert(!CS.getInstruction() || !CS.getCalledFunction() || - !CS.getCalledFunction()->isIntrinsic() || - !Intrinsic::isLeaf(CS.getCalledFunction()->getIntrinsicID())); - CalledFunctions.emplace_back(CS.getInstruction(), M); + void addCalledFunction(CallBase *Call, CallGraphNode *M) { + assert(!Call || !Call->getCalledFunction() || + !Call->getCalledFunction()->isIntrinsic() || + !Intrinsic::isLeaf(Call->getCalledFunction()->getIntrinsicID())); + CalledFunctions.emplace_back(Call, M); M->AddRef(); } @@ -247,7 +246,7 @@ public: /// Removes the edge in the node for the specified call site. /// /// Note that this method takes linear time, so it should be used sparingly. - void removeCallEdgeFor(CallSite CS); + void removeCallEdgeFor(CallBase &Call); /// Removes all call edges from this node to the specified callee /// function. @@ -264,7 +263,8 @@ public: /// new one. /// /// Note that this method takes linear time, so it should be used sparingly. - void replaceCallEdge(CallSite CS, CallSite NewCS, CallGraphNode *NewNode); + void replaceCallEdge(CallBase &Call, CallBase &NewCall, + CallGraphNode *NewNode); private: friend class CallGraph; diff --git a/include/llvm/Analysis/CallGraphSCCPass.h b/include/llvm/Analysis/CallGraphSCCPass.h index ace54607634c..1b5b7e2f039e 100644 --- a/include/llvm/Analysis/CallGraphSCCPass.h +++ b/include/llvm/Analysis/CallGraphSCCPass.h @@ -1,9 +1,8 @@ //===- CallGraphSCCPass.h - Pass that operates BU on call graph -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/CallPrinter.h b/include/llvm/Analysis/CallPrinter.h index 8b697d5aa149..8d4159f3ddc0 100644 --- a/include/llvm/Analysis/CallPrinter.h +++ b/include/llvm/Analysis/CallPrinter.h @@ -1,9 +1,8 @@ //===-- CallPrinter.h - Call graph printer external interface ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/CaptureTracking.h b/include/llvm/Analysis/CaptureTracking.h index aaaaff9ae252..ca7abd34fea2 100644 --- a/include/llvm/Analysis/CaptureTracking.h +++ b/include/llvm/Analysis/CaptureTracking.h @@ -1,9 +1,8 @@ //===----- llvm/Analysis/CaptureTracking.h - Pointer capture ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/CmpInstAnalysis.h b/include/llvm/Analysis/CmpInstAnalysis.h index 0e9c6a96b0f4..3d34cd12aea4 100644 --- a/include/llvm/Analysis/CmpInstAnalysis.h +++ b/include/llvm/Analysis/CmpInstAnalysis.h @@ -1,9 +1,8 @@ //===-- CmpInstAnalysis.h - Utils to help fold compare insts ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/CodeMetrics.h b/include/llvm/Analysis/CodeMetrics.h index 752902238522..1482b66a3080 100644 --- a/include/llvm/Analysis/CodeMetrics.h +++ b/include/llvm/Analysis/CodeMetrics.h @@ -1,9 +1,8 @@ //===- CodeMetrics.h - Code cost measurements -------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -17,7 +16,6 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallPtrSet.h" -#include "llvm/IR/CallSite.h" namespace llvm { class AssumptionCache; @@ -29,14 +27,6 @@ class DataLayout; class TargetTransformInfo; class Value; -/// Check whether a call will lower to something small. -/// -/// This tests checks whether this callsite will lower to something -/// significantly cheaper than a traditional call, often a single -/// instruction. Note that if isInstructionFree(CS.getInstruction()) would -/// return true, so will this function. -bool callIsSmall(ImmutableCallSite CS); - /// Utility to calculate the size and a few similar metrics for a set /// of basic blocks. struct CodeMetrics { diff --git a/include/llvm/Analysis/ConstantFolding.h b/include/llvm/Analysis/ConstantFolding.h index 192c1abddcd2..2385b6f09c40 100644 --- a/include/llvm/Analysis/ConstantFolding.h +++ b/include/llvm/Analysis/ConstantFolding.h @@ -1,9 +1,8 @@ //===-- ConstantFolding.h - Fold instructions into constants ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -23,7 +22,7 @@ namespace llvm { class APInt; template class ArrayRef; -class CallSite; +class CallBase; class Constant; class ConstantExpr; class ConstantVector; @@ -31,7 +30,6 @@ class DataLayout; class Function; class GlobalValue; class Instruction; -class ImmutableCallSite; class TargetLibraryInfo; class Type; @@ -73,6 +71,12 @@ ConstantFoldCompareInstOperands(unsigned Predicate, Constant *LHS, Constant *RHS, const DataLayout &DL, const TargetLibraryInfo *TLI = nullptr); +/// Attempt to constant fold a unary operation with the specified +/// operand. If it fails, it returns a constant expression of the specified +/// operands. +Constant *ConstantFoldUnaryOpOperand(unsigned Opcode, Constant *Op, + const DataLayout &DL); + /// Attempt to constant fold a binary operation with the specified /// operands. If it fails, it returns a constant expression of the specified /// operands. @@ -139,11 +143,11 @@ Constant *ConstantFoldLoadThroughGEPIndices(Constant *C, /// canConstantFoldCallTo - Return true if its even possible to fold a call to /// the specified function. -bool canConstantFoldCallTo(ImmutableCallSite CS, const Function *F); +bool canConstantFoldCallTo(const CallBase *Call, const Function *F); /// ConstantFoldCall - Attempt to constant fold a call to the specified function /// with the specified arguments, returning null if unsuccessful. -Constant *ConstantFoldCall(ImmutableCallSite CS, Function *F, +Constant *ConstantFoldCall(const CallBase *Call, Function *F, ArrayRef Operands, const TargetLibraryInfo *TLI = nullptr); @@ -155,7 +159,7 @@ Constant *ConstantFoldLoadThroughBitcast(Constant *C, Type *DestTy, /// Check whether the given call has no side-effects. /// Specifically checks for math routimes which sometimes set errno. -bool isMathLibCallNoop(CallSite CS, const TargetLibraryInfo *TLI); +bool isMathLibCallNoop(const CallBase *Call, const TargetLibraryInfo *TLI); } #endif diff --git a/include/llvm/Analysis/DOTGraphTraitsPass.h b/include/llvm/Analysis/DOTGraphTraitsPass.h index b7447a0547d5..0410a3314659 100644 --- a/include/llvm/Analysis/DOTGraphTraitsPass.h +++ b/include/llvm/Analysis/DOTGraphTraitsPass.h @@ -1,9 +1,8 @@ //===-- DOTGraphTraitsPass.h - Print/View dotty graphs-----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/DemandedBits.h b/include/llvm/Analysis/DemandedBits.h index 4c4e3f6c99e7..04db3eb57c18 100644 --- a/include/llvm/Analysis/DemandedBits.h +++ b/include/llvm/Analysis/DemandedBits.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/DemandedBits.h - Determine demanded bits ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/DependenceAnalysis.h b/include/llvm/Analysis/DependenceAnalysis.h index 69d0e2c1513e..997013a5fc8e 100644 --- a/include/llvm/Analysis/DependenceAnalysis.h +++ b/include/llvm/Analysis/DependenceAnalysis.h @@ -1,9 +1,8 @@ //===-- llvm/Analysis/DependenceAnalysis.h -------------------- -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -275,6 +274,10 @@ template class ArrayRef; LoopInfo *LI) : AA(AA), SE(SE), LI(LI), F(F) {} + /// Handle transitive invalidation when the cached analysis results go away. + bool invalidate(Function &F, const PreservedAnalyses &PA, + FunctionAnalysisManager::Invalidator &Inv); + /// depends - Tests for a dependence between the Src and Dst instructions. /// Returns NULL if no dependence; otherwise, returns a Dependence (or a /// FullDependence) with as much information as can be gleaned. diff --git a/include/llvm/Analysis/DivergenceAnalysis.h b/include/llvm/Analysis/DivergenceAnalysis.h index d834862db095..3cfb9d13df94 100644 --- a/include/llvm/Analysis/DivergenceAnalysis.h +++ b/include/llvm/Analysis/DivergenceAnalysis.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/DivergenceAnalysis.h - Divergence Analysis -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/DomPrinter.h b/include/llvm/Analysis/DomPrinter.h index 0ed28994995a..a177f877b295 100644 --- a/include/llvm/Analysis/DomPrinter.h +++ b/include/llvm/Analysis/DomPrinter.h @@ -1,9 +1,8 @@ //===-- DomPrinter.h - Dom printer external interface ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/DomTreeUpdater.h b/include/llvm/Analysis/DomTreeUpdater.h new file mode 100644 index 000000000000..5ccce2e064cc --- /dev/null +++ b/include/llvm/Analysis/DomTreeUpdater.h @@ -0,0 +1,309 @@ +//===- DomTreeUpdater.h - DomTree/Post DomTree Updater ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the DomTreeUpdater class, which provides a uniform way to +// update dominator tree related data structures. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_ANALYSIS_DOMTREEUPDATER_H +#define LLVM_ANALYSIS_DOMTREEUPDATER_H + +#include "llvm/Analysis/PostDominators.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/ValueHandle.h" +#include "llvm/Support/GenericDomTree.h" +#include +#include + +namespace llvm { +class DomTreeUpdater { +public: + enum class UpdateStrategy : unsigned char { Eager = 0, Lazy = 1 }; + + explicit DomTreeUpdater(UpdateStrategy Strategy_) : Strategy(Strategy_) {} + DomTreeUpdater(DominatorTree &DT_, UpdateStrategy Strategy_) + : DT(&DT_), Strategy(Strategy_) {} + DomTreeUpdater(DominatorTree *DT_, UpdateStrategy Strategy_) + : DT(DT_), Strategy(Strategy_) {} + DomTreeUpdater(PostDominatorTree &PDT_, UpdateStrategy Strategy_) + : PDT(&PDT_), Strategy(Strategy_) {} + DomTreeUpdater(PostDominatorTree *PDT_, UpdateStrategy Strategy_) + : PDT(PDT_), Strategy(Strategy_) {} + DomTreeUpdater(DominatorTree &DT_, PostDominatorTree &PDT_, + UpdateStrategy Strategy_) + : DT(&DT_), PDT(&PDT_), Strategy(Strategy_) {} + DomTreeUpdater(DominatorTree *DT_, PostDominatorTree *PDT_, + UpdateStrategy Strategy_) + : DT(DT_), PDT(PDT_), Strategy(Strategy_) {} + + ~DomTreeUpdater() { flush(); } + + /// Returns true if the current strategy is Lazy. + bool isLazy() const { return Strategy == UpdateStrategy::Lazy; }; + + /// Returns true if the current strategy is Eager. + bool isEager() const { return Strategy == UpdateStrategy::Eager; }; + + /// Returns true if it holds a DominatorTree. + bool hasDomTree() const { return DT != nullptr; } + + /// Returns true if it holds a PostDominatorTree. + bool hasPostDomTree() const { return PDT != nullptr; } + + /// Returns true if there is BasicBlock awaiting deletion. + /// The deletion will only happen until a flush event and + /// all available trees are up-to-date. + /// Returns false under Eager UpdateStrategy. + bool hasPendingDeletedBB() const { return !DeletedBBs.empty(); } + + /// Returns true if DelBB is awaiting deletion. + /// Returns false under Eager UpdateStrategy. + bool isBBPendingDeletion(BasicBlock *DelBB) const; + + /// Returns true if either of DT or PDT is valid and the tree has at + /// least one update pending. If DT or PDT is nullptr it is treated + /// as having no pending updates. This function does not check + /// whether there is BasicBlock awaiting deletion. + /// Returns false under Eager UpdateStrategy. + bool hasPendingUpdates() const; + + /// Returns true if there are DominatorTree updates queued. + /// Returns false under Eager UpdateStrategy or DT is nullptr. + bool hasPendingDomTreeUpdates() const; + + /// Returns true if there are PostDominatorTree updates queued. + /// Returns false under Eager UpdateStrategy or PDT is nullptr. + bool hasPendingPostDomTreeUpdates() const; + + ///@{ + /// \name Mutation APIs + /// + /// These methods provide APIs for submitting updates to the DominatorTree and + /// the PostDominatorTree. + /// + /// Note: There are two strategies to update the DominatorTree and the + /// PostDominatorTree: + /// 1. Eager UpdateStrategy: Updates are submitted and then flushed + /// immediately. + /// 2. Lazy UpdateStrategy: Updates are submitted but only flushed when you + /// explicitly call Flush APIs. It is recommended to use this update strategy + /// when you submit a bunch of updates multiple times which can then + /// add up to a large number of updates between two queries on the + /// DominatorTree. The incremental updater can reschedule the updates or + /// decide to recalculate the dominator tree in order to speedup the updating + /// process depending on the number of updates. + /// + /// Although GenericDomTree provides several update primitives, + /// it is not encouraged to use these APIs directly. + + /// Submit updates to all available trees. + /// The Eager Strategy flushes updates immediately while the Lazy Strategy + /// queues the updates. + /// + /// Note: The "existence" of an edge in a CFG refers to the CFG which DTU is + /// in sync with + all updates before that single update. + /// + /// CAUTION! + /// 1. It is required for the state of the LLVM IR to be updated + /// *before* submitting the updates because the internal update routine will + /// analyze the current state of the CFG to determine whether an update + /// is valid. + /// 2. It is illegal to submit any update that has already been submitted, + /// i.e., you are supposed not to insert an existent edge or delete a + /// nonexistent edge. + void applyUpdates(ArrayRef Updates); + + /// Submit updates to all available trees. It will also + /// 1. discard duplicated updates, + /// 2. remove invalid updates. (Invalid updates means deletion of an edge that + /// still exists or insertion of an edge that does not exist.) + /// The Eager Strategy flushes updates immediately while the Lazy Strategy + /// queues the updates. + /// + /// Note: The "existence" of an edge in a CFG refers to the CFG which DTU is + /// in sync with + all updates before that single update. + /// + /// CAUTION! + /// 1. It is required for the state of the LLVM IR to be updated + /// *before* submitting the updates because the internal update routine will + /// analyze the current state of the CFG to determine whether an update + /// is valid. + /// 2. It is illegal to submit any update that has already been submitted, + /// i.e., you are supposed not to insert an existent edge or delete a + /// nonexistent edge. + /// 3. It is only legal to submit updates to an edge in the order CFG changes + /// are made. The order you submit updates on different edges is not + /// restricted. + void applyUpdatesPermissive(ArrayRef Updates); + + /// Notify DTU that the entry block was replaced. + /// Recalculate all available trees and flush all BasicBlocks + /// awaiting deletion immediately. + void recalculate(Function &F); + + /// \deprecated { Submit an edge insertion to all available trees. The Eager + /// Strategy flushes this update immediately while the Lazy Strategy queues + /// the update. An internal function checks if the edge exists in the CFG in + /// DEBUG mode. CAUTION! This function has to be called *after* making the + /// update on the actual CFG. It is illegal to submit any update that has + /// already been applied. } + LLVM_ATTRIBUTE_DEPRECATED(void insertEdge(BasicBlock *From, BasicBlock *To), + "Use applyUpdates() instead."); + + /// \deprecated {Submit an edge insertion to all available trees. + /// Under either Strategy, an invalid update will be discard silently. + /// Invalid update means inserting an edge that does not exist in the CFG. + /// The Eager Strategy flushes this update immediately while the Lazy Strategy + /// queues the update. It is only recommended to use this method when you + /// want to discard an invalid update. + /// CAUTION! It is illegal to submit any update that has already been + /// submitted. } + LLVM_ATTRIBUTE_DEPRECATED(void insertEdgeRelaxed(BasicBlock *From, + BasicBlock *To), + "Use applyUpdatesPermissive() instead."); + + /// \deprecated { Submit an edge deletion to all available trees. The Eager + /// Strategy flushes this update immediately while the Lazy Strategy queues + /// the update. An internal function checks if the edge doesn't exist in the + /// CFG in DEBUG mode. + /// CAUTION! This function has to be called *after* making the update on the + /// actual CFG. It is illegal to submit any update that has already been + /// submitted. } + LLVM_ATTRIBUTE_DEPRECATED(void deleteEdge(BasicBlock *From, BasicBlock *To), + "Use applyUpdates() instead."); + + /// \deprecated { Submit an edge deletion to all available trees. + /// Under either Strategy, an invalid update will be discard silently. + /// Invalid update means deleting an edge that exists in the CFG. + /// The Eager Strategy flushes this update immediately while the Lazy Strategy + /// queues the update. It is only recommended to use this method when you + /// want to discard an invalid update. + /// CAUTION! It is illegal to submit any update that has already been + /// submitted. } + LLVM_ATTRIBUTE_DEPRECATED(void deleteEdgeRelaxed(BasicBlock *From, + BasicBlock *To), + "Use applyUpdatesPermissive() instead."); + + /// Delete DelBB. DelBB will be removed from its Parent and + /// erased from available trees if it exists and finally get deleted. + /// Under Eager UpdateStrategy, DelBB will be processed immediately. + /// Under Lazy UpdateStrategy, DelBB will be queued until a flush event and + /// all available trees are up-to-date. Assert if any instruction of DelBB is + /// modified while awaiting deletion. When both DT and PDT are nullptrs, DelBB + /// will be queued until flush() is called. + void deleteBB(BasicBlock *DelBB); + + /// Delete DelBB. DelBB will be removed from its Parent and + /// erased from available trees if it exists. Then the callback will + /// be called. Finally, DelBB will be deleted. + /// Under Eager UpdateStrategy, DelBB will be processed immediately. + /// Under Lazy UpdateStrategy, DelBB will be queued until a flush event and + /// all available trees are up-to-date. Assert if any instruction of DelBB is + /// modified while awaiting deletion. Multiple callbacks can be queued for one + /// DelBB under Lazy UpdateStrategy. + void callbackDeleteBB(BasicBlock *DelBB, + std::function Callback); + + ///@} + + ///@{ + /// \name Flush APIs + /// + /// CAUTION! By the moment these flush APIs are called, the current CFG needs + /// to be the same as the CFG which DTU is in sync with + all updates + /// submitted. + + /// Flush DomTree updates and return DomTree. + /// It flushes Deleted BBs if both trees are up-to-date. + /// It must only be called when it has a DomTree. + DominatorTree &getDomTree(); + + /// Flush PostDomTree updates and return PostDomTree. + /// It flushes Deleted BBs if both trees are up-to-date. + /// It must only be called when it has a PostDomTree. + PostDominatorTree &getPostDomTree(); + + /// Apply all pending updates to available trees and flush all BasicBlocks + /// awaiting deletion. + + void flush(); + + ///@} + + /// Debug method to help view the internal state of this class. + LLVM_DUMP_METHOD void dump() const; + +private: + class CallBackOnDeletion final : public CallbackVH { + public: + CallBackOnDeletion(BasicBlock *V, + std::function Callback) + : CallbackVH(V), DelBB(V), Callback_(Callback) {} + + private: + BasicBlock *DelBB = nullptr; + std::function Callback_; + + void deleted() override { + Callback_(DelBB); + CallbackVH::deleted(); + } + }; + + SmallVector PendUpdates; + size_t PendDTUpdateIndex = 0; + size_t PendPDTUpdateIndex = 0; + DominatorTree *DT = nullptr; + PostDominatorTree *PDT = nullptr; + const UpdateStrategy Strategy; + SmallPtrSet DeletedBBs; + std::vector Callbacks; + bool IsRecalculatingDomTree = false; + bool IsRecalculatingPostDomTree = false; + + /// First remove all the instructions of DelBB and then make sure DelBB has a + /// valid terminator instruction which is necessary to have when DelBB still + /// has to be inside of its parent Function while awaiting deletion under Lazy + /// UpdateStrategy to prevent other routines from asserting the state of the + /// IR is inconsistent. Assert if DelBB is nullptr or has predecessors. + void validateDeleteBB(BasicBlock *DelBB); + + /// Returns true if at least one BasicBlock is deleted. + bool forceFlushDeletedBB(); + + /// Helper function to apply all pending DomTree updates. + void applyDomTreeUpdates(); + + /// Helper function to apply all pending PostDomTree updates. + void applyPostDomTreeUpdates(); + + /// Helper function to flush deleted BasicBlocks if all available + /// trees are up-to-date. + void tryFlushDeletedBB(); + + /// Drop all updates applied by all available trees and delete BasicBlocks if + /// all available trees are up-to-date. + void dropOutOfDateUpdates(); + + /// Erase Basic Block node that has been unlinked from Function + /// in the DomTree and PostDomTree. + void eraseDelBBNode(BasicBlock *DelBB); + + /// Returns true if the update appears in the LLVM IR. + /// It is used to check whether an update is valid in + /// insertEdge/deleteEdge or is unnecessary in the batch update. + bool isUpdateValid(DominatorTree::UpdateType Update) const; + + /// Returns true if the update is self dominance. + bool isSelfDominance(DominatorTree::UpdateType Update) const; +}; +} // namespace llvm + +#endif // LLVM_ANALYSIS_DOMTREEUPDATER_H diff --git a/include/llvm/Analysis/DominanceFrontier.h b/include/llvm/Analysis/DominanceFrontier.h index d94c420d7177..c0bf30e162dd 100644 --- a/include/llvm/Analysis/DominanceFrontier.h +++ b/include/llvm/Analysis/DominanceFrontier.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/DominanceFrontier.h - Dominator Frontiers --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/DominanceFrontierImpl.h b/include/llvm/Analysis/DominanceFrontierImpl.h index 99224c0bf131..aa764be93b91 100644 --- a/include/llvm/Analysis/DominanceFrontierImpl.h +++ b/include/llvm/Analysis/DominanceFrontierImpl.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/DominanceFrontier.h - Dominator Frontiers --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/EHPersonalities.h b/include/llvm/Analysis/EHPersonalities.h index fe0e65b828ca..d89aa11617b5 100644 --- a/include/llvm/Analysis/EHPersonalities.h +++ b/include/llvm/Analysis/EHPersonalities.h @@ -1,9 +1,8 @@ //===- EHPersonalities.h - Compute EH-related information -----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Analysis/GlobalsModRef.h b/include/llvm/Analysis/GlobalsModRef.h index 3a664ca6ef50..d3fcfc2d41ab 100644 --- a/include/llvm/Analysis/GlobalsModRef.h +++ b/include/llvm/Analysis/GlobalsModRef.h @@ -1,9 +1,8 @@ //===- GlobalsModRef.h - Simple Mod/Ref AA for Globals ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -85,10 +84,12 @@ public: //------------------------------------------------ // Implement the AliasAnalysis API // - AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB); + AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB, + AAQueryInfo &AAQI); using AAResultBase::getModRefInfo; - ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc); + ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc, + AAQueryInfo &AAQI); /// getModRefBehavior - Return the behavior of the specified function if /// called from the specified call site. The call site may be null in which @@ -114,7 +115,7 @@ private: bool isNonEscapingGlobalNoAlias(const GlobalValue *GV, const Value *V); ModRefInfo getModRefInfoForArgument(const CallBase *Call, - const GlobalValue *GV); + const GlobalValue *GV, AAQueryInfo &AAQI); }; /// Analysis pass providing a never-invalidated alias analysis result. diff --git a/include/llvm/Analysis/GuardUtils.h b/include/llvm/Analysis/GuardUtils.h index 3b151eeafc81..41e7b7c06c75 100644 --- a/include/llvm/Analysis/GuardUtils.h +++ b/include/llvm/Analysis/GuardUtils.h @@ -1,9 +1,8 @@ //===-- GuardUtils.h - Utils for work with guards ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Utils that are used to perform analyzes related to guards and their @@ -15,12 +14,31 @@ namespace llvm { +class BasicBlock; class User; +class Value; -/// Returns true iff \p U has semantics of a guard. +/// Returns true iff \p U has semantics of a guard expressed in a form of call +/// of llvm.experimental.guard intrinsic. bool isGuard(const User *U); +/// Returns true iff \p U has semantics of a guard expressed in a form of a +/// widenable conditional branch to deopt block. +bool isGuardAsWidenableBranch(const User *U); + +/// If U is widenable branch looking like: +/// %cond = ... +/// %wc = call i1 @llvm.experimental.widenable.condition() +/// %branch_cond = and i1 %cond, %wc +/// br i1 %branch_cond, label %if_true_bb, label %if_false_bb ; <--- U +/// The function returns true, and the values %cond and %wc and blocks +/// %if_true_bb, if_false_bb are returned in +/// the parameters (Condition, WidenableCondition, IfTrueBB and IfFalseFF) +/// respectively. If \p U does not match this pattern, return false. +bool parseWidenableBranch(const User *U, Value *&Condition, + Value *&WidenableCondition, BasicBlock *&IfTrueBB, + BasicBlock *&IfFalseBB); + } // llvm #endif // LLVM_ANALYSIS_GUARDUTILS_H - diff --git a/include/llvm/Analysis/IVDescriptors.h b/include/llvm/Analysis/IVDescriptors.h index 64b4ae23cc59..7be1fd3f5788 100644 --- a/include/llvm/Analysis/IVDescriptors.h +++ b/include/llvm/Analysis/IVDescriptors.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/IVDescriptors.h - IndVar Descriptors -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -90,10 +89,12 @@ public: RecurrenceDescriptor() = default; RecurrenceDescriptor(Value *Start, Instruction *Exit, RecurrenceKind K, - MinMaxRecurrenceKind MK, Instruction *UAI, Type *RT, - bool Signed, SmallPtrSetImpl &CI) - : StartValue(Start), LoopExitInstr(Exit), Kind(K), MinMaxKind(MK), - UnsafeAlgebraInst(UAI), RecurrenceType(RT), IsSigned(Signed) { + FastMathFlags FMF, MinMaxRecurrenceKind MK, + Instruction *UAI, Type *RT, bool Signed, + SmallPtrSetImpl &CI) + : StartValue(Start), LoopExitInstr(Exit), Kind(K), FMF(FMF), + MinMaxKind(MK), UnsafeAlgebraInst(UAI), RecurrenceType(RT), + IsSigned(Signed) { CastInsts.insert(CI.begin(), CI.end()); } @@ -199,6 +200,8 @@ public: MinMaxRecurrenceKind getMinMaxRecurrenceKind() { return MinMaxKind; } + FastMathFlags getFastMathFlags() { return FMF; } + TrackingVH getRecurrenceStartValue() { return StartValue; } Instruction *getLoopExitInstr() { return LoopExitInstr; } @@ -238,6 +241,9 @@ private: Instruction *LoopExitInstr = nullptr; // The kind of the recurrence. RecurrenceKind Kind = RK_NoRecurrence; + // The fast-math flags on the recurrent instructions. We propagate these + // fast-math flags into the vectorized FP instructions we generate. + FastMathFlags FMF; // If this a min/max recurrence the kind of recurrence. MinMaxRecurrenceKind MinMaxKind = MRK_Invalid; // First occurrence of unasfe algebra in the PHI's use-chain. @@ -309,12 +315,16 @@ public: /// not have the "fast-math" property. Such operation requires a relaxed FP /// mode. bool hasUnsafeAlgebra() { - return InductionBinOp && !cast(InductionBinOp)->isFast(); + return (IK == IK_FpInduction) && InductionBinOp && + !cast(InductionBinOp)->isFast(); } /// Returns induction operator that does not have "fast-math" property /// and requires FP unsafe mode. Instruction *getUnsafeAlgebraInst() { + if (IK != IK_FpInduction) + return nullptr; + if (!InductionBinOp || cast(InductionBinOp)->isFast()) return nullptr; return InductionBinOp; diff --git a/include/llvm/Analysis/IVUsers.h b/include/llvm/Analysis/IVUsers.h index 035b974c5c1d..f8ea3bcca229 100644 --- a/include/llvm/Analysis/IVUsers.h +++ b/include/llvm/Analysis/IVUsers.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/IVUsers.h - Induction Variable Users -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/IndirectCallPromotionAnalysis.h b/include/llvm/Analysis/IndirectCallPromotionAnalysis.h index be3a28424cf5..8a05e913a910 100644 --- a/include/llvm/Analysis/IndirectCallPromotionAnalysis.h +++ b/include/llvm/Analysis/IndirectCallPromotionAnalysis.h @@ -1,9 +1,8 @@ //===- IndirectCallPromotionAnalysis.h - Indirect call analysis -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/Analysis/IndirectCallVisitor.h b/include/llvm/Analysis/IndirectCallVisitor.h index d00cf63368f1..1d1f3f4cc5c0 100644 --- a/include/llvm/Analysis/IndirectCallVisitor.h +++ b/include/llvm/Analysis/IndirectCallVisitor.h @@ -1,9 +1,8 @@ //===-- IndirectCallVisitor.h - indirect call visitor ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/InlineCost.h b/include/llvm/Analysis/InlineCost.h index 4c270354b0c4..611c9de24e47 100644 --- a/include/llvm/Analysis/InlineCost.h +++ b/include/llvm/Analysis/InlineCost.h @@ -1,9 +1,8 @@ //===- InlineCost.h - Cost analysis for inliner -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -23,7 +22,7 @@ namespace llvm { class AssumptionCacheTracker; class BlockFrequencyInfo; -class CallSite; +class CallBase; class DataLayout; class Function; class ProfileSummaryInfo; @@ -68,10 +67,10 @@ class InlineCost { }; /// The estimated cost of inlining this callsite. - const int Cost; + int Cost; /// The adjusted threshold against which this cost was computed. - const int Threshold; + int Threshold; /// Must be set for Always and Never instances. const char *Reason = nullptr; @@ -200,7 +199,7 @@ InlineParams getInlineParams(unsigned OptLevel, unsigned SizeOptLevel); /// Return the cost associated with a callsite, including parameter passing /// and the call/return instruction. -int getCallsiteCost(CallSite CS, const DataLayout &DL); +int getCallsiteCost(CallBase &Call, const DataLayout &DL); /// Get an InlineCost object representing the cost of inlining this /// callsite. @@ -214,7 +213,7 @@ int getCallsiteCost(CallSite CS, const DataLayout &DL); /// Also note that calling this function *dynamically* computes the cost of /// inlining the callsite. It is an expensive, heavyweight call. InlineCost getInlineCost( - CallSite CS, const InlineParams &Params, TargetTransformInfo &CalleeTTI, + CallBase &Call, const InlineParams &Params, TargetTransformInfo &CalleeTTI, std::function &GetAssumptionCache, Optional> GetBFI, ProfileSummaryInfo *PSI, OptimizationRemarkEmitter *ORE = nullptr); @@ -225,14 +224,14 @@ InlineCost getInlineCost( /// parameter in all other respects. // InlineCost -getInlineCost(CallSite CS, Function *Callee, const InlineParams &Params, +getInlineCost(CallBase &Call, Function *Callee, const InlineParams &Params, TargetTransformInfo &CalleeTTI, std::function &GetAssumptionCache, Optional> GetBFI, ProfileSummaryInfo *PSI, OptimizationRemarkEmitter *ORE); /// Minimal filter to detect invalid constructs for inlining. -bool isInlineViable(Function &Callee); +InlineResult isInlineViable(Function &Callee); } #endif diff --git a/include/llvm/Analysis/InstructionPrecedenceTracking.h b/include/llvm/Analysis/InstructionPrecedenceTracking.h index 073e6ec3b7f6..3c3981066a49 100644 --- a/include/llvm/Analysis/InstructionPrecedenceTracking.h +++ b/include/llvm/Analysis/InstructionPrecedenceTracking.h @@ -1,9 +1,8 @@ //===-- InstructionPrecedenceTracking.h -------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Implements a class that is able to define some instructions as "special" @@ -93,7 +92,7 @@ public: /// example, throwing calls and guards do not always do this. If we need to know /// for sure that some instruction is guaranteed to execute if the given block /// is reached, then we need to make sure that there is no implicit control flow -/// instruction (ICFI) preceeding it. For example, this check is required if we +/// instruction (ICFI) preceding it. For example, this check is required if we /// perform PRE moving non-speculable instruction to other place. class ImplicitControlFlowTracking : public InstructionPrecedenceTracking { public: diff --git a/include/llvm/Analysis/InstructionSimplify.h b/include/llvm/Analysis/InstructionSimplify.h index 6662e91037e1..054ffca7215e 100644 --- a/include/llvm/Analysis/InstructionSimplify.h +++ b/include/llvm/Analysis/InstructionSimplify.h @@ -1,9 +1,8 @@ //===-- InstructionSimplify.h - Fold instrs into simpler forms --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -41,8 +40,8 @@ class Function; template class AnalysisManager; template class ArrayRef; class AssumptionCache; +class CallBase; class DominatorTree; -class ImmutableCallSite; class DataLayout; class FastMathFlags; struct LoopStandardAnalysisResults; @@ -118,6 +117,10 @@ struct SimplifyQuery { // deprecated. // Please use the SimplifyQuery versions in new code. +/// Given operand for an FNeg, fold the result or return null. +Value *SimplifyFNegInst(Value *Op, FastMathFlags FMF, + const SimplifyQuery &Q); + /// Given operands for an Add, fold the result or return null. Value *SimplifyAddInst(Value *LHS, Value *RHS, bool isNSW, bool isNUW, const SimplifyQuery &Q); @@ -228,6 +231,15 @@ Value *SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask, Value *SimplifyCmpInst(unsigned Predicate, Value *LHS, Value *RHS, const SimplifyQuery &Q); +/// Given operand for a UnaryOperator, fold the result or return null. +Value *SimplifyUnOp(unsigned Opcode, Value *Op, const SimplifyQuery &Q); + +/// Given operand for an FP UnaryOperator, fold the result or return null. +/// In contrast to SimplifyUnOp, try to use FastMathFlag when folding the +/// result. In case we don't need FastMathFlags, simply fall to SimplifyUnOp. +Value *SimplifyFPUnOp(unsigned Opcode, Value *Op, FastMathFlags FMF, + const SimplifyQuery &Q); + /// Given operands for a BinaryOperator, fold the result or return null. Value *SimplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, const SimplifyQuery &Q); @@ -239,16 +251,7 @@ Value *SimplifyFPBinOp(unsigned Opcode, Value *LHS, Value *RHS, FastMathFlags FMF, const SimplifyQuery &Q); /// Given a callsite, fold the result or return null. -Value *SimplifyCall(ImmutableCallSite CS, const SimplifyQuery &Q); - -/// Given a function and iterators over arguments, fold the result or return -/// null. -Value *SimplifyCall(ImmutableCallSite CS, Value *V, User::op_iterator ArgBegin, - User::op_iterator ArgEnd, const SimplifyQuery &Q); - -/// Given a function and set of arguments, fold the result or return null. -Value *SimplifyCall(ImmutableCallSite CS, Value *V, ArrayRef Args, - const SimplifyQuery &Q); +Value *SimplifyCall(CallBase *Call, const SimplifyQuery &Q); /// See if we can compute a simplified version of this instruction. If not, /// return null. diff --git a/include/llvm/Analysis/Interval.h b/include/llvm/Analysis/Interval.h index f3714dddedd5..5c9a4535bc7f 100644 --- a/include/llvm/Analysis/Interval.h +++ b/include/llvm/Analysis/Interval.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/Interval.h - Interval Class Declaration ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/IntervalIterator.h b/include/llvm/Analysis/IntervalIterator.h index 6ffcae592e98..efaaf9715b3d 100644 --- a/include/llvm/Analysis/IntervalIterator.h +++ b/include/llvm/Analysis/IntervalIterator.h @@ -1,9 +1,8 @@ //===- IntervalIterator.h - Interval Iterator Declaration -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/IntervalPartition.h b/include/llvm/Analysis/IntervalPartition.h index 50335165711f..5b127c25a2b8 100644 --- a/include/llvm/Analysis/IntervalPartition.h +++ b/include/llvm/Analysis/IntervalPartition.h @@ -1,9 +1,8 @@ //===- IntervalPartition.h - Interval partition Calculation -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/IteratedDominanceFrontier.h b/include/llvm/Analysis/IteratedDominanceFrontier.h index 3083db75b81c..7c826780c318 100644 --- a/include/llvm/Analysis/IteratedDominanceFrontier.h +++ b/include/llvm/Analysis/IteratedDominanceFrontier.h @@ -1,101 +1,89 @@ //===- IteratedDominanceFrontier.h - Calculate IDF --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -/// \file -/// Compute iterated dominance frontiers using a linear time algorithm. -/// -/// The algorithm used here is based on: -/// -/// Sreedhar and Gao. A linear time algorithm for placing phi-nodes. -/// In Proceedings of the 22nd ACM SIGPLAN-SIGACT Symposium on Principles of -/// Programming Languages -/// POPL '95. ACM, New York, NY, 62-73. -/// -/// It has been modified to not explicitly use the DJ graph data structure and -/// to directly compute pruned SSA using per-variable liveness information. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef LLVM_ANALYSIS_IDF_H #define LLVM_ANALYSIS_IDF_H -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallPtrSet.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFGDiff.h" -#include "llvm/IR/Dominators.h" +#include "llvm/Support/GenericIteratedDominanceFrontier.h" namespace llvm { -/// Determine the iterated dominance frontier, given a set of defining -/// blocks, and optionally, a set of live-in blocks. -/// -/// In turn, the results can be used to place phi nodes. -/// -/// This algorithm is a linear time computation of Iterated Dominance Frontiers, -/// pruned using the live-in set. -/// By default, liveness is not used to prune the IDF computation. -/// The template parameters should be either BasicBlock* or Inverse, depending on if you want the forward or reverse IDF. -template -class IDFCalculator { - public: - IDFCalculator(DominatorTreeBase &DT) - : DT(DT), GD(nullptr), useLiveIn(false) {} - - IDFCalculator(DominatorTreeBase &DT, - const GraphDiff *GD) - : DT(DT), GD(GD), useLiveIn(false) {} - - /// Give the IDF calculator the set of blocks in which the value is - /// defined. This is equivalent to the set of starting blocks it should be - /// calculating the IDF for (though later gets pruned based on liveness). - /// - /// Note: This set *must* live for the entire lifetime of the IDF calculator. - void setDefiningBlocks(const SmallPtrSetImpl &Blocks) { - DefBlocks = &Blocks; - } - - /// Give the IDF calculator the set of blocks in which the value is - /// live on entry to the block. This is used to prune the IDF calculation to - /// not include blocks where any phi insertion would be dead. - /// - /// Note: This set *must* live for the entire lifetime of the IDF calculator. - - void setLiveInBlocks(const SmallPtrSetImpl &Blocks) { - LiveInBlocks = &Blocks; - useLiveIn = true; - } +class BasicBlock; - /// Reset the live-in block set to be empty, and tell the IDF - /// calculator to not use liveness anymore. - void resetLiveInBlocks() { - LiveInBlocks = nullptr; - useLiveIn = false; +namespace IDFCalculatorDetail { + +/// Specialization for BasicBlock for the optional use of GraphDiff. +template struct ChildrenGetterTy { + using NodeRef = BasicBlock *; + using ChildrenTy = SmallVector; + + ChildrenGetterTy() = default; + ChildrenGetterTy(const GraphDiff *GD) : GD(GD) { + assert(GD); } - /// Calculate iterated dominance frontiers - /// - /// This uses the linear-time phi algorithm based on DJ-graphs mentioned in - /// the file-level comment. It performs DF->IDF pruning using the live-in - /// set, to avoid computing the IDF for blocks where an inserted PHI node - /// would be dead. - void calculate(SmallVectorImpl &IDFBlocks); - -private: - DominatorTreeBase &DT; - const GraphDiff *GD; - bool useLiveIn; - const SmallPtrSetImpl *LiveInBlocks; - const SmallPtrSetImpl *DefBlocks; + ChildrenTy get(const NodeRef &N); + + const GraphDiff *GD = nullptr; }; -typedef IDFCalculator ForwardIDFCalculator; -typedef IDFCalculator, true> ReverseIDFCalculator; + +} // end of namespace IDFCalculatorDetail + +template +class IDFCalculator final : public IDFCalculatorBase { +public: + using IDFCalculatorBase = + typename llvm::IDFCalculatorBase; + using ChildrenGetterTy = typename IDFCalculatorBase::ChildrenGetterTy; + + IDFCalculator(DominatorTreeBase &DT) + : IDFCalculatorBase(DT) {} + + IDFCalculator(DominatorTreeBase &DT, + const GraphDiff *GD) + : IDFCalculatorBase(DT, ChildrenGetterTy(GD)) { + assert(GD); + } +}; + +using ForwardIDFCalculator = IDFCalculator; +using ReverseIDFCalculator = IDFCalculator; + +//===----------------------------------------------------------------------===// +// Implementation. +//===----------------------------------------------------------------------===// + +namespace IDFCalculatorDetail { + +template +typename ChildrenGetterTy::ChildrenTy +ChildrenGetterTy::get(const NodeRef &N) { + + using OrderedNodeTy = + typename IDFCalculatorBase::OrderedNodeTy; + + if (!GD) { + auto Children = children(N); + return {Children.begin(), Children.end()}; + } + + using SnapShotBBPairTy = + std::pair *, OrderedNodeTy>; + + ChildrenTy Ret; + for (const auto &SnapShotBBPair : children({GD, N})) + Ret.emplace_back(SnapShotBBPair.second); + return Ret; } + +} // end of namespace IDFCalculatorDetail + +} // end of namespace llvm + #endif diff --git a/include/llvm/Analysis/LazyBlockFrequencyInfo.h b/include/llvm/Analysis/LazyBlockFrequencyInfo.h index d1afb63d7e08..0e7dc943bacf 100644 --- a/include/llvm/Analysis/LazyBlockFrequencyInfo.h +++ b/include/llvm/Analysis/LazyBlockFrequencyInfo.h @@ -1,9 +1,8 @@ //===- LazyBlockFrequencyInfo.h - Lazy Block Frequency Analysis -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/LazyBranchProbabilityInfo.h b/include/llvm/Analysis/LazyBranchProbabilityInfo.h index 9e6bcfedcbb9..cae0778cd16d 100644 --- a/include/llvm/Analysis/LazyBranchProbabilityInfo.h +++ b/include/llvm/Analysis/LazyBranchProbabilityInfo.h @@ -1,9 +1,8 @@ //===- LazyBranchProbabilityInfo.h - Lazy Branch Probability ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/LazyCallGraph.h b/include/llvm/Analysis/LazyCallGraph.h index d1ec6a9dcc55..2d83929211e2 100644 --- a/include/llvm/Analysis/LazyCallGraph.h +++ b/include/llvm/Analysis/LazyCallGraph.h @@ -1,9 +1,8 @@ //===- LazyCallGraph.h - Analysis of a Module's call graph ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -39,6 +38,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/PointerIntPair.h" +#include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" @@ -1083,12 +1083,26 @@ public: continue; } + // The blockaddress constant expression is a weird special case, we can't + // generically walk its operands the way we do for all other constants. if (BlockAddress *BA = dyn_cast(C)) { - // The blockaddress constant expression is a weird special case, we - // can't generically walk its operands the way we do for all other - // constants. - if (Visited.insert(BA->getFunction()).second) - Worklist.push_back(BA->getFunction()); + // If we've already visited the function referred to by the block + // address, we don't need to revisit it. + if (Visited.count(BA->getFunction())) + continue; + + // If all of the blockaddress' users are instructions within the + // referred to function, we don't need to insert a cycle. + if (llvm::all_of(BA->users(), [&](User *U) { + if (Instruction *I = dyn_cast(U)) + return I->getFunction() == BA->getFunction(); + return false; + })) + continue; + + // Otherwise we should go visit the referred to function. + Visited.insert(BA->getFunction()); + Worklist.push_back(BA->getFunction()); continue; } diff --git a/include/llvm/Analysis/LazyValueInfo.h b/include/llvm/Analysis/LazyValueInfo.h index 1a4fdb591427..570a5044f6f8 100644 --- a/include/llvm/Analysis/LazyValueInfo.h +++ b/include/llvm/Analysis/LazyValueInfo.h @@ -1,9 +1,8 @@ //===- LazyValueInfo.h - Value constraint analysis --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/LegacyDivergenceAnalysis.h b/include/llvm/Analysis/LegacyDivergenceAnalysis.h index fc426ad7fb64..0a338b816640 100644 --- a/include/llvm/Analysis/LegacyDivergenceAnalysis.h +++ b/include/llvm/Analysis/LegacyDivergenceAnalysis.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/LegacyDivergenceAnalysis.h - KernelDivergence Analysis -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/Lint.h b/include/llvm/Analysis/Lint.h index db5919fd91c7..0fea81e215c9 100644 --- a/include/llvm/Analysis/Lint.h +++ b/include/llvm/Analysis/Lint.h @@ -1,9 +1,8 @@ //===-- llvm/Analysis/Lint.h - LLVM IR Lint ---------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/Loads.h b/include/llvm/Analysis/Loads.h index f110c28bfc6d..5df6bb02308d 100644 --- a/include/llvm/Analysis/Loads.h +++ b/include/llvm/Analysis/Loads.h @@ -1,9 +1,8 @@ //===- Loads.h - Local load analysis --------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -26,7 +25,8 @@ class MDNode; /// Return true if this is always a dereferenceable pointer. If the context /// instruction is specified perform context-sensitive analysis and return true /// if the pointer is dereferenceable at the specified instruction. -bool isDereferenceablePointer(const Value *V, const DataLayout &DL, +bool isDereferenceablePointer(const Value *V, Type *Ty, + const DataLayout &DL, const Instruction *CtxI = nullptr, const DominatorTree *DT = nullptr); @@ -34,8 +34,8 @@ bool isDereferenceablePointer(const Value *V, const DataLayout &DL, /// greater or equal than requested. If the context instruction is specified /// performs context-sensitive analysis and returns true if the pointer is /// dereferenceable at the specified instruction. -bool isDereferenceableAndAlignedPointer(const Value *V, unsigned Align, - const DataLayout &DL, +bool isDereferenceableAndAlignedPointer(const Value *V, Type *Ty, + unsigned Align, const DataLayout &DL, const Instruction *CtxI = nullptr, const DominatorTree *DT = nullptr); @@ -56,7 +56,20 @@ bool isDereferenceableAndAlignedPointer(const Value *V, unsigned Align, /// If it is not obviously safe to load from the specified pointer, we do a /// quick local scan of the basic block containing ScanFrom, to determine if /// the address is already accessed. -bool isSafeToLoadUnconditionally(Value *V, unsigned Align, +bool isSafeToLoadUnconditionally(Value *V, unsigned Align, APInt &Size, + const DataLayout &DL, + Instruction *ScanFrom = nullptr, + const DominatorTree *DT = nullptr); + +/// Return true if we know that executing a load from this value cannot trap. +/// +/// If DT and ScanFrom are specified this method performs context-sensitive +/// analysis and returns true if it is safe to load immediately before ScanFrom. +/// +/// If it is not obviously safe to load from the specified pointer, we do a +/// quick local scan of the basic block containing ScanFrom, to determine if +/// the address is already accessed. +bool isSafeToLoadUnconditionally(Value *V, Type *Ty, unsigned Align, const DataLayout &DL, Instruction *ScanFrom = nullptr, const DominatorTree *DT = nullptr); diff --git a/include/llvm/Analysis/LoopAccessAnalysis.h b/include/llvm/Analysis/LoopAccessAnalysis.h index 4ed00e207753..9e9aaa32c64f 100644 --- a/include/llvm/Analysis/LoopAccessAnalysis.h +++ b/include/llvm/Analysis/LoopAccessAnalysis.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/LoopAccessAnalysis.h -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -523,6 +522,11 @@ public: /// no memory dependence cycles. bool canVectorizeMemory() const { return CanVecMem; } + /// Return true if there is a convergent operation in the loop. There may + /// still be reported runtime pointer checks that would be required, but it is + /// not legal to insert them. + bool hasConvergentOp() const { return HasConvergentOp; } + const RuntimePointerChecking *getRuntimePointerChecking() const { return PtrRtChecking.get(); } @@ -643,6 +647,7 @@ private: /// Cache the result of analyzeLoop. bool CanVecMem; + bool HasConvergentOp; /// Indicator that there are non vectorizable stores to a uniform address. bool HasDependenceInvolvingLoopInvariantAddress; diff --git a/include/llvm/Analysis/LoopAnalysisManager.h b/include/llvm/Analysis/LoopAnalysisManager.h index 00e562c4f31f..368a810cfa67 100644 --- a/include/llvm/Analysis/LoopAnalysisManager.h +++ b/include/llvm/Analysis/LoopAnalysisManager.h @@ -1,9 +1,8 @@ //===- LoopAnalysisManager.h - Loop analysis management ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -62,9 +61,6 @@ struct LoopStandardAnalysisResults { MemorySSA *MSSA; }; -/// Enables memory ssa as a dependency for loop passes. -extern cl::opt EnableMSSALoopDependency; - /// Extern template declaration for the analysis set for this IR unit. extern template class AllAnalysesOn; diff --git a/include/llvm/Analysis/LoopInfo.h b/include/llvm/Analysis/LoopInfo.h index 72873546a068..584eb3a8c854 100644 --- a/include/llvm/Analysis/LoopInfo.h +++ b/include/llvm/Analysis/LoopInfo.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/LoopInfo.h - Natural Loop Calculator -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -55,8 +54,11 @@ namespace llvm { class DominatorTree; class LoopInfo; class Loop; +class InductionDescriptor; class MDNode; +class MemorySSAUpdater; class PHINode; +class ScalarEvolution; class raw_ostream; template class DominatorTreeBase; template class LoopInfoBase; @@ -199,9 +201,10 @@ public: } /// True if terminator in the block can branch to another block that is - /// outside of the current loop. + /// outside of the current loop. \p BB must be inside the loop. bool isLoopExiting(const BlockT *BB) const { assert(!isInvalid() && "Loop not in a valid state!"); + assert(contains(BB) && "Exiting block must be part of the loop"); for (const auto &Succ : children(BB)) { if (!contains(Succ)) return true; @@ -267,16 +270,20 @@ public: /// Return all unique successor blocks of this loop. /// These are the blocks _outside of the current loop_ which are branched to. - /// This assumes that loop exits are in canonical form, i.e. all exits are - /// dedicated exits. void getUniqueExitBlocks(SmallVectorImpl &ExitBlocks) const; + /// Return all unique successor blocks of this loop except successors from + /// Latch block are not considered. If the exit comes from Latch has also + /// non Latch predecessor in a loop it will be added to ExitBlocks. + /// These are the blocks _outside of the current loop_ which are branched to. + void getUniqueNonLatchExitBlocks(SmallVectorImpl &ExitBlocks) const; + /// If getUniqueExitBlocks would return exactly one block, return that block. /// Otherwise return null. BlockT *getUniqueExitBlock() const; /// Edge type. - typedef std::pair Edge; + typedef std::pair Edge; /// Return all pairs of (_inside_block_,_outside_block_). void getExitEdges(SmallVectorImpl &ExitEdges) const; @@ -309,6 +316,40 @@ public: LoopLatches.push_back(Pred); } + /// Return all inner loops in the loop nest rooted by the loop in preorder, + /// with siblings in forward program order. + template + static void getInnerLoopsInPreorder(const LoopT &L, + SmallVectorImpl &PreOrderLoops) { + SmallVector PreOrderWorklist; + PreOrderWorklist.append(L.rbegin(), L.rend()); + + while (!PreOrderWorklist.empty()) { + LoopT *L = PreOrderWorklist.pop_back_val(); + // Sub-loops are stored in forward program order, but will process the + // worklist backwards so append them in reverse order. + PreOrderWorklist.append(L->rbegin(), L->rend()); + PreOrderLoops.push_back(L); + } + } + + /// Return all loops in the loop nest rooted by the loop in preorder, with + /// siblings in forward program order. + SmallVector getLoopsInPreorder() const { + SmallVector PreOrderLoops; + const LoopT *CurLoop = static_cast(this); + PreOrderLoops.push_back(CurLoop); + getInnerLoopsInPreorder(*CurLoop, PreOrderLoops); + return PreOrderLoops; + } + SmallVector getLoopsInPreorder() { + SmallVector PreOrderLoops; + LoopT *CurLoop = static_cast(this); + PreOrderLoops.push_back(CurLoop); + getInnerLoopsInPreorder(*CurLoop, PreOrderLoops); + return PreOrderLoops; + } + //===--------------------------------------------------------------------===// // APIs for updating loop information after changing the CFG // @@ -471,7 +512,7 @@ public: public: LocRange() {} - LocRange(DebugLoc Start) : Start(std::move(Start)), End(std::move(Start)) {} + LocRange(DebugLoc Start) : Start(Start), End(Start) {} LocRange(DebugLoc Start, DebugLoc End) : Start(std::move(Start)), End(std::move(End)) {} @@ -499,7 +540,8 @@ public: /// If InsertPt is specified, it is the point to hoist instructions to. /// If null, the terminator of the loop preheader is used. bool makeLoopInvariant(Value *V, bool &Changed, - Instruction *InsertPt = nullptr) const; + Instruction *InsertPt = nullptr, + MemorySSAUpdater *MSSAU = nullptr) const; /// If the given instruction is inside of the loop and it can be hoisted, do /// so to make it trivially loop-invariant. @@ -511,7 +553,8 @@ public: /// If null, the terminator of the loop preheader is used. /// bool makeLoopInvariant(Instruction *I, bool &Changed, - Instruction *InsertPt = nullptr) const; + Instruction *InsertPt = nullptr, + MemorySSAUpdater *MSSAU = nullptr) const; /// Check to see if the loop has a canonical induction variable: an integer /// recurrence that starts at 0 and increments by one each time through the @@ -522,6 +565,170 @@ public: /// PHINode *getCanonicalInductionVariable() const; + /// Obtain the unique incoming and back edge. Return false if they are + /// non-unique or the loop is dead; otherwise, return true. + bool getIncomingAndBackEdge(BasicBlock *&Incoming, + BasicBlock *&Backedge) const; + + /// Below are some utilities to get loop bounds and induction variable, and + /// check if a given phinode is an auxiliary induction variable, as well as + /// checking if the loop is canonical. + /// + /// Here is an example: + /// \code + /// for (int i = lb; i < ub; i+=step) + /// + /// --- pseudo LLVMIR --- + /// beforeloop: + /// guardcmp = (lb < ub) + /// if (guardcmp) goto preheader; else goto afterloop + /// preheader: + /// loop: + /// i_1 = phi[{lb, preheader}, {i_2, latch}] + /// + /// i_2 = i_1 + step + /// latch: + /// cmp = (i_2 < ub) + /// if (cmp) goto loop + /// exit: + /// afterloop: + /// \endcode + /// + /// - getBounds + /// - getInitialIVValue --> lb + /// - getStepInst --> i_2 = i_1 + step + /// - getStepValue --> step + /// - getFinalIVValue --> ub + /// - getCanonicalPredicate --> '<' + /// - getDirection --> Increasing + /// + /// - getInductionVariable --> i_1 + /// - isAuxiliaryInductionVariable(x) --> true if x == i_1 + /// - isCanonical --> false + struct LoopBounds { + /// Return the LoopBounds object if + /// - the given \p IndVar is an induction variable + /// - the initial value of the induction variable can be found + /// - the step instruction of the induction variable can be found + /// - the final value of the induction variable can be found + /// + /// Else None. + static Optional getBounds(const Loop &L, PHINode &IndVar, + ScalarEvolution &SE); + + /// Get the initial value of the loop induction variable. + Value &getInitialIVValue() const { return InitialIVValue; } + + /// Get the instruction that updates the loop induction variable. + Instruction &getStepInst() const { return StepInst; } + + /// Get the step that the loop induction variable gets updated by in each + /// loop iteration. Return nullptr if not found. + Value *getStepValue() const { return StepValue; } + + /// Get the final value of the loop induction variable. + Value &getFinalIVValue() const { return FinalIVValue; } + + /// Return the canonical predicate for the latch compare instruction, if + /// able to be calcuated. Else BAD_ICMP_PREDICATE. + /// + /// A predicate is considered as canonical if requirements below are all + /// satisfied: + /// 1. The first successor of the latch branch is the loop header + /// If not, inverse the predicate. + /// 2. One of the operands of the latch comparison is StepInst + /// If not, and + /// - if the current calcuated predicate is not ne or eq, flip the + /// predicate. + /// - else if the loop is increasing, return slt + /// (notice that it is safe to change from ne or eq to sign compare) + /// - else if the loop is decreasing, return sgt + /// (notice that it is safe to change from ne or eq to sign compare) + /// + /// Here is an example when both (1) and (2) are not satisfied: + /// \code + /// loop.header: + /// %iv = phi [%initialiv, %loop.preheader], [%inc, %loop.header] + /// %inc = add %iv, %step + /// %cmp = slt %iv, %finaliv + /// br %cmp, %loop.exit, %loop.header + /// loop.exit: + /// \endcode + /// - The second successor of the latch branch is the loop header instead + /// of the first successor (slt -> sge) + /// - The first operand of the latch comparison (%cmp) is the IndVar (%iv) + /// instead of the StepInst (%inc) (sge -> sgt) + /// + /// The predicate would be sgt if both (1) and (2) are satisfied. + /// getCanonicalPredicate() returns sgt for this example. + /// Note: The IR is not changed. + ICmpInst::Predicate getCanonicalPredicate() const; + + /// An enum for the direction of the loop + /// - for (int i = 0; i < ub; ++i) --> Increasing + /// - for (int i = ub; i > 0; --i) --> Descresing + /// - for (int i = x; i != y; i+=z) --> Unknown + enum class Direction { Increasing, Decreasing, Unknown }; + + /// Get the direction of the loop. + Direction getDirection() const; + + private: + LoopBounds(const Loop &Loop, Value &I, Instruction &SI, Value *SV, Value &F, + ScalarEvolution &SE) + : L(Loop), InitialIVValue(I), StepInst(SI), StepValue(SV), + FinalIVValue(F), SE(SE) {} + + const Loop &L; + + // The initial value of the loop induction variable + Value &InitialIVValue; + + // The instruction that updates the loop induction variable + Instruction &StepInst; + + // The value that the loop induction variable gets updated by in each loop + // iteration + Value *StepValue; + + // The final value of the loop induction variable + Value &FinalIVValue; + + ScalarEvolution &SE; + }; + + /// Return the struct LoopBounds collected if all struct members are found, + /// else None. + Optional getBounds(ScalarEvolution &SE) const; + + /// Return the loop induction variable if found, else return nullptr. + /// An instruction is considered as the loop induction variable if + /// - it is an induction variable of the loop; and + /// - it is used to determine the condition of the branch in the loop latch + /// + /// Note: the induction variable doesn't need to be canonical, i.e. starts at + /// zero and increments by one each time through the loop (but it can be). + PHINode *getInductionVariable(ScalarEvolution &SE) const; + + /// Get the loop induction descriptor for the loop induction variable. Return + /// true if the loop induction variable is found. + bool getInductionDescriptor(ScalarEvolution &SE, + InductionDescriptor &IndDesc) const; + + /// Return true if the given PHINode \p AuxIndVar is + /// - in the loop header + /// - not used outside of the loop + /// - incremented by a loop invariant step for each loop iteration + /// - step instruction opcode should be add or sub + /// Note: auxiliary induction variable is not required to be used in the + /// conditional branch in the loop latch. (but it can be) + bool isAuxiliaryInductionVariable(PHINode &AuxIndVar, + ScalarEvolution &SE) const; + + /// Return true if the loop induction variable starts at zero and increments + /// by one each time through the loop. + bool isCanonical(ScalarEvolution &SE) const; + /// Return true if the Loop is in LCSSA form. bool isLCSSAForm(DominatorTree &DT) const; @@ -1015,6 +1222,26 @@ MDNode *findOptionMDForLoop(const Loop *TheLoop, StringRef Name); /// is representing an access group. bool isValidAsAccessGroup(MDNode *AccGroup); +/// Create a new LoopID after the loop has been transformed. +/// +/// This can be used when no follow-up loop attributes are defined +/// (llvm::makeFollowupLoopID returning None) to stop transformations to be +/// applied again. +/// +/// @param Context The LLVMContext in which to create the new LoopID. +/// @param OrigLoopID The original LoopID; can be nullptr if the original +/// loop has no LoopID. +/// @param RemovePrefixes Remove all loop attributes that have these prefixes. +/// Use to remove metadata of the transformation that has +/// been applied. +/// @param AddAttrs Add these loop attributes to the new LoopID. +/// +/// @return A new LoopID that can be applied using Loop::setLoopID(). +llvm::MDNode * +makePostTransformationMetadata(llvm::LLVMContext &Context, MDNode *OrigLoopID, + llvm::ArrayRef RemovePrefixes, + llvm::ArrayRef AddAttrs); + } // End llvm namespace #endif diff --git a/include/llvm/Analysis/LoopInfoImpl.h b/include/llvm/Analysis/LoopInfoImpl.h index 2b807919fedf..4c33dac9e21e 100644 --- a/include/llvm/Analysis/LoopInfoImpl.h +++ b/include/llvm/Analysis/LoopInfoImpl.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/LoopInfoImpl.h - Natural Loop Calculator ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -96,49 +95,36 @@ bool LoopBase::hasDedicatedExits() const { return true; } +// Helper function to get unique loop exits. Pred is a predicate pointing to +// BasicBlocks in a loop which should be considered to find loop exits. +template +void getUniqueExitBlocksHelper(const LoopT *L, + SmallVectorImpl &ExitBlocks, + PredicateT Pred) { + assert(!L->isInvalid() && "Loop not in a valid state!"); + SmallPtrSet Visited; + auto Filtered = make_filter_range(L->blocks(), Pred); + for (BlockT *BB : Filtered) + for (BlockT *Successor : children(BB)) + if (!L->contains(Successor)) + if (Visited.insert(Successor).second) + ExitBlocks.push_back(Successor); +} + template void LoopBase::getUniqueExitBlocks( SmallVectorImpl &ExitBlocks) const { - typedef GraphTraits BlockTraits; - typedef GraphTraits> InvBlockTraits; - - assert(hasDedicatedExits() && - "getUniqueExitBlocks assumes the loop has canonical form exits!"); - - SmallVector SwitchExitBlocks; - for (BlockT *Block : this->blocks()) { - SwitchExitBlocks.clear(); - for (BlockT *Successor : children(Block)) { - // If block is inside the loop then it is not an exit block. - if (contains(Successor)) - continue; - - BlockT *FirstPred = *InvBlockTraits::child_begin(Successor); - - // If current basic block is this exit block's first predecessor then only - // insert exit block in to the output ExitBlocks vector. This ensures that - // same exit block is not inserted twice into ExitBlocks vector. - if (Block != FirstPred) - continue; - - // If a terminator has more then two successors, for example SwitchInst, - // then it is possible that there are multiple edges from current block to - // one exit block. - if (std::distance(BlockTraits::child_begin(Block), - BlockTraits::child_end(Block)) <= 2) { - ExitBlocks.push_back(Successor); - continue; - } + getUniqueExitBlocksHelper(this, ExitBlocks, + [](const BlockT *BB) { return true; }); +} - // In case of multiple edges from current block to exit block, collect - // only one edge in ExitBlocks. Use switchExitBlocks to keep track of - // duplicate edges. - if (!is_contained(SwitchExitBlocks, Successor)) { - SwitchExitBlocks.push_back(Successor); - ExitBlocks.push_back(Successor); - } - } - } +template +void LoopBase::getUniqueNonLatchExitBlocks( + SmallVectorImpl &ExitBlocks) const { + const BlockT *Latch = getLoopLatch(); + assert(Latch && "Latch block must exists"); + getUniqueExitBlocksHelper(this, ExitBlocks, + [Latch](const BlockT *BB) { return BB != Latch; }); } template @@ -588,16 +574,9 @@ SmallVector LoopInfoBase::getLoopsInPreorder() { // FIXME: If we change the order of LoopInfo we will want to remove the // reverse here. for (LoopT *RootL : reverse(*this)) { - assert(PreOrderWorklist.empty() && - "Must start with an empty preorder walk worklist."); - PreOrderWorklist.push_back(RootL); - do { - LoopT *L = PreOrderWorklist.pop_back_val(); - // Sub-loops are stored in forward program order, but will process the - // worklist backwards so append them in reverse order. - PreOrderWorklist.append(L->rbegin(), L->rend()); - PreOrderLoops.push_back(L); - } while (!PreOrderWorklist.empty()); + auto PreOrderLoopsInRootL = RootL->getLoopsInPreorder(); + PreOrderLoops.append(PreOrderLoopsInRootL.begin(), + PreOrderLoopsInRootL.end()); } return PreOrderLoops; diff --git a/include/llvm/Analysis/LoopIterator.h b/include/llvm/Analysis/LoopIterator.h index 91c54b23029b..fa4da4283f55 100644 --- a/include/llvm/Analysis/LoopIterator.h +++ b/include/llvm/Analysis/LoopIterator.h @@ -1,9 +1,8 @@ //===--------- LoopIterator.h - Iterate over loop blocks --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This file defines iterators to visit the basic blocks within a loop. diff --git a/include/llvm/Analysis/LoopPass.h b/include/llvm/Analysis/LoopPass.h index 86cfecd9df11..9215ab34ec6d 100644 --- a/include/llvm/Analysis/LoopPass.h +++ b/include/llvm/Analysis/LoopPass.h @@ -1,9 +1,8 @@ //===- LoopPass.h - LoopPass class ----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/LoopUnrollAnalyzer.h b/include/llvm/Analysis/LoopUnrollAnalyzer.h index f45bf0b223b8..5f332e3cac16 100644 --- a/include/llvm/Analysis/LoopUnrollAnalyzer.h +++ b/include/llvm/Analysis/LoopUnrollAnalyzer.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/LoopUnrollAnalyzer.h - Loop Unroll Analyzer-*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/MemoryBuiltins.h b/include/llvm/Analysis/MemoryBuiltins.h index 5418128f16ef..49f9e58ffad7 100644 --- a/include/llvm/Analysis/MemoryBuiltins.h +++ b/include/llvm/Analysis/MemoryBuiltins.h @@ -1,9 +1,8 @@ //==- llvm/Analysis/MemoryBuiltins.h - Calls to memory builtins --*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,6 +18,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/Analysis/TargetFolder.h" +#include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstVisitor.h" @@ -84,6 +84,15 @@ bool isMallocOrCallocLikeFn(const Value *V, const TargetLibraryInfo *TLI, bool isAllocLikeFn(const Value *V, const TargetLibraryInfo *TLI, bool LookThroughBitCast = false); +/// Tests if a value is a call or invoke to a library function that +/// reallocates memory (e.g., realloc). +bool isReallocLikeFn(const Value *V, const TargetLibraryInfo *TLI, + bool LookThroughBitCast = false); + +/// Tests if a function is a call or invoke to a library function that +/// reallocates memory (e.g., realloc). +bool isReallocLikeFn(const Function *F, const TargetLibraryInfo *TLI); + //===----------------------------------------------------------------------===// // malloc Call Utility Functions. // @@ -135,6 +144,9 @@ inline CallInst *extractCallocCall(Value *I, const TargetLibraryInfo *TLI) { // free Call Utility Functions. // +/// isLibFreeFunction - Returns true if the function is a builtin free() +bool isLibFreeFunction(const Function *F, const LibFunc TLIFn); + /// isFreeCall - Returns non-null if the value is a call to the builtin free() const CallInst *isFreeCall(const Value *I, const TargetLibraryInfo *TLI); @@ -178,14 +190,13 @@ bool getObjectSize(const Value *Ptr, uint64_t &Size, const DataLayout &DL, const TargetLibraryInfo *TLI, ObjectSizeOpts Opts = {}); /// Try to turn a call to \@llvm.objectsize into an integer value of the given -/// Type. Returns null on failure. -/// If MustSucceed is true, this function will not return null, and may return -/// conservative values governed by the second argument of the call to -/// objectsize. -ConstantInt *lowerObjectSizeCall(IntrinsicInst *ObjectSize, - const DataLayout &DL, - const TargetLibraryInfo *TLI, - bool MustSucceed); +/// Type. Returns null on failure. If MustSucceed is true, this function will +/// not return null, and may return conservative values governed by the second +/// argument of the call to objectsize. +Value *lowerObjectSizeCall(IntrinsicInst *ObjectSize, const DataLayout &DL, + const TargetLibraryInfo *TLI, bool MustSucceed); + + using SizeOffsetType = std::pair; @@ -252,7 +263,7 @@ using SizeOffsetEvalType = std::pair; /// May create code to compute the result at run-time. class ObjectSizeOffsetEvaluator : public InstVisitor { - using BuilderTy = IRBuilder; + using BuilderTy = IRBuilder; using WeakEvalType = std::pair; using CacheMapTy = DenseMap; using PtrSetTy = SmallPtrSet; @@ -265,17 +276,18 @@ class ObjectSizeOffsetEvaluator Value *Zero; CacheMapTy CacheMap; PtrSetTy SeenVals; - bool RoundToAlign; - - SizeOffsetEvalType unknown() { - return std::make_pair(nullptr, nullptr); - } + ObjectSizeOpts EvalOpts; + SmallPtrSet InsertedInstructions; SizeOffsetEvalType compute_(Value *V); public: + static SizeOffsetEvalType unknown() { + return std::make_pair(nullptr, nullptr); + } + ObjectSizeOffsetEvaluator(const DataLayout &DL, const TargetLibraryInfo *TLI, - LLVMContext &Context, bool RoundToAlign = false); + LLVMContext &Context, ObjectSizeOpts EvalOpts = {}); SizeOffsetEvalType compute(Value *V); diff --git a/include/llvm/Analysis/MemoryDependenceAnalysis.h b/include/llvm/Analysis/MemoryDependenceAnalysis.h index 958d4fe4b832..e2669c2fa601 100644 --- a/include/llvm/Analysis/MemoryDependenceAnalysis.h +++ b/include/llvm/Analysis/MemoryDependenceAnalysis.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/MemoryDependenceAnalysis.h - Memory Deps ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -382,7 +381,8 @@ public: /// /// See the class comment for more details. It is illegal to call this on /// non-memory instructions. - MemDepResult getDependency(Instruction *QueryInst); + MemDepResult getDependency(Instruction *QueryInst, + OrderedBasicBlock *OBB = nullptr); /// Perform a full dependency query for the specified call, returning the set /// of blocks that the value is potentially live across. @@ -448,14 +448,14 @@ public: BasicBlock::iterator ScanIt, BasicBlock *BB, Instruction *QueryInst = nullptr, - unsigned *Limit = nullptr); - - MemDepResult getSimplePointerDependencyFrom(const MemoryLocation &MemLoc, - bool isLoad, - BasicBlock::iterator ScanIt, - BasicBlock *BB, - Instruction *QueryInst, - unsigned *Limit = nullptr); + unsigned *Limit = nullptr, + OrderedBasicBlock *OBB = nullptr); + + MemDepResult + getSimplePointerDependencyFrom(const MemoryLocation &MemLoc, bool isLoad, + BasicBlock::iterator ScanIt, BasicBlock *BB, + Instruction *QueryInst, unsigned *Limit, + OrderedBasicBlock *OBB); /// This analysis looks for other loads and stores with invariant.group /// metadata and the same pointer operand. Returns Unknown if it does not diff --git a/include/llvm/Analysis/MemoryLocation.h b/include/llvm/Analysis/MemoryLocation.h index fca18c1b5999..7c26353e618b 100644 --- a/include/llvm/Analysis/MemoryLocation.h +++ b/include/llvm/Analysis/MemoryLocation.h @@ -1,9 +1,8 @@ //===- MemoryLocation.h - Memory location descriptions ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/Analysis/MemorySSA.h b/include/llvm/Analysis/MemorySSA.h index 17e2d0c73977..b7730be75354 100644 --- a/include/llvm/Analysis/MemorySSA.h +++ b/include/llvm/Analysis/MemorySSA.h @@ -1,9 +1,8 @@ //===- MemorySSA.h - Build Memory SSA ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -105,6 +104,9 @@ namespace llvm { +/// Enables memory ssa as a dependency for loop passes. +extern cl::opt EnableMSSALoopDependency; + class Function; class Instruction; class MemoryAccess; @@ -701,6 +703,11 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(MemoryPhi, MemoryAccess) class MemorySSA { public: MemorySSA(Function &, AliasAnalysis *, DominatorTree *); + + // MemorySSA must remain where it's constructed; Walkers it creates store + // pointers to it. + MemorySSA(MemorySSA &&) = delete; + ~MemorySSA(); MemorySSAWalker *getWalker(); @@ -776,9 +783,6 @@ public: /// all uses, uses appear in the right places). This is used by unit tests. void verifyMemorySSA() const; - /// Check clobber sanity for an access. - void checkClobberSanityAccess(const MemoryAccess *MA) const; - /// Used in various insertion functions to specify whether we are talking /// about the beginning or end of a block. enum InsertionPlace { Beginning, End }; @@ -793,7 +797,6 @@ protected: void verifyDomination(Function &F) const; void verifyOrdering(Function &F) const; void verifyDominationNumbers(const Function &F) const; - void verifyClobberSanity(const Function &F) const; // This is used by the use optimizer and updater. AccessList *getWritableBlockAccesses(const BasicBlock *BB) const { @@ -830,13 +833,13 @@ protected: const MemoryUseOrDef *Template = nullptr); private: - class ClobberWalkerBase; - class CachingWalker; - class SkipSelfWalker; + template class ClobberWalkerBase; + template class CachingWalker; + template class SkipSelfWalker; class OptimizeUses; - CachingWalker *getWalkerImpl(); - void buildMemorySSA(); + CachingWalker *getWalkerImpl(); + void buildMemorySSA(BatchAAResults &BAA); void optimizeUses(); void prepareForMoveTo(MemoryAccess *, BasicBlock *); @@ -850,7 +853,8 @@ private: void markUnreachableAsLiveOnEntry(BasicBlock *BB); bool dominatesUse(const MemoryAccess *, const MemoryAccess *) const; MemoryPhi *createMemoryPhi(BasicBlock *BB); - MemoryUseOrDef *createNewAccess(Instruction *, + template + MemoryUseOrDef *createNewAccess(Instruction *, AliasAnalysisType *, const MemoryUseOrDef *Template = nullptr); MemoryAccess *findDominatingDef(BasicBlock *, enum InsertionPlace); void placePHINodes(const SmallPtrSetImpl &); @@ -886,9 +890,9 @@ private: mutable DenseMap BlockNumbering; // Memory SSA building info - std::unique_ptr WalkerBase; - std::unique_ptr Walker; - std::unique_ptr SkipWalker; + std::unique_ptr> WalkerBase; + std::unique_ptr> Walker; + std::unique_ptr> SkipWalker; unsigned NextID; }; @@ -932,6 +936,9 @@ public: MemorySSA &getMSSA() { return *MSSA.get(); } std::unique_ptr MSSA; + + bool invalidate(Function &F, const PreservedAnalyses &PA, + FunctionAnalysisManager::Invalidator &Inv); }; Result run(Function &F, FunctionAnalysisManager &AM); @@ -1044,8 +1051,6 @@ public: /// the walker it uses or returns. virtual void invalidateInfo(MemoryAccess *) {} - virtual void verify(const MemorySSA *MSSA) { assert(MSSA == this->MSSA); } - protected: friend class MemorySSA; // For updating MSSA pointer in MemorySSA move // constructor. @@ -1101,15 +1106,15 @@ public: assert(Access && "Tried to access past the end of our iterator"); // Go to the first argument for phis, and the defining access for everything // else. - if (MemoryPhi *MP = dyn_cast(Access)) + if (const MemoryPhi *MP = dyn_cast(Access)) return MP->getIncomingValue(ArgNo); return cast(Access)->getDefiningAccess(); } using BaseT::operator++; - memoryaccess_def_iterator &operator++() { + memoryaccess_def_iterator_base &operator++() { assert(Access && "Hit end of iterator"); - if (MemoryPhi *MP = dyn_cast(Access)) { + if (const MemoryPhi *MP = dyn_cast(Access)) { if (++ArgNo >= MP->getNumIncomingValues()) { ArgNo = 0; Access = nullptr; diff --git a/include/llvm/Analysis/MemorySSAUpdater.h b/include/llvm/Analysis/MemorySSAUpdater.h index 169d5bd9fa8b..d4d8040c1ff6 100644 --- a/include/llvm/Analysis/MemorySSAUpdater.h +++ b/include/llvm/Analysis/MemorySSAUpdater.h @@ -1,9 +1,8 @@ //===- MemorySSAUpdater.h - Memory SSA Updater-------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -32,6 +31,7 @@ #ifndef LLVM_ANALYSIS_MEMORYSSAUPDATER_H #define LLVM_ANALYSIS_MEMORYSSAUPDATER_H +#include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" @@ -106,7 +106,12 @@ public: /// Update the MemoryPhi in `To` to have a single incoming edge from `From`, /// following a CFG change that replaced multiple edges (switch) with a direct /// branch. - void removeDuplicatePhiEdgesBetween(BasicBlock *From, BasicBlock *To); + void removeDuplicatePhiEdgesBetween(const BasicBlock *From, + const BasicBlock *To); + /// Update MemorySSA when inserting a unique backedge block for a loop. + void updatePhisWhenInsertingUniqueBackedgeBlock(BasicBlock *LoopHeader, + BasicBlock *LoopPreheader, + BasicBlock *BackedgeBlock); /// Update MemorySSA after a loop was cloned, given the blocks in RPO order, /// the exit blocks and a 1:1 mapping of all blocks and instructions /// cloned. This involves duplicating all defs and uses in the cloned blocks @@ -222,14 +227,14 @@ public: /// associated with it is erased from the program. For example, if a store or /// load is simply erased (not replaced), removeMemoryAccess should be called /// on the MemoryAccess for that store/load. - void removeMemoryAccess(MemoryAccess *); + void removeMemoryAccess(MemoryAccess *, bool OptimizePhis = false); /// Remove MemoryAccess for a given instruction, if a MemoryAccess exists. /// This should be called when an instruction (load/store) is deleted from /// the program. - void removeMemoryAccess(const Instruction *I) { + void removeMemoryAccess(const Instruction *I, bool OptimizePhis = false) { if (MemoryAccess *MA = MSSA->getMemoryAccess(I)) - removeMemoryAccess(MA); + removeMemoryAccess(MA, OptimizePhis); } /// Remove all MemoryAcceses in a set of BasicBlocks about to be deleted. @@ -239,7 +244,17 @@ public: /// Deleted blocks still have successor info, but their predecessor edges and /// Phi nodes may already be updated. Instructions in DeadBlocks should be /// deleted after this call. - void removeBlocks(const SmallPtrSetImpl &DeadBlocks); + void removeBlocks(const SmallSetVector &DeadBlocks); + + /// Instruction I will be changed to an unreachable. Remove all accesses in + /// I's block that follow I (inclusive), and update the Phis in the blocks' + /// successors. + void changeToUnreachable(const Instruction *I); + + /// Conditional branch BI is changed or replaced with an unconditional branch + /// to `To`. Update Phis in BI's successors to remove BI's BB. + void changeCondBranchToUnconditionalTo(const BranchInst *BI, + const BasicBlock *To); /// Get handle on MemorySSA. MemorySSA* getMemorySSA() const { return MSSA; } @@ -262,6 +277,7 @@ private: MemoryAccess *recursePhi(MemoryAccess *Phi); template MemoryAccess *tryRemoveTrivialPhi(MemoryPhi *Phi, RangeType &Operands); + void tryRemoveTrivialPhis(ArrayRef UpdatedPHIs); void fixupDefs(const SmallVectorImpl &); // Clone all uses and defs from BB to NewBB given a 1:1 map of all // instructions and blocks cloned, and a map of MemoryPhi : Definition @@ -272,8 +288,14 @@ private: // not necessarily be MemoryPhis themselves, they may be MemoryDefs. As such, // the map is between MemoryPhis and MemoryAccesses, where the MemoryAccesses // may be MemoryPhis or MemoryDefs and not MemoryUses. + // If CloneWasSimplified = true, the clone was exact. Otherwise, assume that + // the clone involved simplifications that may have: (1) turned a MemoryUse + // into an instruction that MemorySSA has no representation for, or (2) turned + // a MemoryDef into a MemoryUse or an instruction that MemorySSA has no + // representation for. No other cases are supported. void cloneUsesAndDefs(BasicBlock *BB, BasicBlock *NewBB, - const ValueToValueMapTy &VMap, PhiToDefMap &MPhiMap); + const ValueToValueMapTy &VMap, PhiToDefMap &MPhiMap, + bool CloneWasSimplified = false); template void privateUpdateExitBlocksForClonedLoop(ArrayRef ExitBlocks, Iter ValuesBegin, Iter ValuesEnd, diff --git a/include/llvm/Analysis/ModuleSummaryAnalysis.h b/include/llvm/Analysis/ModuleSummaryAnalysis.h index 9af7859cb4bf..1572a49e3384 100644 --- a/include/llvm/Analysis/ModuleSummaryAnalysis.h +++ b/include/llvm/Analysis/ModuleSummaryAnalysis.h @@ -1,9 +1,8 @@ //===- ModuleSummaryAnalysis.h - Module summary index builder ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/Analysis/MustExecute.h b/include/llvm/Analysis/MustExecute.h index ad3222c17e62..3ef539c89d97 100644 --- a/include/llvm/Analysis/MustExecute.h +++ b/include/llvm/Analysis/MustExecute.h @@ -1,9 +1,8 @@ //===- MustExecute.h - Is an instruction known to execute--------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/Analysis/ObjCARCAliasAnalysis.h b/include/llvm/Analysis/ObjCARCAliasAnalysis.h index 58a67042ea2d..b4f4e5f29768 100644 --- a/include/llvm/Analysis/ObjCARCAliasAnalysis.h +++ b/include/llvm/Analysis/ObjCARCAliasAnalysis.h @@ -1,9 +1,8 @@ //===- ObjCARCAliasAnalysis.h - ObjC ARC Alias Analysis ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -53,14 +52,17 @@ public: return false; } - AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB); - bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal); + AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB, + AAQueryInfo &AAQI); + bool pointsToConstantMemory(const MemoryLocation &Loc, AAQueryInfo &AAQI, + bool OrLocal); using AAResultBase::getModRefBehavior; FunctionModRefBehavior getModRefBehavior(const Function *F); using AAResultBase::getModRefInfo; - ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc); + ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc, + AAQueryInfo &AAQI); }; /// Analysis pass providing a never-invalidated alias analysis result. diff --git a/include/llvm/Analysis/ObjCARCAnalysisUtils.h b/include/llvm/Analysis/ObjCARCAnalysisUtils.h index 1f497fab35da..522abd756c9f 100644 --- a/include/llvm/Analysis/ObjCARCAnalysisUtils.h +++ b/include/llvm/Analysis/ObjCARCAnalysisUtils.h @@ -1,9 +1,8 @@ //===- ObjCARCAnalysisUtils.h - ObjC ARC Analysis Utilities -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/Analysis/ObjCARCInstKind.h b/include/llvm/Analysis/ObjCARCInstKind.h index 018ea1f851be..dc6093a7b86c 100644 --- a/include/llvm/Analysis/ObjCARCInstKind.h +++ b/include/llvm/Analysis/ObjCARCInstKind.h @@ -1,9 +1,8 @@ //===- ObjCARCInstKind.h - ARC instruction equivalence classes --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -75,6 +74,10 @@ bool IsForwarding(ARCInstKind Class); /// passed a null pointer. bool IsNoopOnNull(ARCInstKind Class); +/// Test if the given class represents instructions which do nothing if +/// passed a global variable. +bool IsNoopOnGlobal(ARCInstKind Class); + /// Test if the given class represents instructions which are always safe /// to mark with the "tail" keyword. bool IsAlwaysTail(ARCInstKind Class); diff --git a/include/llvm/Analysis/OptimizationRemarkEmitter.h b/include/llvm/Analysis/OptimizationRemarkEmitter.h index fa838696e2f8..7b8404404ce7 100644 --- a/include/llvm/Analysis/OptimizationRemarkEmitter.h +++ b/include/llvm/Analysis/OptimizationRemarkEmitter.h @@ -1,9 +1,8 @@ //===- OptimizationRemarkEmitter.h - Optimization Diagnostic ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -78,7 +77,7 @@ public: // remarks enabled. We can't currently check whether remarks are requested // for the calling pass since that requires actually building the remark. - if (F->getContext().getDiagnosticsOutputFile() || + if (F->getContext().getRemarkStreamer() || F->getContext().getDiagHandlerPtr()->isAnyRemarkEnabled()) { auto R = RemarkBuilder(); emit((DiagnosticInfoOptimizationBase &)R); @@ -93,7 +92,7 @@ public: /// provide more context so that non-trivial false positives can be quickly /// detected by the user. bool allowExtraAnalysis(StringRef PassName) const { - return (F->getContext().getDiagnosticsOutputFile() || + return (F->getContext().getRemarkStreamer() || F->getContext().getDiagHandlerPtr()->isAnyRemarkEnabled(PassName)); } diff --git a/include/llvm/Analysis/OrderedBasicBlock.h b/include/llvm/Analysis/OrderedBasicBlock.h index 0776aa626005..ae64c0189f5e 100644 --- a/include/llvm/Analysis/OrderedBasicBlock.h +++ b/include/llvm/Analysis/OrderedBasicBlock.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/OrderedBasicBlock.h --------------------- -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -60,6 +59,14 @@ public: /// only relevant to compare relative instructions positions inside \p BB. /// Returns false for A == B. bool dominates(const Instruction *A, const Instruction *B); + + /// Remove \p from the ordering, if it is present. + void eraseInstruction(const Instruction *I); + + /// Replace \p Old with \p New in the ordering. \p New is assigned the + /// numbering of \p Old, so it must be inserted at the same position in the + /// IR. + void replaceInstruction(const Instruction *Old, const Instruction *New); }; } // End llvm namespace diff --git a/include/llvm/Analysis/OrderedInstructions.h b/include/llvm/Analysis/OrderedInstructions.h index 7e3850b87c57..967b146b52de 100644 --- a/include/llvm/Analysis/OrderedInstructions.h +++ b/include/llvm/Analysis/OrderedInstructions.h @@ -1,9 +1,8 @@ //===- llvm/Transforms/Utils/OrderedInstructions.h -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/PHITransAddr.h b/include/llvm/Analysis/PHITransAddr.h index 0a335b6be6c7..54a07f053478 100644 --- a/include/llvm/Analysis/PHITransAddr.h +++ b/include/llvm/Analysis/PHITransAddr.h @@ -1,9 +1,8 @@ //===- PHITransAddr.h - PHI Translation for Addresses -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/Passes.h b/include/llvm/Analysis/Passes.h index 081dd5000835..d9c97dff8c6e 100644 --- a/include/llvm/Analysis/Passes.h +++ b/include/llvm/Analysis/Passes.h @@ -1,9 +1,8 @@ //===-- llvm/Analysis/Passes.h - Constructors for analyses ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/PhiValues.h b/include/llvm/Analysis/PhiValues.h index 76204ac1bc6c..124fa2191694 100644 --- a/include/llvm/Analysis/PhiValues.h +++ b/include/llvm/Analysis/PhiValues.h @@ -1,9 +1,8 @@ //===- PhiValues.h - Phi Value Analysis -------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/PostDominators.h b/include/llvm/Analysis/PostDominators.h index f2dc8d135d71..87d2e0318d0a 100644 --- a/include/llvm/Analysis/PostDominators.h +++ b/include/llvm/Analysis/PostDominators.h @@ -1,9 +1,8 @@ //=- llvm/Analysis/PostDominators.h - Post Dominator Calculation --*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/ProfileSummaryInfo.h b/include/llvm/Analysis/ProfileSummaryInfo.h index 3aef4be72d71..f309d344b8d1 100644 --- a/include/llvm/Analysis/ProfileSummaryInfo.h +++ b/include/llvm/Analysis/ProfileSummaryInfo.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/ProfileSummaryInfo.h - profile summary ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -74,6 +73,12 @@ public: Summary->getKind() == ProfileSummary::PSK_Instr; } + /// Returns true if module \c M has context sensitive instrumentation profile. + bool hasCSInstrumentationProfile() { + return hasProfileSummary() && + Summary->getKind() == ProfileSummary::PSK_CSInstr; + } + /// Handle the invalidation of this information. /// /// When used as a result of \c ProfileSummaryAnalysis this method will be @@ -87,7 +92,8 @@ public: /// Returns the profile count for \p CallInst. Optional getProfileCount(const Instruction *CallInst, - BlockFrequencyInfo *BFI); + BlockFrequencyInfo *BFI, + bool AllowSynthetic = false); /// Returns true if the working set size of the code is considered huge. bool hasHugeWorkingSetSize(); /// Returns true if \p F has hot function entry. diff --git a/include/llvm/Analysis/PtrUseVisitor.h b/include/llvm/Analysis/PtrUseVisitor.h index b34b25c75040..fbf04c841d30 100644 --- a/include/llvm/Analysis/PtrUseVisitor.h +++ b/include/llvm/Analysis/PtrUseVisitor.h @@ -1,9 +1,8 @@ //===- PtrUseVisitor.h - InstVisitors over a pointers uses ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -257,6 +256,10 @@ protected: enqueueUsers(BC); } + void visitAddrSpaceCastInst(AddrSpaceCastInst &ASC) { + enqueueUsers(ASC); + } + void visitPtrToIntInst(PtrToIntInst &I) { PI.setEscaped(&I); } diff --git a/include/llvm/Analysis/RegionInfo.h b/include/llvm/Analysis/RegionInfo.h index 27f6cc197927..8bcc3e851200 100644 --- a/include/llvm/Analysis/RegionInfo.h +++ b/include/llvm/Analysis/RegionInfo.h @@ -1,9 +1,8 @@ //===- RegionInfo.h - SESE region analysis ----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/RegionInfoImpl.h b/include/llvm/Analysis/RegionInfoImpl.h index 5904214aa925..c59c09dd2095 100644 --- a/include/llvm/Analysis/RegionInfoImpl.h +++ b/include/llvm/Analysis/RegionInfoImpl.h @@ -1,9 +1,8 @@ //===- RegionInfoImpl.h - SESE region detection analysis --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Detects single entry single exit regions in the control flow graph. diff --git a/include/llvm/Analysis/RegionIterator.h b/include/llvm/Analysis/RegionIterator.h index 4fd92fcde20b..72bc5bbcb506 100644 --- a/include/llvm/Analysis/RegionIterator.h +++ b/include/llvm/Analysis/RegionIterator.h @@ -1,9 +1,8 @@ //===- RegionIterator.h - Iterators to iteratate over Regions ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This file defines the iterators to iterate over the elements of a Region. diff --git a/include/llvm/Analysis/RegionPass.h b/include/llvm/Analysis/RegionPass.h index b3da91c89cbd..5b1864a37629 100644 --- a/include/llvm/Analysis/RegionPass.h +++ b/include/llvm/Analysis/RegionPass.h @@ -1,9 +1,8 @@ //===- RegionPass.h - RegionPass class --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/RegionPrinter.h b/include/llvm/Analysis/RegionPrinter.h index e132eaea5674..154ac35c486a 100644 --- a/include/llvm/Analysis/RegionPrinter.h +++ b/include/llvm/Analysis/RegionPrinter.h @@ -1,9 +1,8 @@ //===-- RegionPrinter.h - Region printer external interface -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/ScalarEvolution.h b/include/llvm/Analysis/ScalarEvolution.h index 8f4200b07e5c..0bd98ef37e7a 100644 --- a/include/llvm/Analysis/ScalarEvolution.h +++ b/include/llvm/Analysis/ScalarEvolution.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/ScalarEvolution.h - Scalar Evolution -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -85,6 +84,9 @@ class SCEV : public FoldingSetNode { const unsigned short SCEVType; protected: + // Estimated complexity of this node's expression tree size. + const unsigned short ExpressionSize; + /// This field is initialized to zero and may be used in subclasses to store /// miscellaneous information. unsigned short SubclassData = 0; @@ -116,8 +118,9 @@ public: NoWrapMask = (1 << 3) - 1 }; - explicit SCEV(const FoldingSetNodeIDRef ID, unsigned SCEVTy) - : FastID(ID), SCEVType(SCEVTy) {} + explicit SCEV(const FoldingSetNodeIDRef ID, unsigned SCEVTy, + unsigned short ExpressionSize) + : FastID(ID), SCEVType(SCEVTy), ExpressionSize(ExpressionSize) {} SCEV(const SCEV &) = delete; SCEV &operator=(const SCEV &) = delete; @@ -138,6 +141,19 @@ public: /// Return true if the specified scev is negated, but not a constant. bool isNonConstantNegative() const; + // Returns estimated size of the mathematical expression represented by this + // SCEV. The rules of its calculation are following: + // 1) Size of a SCEV without operands (like constants and SCEVUnknown) is 1; + // 2) Size SCEV with operands Op1, Op2, ..., OpN is calculated by formula: + // (1 + Size(Op1) + ... + Size(OpN)). + // This value gives us an estimation of time we need to traverse through this + // SCEV and all its operands recursively. We may use it to avoid performing + // heavy transformations on SCEVs of excessive size for sake of saving the + // compilation time. + unsigned short getExpressionSize() const { + return ExpressionSize; + } + /// Print out the internal representation of this scalar to the specified /// stream. This should really only be used for debugging purposes. void print(raw_ostream &OS) const; @@ -521,7 +537,7 @@ public: const SCEV *getConstant(ConstantInt *V); const SCEV *getConstant(const APInt &Val); const SCEV *getConstant(Type *Ty, uint64_t V, bool isSigned = false); - const SCEV *getTruncateExpr(const SCEV *Op, Type *Ty); + const SCEV *getTruncateExpr(const SCEV *Op, Type *Ty, unsigned Depth = 0); const SCEV *getZeroExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth = 0); const SCEV *getSignExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth = 0); const SCEV *getAnyExtendExpr(const SCEV *Op, Type *Ty); @@ -582,6 +598,8 @@ public: /// \p IndexExprs The expressions for the indices. const SCEV *getGEPExpr(GEPOperator *GEP, const SmallVectorImpl &IndexExprs); + const SCEV *getMinMaxExpr(unsigned Kind, + SmallVectorImpl &Operands); const SCEV *getSMaxExpr(const SCEV *LHS, const SCEV *RHS); const SCEV *getSMaxExpr(SmallVectorImpl &Operands); const SCEV *getUMaxExpr(const SCEV *LHS, const SCEV *RHS); @@ -619,11 +637,13 @@ public: /// Return a SCEV corresponding to a conversion of the input value to the /// specified type. If the type must be extended, it is zero extended. - const SCEV *getTruncateOrZeroExtend(const SCEV *V, Type *Ty); + const SCEV *getTruncateOrZeroExtend(const SCEV *V, Type *Ty, + unsigned Depth = 0); /// Return a SCEV corresponding to a conversion of the input value to the /// specified type. If the type must be extended, it is sign extended. - const SCEV *getTruncateOrSignExtend(const SCEV *V, Type *Ty); + const SCEV *getTruncateOrSignExtend(const SCEV *V, Type *Ty, + unsigned Depth = 0); /// Return a SCEV corresponding to a conversion of the input value to the /// specified type. If the type must be extended, it is zero extended. The @@ -726,9 +746,12 @@ public: unsigned getSmallConstantTripMultiple(const Loop *L, BasicBlock *ExitingBlock); - /// Get the expression for the number of loop iterations for which this loop - /// is guaranteed not to exit via ExitingBlock. Otherwise return - /// SCEVCouldNotCompute. + /// Return the number of times the backedge executes before the given exit + /// would be taken; if not exactly computable, return SCEVCouldNotCompute. + /// For a single exit loop, this value is equivelent to the result of + /// getBackedgeTakenCount. The loop is guaranteed to exit (via *some* exit) + /// before the backedge is executed (ExitCount + 1) times. Note that there + /// is no guarantee about *which* exit is taken on the exiting iteration. const SCEV *getExitCount(const Loop *L, BasicBlock *ExitingBlock); /// If the specified loop has a predictable backedge-taken count, return it, @@ -764,6 +787,13 @@ public: /// backedge-taken count. bool hasLoopInvariantBackedgeTakenCount(const Loop *L); + // This method should be called by the client when it made any change that + // would invalidate SCEV's answers, and the client wants to remove all loop + // information held internally by ScalarEvolution. This is intended to be used + // when the alternative to forget a loop is too expensive (i.e. large loop + // bodies). + void forgetAllLoops(); + /// This method should be called by the client when it has changed a loop in /// a way that may effect ScalarEvolution's ability to compute a trip count, /// or if the loop is deleted. This call is potentially expensive for large @@ -1273,7 +1303,7 @@ private: using EdgeExitInfo = std::pair; /// Initialize BackedgeTakenInfo from a list of exact exit counts. - BackedgeTakenInfo(SmallVectorImpl &&ExitCounts, bool Complete, + BackedgeTakenInfo(ArrayRef ExitCounts, bool Complete, const SCEV *MaxCount, bool MaxOrZero); /// Test whether this BackedgeTakenInfo contains any computed information, @@ -1826,15 +1856,15 @@ private: bool NoWrap); /// Get add expr already created or create a new one. - const SCEV *getOrCreateAddExpr(SmallVectorImpl &Ops, + const SCEV *getOrCreateAddExpr(ArrayRef Ops, SCEV::NoWrapFlags Flags); /// Get mul expr already created or create a new one. - const SCEV *getOrCreateMulExpr(SmallVectorImpl &Ops, + const SCEV *getOrCreateMulExpr(ArrayRef Ops, SCEV::NoWrapFlags Flags); // Get addrec expr already created or create a new one. - const SCEV *getOrCreateAddRecExpr(SmallVectorImpl &Ops, + const SCEV *getOrCreateAddRecExpr(ArrayRef Ops, const Loop *L, SCEV::NoWrapFlags Flags); /// Return x if \p Val is f(x) where f is a 1-1 function. @@ -1853,6 +1883,16 @@ private: /// Assign A and B to LHS and RHS, respectively. bool matchURem(const SCEV *Expr, const SCEV *&LHS, const SCEV *&RHS); + /// Look for a SCEV expression with type `SCEVType` and operands `Ops` in + /// `UniqueSCEVs`. + /// + /// The first component of the returned tuple is the SCEV if found and null + /// otherwise. The second component is the `FoldingSetNodeID` that was + /// constructed to look up the SCEV and the third component is the insertion + /// point. + std::tuple + findExistingSCEVInCache(int SCEVType, ArrayRef Ops); + FoldingSet UniqueSCEVs; FoldingSet UniquePreds; BumpPtrAllocator SCEVAllocator; diff --git a/include/llvm/Analysis/ScalarEvolutionAliasAnalysis.h b/include/llvm/Analysis/ScalarEvolutionAliasAnalysis.h index 329be51e5eac..98d53237d4a0 100644 --- a/include/llvm/Analysis/ScalarEvolutionAliasAnalysis.h +++ b/include/llvm/Analysis/ScalarEvolutionAliasAnalysis.h @@ -1,9 +1,8 @@ //===- ScalarEvolutionAliasAnalysis.h - SCEV-based AA -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -31,7 +30,8 @@ public: explicit SCEVAAResult(ScalarEvolution &SE) : AAResultBase(), SE(SE) {} SCEVAAResult(SCEVAAResult &&Arg) : AAResultBase(std::move(Arg)), SE(Arg.SE) {} - AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB); + AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB, + AAQueryInfo &AAQI); private: Value *GetBaseValue(const SCEV *S); diff --git a/include/llvm/Analysis/ScalarEvolutionExpander.h b/include/llvm/Analysis/ScalarEvolutionExpander.h index 58d42680d6bc..a519f93216b3 100644 --- a/include/llvm/Analysis/ScalarEvolutionExpander.h +++ b/include/llvm/Analysis/ScalarEvolutionExpander.h @@ -1,9 +1,8 @@ //===---- llvm/Analysis/ScalarEvolutionExpander.h - SCEV Exprs --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -316,8 +315,10 @@ namespace llvm { SmallPtrSetImpl &Processed); /// Insert the specified binary operator, doing a small amount of work to - /// avoid inserting an obviously redundant operation. - Value *InsertBinop(Instruction::BinaryOps Opcode, Value *LHS, Value *RHS); + /// avoid inserting an obviously redundant operation, and hoisting to an + /// outer loop when the opportunity is there and it is safe. + Value *InsertBinop(Instruction::BinaryOps Opcode, Value *LHS, Value *RHS, + SCEV::NoWrapFlags Flags, bool IsSafeToHoist); /// Arrange for there to be a cast of V to Ty at IP, reusing an existing /// cast if a suitable one exists, moving an existing cast if a suitable one @@ -368,6 +369,10 @@ namespace llvm { Value *visitUMaxExpr(const SCEVUMaxExpr *S); + Value *visitSMinExpr(const SCEVSMinExpr *S); + + Value *visitUMinExpr(const SCEVUMinExpr *S); + Value *visitUnknown(const SCEVUnknown *S) { return S->getValue(); } diff --git a/include/llvm/Analysis/ScalarEvolutionExpressions.h b/include/llvm/Analysis/ScalarEvolutionExpressions.h index 42e76094eb2b..d008af7b7e6f 100644 --- a/include/llvm/Analysis/ScalarEvolutionExpressions.h +++ b/include/llvm/Analysis/ScalarEvolutionExpressions.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/ScalarEvolutionExpressions.h - SCEV Exprs --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -40,7 +39,7 @@ class Type; // These should be ordered in terms of increasing complexity to make the // folders simpler. scConstant, scTruncate, scZeroExtend, scSignExtend, scAddExpr, scMulExpr, - scUDivExpr, scAddRecExpr, scUMaxExpr, scSMaxExpr, + scUDivExpr, scAddRecExpr, scUMaxExpr, scSMaxExpr, scUMinExpr, scSMinExpr, scUnknown, scCouldNotCompute }; @@ -51,7 +50,7 @@ class Type; ConstantInt *V; SCEVConstant(const FoldingSetNodeIDRef ID, ConstantInt *v) : - SCEV(ID, scConstant), V(v) {} + SCEV(ID, scConstant, 1), V(v) {} public: ConstantInt *getValue() const { return V; } @@ -65,6 +64,13 @@ class Type; } }; + static unsigned short computeExpressionSize(ArrayRef Args) { + APInt Size(16, 1); + for (auto *Arg : Args) + Size = Size.uadd_sat(APInt(16, Arg->getExpressionSize())); + return (unsigned short)Size.getZExtValue(); + } + /// This is the base class for unary cast operator classes. class SCEVCastExpr : public SCEV { protected: @@ -142,9 +148,10 @@ class Type; const SCEV *const *Operands; size_t NumOperands; - SCEVNAryExpr(const FoldingSetNodeIDRef ID, - enum SCEVTypes T, const SCEV *const *O, size_t N) - : SCEV(ID, T), Operands(O), NumOperands(N) {} + SCEVNAryExpr(const FoldingSetNodeIDRef ID, enum SCEVTypes T, + const SCEV *const *O, size_t N) + : SCEV(ID, T, computeExpressionSize(makeArrayRef(O, N))), Operands(O), + NumOperands(N) {} public: size_t getNumOperands() const { return NumOperands; } @@ -183,10 +190,9 @@ class Type; /// Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const SCEV *S) { - return S->getSCEVType() == scAddExpr || - S->getSCEVType() == scMulExpr || - S->getSCEVType() == scSMaxExpr || - S->getSCEVType() == scUMaxExpr || + return S->getSCEVType() == scAddExpr || S->getSCEVType() == scMulExpr || + S->getSCEVType() == scSMaxExpr || S->getSCEVType() == scUMaxExpr || + S->getSCEVType() == scSMinExpr || S->getSCEVType() == scUMinExpr || S->getSCEVType() == scAddRecExpr; } }; @@ -201,10 +207,9 @@ class Type; public: /// Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const SCEV *S) { - return S->getSCEVType() == scAddExpr || - S->getSCEVType() == scMulExpr || - S->getSCEVType() == scSMaxExpr || - S->getSCEVType() == scUMaxExpr; + return S->getSCEVType() == scAddExpr || S->getSCEVType() == scMulExpr || + S->getSCEVType() == scSMaxExpr || S->getSCEVType() == scUMaxExpr || + S->getSCEVType() == scSMinExpr || S->getSCEVType() == scUMinExpr; } /// Set flags for a non-recurrence without clearing previously set flags. @@ -258,7 +263,8 @@ class Type; const SCEV *RHS; SCEVUDivExpr(const FoldingSetNodeIDRef ID, const SCEV *lhs, const SCEV *rhs) - : SCEV(ID, scUDivExpr), LHS(lhs), RHS(rhs) {} + : SCEV(ID, scUDivExpr, computeExpressionSize({lhs, rhs})), LHS(lhs), + RHS(rhs) {} public: const SCEV *getLHS() const { return LHS; } @@ -358,17 +364,53 @@ class Type; } }; - /// This class represents a signed maximum selection. - class SCEVSMaxExpr : public SCEVCommutativeExpr { + /// This node is the base class min/max selections. + class SCEVMinMaxExpr : public SCEVCommutativeExpr { friend class ScalarEvolution; - SCEVSMaxExpr(const FoldingSetNodeIDRef ID, - const SCEV *const *O, size_t N) - : SCEVCommutativeExpr(ID, scSMaxExpr, O, N) { - // Max never overflows. + static bool isMinMaxType(enum SCEVTypes T) { + return T == scSMaxExpr || T == scUMaxExpr || T == scSMinExpr || + T == scUMinExpr; + } + + protected: + /// Note: Constructing subclasses via this constructor is allowed + SCEVMinMaxExpr(const FoldingSetNodeIDRef ID, enum SCEVTypes T, + const SCEV *const *O, size_t N) + : SCEVCommutativeExpr(ID, T, O, N) { + assert(isMinMaxType(T)); + // Min and max never overflow setNoWrapFlags((NoWrapFlags)(FlagNUW | FlagNSW)); } + public: + static bool classof(const SCEV *S) { + return isMinMaxType(static_cast(S->getSCEVType())); + } + + static enum SCEVTypes negate(enum SCEVTypes T) { + switch (T) { + case scSMaxExpr: + return scSMinExpr; + case scSMinExpr: + return scSMaxExpr; + case scUMaxExpr: + return scUMinExpr; + case scUMinExpr: + return scUMaxExpr; + default: + llvm_unreachable("Not a min or max SCEV type!"); + } + } + }; + + /// This class represents a signed maximum selection. + class SCEVSMaxExpr : public SCEVMinMaxExpr { + friend class ScalarEvolution; + + SCEVSMaxExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N) + : SCEVMinMaxExpr(ID, scSMaxExpr, O, N) {} + public: /// Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const SCEV *S) { @@ -377,15 +419,11 @@ class Type; }; /// This class represents an unsigned maximum selection. - class SCEVUMaxExpr : public SCEVCommutativeExpr { + class SCEVUMaxExpr : public SCEVMinMaxExpr { friend class ScalarEvolution; - SCEVUMaxExpr(const FoldingSetNodeIDRef ID, - const SCEV *const *O, size_t N) - : SCEVCommutativeExpr(ID, scUMaxExpr, O, N) { - // Max never overflows. - setNoWrapFlags((NoWrapFlags)(FlagNUW | FlagNSW)); - } + SCEVUMaxExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N) + : SCEVMinMaxExpr(ID, scUMaxExpr, O, N) {} public: /// Methods for support type inquiry through isa, cast, and dyn_cast: @@ -394,6 +432,34 @@ class Type; } }; + /// This class represents a signed minimum selection. + class SCEVSMinExpr : public SCEVMinMaxExpr { + friend class ScalarEvolution; + + SCEVSMinExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N) + : SCEVMinMaxExpr(ID, scSMinExpr, O, N) {} + + public: + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const SCEV *S) { + return S->getSCEVType() == scSMinExpr; + } + }; + + /// This class represents an unsigned minimum selection. + class SCEVUMinExpr : public SCEVMinMaxExpr { + friend class ScalarEvolution; + + SCEVUMinExpr(const FoldingSetNodeIDRef ID, const SCEV *const *O, size_t N) + : SCEVMinMaxExpr(ID, scUMinExpr, O, N) {} + + public: + /// Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const SCEV *S) { + return S->getSCEVType() == scUMinExpr; + } + }; + /// This means that we are dealing with an entirely unknown SCEV /// value, and only represent it as its LLVM Value. This is the /// "bottom" value for the analysis. @@ -411,7 +477,7 @@ class Type; SCEVUnknown(const FoldingSetNodeIDRef ID, Value *V, ScalarEvolution *se, SCEVUnknown *next) : - SCEV(ID, scUnknown), CallbackVH(V), SE(se), Next(next) {} + SCEV(ID, scUnknown, 1), CallbackVH(V), SE(se), Next(next) {} // Implement CallbackVH. void deleted() override; @@ -466,6 +532,10 @@ class Type; return ((SC*)this)->visitSMaxExpr((const SCEVSMaxExpr*)S); case scUMaxExpr: return ((SC*)this)->visitUMaxExpr((const SCEVUMaxExpr*)S); + case scSMinExpr: + return ((SC *)this)->visitSMinExpr((const SCEVSMinExpr *)S); + case scUMinExpr: + return ((SC *)this)->visitUMinExpr((const SCEVUMinExpr *)S); case scUnknown: return ((SC*)this)->visitUnknown((const SCEVUnknown*)S); case scCouldNotCompute: @@ -519,6 +589,8 @@ class Type; case scMulExpr: case scSMaxExpr: case scUMaxExpr: + case scSMinExpr: + case scUMinExpr: case scAddRecExpr: for (const auto *Op : cast(S)->operands()) push(Op); @@ -681,6 +753,26 @@ class Type; return !Changed ? Expr : SE.getUMaxExpr(Operands); } + const SCEV *visitSMinExpr(const SCEVSMinExpr *Expr) { + SmallVector Operands; + bool Changed = false; + for (auto *Op : Expr->operands()) { + Operands.push_back(((SC *)this)->visit(Op)); + Changed |= Op != Operands.back(); + } + return !Changed ? Expr : SE.getSMinExpr(Operands); + } + + const SCEV *visitUMinExpr(const SCEVUMinExpr *Expr) { + SmallVector Operands; + bool Changed = false; + for (auto *Op : Expr->operands()) { + Operands.push_back(((SC *)this)->visit(Op)); + Changed |= Op != Operands.back(); + } + return !Changed ? Expr : SE.getUMinExpr(Operands); + } + const SCEV *visitUnknown(const SCEVUnknown *Expr) { return Expr; } diff --git a/include/llvm/Analysis/ScalarEvolutionNormalization.h b/include/llvm/Analysis/ScalarEvolutionNormalization.h index 51c92121c8f0..1a05594a46ec 100644 --- a/include/llvm/Analysis/ScalarEvolutionNormalization.h +++ b/include/llvm/Analysis/ScalarEvolutionNormalization.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/ScalarEvolutionNormalization.h - See below -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/ScopedNoAliasAA.h b/include/llvm/Analysis/ScopedNoAliasAA.h index 1356c6e9198a..dae733bd2015 100644 --- a/include/llvm/Analysis/ScopedNoAliasAA.h +++ b/include/llvm/Analysis/ScopedNoAliasAA.h @@ -1,9 +1,8 @@ //===- ScopedNoAliasAA.h - Scoped No-Alias Alias Analysis -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -40,9 +39,12 @@ public: return false; } - AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB); - ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc); - ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2); + AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB, + AAQueryInfo &AAQI); + ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc, + AAQueryInfo &AAQI); + ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2, + AAQueryInfo &AAQI); private: bool mayAliasInScopes(const MDNode *Scopes, const MDNode *NoAlias) const; diff --git a/include/llvm/Analysis/SparsePropagation.h b/include/llvm/Analysis/SparsePropagation.h index 02a2e64268b7..fac92e4a25a4 100644 --- a/include/llvm/Analysis/SparsePropagation.h +++ b/include/llvm/Analysis/SparsePropagation.h @@ -1,9 +1,8 @@ //===- SparsePropagation.h - Sparse Conditional Property Propagation ------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -330,12 +329,8 @@ void SparseSolver::getFeasibleSuccessors( return; } - if (TI.isExceptionalTerminator()) { - Succs.assign(Succs.size(), true); - return; - } - - if (isa(TI)) { + if (TI.isExceptionalTerminator() || + TI.isIndirectTerminator()) { Succs.assign(Succs.size(), true); return; } diff --git a/include/llvm/Analysis/StackSafetyAnalysis.h b/include/llvm/Analysis/StackSafetyAnalysis.h index 8a151650a34c..f9d8b08ac142 100644 --- a/include/llvm/Analysis/StackSafetyAnalysis.h +++ b/include/llvm/Analysis/StackSafetyAnalysis.h @@ -1,9 +1,8 @@ //===- StackSafetyAnalysis.h - Stack memory safety analysis -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/SyncDependenceAnalysis.h b/include/llvm/Analysis/SyncDependenceAnalysis.h index df693d9d8e8c..099403b47757 100644 --- a/include/llvm/Analysis/SyncDependenceAnalysis.h +++ b/include/llvm/Analysis/SyncDependenceAnalysis.h @@ -1,9 +1,8 @@ //===- SyncDependenceAnalysis.h - Divergent Branch Dependence -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/SyntheticCountsUtils.h b/include/llvm/Analysis/SyntheticCountsUtils.h index db80bef001e2..b9b4c98bfc35 100644 --- a/include/llvm/Analysis/SyntheticCountsUtils.h +++ b/include/llvm/Analysis/SyntheticCountsUtils.h @@ -1,9 +1,8 @@ //===- SyntheticCountsUtils.h - utilities for count propagation--*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/TargetFolder.h b/include/llvm/Analysis/TargetFolder.h index ae75d3773362..7ab6562be440 100644 --- a/include/llvm/Analysis/TargetFolder.h +++ b/include/llvm/Analysis/TargetFolder.h @@ -1,9 +1,8 @@ //====- TargetFolder.h - Constant folding helper ---------------*- C++ -*-====// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -125,6 +124,10 @@ public: return Fold(ConstantExpr::getNot(C)); } + Constant *CreateUnOp(Instruction::UnaryOps Opc, Constant *C) const { + return Fold(ConstantExpr::get(Opc, C)); + } + //===--------------------------------------------------------------------===// // Memory Instructions //===--------------------------------------------------------------------===// diff --git a/include/llvm/Analysis/TargetLibraryInfo.def b/include/llvm/Analysis/TargetLibraryInfo.def index 518a85ee1a01..afed404f04c0 100644 --- a/include/llvm/Analysis/TargetLibraryInfo.def +++ b/include/llvm/Analysis/TargetLibraryInfo.def @@ -1,9 +1,8 @@ //===-- TargetLibraryInfo.def - Library information -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -12,6 +11,15 @@ // Which is defined depends on whether TLI_DEFINE_ENUM is defined or // TLI_DEFINE_STRING is defined. Only one should be defined at a time. +// NOTE: The nofree attribute is added to Libfuncs which are not +// listed as free or realloc functions in MemoryBuiltins.cpp +// +// When adding a function which frees memory include the LibFunc +// in lib/Analysis/MemoryBuiltins.cpp "isLibFreeFunction". +// +// When adding a LibFunc which reallocates memory include the LibFunc +// in lib/Analysis/MemoryBuiltins.cpp "AllocationFnData[]". + #if !(defined(TLI_DEFINE_ENUM) || defined(TLI_DEFINE_STRING)) #error "Must define TLI_DEFINE_ENUM or TLI_DEFINE_STRING for TLI .def." #elif defined(TLI_DEFINE_ENUM) && defined(TLI_DEFINE_STRING) @@ -330,6 +338,10 @@ TLI_DEFINE_STRING_INTERNAL("__logf_finite") /// long double __logl_finite(long double x); TLI_DEFINE_ENUM_INTERNAL(logl_finite) TLI_DEFINE_STRING_INTERNAL("__logl_finite") +/// void *__memccpy_chk(void *dst, const void *src, int c, size_t n, +/// size_t dstsize) +TLI_DEFINE_ENUM_INTERNAL(memccpy_chk) +TLI_DEFINE_STRING_INTERNAL("__memccpy_chk") /// void *__memcpy_chk(void *s1, const void *s2, size_t n, size_t s1size); TLI_DEFINE_ENUM_INTERNAL(memcpy_chk) TLI_DEFINE_STRING_INTERNAL("__memcpy_chk") @@ -373,6 +385,23 @@ TLI_DEFINE_STRING_INTERNAL("__sinpi") /// float __sinpif(float x); TLI_DEFINE_ENUM_INTERNAL(sinpif) TLI_DEFINE_STRING_INTERNAL("__sinpif") +/// int __small_fprintf(FILE *stream, const char *format, ...); +TLI_DEFINE_ENUM_INTERNAL(small_fprintf) +TLI_DEFINE_STRING_INTERNAL("__small_fprintf") +/// int __small_printf(const char *format, ...); +TLI_DEFINE_ENUM_INTERNAL(small_printf) +TLI_DEFINE_STRING_INTERNAL("__small_printf") +/// int __small_sprintf(char *str, const char *format, ...); +TLI_DEFINE_ENUM_INTERNAL(small_sprintf) +TLI_DEFINE_STRING_INTERNAL("__small_sprintf") +/// int __snprintf_chk(char *s, size_t n, int flags, size_t slen, +/// const char *format, ...); +TLI_DEFINE_ENUM_INTERNAL(snprintf_chk) +TLI_DEFINE_STRING_INTERNAL("__snprintf_chk") +/// int __sprintf_chk(char *str, int flags, size_t str_len, +/// const char *format, ...); +TLI_DEFINE_ENUM_INTERNAL(sprintf_chk) +TLI_DEFINE_STRING_INTERNAL("__sprintf_chk") /// double __sqrt_finite(double x); TLI_DEFINE_ENUM_INTERNAL(sqrt_finite) TLI_DEFINE_STRING_INTERNAL("__sqrt_finite") @@ -388,12 +417,26 @@ TLI_DEFINE_STRING_INTERNAL("__stpcpy_chk") /// char *__stpncpy_chk(char *s1, const char *s2, size_t n, size_t s1size); TLI_DEFINE_ENUM_INTERNAL(stpncpy_chk) TLI_DEFINE_STRING_INTERNAL("__stpncpy_chk") +/// char *__strcat_chk(char *s1, const char *s2, size_t s1size); +TLI_DEFINE_ENUM_INTERNAL(strcat_chk) +TLI_DEFINE_STRING_INTERNAL("__strcat_chk") /// char *__strcpy_chk(char *s1, const char *s2, size_t s1size); TLI_DEFINE_ENUM_INTERNAL(strcpy_chk) TLI_DEFINE_STRING_INTERNAL("__strcpy_chk") /// char * __strdup(const char *s); TLI_DEFINE_ENUM_INTERNAL(dunder_strdup) TLI_DEFINE_STRING_INTERNAL("__strdup") +/// size_t __strlcat_chk(char *dst, const char *src, size_t size, +/// size_t dstsize); +TLI_DEFINE_ENUM_INTERNAL(strlcat_chk) +TLI_DEFINE_STRING_INTERNAL("__strlcat_chk") +/// size_t __strlcpy_chk(char *dst, const char *src, size_t size, +/// size_t dstsize); +TLI_DEFINE_ENUM_INTERNAL(strlcpy_chk) +TLI_DEFINE_STRING_INTERNAL("__strlcpy_chk") +/// char *strncat_chk(char *s1, const char *s2, size_t n, size_t s1size); +TLI_DEFINE_ENUM_INTERNAL(strncat_chk) +TLI_DEFINE_STRING_INTERNAL("__strncat_chk") /// char *__strncpy_chk(char *s1, const char *s2, size_t n, size_t s1size); TLI_DEFINE_ENUM_INTERNAL(strncpy_chk) TLI_DEFINE_STRING_INTERNAL("__strncpy_chk") @@ -403,6 +446,14 @@ TLI_DEFINE_STRING_INTERNAL("__strndup") /// char * __strtok_r(char *s, const char *delim, char **save_ptr); TLI_DEFINE_ENUM_INTERNAL(dunder_strtok_r) TLI_DEFINE_STRING_INTERNAL("__strtok_r") +/// int __vsnprintf_chk(char *s, size_t n, int flags, size_t slen, +/// const char *format, va_list ap); +TLI_DEFINE_ENUM_INTERNAL(vsnprintf_chk) +TLI_DEFINE_STRING_INTERNAL("__vsnprintf_chk") +/// int __vsprintf_chk(char *s, int flags, size_t slen, const char *format, +/// va_list ap); +TLI_DEFINE_ENUM_INTERNAL(vsprintf_chk) +TLI_DEFINE_STRING_INTERNAL("__vsprintf_chk") /// int abs(int j); TLI_DEFINE_ENUM_INTERNAL(abs) TLI_DEFINE_STRING_INTERNAL("abs") @@ -1192,6 +1243,12 @@ TLI_DEFINE_STRING_INTERNAL("strcspn") /// char *strdup(const char *s1); TLI_DEFINE_ENUM_INTERNAL(strdup) TLI_DEFINE_STRING_INTERNAL("strdup") +/// size_t strlcat(char *dst, const char *src, size_t size); +TLI_DEFINE_ENUM_INTERNAL(strlcat) +TLI_DEFINE_STRING_INTERNAL("strlcat") +/// size_t strlcpy(char *dst, const char *src, size_t size); +TLI_DEFINE_ENUM_INTERNAL(strlcpy) +TLI_DEFINE_STRING_INTERNAL("strlcpy") /// size_t strlen(const char *s); TLI_DEFINE_ENUM_INTERNAL(strlen) TLI_DEFINE_STRING_INTERNAL("strlen") diff --git a/include/llvm/Analysis/TargetLibraryInfo.h b/include/llvm/Analysis/TargetLibraryInfo.h index a3fe834022f7..4b5200f5a838 100644 --- a/include/llvm/Analysis/TargetLibraryInfo.h +++ b/include/llvm/Analysis/TargetLibraryInfo.h @@ -1,9 +1,8 @@ //===-- TargetLibraryInfo.h - Library information ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -87,6 +86,7 @@ public: enum VectorLibrary { NoLibrary, // Don't use any vector library. Accelerate, // Use Accelerate framework. + MASSV, // IBM MASS vector library. SVML // Intel short vector math library. }; @@ -281,9 +281,9 @@ public: case LibFunc_trunc: case LibFunc_truncf: case LibFunc_truncl: case LibFunc_log2: case LibFunc_log2f: case LibFunc_log2l: case LibFunc_exp2: case LibFunc_exp2f: case LibFunc_exp2l: - case LibFunc_memcmp: case LibFunc_strcmp: case LibFunc_strcpy: - case LibFunc_stpcpy: case LibFunc_strlen: case LibFunc_strnlen: - case LibFunc_memchr: case LibFunc_mempcpy: + case LibFunc_memcmp: case LibFunc_bcmp: case LibFunc_strcmp: + case LibFunc_strcpy: case LibFunc_stpcpy: case LibFunc_strlen: + case LibFunc_strnlen: case LibFunc_memchr: case LibFunc_mempcpy: return true; } return false; diff --git a/include/llvm/Analysis/TargetTransformInfo.h b/include/llvm/Analysis/TargetTransformInfo.h index 223175d17c2d..7574b811bc1c 100644 --- a/include/llvm/Analysis/TargetTransformInfo.h +++ b/include/llvm/Analysis/TargetTransformInfo.h @@ -1,9 +1,8 @@ //===- TargetTransformInfo.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -28,6 +27,10 @@ #include "llvm/Pass.h" #include "llvm/Support/AtomicOrdering.h" #include "llvm/Support/DataTypes.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/ScalarEvolution.h" +#include "llvm/IR/Dominators.h" +#include "llvm/Analysis/AssumptionCache.h" #include namespace llvm { @@ -36,6 +39,8 @@ namespace Intrinsic { enum ID : unsigned; } +class AssumptionCache; +class BranchInst; class Function; class GlobalValue; class IntrinsicInst; @@ -45,6 +50,7 @@ class SCEV; class ScalarEvolution; class StoreInst; class SwitchInst; +class TargetLibraryInfo; class Type; class User; class Value; @@ -73,6 +79,30 @@ struct MemIntrinsicInfo { } }; +/// Attributes of a target dependent hardware loop. +struct HardwareLoopInfo { + HardwareLoopInfo() = delete; + HardwareLoopInfo(Loop *L) : L(L) {} + Loop *L = nullptr; + BasicBlock *ExitBlock = nullptr; + BranchInst *ExitBranch = nullptr; + const SCEV *ExitCount = nullptr; + IntegerType *CountType = nullptr; + Value *LoopDecrement = nullptr; // Decrement the loop counter by this + // value in every iteration. + bool IsNestingLegal = false; // Can a hardware loop be a parent to + // another hardware loop? + bool CounterInReg = false; // Should loop counter be updated in + // the loop via a phi? + bool PerformEntryTest = false; // Generate the intrinsic which also performs + // icmp ne zero on the loop counter value and + // produces an i1 to guard the loop entry. + bool isHardwareLoopCandidate(ScalarEvolution &SE, LoopInfo &LI, + DominatorTree &DT, bool ForceNestedLoop = false, + bool ForceHardwareLoopPHI = false); + bool canAnalyze(LoopInfo &LI); +}; + /// This pass provides access to the codegen interfaces that are needed /// for IR-level transformations. class TargetTransformInfo { @@ -81,7 +111,7 @@ public: /// API below. /// /// This is used by targets to construct a TTI wrapping their target-specific - /// implementaion that encodes appropriate costs for their target. + /// implementation that encodes appropriate costs for their target. template TargetTransformInfo(T Impl); /// Construct a baseline TTI object using a minimal implementation of @@ -209,18 +239,21 @@ public: /// This is the most basic query for estimating call cost: it only knows the /// function type and (potentially) the number of arguments at the call site. /// The latter is only interesting for varargs function types. - int getCallCost(FunctionType *FTy, int NumArgs = -1) const; + int getCallCost(FunctionType *FTy, int NumArgs = -1, + const User *U = nullptr) const; /// Estimate the cost of calling a specific function when lowered. /// /// This overload adds the ability to reason about the particular function /// being called in the event it is a library call with special lowering. - int getCallCost(const Function *F, int NumArgs = -1) const; + int getCallCost(const Function *F, int NumArgs = -1, + const User *U = nullptr) const; /// Estimate the cost of calling a specific function when lowered. /// /// This overload allows specifying a set of candidate argument values. - int getCallCost(const Function *F, ArrayRef Arguments) const; + int getCallCost(const Function *F, ArrayRef Arguments, + const User *U = nullptr) const; /// \returns A value by which our inlining threshold should be multiplied. /// This is primarily used to bump up the inlining threshold wholesale on @@ -230,17 +263,35 @@ public: /// individual classes of instructions would be better. unsigned getInliningThresholdMultiplier() const; + /// \returns Vector bonus in percent. + /// + /// Vector bonuses: We want to more aggressively inline vector-dense kernels + /// and apply this bonus based on the percentage of vector instructions. A + /// bonus is applied if the vector instructions exceed 50% and half that amount + /// is applied if it exceeds 10%. Note that these bonuses are some what + /// arbitrary and evolved over time by accident as much as because they are + /// principled bonuses. + /// FIXME: It would be nice to base the bonus values on something more + /// scientific. A target may has no bonus on vector instructions. + int getInlinerVectorBonusPercent() const; + /// Estimate the cost of an intrinsic when lowered. /// /// Mirrors the \c getCallCost method but uses an intrinsic identifier. int getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, - ArrayRef ParamTys) const; + ArrayRef ParamTys, + const User *U = nullptr) const; /// Estimate the cost of an intrinsic when lowered. /// /// Mirrors the \c getCallCost method but uses an intrinsic identifier. int getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, - ArrayRef Arguments) const; + ArrayRef Arguments, + const User *U = nullptr) const; + + /// \return the expected cost of a memcpy, which could e.g. depend on the + /// source/destination type and alignment and the number of bytes copied. + int getMemcpyCost(const Instruction *I) const; /// \return The estimated number of case clusters when lowering \p 'SI'. /// \p JTSize Set a jump table size only when \p SI is suitable for a jump @@ -296,7 +347,7 @@ public: // Returns true for the target specific // set of operations which produce uniform result - // even taking non-unform arguments + // even taking non-uniform arguments bool isAlwaysUniform(const Value *V) const; /// Returns the address space ID for a target's 'flat' address space. Note @@ -437,6 +488,13 @@ public: void getUnrollingPreferences(Loop *L, ScalarEvolution &, UnrollingPreferences &UP) const; + /// Query the target whether it would be profitable to convert the given loop + /// into a hardware loop. + bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE, + AssumptionCache &AC, + TargetLibraryInfo *LibInfo, + HardwareLoopInfo &HWLoopInfo) const; + /// @} /// \name Scalar Target Information @@ -483,21 +541,40 @@ public: /// calculation for the instructions in a loop. bool canMacroFuseCmp() const; + /// Return true if the target can save a compare for loop count, for example + /// hardware loop saves a compare. + bool canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, LoopInfo *LI, + DominatorTree *DT, AssumptionCache *AC, + TargetLibraryInfo *LibInfo) const; + /// \return True is LSR should make efforts to create/preserve post-inc /// addressing mode expressions. bool shouldFavorPostInc() const; - /// Return true if the target supports masked load/store - /// AVX2 and AVX-512 targets allow masks for consecutive load and store + /// Return true if LSR should make efforts to generate indexed addressing + /// modes that operate across loop iterations. + bool shouldFavorBackedgeIndex(const Loop *L) const; + + /// Return true if the target supports masked load. bool isLegalMaskedStore(Type *DataType) const; + /// Return true if the target supports masked store. bool isLegalMaskedLoad(Type *DataType) const; - /// Return true if the target supports masked gather/scatter - /// AVX-512 fully supports gather and scatter for vectors with 32 and 64 - /// bits scalar type. + /// Return true if the target supports nontemporal store. + bool isLegalNTStore(Type *DataType, unsigned Alignment) const; + /// Return true if the target supports nontemporal load. + bool isLegalNTLoad(Type *DataType, unsigned Alignment) const; + + /// Return true if the target supports masked scatter. bool isLegalMaskedScatter(Type *DataType) const; + /// Return true if the target supports masked gather. bool isLegalMaskedGather(Type *DataType) const; + /// Return true if the target supports masked compress store. + bool isLegalMaskedCompressStore(Type *DataType) const; + /// Return true if the target supports masked expand load. + bool isLegalMaskedExpandLoad(Type *DataType) const; + /// Return true if the target has a unified operation to calculate division /// and remainder. If so, the additional implicit multiplication and /// subtraction required to calculate a remainder from division are free. This @@ -576,17 +653,35 @@ public: /// Don't restrict interleaved unrolling to small loops. bool enableAggressiveInterleaving(bool LoopHasReductions) const; - /// If not nullptr, enable inline expansion of memcmp. IsZeroCmp is - /// true if this is the expansion of memcmp(p1, p2, s) == 0. + /// Returns options for expansion of memcmp. IsZeroCmp is + // true if this is the expansion of memcmp(p1, p2, s) == 0. struct MemCmpExpansionOptions { + // Return true if memcmp expansion is enabled. + operator bool() const { return MaxNumLoads > 0; } + + // Maximum number of load operations. + unsigned MaxNumLoads = 0; + // The list of available load sizes (in bytes), sorted in decreasing order. SmallVector LoadSizes; + + // For memcmp expansion when the memcmp result is only compared equal or + // not-equal to 0, allow up to this number of load pairs per block. As an + // example, this may allow 'memcmp(a, b, 3) == 0' in a single block: + // a0 = load2bytes &a[0] + // b0 = load2bytes &b[0] + // a2 = load1byte &a[2] + // b2 = load1byte &b[2] + // r = cmp eq (a0 ^ b0 | a2 ^ b2), 0 + unsigned NumLoadsPerBlock = 1; + // Set to true to allow overlapping loads. For example, 7-byte compares can // be done with two 4-byte compares instead of 4+2+1-byte compares. This // requires all loads in LoadSizes to be doable in an unaligned way. bool AllowOverlappingLoads = false; }; - const MemCmpExpansionOptions *enableMemCmpExpansion(bool IsZeroCmp) const; + MemCmpExpansionOptions enableMemCmpExpansion(bool OptSize, + bool IsZeroCmp) const; /// Enable matching of interleaved access groups. bool enableInterleavedAccessVectorization() const; @@ -700,7 +795,7 @@ public: bool shouldMaximizeVectorBandwidth(bool OptSize) const; /// \return The minimum vectorization factor for types of given element - /// bit width, or 0 if there is no mimimum VF. The returned value only + /// bit width, or 0 if there is no minimum VF. The returned value only /// applies when shouldMaximizeVectorBandwidth returns true. unsigned getMinimumVF(unsigned ElemWidth) const; @@ -1005,6 +1100,11 @@ public: /// \returns True if the target wants to expand the given reduction intrinsic /// into a shuffle sequence. bool shouldExpandReduction(const IntrinsicInst *II) const; + + /// \returns the size cost of rematerializing a GlobalValue address relative + /// to a stack reload. + unsigned getGISelRematGlobalCost() const; + /// @} private: @@ -1035,15 +1135,18 @@ public: virtual int getGEPCost(Type *PointeeType, const Value *Ptr, ArrayRef Operands) = 0; virtual int getExtCost(const Instruction *I, const Value *Src) = 0; - virtual int getCallCost(FunctionType *FTy, int NumArgs) = 0; - virtual int getCallCost(const Function *F, int NumArgs) = 0; + virtual int getCallCost(FunctionType *FTy, int NumArgs, const User *U) = 0; + virtual int getCallCost(const Function *F, int NumArgs, const User *U) = 0; virtual int getCallCost(const Function *F, - ArrayRef Arguments) = 0; + ArrayRef Arguments, const User *U) = 0; virtual unsigned getInliningThresholdMultiplier() = 0; + virtual int getInlinerVectorBonusPercent() = 0; virtual int getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, - ArrayRef ParamTys) = 0; + ArrayRef ParamTys, const User *U) = 0; virtual int getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, - ArrayRef Arguments) = 0; + ArrayRef Arguments, + const User *U) = 0; + virtual int getMemcpyCost(const Instruction *I) = 0; virtual unsigned getEstimatedNumberOfCaseClusters(const SwitchInst &SI, unsigned &JTSize) = 0; virtual int @@ -1055,6 +1158,10 @@ public: virtual bool isLoweredToCall(const Function *F) = 0; virtual void getUnrollingPreferences(Loop *L, ScalarEvolution &, UnrollingPreferences &UP) = 0; + virtual bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE, + AssumptionCache &AC, + TargetLibraryInfo *LibInfo, + HardwareLoopInfo &HWLoopInfo) = 0; virtual bool isLegalAddImmediate(int64_t Imm) = 0; virtual bool isLegalICmpImmediate(int64_t Imm) = 0; virtual bool isLegalAddressingMode(Type *Ty, GlobalValue *BaseGV, @@ -1065,11 +1172,19 @@ public: virtual bool isLSRCostLess(TargetTransformInfo::LSRCost &C1, TargetTransformInfo::LSRCost &C2) = 0; virtual bool canMacroFuseCmp() = 0; + virtual bool canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, + LoopInfo *LI, DominatorTree *DT, AssumptionCache *AC, + TargetLibraryInfo *LibInfo) = 0; virtual bool shouldFavorPostInc() const = 0; + virtual bool shouldFavorBackedgeIndex(const Loop *L) const = 0; virtual bool isLegalMaskedStore(Type *DataType) = 0; virtual bool isLegalMaskedLoad(Type *DataType) = 0; + virtual bool isLegalNTStore(Type *DataType, unsigned Alignment) = 0; + virtual bool isLegalNTLoad(Type *DataType, unsigned Alignment) = 0; virtual bool isLegalMaskedScatter(Type *DataType) = 0; virtual bool isLegalMaskedGather(Type *DataType) = 0; + virtual bool isLegalMaskedCompressStore(Type *DataType) = 0; + virtual bool isLegalMaskedExpandLoad(Type *DataType) = 0; virtual bool hasDivRemOp(Type *DataType, bool IsSigned) = 0; virtual bool hasVolatileVariant(Instruction *I, unsigned AddrSpace) = 0; virtual bool prefersVectorizedAddressing() = 0; @@ -1092,8 +1207,8 @@ public: unsigned VF) = 0; virtual bool supportsEfficientVectorElementLoadStore() = 0; virtual bool enableAggressiveInterleaving(bool LoopHasReductions) = 0; - virtual const MemCmpExpansionOptions *enableMemCmpExpansion( - bool IsZeroCmp) const = 0; + virtual MemCmpExpansionOptions + enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const = 0; virtual bool enableInterleavedAccessVectorization() = 0; virtual bool enableMaskedInterleavedAccessVectorization() = 0; virtual bool isFPVectorizationPotentiallyUnsafe() = 0; @@ -1210,6 +1325,7 @@ public: virtual bool useReductionIntrinsic(unsigned Opcode, Type *Ty, ReductionFlags) const = 0; virtual bool shouldExpandReduction(const IntrinsicInst *II) const = 0; + virtual unsigned getGISelRematGlobalCost() const = 0; virtual int getInstructionLatency(const Instruction *I) = 0; }; @@ -1235,26 +1351,33 @@ public: int getExtCost(const Instruction *I, const Value *Src) override { return Impl.getExtCost(I, Src); } - int getCallCost(FunctionType *FTy, int NumArgs) override { - return Impl.getCallCost(FTy, NumArgs); + int getCallCost(FunctionType *FTy, int NumArgs, const User *U) override { + return Impl.getCallCost(FTy, NumArgs, U); } - int getCallCost(const Function *F, int NumArgs) override { - return Impl.getCallCost(F, NumArgs); + int getCallCost(const Function *F, int NumArgs, const User *U) override { + return Impl.getCallCost(F, NumArgs, U); } int getCallCost(const Function *F, - ArrayRef Arguments) override { - return Impl.getCallCost(F, Arguments); + ArrayRef Arguments, const User *U) override { + return Impl.getCallCost(F, Arguments, U); } unsigned getInliningThresholdMultiplier() override { return Impl.getInliningThresholdMultiplier(); } + int getInlinerVectorBonusPercent() override { + return Impl.getInlinerVectorBonusPercent(); + } int getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, - ArrayRef ParamTys) override { - return Impl.getIntrinsicCost(IID, RetTy, ParamTys); + ArrayRef ParamTys, const User *U = nullptr) override { + return Impl.getIntrinsicCost(IID, RetTy, ParamTys, U); } int getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, - ArrayRef Arguments) override { - return Impl.getIntrinsicCost(IID, RetTy, Arguments); + ArrayRef Arguments, + const User *U = nullptr) override { + return Impl.getIntrinsicCost(IID, RetTy, Arguments, U); + } + int getMemcpyCost(const Instruction *I) override { + return Impl.getMemcpyCost(I); } int getUserCost(const User *U, ArrayRef Operands) override { return Impl.getUserCost(U, Operands); @@ -1279,6 +1402,12 @@ public: UnrollingPreferences &UP) override { return Impl.getUnrollingPreferences(L, SE, UP); } + bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE, + AssumptionCache &AC, + TargetLibraryInfo *LibInfo, + HardwareLoopInfo &HWLoopInfo) override { + return Impl.isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo); + } bool isLegalAddImmediate(int64_t Imm) override { return Impl.isLegalAddImmediate(Imm); } @@ -1299,21 +1428,42 @@ public: bool canMacroFuseCmp() override { return Impl.canMacroFuseCmp(); } + bool canSaveCmp(Loop *L, BranchInst **BI, + ScalarEvolution *SE, + LoopInfo *LI, DominatorTree *DT, AssumptionCache *AC, + TargetLibraryInfo *LibInfo) override { + return Impl.canSaveCmp(L, BI, SE, LI, DT, AC, LibInfo); + } bool shouldFavorPostInc() const override { return Impl.shouldFavorPostInc(); } + bool shouldFavorBackedgeIndex(const Loop *L) const override { + return Impl.shouldFavorBackedgeIndex(L); + } bool isLegalMaskedStore(Type *DataType) override { return Impl.isLegalMaskedStore(DataType); } bool isLegalMaskedLoad(Type *DataType) override { return Impl.isLegalMaskedLoad(DataType); } + bool isLegalNTStore(Type *DataType, unsigned Alignment) override { + return Impl.isLegalNTStore(DataType, Alignment); + } + bool isLegalNTLoad(Type *DataType, unsigned Alignment) override { + return Impl.isLegalNTLoad(DataType, Alignment); + } bool isLegalMaskedScatter(Type *DataType) override { return Impl.isLegalMaskedScatter(DataType); } bool isLegalMaskedGather(Type *DataType) override { return Impl.isLegalMaskedGather(DataType); } + bool isLegalMaskedCompressStore(Type *DataType) override { + return Impl.isLegalMaskedCompressStore(DataType); + } + bool isLegalMaskedExpandLoad(Type *DataType) override { + return Impl.isLegalMaskedExpandLoad(DataType); + } bool hasDivRemOp(Type *DataType, bool IsSigned) override { return Impl.hasDivRemOp(DataType, IsSigned); } @@ -1368,9 +1518,9 @@ public: bool enableAggressiveInterleaving(bool LoopHasReductions) override { return Impl.enableAggressiveInterleaving(LoopHasReductions); } - const MemCmpExpansionOptions *enableMemCmpExpansion( - bool IsZeroCmp) const override { - return Impl.enableMemCmpExpansion(IsZeroCmp); + MemCmpExpansionOptions enableMemCmpExpansion(bool OptSize, + bool IsZeroCmp) const override { + return Impl.enableMemCmpExpansion(OptSize, IsZeroCmp); } bool enableInterleavedAccessVectorization() override { return Impl.enableInterleavedAccessVectorization(); @@ -1617,6 +1767,11 @@ public: bool shouldExpandReduction(const IntrinsicInst *II) const override { return Impl.shouldExpandReduction(II); } + + unsigned getGISelRematGlobalCost() const override { + return Impl.getGISelRematGlobalCost(); + } + int getInstructionLatency(const Instruction *I) override { return Impl.getInstructionLatency(I); } diff --git a/include/llvm/Analysis/TargetTransformInfoImpl.h b/include/llvm/Analysis/TargetTransformInfoImpl.h index c9a234deeb7d..b99e1eb9adf0 100644 --- a/include/llvm/Analysis/TargetTransformInfoImpl.h +++ b/include/llvm/Analysis/TargetTransformInfoImpl.h @@ -1,9 +1,8 @@ //===- TargetTransformInfoImpl.h --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -124,7 +123,7 @@ public: return TTI::TCC_Basic; } - unsigned getCallCost(FunctionType *FTy, int NumArgs) { + unsigned getCallCost(FunctionType *FTy, int NumArgs, const User *U) { assert(FTy && "FunctionType must be provided to this routine."); // The target-independent implementation just measures the size of the @@ -141,45 +140,10 @@ public: unsigned getInliningThresholdMultiplier() { return 1; } - unsigned getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, - ArrayRef ParamTys) { - switch (IID) { - default: - // Intrinsics rarely (if ever) have normal argument setup constraints. - // Model them as having a basic instruction cost. - // FIXME: This is wrong for libc intrinsics. - return TTI::TCC_Basic; + int getInlinerVectorBonusPercent() { return 150; } - case Intrinsic::annotation: - case Intrinsic::assume: - case Intrinsic::sideeffect: - case Intrinsic::dbg_declare: - case Intrinsic::dbg_value: - case Intrinsic::dbg_label: - case Intrinsic::invariant_start: - case Intrinsic::invariant_end: - case Intrinsic::launder_invariant_group: - case Intrinsic::strip_invariant_group: - case Intrinsic::is_constant: - case Intrinsic::lifetime_start: - case Intrinsic::lifetime_end: - case Intrinsic::objectsize: - case Intrinsic::ptr_annotation: - case Intrinsic::var_annotation: - case Intrinsic::experimental_gc_result: - case Intrinsic::experimental_gc_relocate: - case Intrinsic::coro_alloc: - case Intrinsic::coro_begin: - case Intrinsic::coro_free: - case Intrinsic::coro_end: - case Intrinsic::coro_frame: - case Intrinsic::coro_size: - case Intrinsic::coro_suspend: - case Intrinsic::coro_param: - case Intrinsic::coro_subfn_addr: - // These intrinsics don't actually represent code after lowering. - return TTI::TCC_Free; - } + unsigned getMemcpyCost(const Instruction *I) { + return TTI::TCC_Expensive; } bool hasBranchDivergence() { return false; } @@ -228,6 +192,13 @@ public: return true; } + bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE, + AssumptionCache &AC, + TargetLibraryInfo *LibInfo, + HardwareLoopInfo &HWLoopInfo) { + return false; + } + void getUnrollingPreferences(Loop *, ScalarEvolution &, TTI::UnrollingPreferences &) {} @@ -252,16 +223,42 @@ public: bool canMacroFuseCmp() { return false; } + bool canSaveCmp(Loop *L, BranchInst **BI, ScalarEvolution *SE, LoopInfo *LI, + DominatorTree *DT, AssumptionCache *AC, + TargetLibraryInfo *LibInfo) { + return false; + } + bool shouldFavorPostInc() const { return false; } + bool shouldFavorBackedgeIndex(const Loop *L) const { return false; } + bool isLegalMaskedStore(Type *DataType) { return false; } bool isLegalMaskedLoad(Type *DataType) { return false; } + bool isLegalNTStore(Type *DataType, unsigned Alignment) { + // By default, assume nontemporal memory stores are available for stores + // that are aligned and have a size that is a power of 2. + unsigned DataSize = DL.getTypeStoreSize(DataType); + return Alignment >= DataSize && isPowerOf2_32(DataSize); + } + + bool isLegalNTLoad(Type *DataType, unsigned Alignment) { + // By default, assume nontemporal memory loads are available for loads that + // are aligned and have a size that is a power of 2. + unsigned DataSize = DL.getTypeStoreSize(DataType); + return Alignment >= DataSize && isPowerOf2_32(DataSize); + } + bool isLegalMaskedScatter(Type *DataType) { return false; } bool isLegalMaskedGather(Type *DataType) { return false; } + bool isLegalMaskedCompressStore(Type *DataType) { return false; } + + bool isLegalMaskedExpandLoad(Type *DataType) { return false; } + bool hasDivRemOp(Type *DataType, bool IsSigned) { return false; } bool hasVolatileVariant(Instruction *I, unsigned AddrSpace) { return false; } @@ -307,9 +304,9 @@ public: bool enableAggressiveInterleaving(bool LoopHasReductions) { return false; } - const TTI::MemCmpExpansionOptions *enableMemCmpExpansion( - bool IsZeroCmp) const { - return nullptr; + TTI::MemCmpExpansionOptions enableMemCmpExpansion(bool OptSize, + bool IsZeroCmp) const { + return {}; } bool enableInterleavedAccessVectorization() { return false; } @@ -583,6 +580,10 @@ public: return true; } + unsigned getGISelRematGlobalCost() const { + return 1; + } + protected: // Obtain the minimum required size to hold the value (without the sign) // In case of a vector it returns the min required size for one element. @@ -679,7 +680,7 @@ protected: public: using BaseT::getCallCost; - unsigned getCallCost(const Function *F, int NumArgs) { + unsigned getCallCost(const Function *F, int NumArgs, const User *U) { assert(F && "A concrete function must be provided to this routine."); if (NumArgs < 0) @@ -691,35 +692,34 @@ public: FunctionType *FTy = F->getFunctionType(); SmallVector ParamTys(FTy->param_begin(), FTy->param_end()); return static_cast(this) - ->getIntrinsicCost(IID, FTy->getReturnType(), ParamTys); + ->getIntrinsicCost(IID, FTy->getReturnType(), ParamTys, U); } if (!static_cast(this)->isLoweredToCall(F)) return TTI::TCC_Basic; // Give a basic cost if it will be lowered // directly. - return static_cast(this)->getCallCost(F->getFunctionType(), NumArgs); + return static_cast(this)->getCallCost(F->getFunctionType(), NumArgs, U); } - unsigned getCallCost(const Function *F, ArrayRef Arguments) { + unsigned getCallCost(const Function *F, ArrayRef Arguments, + const User *U) { // Simply delegate to generic handling of the call. // FIXME: We should use instsimplify or something else to catch calls which // will constant fold with these arguments. - return static_cast(this)->getCallCost(F, Arguments.size()); + return static_cast(this)->getCallCost(F, Arguments.size(), U); } using BaseT::getGEPCost; int getGEPCost(Type *PointeeType, const Value *Ptr, ArrayRef Operands) { - const GlobalValue *BaseGV = nullptr; - if (Ptr != nullptr) { - // TODO: will remove this when pointers have an opaque type. - assert(Ptr->getType()->getScalarType()->getPointerElementType() == - PointeeType && - "explicit pointee type doesn't match operand's pointee type"); - BaseGV = dyn_cast(Ptr->stripPointerCasts()); - } + assert(PointeeType && Ptr && "can't get GEPCost of nullptr"); + // TODO: will remove this when pointers have an opaque type. + assert(Ptr->getType()->getScalarType()->getPointerElementType() == + PointeeType && + "explicit pointee type doesn't match operand's pointee type"); + auto *BaseGV = dyn_cast(Ptr->stripPointerCasts()); bool HasBaseReg = (BaseGV == nullptr); auto PtrSizeBits = DL.getPointerTypeSizeInBits(Ptr->getType()); @@ -762,21 +762,60 @@ public: } } - // Assumes the address space is 0 when Ptr is nullptr. - unsigned AS = - (Ptr == nullptr ? 0 : Ptr->getType()->getPointerAddressSpace()); - if (static_cast(this)->isLegalAddressingMode( TargetType, const_cast(BaseGV), - BaseOffset.sextOrTrunc(64).getSExtValue(), HasBaseReg, Scale, AS)) + BaseOffset.sextOrTrunc(64).getSExtValue(), HasBaseReg, Scale, + Ptr->getType()->getPointerAddressSpace())) return TTI::TCC_Free; return TTI::TCC_Basic; } - using BaseT::getIntrinsicCost; + unsigned getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, + ArrayRef ParamTys, const User *U) { + switch (IID) { + default: + // Intrinsics rarely (if ever) have normal argument setup constraints. + // Model them as having a basic instruction cost. + return TTI::TCC_Basic; + + // TODO: other libc intrinsics. + case Intrinsic::memcpy: + return static_cast(this)->getMemcpyCost(dyn_cast(U)); + + case Intrinsic::annotation: + case Intrinsic::assume: + case Intrinsic::sideeffect: + case Intrinsic::dbg_declare: + case Intrinsic::dbg_value: + case Intrinsic::dbg_label: + case Intrinsic::invariant_start: + case Intrinsic::invariant_end: + case Intrinsic::launder_invariant_group: + case Intrinsic::strip_invariant_group: + case Intrinsic::is_constant: + case Intrinsic::lifetime_start: + case Intrinsic::lifetime_end: + case Intrinsic::objectsize: + case Intrinsic::ptr_annotation: + case Intrinsic::var_annotation: + case Intrinsic::experimental_gc_result: + case Intrinsic::experimental_gc_relocate: + case Intrinsic::coro_alloc: + case Intrinsic::coro_begin: + case Intrinsic::coro_free: + case Intrinsic::coro_end: + case Intrinsic::coro_frame: + case Intrinsic::coro_size: + case Intrinsic::coro_suspend: + case Intrinsic::coro_param: + case Intrinsic::coro_subfn_addr: + // These intrinsics don't actually represent code after lowering. + return TTI::TCC_Free; + } + } unsigned getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, - ArrayRef Arguments) { + ArrayRef Arguments, const User *U) { // Delegate to the generic intrinsic handling code. This mostly provides an // opportunity for targets to (for example) special case the cost of // certain intrinsics based on constants used as arguments. @@ -784,7 +823,7 @@ public: ParamTys.reserve(Arguments.size()); for (unsigned Idx = 0, Size = Arguments.size(); Idx != Size; ++Idx) ParamTys.push_back(Arguments[Idx]->getType()); - return static_cast(this)->getIntrinsicCost(IID, RetTy, ParamTys); + return static_cast(this)->getIntrinsicCost(IID, RetTy, ParamTys, U); } unsigned getUserCost(const User *U, ArrayRef Operands) { @@ -808,22 +847,18 @@ public: // Just use the called value type. Type *FTy = CS.getCalledValue()->getType()->getPointerElementType(); return static_cast(this) - ->getCallCost(cast(FTy), CS.arg_size()); + ->getCallCost(cast(FTy), CS.arg_size(), U); } SmallVector Arguments(CS.arg_begin(), CS.arg_end()); - return static_cast(this)->getCallCost(F, Arguments); + return static_cast(this)->getCallCost(F, Arguments, U); } - if (const CastInst *CI = dyn_cast(U)) { - // Result of a cmp instruction is often extended (to be used by other - // cmp instructions, logical or return instructions). These are usually - // nop on most sane targets. - if (isa(CI->getOperand(0))) - return TTI::TCC_Free; - if (isa(CI) || isa(CI) || isa(CI)) - return static_cast(this)->getExtCost(CI, Operands.back()); - } + if (isa(U) || isa(U) || isa(U)) + // The old behaviour of generally treating extensions of icmp to be free + // has been removed. A target that needs it should override getUserCost(). + return static_cast(this)->getExtCost(cast(U), + Operands.back()); return static_cast(this)->getOperationCost( Operator::getOpcode(U), U->getType(), diff --git a/include/llvm/Analysis/Trace.h b/include/llvm/Analysis/Trace.h index b05d384ab1a3..a1ffd03c4053 100644 --- a/include/llvm/Analysis/Trace.h +++ b/include/llvm/Analysis/Trace.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/Trace.h - Represent one trace of LLVM code -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/TypeBasedAliasAnalysis.h b/include/llvm/Analysis/TypeBasedAliasAnalysis.h index d2e6df22425e..344f26806618 100644 --- a/include/llvm/Analysis/TypeBasedAliasAnalysis.h +++ b/include/llvm/Analysis/TypeBasedAliasAnalysis.h @@ -1,9 +1,8 @@ //===- TypeBasedAliasAnalysis.h - Type-Based Alias Analysis -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -41,12 +40,16 @@ public: return false; } - AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB); - bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal); + AliasResult alias(const MemoryLocation &LocA, const MemoryLocation &LocB, + AAQueryInfo &AAQI); + bool pointsToConstantMemory(const MemoryLocation &Loc, AAQueryInfo &AAQI, + bool OrLocal); FunctionModRefBehavior getModRefBehavior(const CallBase *Call); FunctionModRefBehavior getModRefBehavior(const Function *F); - ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc); - ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2); + ModRefInfo getModRefInfo(const CallBase *Call, const MemoryLocation &Loc, + AAQueryInfo &AAQI); + ModRefInfo getModRefInfo(const CallBase *Call1, const CallBase *Call2, + AAQueryInfo &AAQI); private: bool Aliases(const MDNode *A, const MDNode *B) const; diff --git a/include/llvm/Analysis/TypeMetadataUtils.h b/include/llvm/Analysis/TypeMetadataUtils.h index 3bf9c5d20741..82cf8efeea54 100644 --- a/include/llvm/Analysis/TypeMetadataUtils.h +++ b/include/llvm/Analysis/TypeMetadataUtils.h @@ -1,9 +1,8 @@ //===- TypeMetadataUtils.h - Utilities related to type metadata --*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/Utils/Local.h b/include/llvm/Analysis/Utils/Local.h index b4141bbff28d..acbdf5dca32c 100644 --- a/include/llvm/Analysis/Utils/Local.h +++ b/include/llvm/Analysis/Utils/Local.h @@ -1,9 +1,8 @@ //===- Local.h - Functions to perform local transformations -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/ValueLattice.h b/include/llvm/Analysis/ValueLattice.h index 0744ca617e48..56519d7d0857 100644 --- a/include/llvm/Analysis/ValueLattice.h +++ b/include/llvm/Analysis/ValueLattice.h @@ -1,9 +1,8 @@ //===- ValueLattice.h - Value constraint analysis ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Analysis/ValueLatticeUtils.h b/include/llvm/Analysis/ValueLatticeUtils.h index 02072672e56e..a3bbb96129bf 100644 --- a/include/llvm/Analysis/ValueLatticeUtils.h +++ b/include/llvm/Analysis/ValueLatticeUtils.h @@ -1,9 +1,8 @@ //===-- ValueLatticeUtils.h - Utils for solving lattices --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Analysis/ValueTracking.h b/include/llvm/Analysis/ValueTracking.h index f46fdfcb608e..fa7e0e0eef7e 100644 --- a/include/llvm/Analysis/ValueTracking.h +++ b/include/llvm/Analysis/ValueTracking.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/ValueTracking.h - Walk computations --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -17,8 +16,10 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/Constants.h" +#include "llvm/IR/DataLayout.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Intrinsics.h" #include @@ -29,10 +30,10 @@ namespace llvm { class AddOperator; class APInt; class AssumptionCache; -class DataLayout; class DominatorTree; class GEPOperator; class IntrinsicInst; +class WithOverflowInst; struct KnownBits; class Loop; class LoopInfo; @@ -223,7 +224,7 @@ class Value; /// 0.0 etc. If the value can't be handled with a repeated byte store (e.g. /// i16 0x1234), return null. If the value is entirely undef and padding, /// return undef. - Value *isBytewiseValue(Value *V); + Value *isBytewiseValue(Value *V, const DataLayout &DL); /// Given an aggregrate and an sequence of indices, see if the scalar value /// indexed is already around as a register, for example if it were inserted @@ -237,8 +238,18 @@ class Value; /// Analyze the specified pointer to see if it can be expressed as a base /// pointer plus a constant offset. Return the base and offset to the caller. - Value *GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, - const DataLayout &DL); + /// + /// This is a wrapper around Value::stripAndAccumulateConstantOffsets that + /// creates and later unpacks the required APInt. + inline Value *GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, + const DataLayout &DL) { + APInt OffsetAPInt(DL.getIndexTypeSizeInBits(Ptr->getType()), 0); + Value *Base = + Ptr->stripAndAccumulateConstantOffsets(DL, OffsetAPInt, + /* AllowNonInbounds */ true); + Offset = OffsetAPInt.getSExtValue(); + return Base; + } inline const Value *GetPointerBaseWithConstantOffset(const Value *Ptr, int64_t &Offset, const DataLayout &DL) { @@ -351,7 +362,8 @@ class Value; /// Since A[i] and A[i-1] are independent pointers, getUnderlyingObjects /// should not assume that Curr and Prev share the same underlying object thus /// it shouldn't look through the phi above. - void GetUnderlyingObjects(Value *V, SmallVectorImpl &Objects, + void GetUnderlyingObjects(const Value *V, + SmallVectorImpl &Objects, const DataLayout &DL, LoopInfo *LI = nullptr, unsigned MaxLookup = 6); @@ -411,7 +423,16 @@ class Value; bool isValidAssumeForContext(const Instruction *I, const Instruction *CxtI, const DominatorTree *DT = nullptr); - enum class OverflowResult { AlwaysOverflows, MayOverflow, NeverOverflows }; + enum class OverflowResult { + /// Always overflows in the direction of signed/unsigned min value. + AlwaysOverflowsLow, + /// Always overflows in the direction of signed/unsigned max value. + AlwaysOverflowsHigh, + /// May or may not overflow. + MayOverflow, + /// Never overflows. + NeverOverflows, + }; OverflowResult computeOverflowForUnsignedMul(const Value *LHS, const Value *RHS, @@ -455,12 +476,17 @@ class Value; const Instruction *CxtI, const DominatorTree *DT); - /// Returns true if the arithmetic part of the \p II 's result is + /// Returns true if the arithmetic part of the \p WO 's result is /// used only along the paths control dependent on the computation - /// not overflowing, \p II being an .with.overflow intrinsic. - bool isOverflowIntrinsicNoWrap(const IntrinsicInst *II, + /// not overflowing, \p WO being an .with.overflow intrinsic. + bool isOverflowIntrinsicNoWrap(const WithOverflowInst *WO, const DominatorTree &DT); + + /// Determine the possible constant range of an integer or vector of integer + /// value. This is intended as a cheap, non-recursive check. + ConstantRange computeConstantRange(const Value *V, bool UseInstrInfo = true); + /// Return true if this function can prove that the instruction I will /// always transfer execution to one of its successors (including the next /// instruction that follows within a basic block). E.g. this is not @@ -506,6 +532,12 @@ class Value; /// value (all bits poison). const Value *getGuaranteedNonFullPoisonOp(const Instruction *I); + /// Return true if the given instruction must trigger undefined behavior. + /// when I is executed with any operands which appear in KnownPoison holding + /// a full-poison value at the point of execution. + bool mustTriggerUB(const Instruction *I, + const SmallSet& KnownPoison); + /// Return true if this function can prove that if PoisonI is executed /// and yields a full-poison value (all bits poison), then that will /// trigger undefined behavior. @@ -584,6 +616,12 @@ class Value; return Result; } + /// Determine the pattern that a select with the given compare as its + /// predicate and given values as its true/false operands would match. + SelectPatternResult matchDecomposedSelectPattern( + CmpInst *CmpI, Value *TrueVal, Value *FalseVal, Value *&LHS, Value *&RHS, + Instruction::CastOps *CastOp = nullptr, unsigned Depth = 0); + /// Return the canonical comparison predicate for the specified /// minimum/maximum flavor. CmpInst::Predicate getMinMaxPred(SelectPatternFlavor SPF, diff --git a/include/llvm/Analysis/VecFuncs.def b/include/llvm/Analysis/VecFuncs.def new file mode 100644 index 000000000000..4c9206266d9a --- /dev/null +++ b/include/llvm/Analysis/VecFuncs.def @@ -0,0 +1,250 @@ +//===-- VecFuncs.def - Library information -------------*- C++ -*-----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// This .def file will create mappings from scalar math functions to vector +// functions along with their vectorization factor. The current support includes +// such mappings for Accelerate framework, MASS vector library, and SVML library. + +#if !(defined(TLI_DEFINE_VECFUNC)) +#define TLI_DEFINE_VECFUNC(SCAL, VEC, VF) {SCAL, VEC, VF}, +#endif + +#if defined(TLI_DEFINE_ACCELERATE_VECFUNCS) +// Accelerate framework's Vector Functions + +// Floating-Point Arithmetic and Auxiliary Functions +TLI_DEFINE_VECFUNC("ceilf", "vceilf", 4) +TLI_DEFINE_VECFUNC("fabsf", "vfabsf", 4) +TLI_DEFINE_VECFUNC("llvm.fabs.f32", "vfabsf", 4) +TLI_DEFINE_VECFUNC("floorf", "vfloorf", 4) +TLI_DEFINE_VECFUNC("sqrtf", "vsqrtf", 4) +TLI_DEFINE_VECFUNC("llvm.sqrt.f32", "vsqrtf", 4) + +// Exponential and Logarithmic Functions +TLI_DEFINE_VECFUNC("expf", "vexpf", 4) +TLI_DEFINE_VECFUNC("llvm.exp.f32", "vexpf", 4) +TLI_DEFINE_VECFUNC("expm1f", "vexpm1f", 4) +TLI_DEFINE_VECFUNC("logf", "vlogf", 4) +TLI_DEFINE_VECFUNC("llvm.log.f32", "vlogf", 4) +TLI_DEFINE_VECFUNC("log1pf", "vlog1pf", 4) +TLI_DEFINE_VECFUNC("log10f", "vlog10f", 4) +TLI_DEFINE_VECFUNC("llvm.log10.f32", "vlog10f", 4) +TLI_DEFINE_VECFUNC("logbf", "vlogbf", 4) + +// Trigonometric Functions +TLI_DEFINE_VECFUNC("sinf", "vsinf", 4) +TLI_DEFINE_VECFUNC("llvm.sin.f32", "vsinf", 4) +TLI_DEFINE_VECFUNC("cosf", "vcosf", 4) +TLI_DEFINE_VECFUNC("llvm.cos.f32", "vcosf", 4) +TLI_DEFINE_VECFUNC("tanf", "vtanf", 4) +TLI_DEFINE_VECFUNC("asinf", "vasinf", 4) +TLI_DEFINE_VECFUNC("acosf", "vacosf", 4) +TLI_DEFINE_VECFUNC("atanf", "vatanf", 4) + +// Hyperbolic Functions +TLI_DEFINE_VECFUNC("sinhf", "vsinhf", 4) +TLI_DEFINE_VECFUNC("coshf", "vcoshf", 4) +TLI_DEFINE_VECFUNC("tanhf", "vtanhf", 4) +TLI_DEFINE_VECFUNC("asinhf", "vasinhf", 4) +TLI_DEFINE_VECFUNC("acoshf", "vacoshf", 4) +TLI_DEFINE_VECFUNC("atanhf", "vatanhf", 4) + + +#elif defined(TLI_DEFINE_MASSV_VECFUNCS) +// IBM MASS library's vector Functions + +// Floating-Point Arithmetic and Auxiliary Functions +TLI_DEFINE_VECFUNC("cbrt", "__cbrtd2_massv", 2) +TLI_DEFINE_VECFUNC("cbrtf", "__cbrtf4_massv", 4) +TLI_DEFINE_VECFUNC("pow", "__powd2_massv", 2) +TLI_DEFINE_VECFUNC("llvm.pow.f64", "__powd2_massv", 2) +TLI_DEFINE_VECFUNC("powf", "__powf4_massv", 4) +TLI_DEFINE_VECFUNC("llvm.pow.f32", "__powf4_massv", 4) +TLI_DEFINE_VECFUNC("sqrt", "__sqrtd2_massv", 2) +TLI_DEFINE_VECFUNC("llvm.sqrt.f64", "__sqrtd2_massv", 2) +TLI_DEFINE_VECFUNC("sqrtf", "__sqrtf4_massv", 4) +TLI_DEFINE_VECFUNC("llvm.sqrt.f32", "__sqrtf4_massv", 4) + +// Exponential and Logarithmic Functions +TLI_DEFINE_VECFUNC("exp", "__expd2_massv", 2) +TLI_DEFINE_VECFUNC("llvm.exp.f64", "__expd2_massv", 2) +TLI_DEFINE_VECFUNC("expf", "__expf4_massv", 4) +TLI_DEFINE_VECFUNC("llvm.exp.f32", "__expf4_massv", 4) +TLI_DEFINE_VECFUNC("exp2", "__exp2d2_massv", 2) +TLI_DEFINE_VECFUNC("llvm.exp2.f64", "__exp2d2_massv", 2) +TLI_DEFINE_VECFUNC("exp2f", "__exp2f4_massv", 4) +TLI_DEFINE_VECFUNC("llvm.exp2.f32", "__exp2f4_massv", 4) +TLI_DEFINE_VECFUNC("expm1", "__expm1d2_massv", 2) +TLI_DEFINE_VECFUNC("expm1f", "__expm1f4_massv", 4) +TLI_DEFINE_VECFUNC("log", "__logd2_massv", 2) +TLI_DEFINE_VECFUNC("llvm.log.f64", "__logd2_massv", 2) +TLI_DEFINE_VECFUNC("logf", "__logf4_massv", 4) +TLI_DEFINE_VECFUNC("llvm.log.f32", "__logf4_massv", 4) +TLI_DEFINE_VECFUNC("log1p", "__log1pd2_massv", 2) +TLI_DEFINE_VECFUNC("log1pf", "__log1pf4_massv", 4) +TLI_DEFINE_VECFUNC("log10", "__log10d2_massv", 2) +TLI_DEFINE_VECFUNC("llvm.log10.f64", "__log10d2_massv", 2) +TLI_DEFINE_VECFUNC("log10f", "__log10f4_massv", 4) +TLI_DEFINE_VECFUNC("llvm.log10.f32", "__log10f4_massv", 4) +TLI_DEFINE_VECFUNC("log2", "__log2d2_massv", 2) +TLI_DEFINE_VECFUNC("llvm.log2.f64", "__log2d2_massv", 2) +TLI_DEFINE_VECFUNC("log2f", "__log2f4_massv", 4) +TLI_DEFINE_VECFUNC("llvm.log2.f32", "__log2f4_massv", 4) + +// Trigonometric Functions +TLI_DEFINE_VECFUNC("sin", "__sind2_massv", 2) +TLI_DEFINE_VECFUNC("llvm.sin.f64", "__sind2_massv", 2) +TLI_DEFINE_VECFUNC("sinf", "__sinf4_massv", 4) +TLI_DEFINE_VECFUNC("llvm.sin.f32", "__sinf4_massv", 4) +TLI_DEFINE_VECFUNC("cos", "__cosd2_massv", 2) +TLI_DEFINE_VECFUNC("llvm.cos.f64", "__cosd2_massv", 2) +TLI_DEFINE_VECFUNC("cosf", "__cosf4_massv", 4) +TLI_DEFINE_VECFUNC("llvm.cos.f32", "__cosf4_massv", 4) +TLI_DEFINE_VECFUNC("tan", "__tand2_massv", 2) +TLI_DEFINE_VECFUNC("tanf", "__tanf4_massv", 4) +TLI_DEFINE_VECFUNC("asin", "__asind2_massv", 2) +TLI_DEFINE_VECFUNC("asinf", "__asinf4_massv", 4) +TLI_DEFINE_VECFUNC("acos", "__acosd2_massv", 2) +TLI_DEFINE_VECFUNC("acosf", "__acosf4_massv", 4) +TLI_DEFINE_VECFUNC("atan", "__atand2_massv", 2) +TLI_DEFINE_VECFUNC("atanf", "__atanf4_massv", 4) +TLI_DEFINE_VECFUNC("atan2", "__atan2d2_massv", 2) +TLI_DEFINE_VECFUNC("atan2f", "__atan2f4_massv", 4) + +// Hyperbolic Functions +TLI_DEFINE_VECFUNC("sinh", "__sinhd2_massv", 2) +TLI_DEFINE_VECFUNC("sinhf", "__sinhf4_massv", 4) +TLI_DEFINE_VECFUNC("cosh", "__coshd2_massv", 2) +TLI_DEFINE_VECFUNC("coshf", "__coshf4_massv", 4) +TLI_DEFINE_VECFUNC("tanh", "__tanhd2_massv", 2) +TLI_DEFINE_VECFUNC("tanhf", "__tanhf4_massv", 4) +TLI_DEFINE_VECFUNC("asinh", "__asinhd2_massv", 2) +TLI_DEFINE_VECFUNC("asinhf", "__asinhf4_massv", 4) +TLI_DEFINE_VECFUNC("acosh", "__acoshd2_massv", 2) +TLI_DEFINE_VECFUNC("acoshf", "__acoshf4_massv", 4) +TLI_DEFINE_VECFUNC("atanh", "__atanhd2_massv", 2) +TLI_DEFINE_VECFUNC("atanhf", "__atanhf4_massv", 4) + + +#elif defined(TLI_DEFINE_SVML_VECFUNCS) +// Intel SVM library's Vector Functions + +TLI_DEFINE_VECFUNC("sin", "__svml_sin2", 2) +TLI_DEFINE_VECFUNC("sin", "__svml_sin4", 4) +TLI_DEFINE_VECFUNC("sin", "__svml_sin8", 8) + +TLI_DEFINE_VECFUNC("sinf", "__svml_sinf4", 4) +TLI_DEFINE_VECFUNC("sinf", "__svml_sinf8", 8) +TLI_DEFINE_VECFUNC("sinf", "__svml_sinf16", 16) + +TLI_DEFINE_VECFUNC("llvm.sin.f64", "__svml_sin2", 2) +TLI_DEFINE_VECFUNC("llvm.sin.f64", "__svml_sin4", 4) +TLI_DEFINE_VECFUNC("llvm.sin.f64", "__svml_sin8", 8) + +TLI_DEFINE_VECFUNC("llvm.sin.f32", "__svml_sinf4", 4) +TLI_DEFINE_VECFUNC("llvm.sin.f32", "__svml_sinf8", 8) +TLI_DEFINE_VECFUNC("llvm.sin.f32", "__svml_sinf16", 16) + +TLI_DEFINE_VECFUNC("cos", "__svml_cos2", 2) +TLI_DEFINE_VECFUNC("cos", "__svml_cos4", 4) +TLI_DEFINE_VECFUNC("cos", "__svml_cos8", 8) + +TLI_DEFINE_VECFUNC("cosf", "__svml_cosf4", 4) +TLI_DEFINE_VECFUNC("cosf", "__svml_cosf8", 8) +TLI_DEFINE_VECFUNC("cosf", "__svml_cosf16", 16) + +TLI_DEFINE_VECFUNC("llvm.cos.f64", "__svml_cos2", 2) +TLI_DEFINE_VECFUNC("llvm.cos.f64", "__svml_cos4", 4) +TLI_DEFINE_VECFUNC("llvm.cos.f64", "__svml_cos8", 8) + +TLI_DEFINE_VECFUNC("llvm.cos.f32", "__svml_cosf4", 4) +TLI_DEFINE_VECFUNC("llvm.cos.f32", "__svml_cosf8", 8) +TLI_DEFINE_VECFUNC("llvm.cos.f32", "__svml_cosf16", 16) + +TLI_DEFINE_VECFUNC("pow", "__svml_pow2", 2) +TLI_DEFINE_VECFUNC("pow", "__svml_pow4", 4) +TLI_DEFINE_VECFUNC("pow", "__svml_pow8", 8) + +TLI_DEFINE_VECFUNC("powf", "__svml_powf4", 4) +TLI_DEFINE_VECFUNC("powf", "__svml_powf8", 8) +TLI_DEFINE_VECFUNC("powf", "__svml_powf16", 16) + +TLI_DEFINE_VECFUNC("__pow_finite", "__svml_pow2", 2) +TLI_DEFINE_VECFUNC("__pow_finite", "__svml_pow4", 4) +TLI_DEFINE_VECFUNC("__pow_finite", "__svml_pow8", 8) + +TLI_DEFINE_VECFUNC("__powf_finite", "__svml_powf4", 4) +TLI_DEFINE_VECFUNC("__powf_finite", "__svml_powf8", 8) +TLI_DEFINE_VECFUNC("__powf_finite", "__svml_powf16", 16) + +TLI_DEFINE_VECFUNC("llvm.pow.f64", "__svml_pow2", 2) +TLI_DEFINE_VECFUNC("llvm.pow.f64", "__svml_pow4", 4) +TLI_DEFINE_VECFUNC("llvm.pow.f64", "__svml_pow8", 8) + +TLI_DEFINE_VECFUNC("llvm.pow.f32", "__svml_powf4", 4) +TLI_DEFINE_VECFUNC("llvm.pow.f32", "__svml_powf8", 8) +TLI_DEFINE_VECFUNC("llvm.pow.f32", "__svml_powf16", 16) + +TLI_DEFINE_VECFUNC("exp", "__svml_exp2", 2) +TLI_DEFINE_VECFUNC("exp", "__svml_exp4", 4) +TLI_DEFINE_VECFUNC("exp", "__svml_exp8", 8) + +TLI_DEFINE_VECFUNC("expf", "__svml_expf4", 4) +TLI_DEFINE_VECFUNC("expf", "__svml_expf8", 8) +TLI_DEFINE_VECFUNC("expf", "__svml_expf16", 16) + +TLI_DEFINE_VECFUNC("__exp_finite", "__svml_exp2", 2) +TLI_DEFINE_VECFUNC("__exp_finite", "__svml_exp4", 4) +TLI_DEFINE_VECFUNC("__exp_finite", "__svml_exp8", 8) + +TLI_DEFINE_VECFUNC("__expf_finite", "__svml_expf4", 4) +TLI_DEFINE_VECFUNC("__expf_finite", "__svml_expf8", 8) +TLI_DEFINE_VECFUNC("__expf_finite", "__svml_expf16", 16) + +TLI_DEFINE_VECFUNC("llvm.exp.f64", "__svml_exp2", 2) +TLI_DEFINE_VECFUNC("llvm.exp.f64", "__svml_exp4", 4) +TLI_DEFINE_VECFUNC("llvm.exp.f64", "__svml_exp8", 8) + +TLI_DEFINE_VECFUNC("llvm.exp.f32", "__svml_expf4", 4) +TLI_DEFINE_VECFUNC("llvm.exp.f32", "__svml_expf8", 8) +TLI_DEFINE_VECFUNC("llvm.exp.f32", "__svml_expf16", 16) + +TLI_DEFINE_VECFUNC("log", "__svml_log2", 2) +TLI_DEFINE_VECFUNC("log", "__svml_log4", 4) +TLI_DEFINE_VECFUNC("log", "__svml_log8", 8) + +TLI_DEFINE_VECFUNC("logf", "__svml_logf4", 4) +TLI_DEFINE_VECFUNC("logf", "__svml_logf8", 8) +TLI_DEFINE_VECFUNC("logf", "__svml_logf16", 16) + +TLI_DEFINE_VECFUNC("__log_finite", "__svml_log2", 2) +TLI_DEFINE_VECFUNC("__log_finite", "__svml_log4", 4) +TLI_DEFINE_VECFUNC("__log_finite", "__svml_log8", 8) + +TLI_DEFINE_VECFUNC("__logf_finite", "__svml_logf4", 4) +TLI_DEFINE_VECFUNC("__logf_finite", "__svml_logf8", 8) +TLI_DEFINE_VECFUNC("__logf_finite", "__svml_logf16", 16) + +TLI_DEFINE_VECFUNC("llvm.log.f64", "__svml_log2", 2) +TLI_DEFINE_VECFUNC("llvm.log.f64", "__svml_log4", 4) +TLI_DEFINE_VECFUNC("llvm.log.f64", "__svml_log8", 8) + +TLI_DEFINE_VECFUNC("llvm.log.f32", "__svml_logf4", 4) +TLI_DEFINE_VECFUNC("llvm.log.f32", "__svml_logf8", 8) +TLI_DEFINE_VECFUNC("llvm.log.f32", "__svml_logf16", 16) + + +#else +#error "Must choose which vector library functions are to be defined." +#endif + +#undef TLI_DEFINE_VECFUNC +#undef TLI_DEFINE_ACCELERATE_VECFUNCS +#undef TLI_DEFINE_MASSV_VECFUNCS +#undef TLI_DEFINE_SVML_VECFUNCS + diff --git a/include/llvm/Analysis/VectorUtils.h b/include/llvm/Analysis/VectorUtils.h index be4d4f17b9ad..d93d2bc4570b 100644 --- a/include/llvm/Analysis/VectorUtils.h +++ b/include/llvm/Analysis/VectorUtils.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/VectorUtils.h - Vector utilities -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,6 +17,7 @@ #include "llvm/Analysis/LoopAccessAnalysis.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/IRBuilder.h" +#include "llvm/Support/CheckedArithmetic.h" namespace llvm { @@ -36,13 +36,12 @@ enum ID : unsigned; } /// Identify if the intrinsic is trivially vectorizable. -/// This method returns true if the intrinsic's argument types are all -/// scalars for the scalar form of the intrinsic and all vectors for -/// the vector form of the intrinsic. +/// This method returns true if the intrinsic's argument types are all scalars +/// for the scalar form of the intrinsic and all vectors (or scalars handled by +/// hasVectorInstrinsicScalarOpd) for the vector form of the intrinsic. bool isTriviallyVectorizable(Intrinsic::ID ID); -/// Identifies if the intrinsic has a scalar operand. It checks for -/// ctlz,cttz and powi special intrinsics whose argument is scalar. +/// Identifies if the vector form of the intrinsic has a scalar operand. bool hasVectorInstrinsicScalarOpd(Intrinsic::ID ID, unsigned ScalarOpdIdx); /// Returns intrinsic ID for call. @@ -78,6 +77,12 @@ Value *findScalarElement(Value *V, unsigned EltNo); /// a sequence of instructions that broadcast a single value into a vector. const Value *getSplatValue(const Value *V); +/// Return true if the input value is known to be a vector with all identical +/// elements (potentially including undefined elements). +/// This may be more powerful than the related getSplatValue() because it is +/// not limited by finding a scalar source value to a splatted vector. +bool isSplatValue(const Value *V, unsigned Depth = 0); + /// Compute a map of integer instructions to their minimum legal type /// size. /// @@ -223,6 +228,20 @@ Constant *createSequentialMask(IRBuilder<> &Builder, unsigned Start, /// elements, it will be padded with undefs. Value *concatenateVectors(IRBuilder<> &Builder, ArrayRef Vecs); +/// Given a mask vector of the form , Return true if all of the +/// elements of this predicate mask are false or undef. That is, return true +/// if all lanes can be assumed inactive. +bool maskIsAllZeroOrUndef(Value *Mask); + +/// Given a mask vector of the form , Return true if all of the +/// elements of this predicate mask are true or undef. That is, return true +/// if all lanes can be assumed active. +bool maskIsAllOneOrUndef(Value *Mask); + +/// Given a mask vector of the form , return an APInt (of bitwidth Y) +/// for each lane which may be active. +APInt possiblyDemandedEltsInMask(Value *Mask); + /// The group of interleaved loads/stores sharing the same stride and /// close to each other. /// @@ -251,10 +270,10 @@ Value *concatenateVectors(IRBuilder<> &Builder, ArrayRef Vecs); /// the interleaved store group doesn't allow gaps. template class InterleaveGroup { public: - InterleaveGroup(unsigned Factor, bool Reverse, unsigned Align) + InterleaveGroup(uint32_t Factor, bool Reverse, uint32_t Align) : Factor(Factor), Reverse(Reverse), Align(Align), InsertPos(nullptr) {} - InterleaveGroup(InstTy *Instr, int Stride, unsigned Align) + InterleaveGroup(InstTy *Instr, int32_t Stride, uint32_t Align) : Align(Align), InsertPos(Instr) { assert(Align && "The alignment should be non-zero"); @@ -266,19 +285,23 @@ public: } bool isReverse() const { return Reverse; } - unsigned getFactor() const { return Factor; } - unsigned getAlignment() const { return Align; } - unsigned getNumMembers() const { return Members.size(); } + uint32_t getFactor() const { return Factor; } + uint32_t getAlignment() const { return Align; } + uint32_t getNumMembers() const { return Members.size(); } /// Try to insert a new member \p Instr with index \p Index and /// alignment \p NewAlign. The index is related to the leader and it could be /// negative if it is the new leader. /// /// \returns false if the instruction doesn't belong to the group. - bool insertMember(InstTy *Instr, int Index, unsigned NewAlign) { + bool insertMember(InstTy *Instr, int32_t Index, uint32_t NewAlign) { assert(NewAlign && "The new member's alignment should be non-zero"); - int Key = Index + SmallestKey; + // Make sure the key fits in an int32_t. + Optional MaybeKey = checkedAdd(Index, SmallestKey); + if (!MaybeKey) + return false; + int32_t Key = *MaybeKey; // Skip if there is already a member with the same index. if (Members.find(Key) != Members.end()) @@ -286,13 +309,19 @@ public: if (Key > LargestKey) { // The largest index is always less than the interleave factor. - if (Index >= static_cast(Factor)) + if (Index >= static_cast(Factor)) return false; LargestKey = Key; } else if (Key < SmallestKey) { + + // Make sure the largest index fits in an int32_t. + Optional MaybeLargestIndex = checkedSub(LargestKey, Key); + if (!MaybeLargestIndex) + return false; + // The largest index is always less than the interleave factor. - if (LargestKey - Key >= static_cast(Factor)) + if (*MaybeLargestIndex >= static_cast(Factor)) return false; SmallestKey = Key; @@ -307,8 +336,8 @@ public: /// Get the member with the given index \p Index /// /// \returns nullptr if contains no such member. - InstTy *getMember(unsigned Index) const { - int Key = SmallestKey + Index; + InstTy *getMember(uint32_t Index) const { + int32_t Key = SmallestKey + Index; auto Member = Members.find(Key); if (Member == Members.end()) return nullptr; @@ -318,7 +347,7 @@ public: /// Get the index for the given member. Unlike the key in the member /// map, the index starts from 0. - unsigned getIndex(const InstTy *Instr) const { + uint32_t getIndex(const InstTy *Instr) const { for (auto I : Members) { if (I.second == Instr) return I.first - SmallestKey; @@ -356,12 +385,12 @@ public: } private: - unsigned Factor; // Interleave Factor. + uint32_t Factor; // Interleave Factor. bool Reverse; - unsigned Align; - DenseMap Members; - int SmallestKey = 0; - int LargestKey = 0; + uint32_t Align; + DenseMap Members; + int32_t SmallestKey = 0; + int32_t LargestKey = 0; // To avoid breaking dependences, vectorized instructions of an interleave // group should be inserted at either the first load or the last store in diff --git a/include/llvm/AsmParser/Parser.h b/include/llvm/AsmParser/Parser.h index 285a7c022a24..b0c603497805 100644 --- a/include/llvm/AsmParser/Parser.h +++ b/include/llvm/AsmParser/Parser.h @@ -1,9 +1,8 @@ //===-- Parser.h - Parser for LLVM IR text assembly files -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/AsmParser/SlotMapping.h b/include/llvm/AsmParser/SlotMapping.h index bd7e8fcad8bc..0e95eb816b4c 100644 --- a/include/llvm/AsmParser/SlotMapping.h +++ b/include/llvm/AsmParser/SlotMapping.h @@ -1,9 +1,8 @@ //===-- SlotMapping.h - Slot number mapping for unnamed values --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/BinaryFormat/AMDGPUMetadataVerifier.h b/include/llvm/BinaryFormat/AMDGPUMetadataVerifier.h index de44f41720ed..7332b2a7ea89 100644 --- a/include/llvm/BinaryFormat/AMDGPUMetadataVerifier.h +++ b/include/llvm/BinaryFormat/AMDGPUMetadataVerifier.h @@ -1,9 +1,8 @@ //===- AMDGPUMetadataVerifier.h - MsgPack Types -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -17,7 +16,7 @@ #ifndef LLVM_BINARYFORMAT_AMDGPUMETADATAVERIFIER_H #define LLVM_BINARYFORMAT_AMDGPUMETADATAVERIFIER_H -#include "llvm/BinaryFormat/MsgPackTypes.h" +#include "llvm/BinaryFormat/MsgPackDocument.h" namespace llvm { namespace AMDGPU { @@ -34,22 +33,22 @@ namespace V3 { class MetadataVerifier { bool Strict; - bool verifyScalar(msgpack::Node &Node, msgpack::ScalarNode::ScalarKind SKind, - function_ref verifyValue = {}); - bool verifyInteger(msgpack::Node &Node); - bool verifyArray(msgpack::Node &Node, - function_ref verifyNode, + bool verifyScalar(msgpack::DocNode &Node, msgpack::Type SKind, + function_ref verifyValue = {}); + bool verifyInteger(msgpack::DocNode &Node); + bool verifyArray(msgpack::DocNode &Node, + function_ref verifyNode, Optional Size = None); - bool verifyEntry(msgpack::MapNode &MapNode, StringRef Key, bool Required, - function_ref verifyNode); + bool verifyEntry(msgpack::MapDocNode &MapNode, StringRef Key, bool Required, + function_ref verifyNode); bool - verifyScalarEntry(msgpack::MapNode &MapNode, StringRef Key, bool Required, - msgpack::ScalarNode::ScalarKind SKind, - function_ref verifyValue = {}); - bool verifyIntegerEntry(msgpack::MapNode &MapNode, StringRef Key, + verifyScalarEntry(msgpack::MapDocNode &MapNode, StringRef Key, bool Required, + msgpack::Type SKind, + function_ref verifyValue = {}); + bool verifyIntegerEntry(msgpack::MapDocNode &MapNode, StringRef Key, bool Required); - bool verifyKernelArgs(msgpack::Node &Node); - bool verifyKernel(msgpack::Node &Node); + bool verifyKernelArgs(msgpack::DocNode &Node); + bool verifyKernel(msgpack::DocNode &Node); public: /// Construct a MetadataVerifier, specifying whether it will operate in \p @@ -59,7 +58,7 @@ public: /// Verify given HSA metadata. /// /// \returns True when successful, false when metadata is invalid. - bool verify(msgpack::Node &HSAMetadataRoot); + bool verify(msgpack::DocNode &HSAMetadataRoot); }; } // end namespace V3 diff --git a/include/llvm/BinaryFormat/COFF.h b/include/llvm/BinaryFormat/COFF.h index 7b973c03cc80..0fe38a437725 100644 --- a/include/llvm/BinaryFormat/COFF.h +++ b/include/llvm/BinaryFormat/COFF.h @@ -1,9 +1,8 @@ //===-- llvm/BinaryFormat/COFF.h --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -371,13 +370,15 @@ enum RelocationTypesARM : unsigned { IMAGE_REL_ARM_TOKEN = 0x0005, IMAGE_REL_ARM_BLX24 = 0x0008, IMAGE_REL_ARM_BLX11 = 0x0009, + IMAGE_REL_ARM_REL32 = 0x000A, IMAGE_REL_ARM_SECTION = 0x000E, IMAGE_REL_ARM_SECREL = 0x000F, IMAGE_REL_ARM_MOV32A = 0x0010, IMAGE_REL_ARM_MOV32T = 0x0011, IMAGE_REL_ARM_BRANCH20T = 0x0012, IMAGE_REL_ARM_BRANCH24T = 0x0014, - IMAGE_REL_ARM_BLX23T = 0x0015 + IMAGE_REL_ARM_BLX23T = 0x0015, + IMAGE_REL_ARM_PAIR = 0x0016, }; enum RelocationTypesARM64 : unsigned { @@ -398,9 +399,10 @@ enum RelocationTypesARM64 : unsigned { IMAGE_REL_ARM64_ADDR64 = 0x000E, IMAGE_REL_ARM64_BRANCH19 = 0x000F, IMAGE_REL_ARM64_BRANCH14 = 0x0010, + IMAGE_REL_ARM64_REL32 = 0x0011, }; -enum COMDATType : unsigned { +enum COMDATType : uint8_t { IMAGE_COMDAT_SELECT_NODUPLICATES = 1, IMAGE_COMDAT_SELECT_ANY, IMAGE_COMDAT_SELECT_SAME_SIZE, diff --git a/include/llvm/BinaryFormat/Dwarf.def b/include/llvm/BinaryFormat/Dwarf.def index 6ad3cb57f62f..b0f78d0fd61f 100644 --- a/include/llvm/BinaryFormat/Dwarf.def +++ b/include/llvm/BinaryFormat/Dwarf.def @@ -1,9 +1,8 @@ //===- llvm/Support/Dwarf.def - Dwarf definitions ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -355,7 +354,13 @@ HANDLE_DW_AT(0x2107, GNU_vector, 0, GNU) HANDLE_DW_AT(0x2110, GNU_template_name, 0, GNU) HANDLE_DW_AT(0x210f, GNU_odr_signature, 0, GNU) HANDLE_DW_AT(0x2111, GNU_call_site_value, 0, GNU) +HANDLE_DW_AT (0x2112, GNU_call_site_data_value, 0, GNU) +HANDLE_DW_AT (0x2113, GNU_call_site_target, 0, GNU) +HANDLE_DW_AT (0x2114, GNU_call_site_target_clobbered, 0, GNU) +HANDLE_DW_AT (0x2115, GNU_tail_call, 0, GNU) +HANDLE_DW_AT (0x2116, GNU_all_tail_call_sites, 0, GNU) HANDLE_DW_AT(0x2117, GNU_all_call_sites, 0, GNU) +HANDLE_DW_AT (0x2118, GNU_all_source_call_sites, 0, GNU) HANDLE_DW_AT(0x2119, GNU_macros, 0, GNU) // Extensions for Fission proposal. HANDLE_DW_AT(0x2130, GNU_dwo_name, 0, GNU) @@ -387,6 +392,7 @@ HANDLE_DW_AT(0x3b31, BORLAND_closure, 0, BORLAND) HANDLE_DW_AT(0x3e00, LLVM_include_path, 0, LLVM) HANDLE_DW_AT(0x3e01, LLVM_config_macros, 0, LLVM) HANDLE_DW_AT(0x3e02, LLVM_isysroot, 0, LLVM) +HANDLE_DW_AT(0x3e03, LLVM_tag_offset, 0, LLVM) // Apple extensions. HANDLE_DW_AT(0x3fe1, APPLE_optimized, 0, APPLE) HANDLE_DW_AT(0x3fe2, APPLE_flags, 0, APPLE) @@ -627,6 +633,8 @@ HANDLE_DW_OP(0xa9, reinterpret, 5, DWARF) // Vendor extensions: // Extensions for GNU-style thread-local storage. HANDLE_DW_OP(0xe0, GNU_push_tls_address, 0, GNU) +// The GNU entry value extension. +HANDLE_DW_OP(0xf3, GNU_entry_value, 0, GNU) // Extensions for Fission proposal. HANDLE_DW_OP(0xfb, GNU_addr_index, 0, GNU) HANDLE_DW_OP(0xfc, GNU_const_index, 0, GNU) diff --git a/include/llvm/BinaryFormat/Dwarf.h b/include/llvm/BinaryFormat/Dwarf.h index 525a04d5e6cf..76d9c365c0a8 100644 --- a/include/llvm/BinaryFormat/Dwarf.h +++ b/include/llvm/BinaryFormat/Dwarf.h @@ -1,9 +1,8 @@ //===-- llvm/BinaryFormat/Dwarf.h ---Dwarf Constants-------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -130,7 +129,9 @@ enum LocationAtom { #include "llvm/BinaryFormat/Dwarf.def" DW_OP_lo_user = 0xe0, DW_OP_hi_user = 0xff, - DW_OP_LLVM_fragment = 0x1000 ///< Only used in LLVM metadata. + DW_OP_LLVM_fragment = 0x1000, ///< Only used in LLVM metadata. + DW_OP_LLVM_convert = 0x1001, ///< Only used in LLVM metadata. + DW_OP_LLVM_tag_offset = 0x1002, ///< Only used in LLVM metadata. }; enum TypeKind : uint8_t { diff --git a/include/llvm/BinaryFormat/DynamicTags.def b/include/llvm/BinaryFormat/DynamicTags.def index 2e15cc30fca7..aec408bd2d72 100644 --- a/include/llvm/BinaryFormat/DynamicTags.def +++ b/include/llvm/BinaryFormat/DynamicTags.def @@ -6,6 +6,11 @@ // such as DT_HIOS, etc. to allow using this file to in other contexts. // For example we can use it to generate a stringification switch statement. +#ifndef AARCH64_DYNAMIC_TAG +#define AARCH64_DYNAMIC_TAG(name, value) DYNAMIC_TAG(name, value) +#define AARCH64_DYNAMIC_TAG_DEFINED +#endif + #ifndef HEXAGON_DYNAMIC_TAG #define HEXAGON_DYNAMIC_TAG(name, value) DYNAMIC_TAG(name, value) #define HEXAGON_DYNAMIC_TAG_DEFINED @@ -16,6 +21,11 @@ #define MIPS_DYNAMIC_TAG_DEFINED #endif +#ifndef PPC_DYNAMIC_TAG +#define PPC_DYNAMIC_TAG(name, value) DYNAMIC_TAG(name, value) +#define PPC_DYNAMIC_TAG_DEFINED +#endif + #ifndef PPC64_DYNAMIC_TAG #define PPC64_DYNAMIC_TAG(name, value) DYNAMIC_TAG(name, value) #define PPC64_DYNAMIC_TAG_DEFINED @@ -107,6 +117,10 @@ DYNAMIC_TAG(VERNEED, 0X6FFFFFFE) // The address of the version dependency // table. DYNAMIC_TAG(VERNEEDNUM, 0X6FFFFFFF) // The number of entries in DT_VERNEED. +// AArch64 specific dynamic table entries +AARCH64_DYNAMIC_TAG(AARCH64_BTI_PLT, 0x70000001) +AARCH64_DYNAMIC_TAG(AARCH64_PAC_PLT, 0x70000003) + // Hexagon specific dynamic table entries HEXAGON_DYNAMIC_TAG(HEXAGON_SYMSZ, 0x70000000) HEXAGON_DYNAMIC_TAG(HEXAGON_VER, 0x70000001) @@ -190,17 +204,27 @@ MIPS_DYNAMIC_TAG(MIPS_RWPLT, 0x70000034) // Points to the base MIPS_DYNAMIC_TAG(MIPS_RLD_MAP_REL, 0x70000035) // Relative offset of run time loader // map, used for debugging. +// PPC specific dynamic table entries. +PPC_DYNAMIC_TAG(PPC_GOT, 0x70000000) // Uses Secure PLT ABI. +PPC_DYNAMIC_TAG(PPC_OPT, 0x70000001) // Has TLS optimization. + // PPC64 specific dynamic table entries. PPC64_DYNAMIC_TAG(PPC64_GLINK, 0x70000000) // Address of 32 bytes before the // first glink lazy resolver stub. // Sun machine-independent extensions. DYNAMIC_TAG(AUXILIARY, 0x7FFFFFFD) // Shared object to load before self +DYNAMIC_TAG(USED, 0x7FFFFFFE) // Same as DT_NEEDED DYNAMIC_TAG(FILTER, 0x7FFFFFFF) // Shared object to get values from #ifdef DYNAMIC_TAG_MARKER_DEFINED #undef DYNAMIC_TAG_MARKER +#undef DYNAMIC_TAG_MARKER_DEFINED +#endif +#ifdef AARCH64_DYNAMIC_TAG_DEFINED +#undef AARCH64_DYNAMIC_TAG +#undef AARCH64_DYNAMIC_TAG_DEFINED #endif #ifdef MIPS_DYNAMIC_TAG_DEFINED #undef MIPS_DYNAMIC_TAG @@ -210,6 +234,10 @@ DYNAMIC_TAG(FILTER, 0x7FFFFFFF) // Shared object to get values from #undef HEXAGON_DYNAMIC_TAG #undef HEXAGON_DYNAMIC_TAG_DEFINED #endif +#ifdef PPC_DYNAMIC_TAG_DEFINED +#undef PPC_DYNAMIC_TAG +#undef PPC_DYNAMIC_TAG_DEFINED +#endif #ifdef PPC64_DYNAMIC_TAG_DEFINED #undef PPC64_DYNAMIC_TAG #undef PPC64_DYNAMIC_TAG_DEFINED diff --git a/include/llvm/BinaryFormat/ELF.h b/include/llvm/BinaryFormat/ELF.h index ce35d127d433..2bd711137845 100644 --- a/include/llvm/BinaryFormat/ELF.h +++ b/include/llvm/BinaryFormat/ELF.h @@ -1,9 +1,8 @@ //===- llvm/BinaryFormat/ELF.h - ELF constants and structures ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -703,15 +702,20 @@ enum : unsigned { EF_AMDGPU_MACH_AMDGCN_GFX902 = 0x02d, EF_AMDGPU_MACH_AMDGCN_GFX904 = 0x02e, EF_AMDGPU_MACH_AMDGCN_GFX906 = 0x02f, + EF_AMDGPU_MACH_AMDGCN_GFX908 = 0x030, EF_AMDGPU_MACH_AMDGCN_GFX909 = 0x031, + // AMDGCN GFX10. + EF_AMDGPU_MACH_AMDGCN_GFX1010 = 0x033, + EF_AMDGPU_MACH_AMDGCN_GFX1011 = 0x034, + EF_AMDGPU_MACH_AMDGCN_GFX1012 = 0x035, // Reserved for AMDGCN-based processors. EF_AMDGPU_MACH_AMDGCN_RESERVED0 = 0x027, - EF_AMDGPU_MACH_AMDGCN_RESERVED1 = 0x030, + EF_AMDGPU_MACH_AMDGCN_RESERVED1 = 0x032, // First/last AMDGCN-based processors. EF_AMDGPU_MACH_AMDGCN_FIRST = EF_AMDGPU_MACH_AMDGCN_GFX600, - EF_AMDGPU_MACH_AMDGCN_LAST = EF_AMDGPU_MACH_AMDGCN_GFX909, + EF_AMDGPU_MACH_AMDGCN_LAST = EF_AMDGPU_MACH_AMDGCN_GFX1012, // Indicates if the "xnack" target feature is enabled for all code contained // in the object. @@ -839,6 +843,10 @@ enum : unsigned { SHT_LLVM_CALL_GRAPH_PROFILE = 0x6fff4c02, // LLVM Call Graph Profile. SHT_LLVM_ADDRSIG = 0x6fff4c03, // List of address-significant symbols // for safe ICF. + SHT_LLVM_DEPENDENT_LIBRARIES = 0x6fff4c04, // LLVM Dependent Library Specifiers. + SHT_LLVM_SYMPART = 0x6fff4c05, // Symbol partition specification. + SHT_LLVM_PART_EHDR = 0x6fff4c06, // ELF header for loadable partition. + SHT_LLVM_PART_PHDR = 0x6fff4c07, // Phdrs for loadable partition. // Android's experimental support for SHT_RELR sections. // https://android.googlesource.com/platform/bionic/+/b7feec74547f84559a1467aca02708ff61346d2a/libc/include/elf.h#512 SHT_ANDROID_RELR = 0x6fffff00, // Relocation entries; only offsets. @@ -1340,6 +1348,14 @@ enum { NT_FREEBSD_PROCSTAT_AUXV = 16, }; +// Generic note types +enum : unsigned { + NT_VERSION = 1, + NT_ARCH = 2, + NT_GNU_BUILD_ATTRIBUTE_OPEN = 0x100, + NT_GNU_BUILD_ATTRIBUTE_FUNC = 0x101, +}; + enum { NT_GNU_ABI_TAG = 1, NT_GNU_HWCAP = 2, @@ -1352,13 +1368,65 @@ enum { enum : unsigned { GNU_PROPERTY_STACK_SIZE = 1, GNU_PROPERTY_NO_COPY_ON_PROTECTED = 2, - GNU_PROPERTY_X86_FEATURE_1_AND = 0xc0000002 + GNU_PROPERTY_AARCH64_FEATURE_1_AND = 0xc0000000, + GNU_PROPERTY_X86_FEATURE_1_AND = 0xc0000002, + GNU_PROPERTY_X86_ISA_1_NEEDED = 0xc0008000, + GNU_PROPERTY_X86_FEATURE_2_NEEDED = 0xc0008001, + GNU_PROPERTY_X86_ISA_1_USED = 0xc0010000, + GNU_PROPERTY_X86_FEATURE_2_USED = 0xc0010001, }; -// CET properties -enum { +// aarch64 processor feature bits. +enum : unsigned { + GNU_PROPERTY_AARCH64_FEATURE_1_BTI = 1 << 0, + GNU_PROPERTY_AARCH64_FEATURE_1_PAC = 1 << 1, +}; + +// x86 processor feature bits. +enum : unsigned { GNU_PROPERTY_X86_FEATURE_1_IBT = 1 << 0, - GNU_PROPERTY_X86_FEATURE_1_SHSTK = 1 << 1 + GNU_PROPERTY_X86_FEATURE_1_SHSTK = 1 << 1, + + GNU_PROPERTY_X86_ISA_1_CMOV = 1 << 0, + GNU_PROPERTY_X86_ISA_1_SSE = 1 << 1, + GNU_PROPERTY_X86_ISA_1_SSE2 = 1 << 2, + GNU_PROPERTY_X86_ISA_1_SSE3 = 1 << 3, + GNU_PROPERTY_X86_ISA_1_SSSE3 = 1 << 4, + GNU_PROPERTY_X86_ISA_1_SSE4_1 = 1 << 5, + GNU_PROPERTY_X86_ISA_1_SSE4_2 = 1 << 6, + GNU_PROPERTY_X86_ISA_1_AVX = 1 << 7, + GNU_PROPERTY_X86_ISA_1_AVX2 = 1 << 8, + GNU_PROPERTY_X86_ISA_1_FMA = 1 << 9, + GNU_PROPERTY_X86_ISA_1_AVX512F = 1 << 10, + GNU_PROPERTY_X86_ISA_1_AVX512CD = 1 << 11, + GNU_PROPERTY_X86_ISA_1_AVX512ER = 1 << 12, + GNU_PROPERTY_X86_ISA_1_AVX512PF = 1 << 13, + GNU_PROPERTY_X86_ISA_1_AVX512VL = 1 << 14, + GNU_PROPERTY_X86_ISA_1_AVX512DQ = 1 << 15, + GNU_PROPERTY_X86_ISA_1_AVX512BW = 1 << 16, + GNU_PROPERTY_X86_ISA_1_AVX512_4FMAPS = 1 << 17, + GNU_PROPERTY_X86_ISA_1_AVX512_4VNNIW = 1 << 18, + GNU_PROPERTY_X86_ISA_1_AVX512_BITALG = 1 << 19, + GNU_PROPERTY_X86_ISA_1_AVX512_IFMA = 1 << 20, + GNU_PROPERTY_X86_ISA_1_AVX512_VBMI = 1 << 21, + GNU_PROPERTY_X86_ISA_1_AVX512_VBMI2 = 1 << 22, + GNU_PROPERTY_X86_ISA_1_AVX512_VNNI = 1 << 23, + + GNU_PROPERTY_X86_FEATURE_2_X86 = 1 << 0, + GNU_PROPERTY_X86_FEATURE_2_X87 = 1 << 1, + GNU_PROPERTY_X86_FEATURE_2_MMX = 1 << 2, + GNU_PROPERTY_X86_FEATURE_2_XMM = 1 << 3, + GNU_PROPERTY_X86_FEATURE_2_YMM = 1 << 4, + GNU_PROPERTY_X86_FEATURE_2_ZMM = 1 << 5, + GNU_PROPERTY_X86_FEATURE_2_FXSR = 1 << 6, + GNU_PROPERTY_X86_FEATURE_2_XSAVE = 1 << 7, + GNU_PROPERTY_X86_FEATURE_2_XSAVEOPT = 1 << 8, + GNU_PROPERTY_X86_FEATURE_2_XSAVEC = 1 << 9, +}; + +// AMDGPU-specific section indices. +enum { + SHN_AMDGPU_LDS = 0xff00, // Variable in LDS; symbol encoded like SHN_COMMON }; // AMD specific notes. (Code Object V2) diff --git a/include/llvm/BinaryFormat/ELFRelocs/ARM.def b/include/llvm/BinaryFormat/ELFRelocs/ARM.def index 730fc5b8836c..e0709fb81813 100644 --- a/include/llvm/BinaryFormat/ELFRelocs/ARM.def +++ b/include/llvm/BinaryFormat/ELFRelocs/ARM.def @@ -135,4 +135,7 @@ ELF_RELOC(R_ARM_PRIVATE_15, 0x7f) ELF_RELOC(R_ARM_ME_TOO, 0x80) ELF_RELOC(R_ARM_THM_TLS_DESCSEQ16, 0x81) ELF_RELOC(R_ARM_THM_TLS_DESCSEQ32, 0x82) +ELF_RELOC(R_ARM_THM_BF16, 0x88) +ELF_RELOC(R_ARM_THM_BF12, 0x89) +ELF_RELOC(R_ARM_THM_BF18, 0x8a) ELF_RELOC(R_ARM_IRELATIVE, 0xa0) diff --git a/include/llvm/BinaryFormat/ELFRelocs/PowerPC.def b/include/llvm/BinaryFormat/ELFRelocs/PowerPC.def index e4f8ee0ebe2b..28036889cca6 100644 --- a/include/llvm/BinaryFormat/ELFRelocs/PowerPC.def +++ b/include/llvm/BinaryFormat/ELFRelocs/PowerPC.def @@ -27,9 +27,25 @@ #undef R_PPC_GOT16_HI #undef R_PPC_GOT16_HA #undef R_PPC_PLTREL24 +#undef R_PPC_COPY +#undef R_PPC_GLOB_DAT #undef R_PPC_JMP_SLOT +#undef R_PPC_RELATIVE #undef R_PPC_LOCAL24PC +#undef R_PPC_UADDR32 +#undef R_PPC_UADDR16 #undef R_PPC_REL32 +#undef R_PPC_PLT32 +#undef R_PPC_PLTREL32 +#undef R_PPC_PLT16_LO +#undef R_PPC_PLT16_HI +#undef R_PPC_PLT16_HA +#undef R_PPC_SDAREL16 +#undef R_PPC_SECTOFF +#undef R_PPC_SECTOFF_LO +#undef R_PPC_SECTOFF_HI +#undef R_PPC_SECTOFF_HA +#undef R_PPC_ADDR30 #undef R_PPC_TLS #undef R_PPC_DTPMOD32 #undef R_PPC_TPREL16 @@ -84,9 +100,25 @@ ELF_RELOC(R_PPC_GOT16_LO, 15) ELF_RELOC(R_PPC_GOT16_HI, 16) ELF_RELOC(R_PPC_GOT16_HA, 17) ELF_RELOC(R_PPC_PLTREL24, 18) +ELF_RELOC(R_PPC_COPY, 19) +ELF_RELOC(R_PPC_GLOB_DAT, 20) ELF_RELOC(R_PPC_JMP_SLOT, 21) +ELF_RELOC(R_PPC_RELATIVE, 22) ELF_RELOC(R_PPC_LOCAL24PC, 23) +ELF_RELOC(R_PPC_UADDR32, 24) +ELF_RELOC(R_PPC_UADDR16, 25) ELF_RELOC(R_PPC_REL32, 26) +ELF_RELOC(R_PPC_PLT32, 27) +ELF_RELOC(R_PPC_PLTREL32, 28) +ELF_RELOC(R_PPC_PLT16_LO, 29) +ELF_RELOC(R_PPC_PLT16_HI, 30) +ELF_RELOC(R_PPC_PLT16_HA, 31) +ELF_RELOC(R_PPC_SDAREL16, 32) +ELF_RELOC(R_PPC_SECTOFF, 33) +ELF_RELOC(R_PPC_SECTOFF_LO, 34) +ELF_RELOC(R_PPC_SECTOFF_HI, 35) +ELF_RELOC(R_PPC_SECTOFF_HA, 36) +ELF_RELOC(R_PPC_ADDR30, 37) ELF_RELOC(R_PPC_TLS, 67) ELF_RELOC(R_PPC_DTPMOD32, 68) ELF_RELOC(R_PPC_TPREL16, 69) @@ -117,6 +149,7 @@ ELF_RELOC(R_PPC_GOT_DTPREL16_HI, 93) ELF_RELOC(R_PPC_GOT_DTPREL16_HA, 94) ELF_RELOC(R_PPC_TLSGD, 95) ELF_RELOC(R_PPC_TLSLD, 96) +ELF_RELOC(R_PPC_IRELATIVE, 248) ELF_RELOC(R_PPC_REL16, 249) ELF_RELOC(R_PPC_REL16_LO, 250) ELF_RELOC(R_PPC_REL16_HI, 251) diff --git a/include/llvm/BinaryFormat/MachO.def b/include/llvm/BinaryFormat/MachO.def index 95de48d2b19e..76dcc58ba048 100644 --- a/include/llvm/BinaryFormat/MachO.def +++ b/include/llvm/BinaryFormat/MachO.def @@ -1,9 +1,8 @@ //,,,-- llvm/Support/MachO.def - The MachO file definitions -----*- C++ -*-,,,// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //,,,----------------------------------------------------------------------,,,// // diff --git a/include/llvm/BinaryFormat/MachO.h b/include/llvm/BinaryFormat/MachO.h index b3d60984249f..a01393a3b303 100644 --- a/include/llvm/BinaryFormat/MachO.h +++ b/include/llvm/BinaryFormat/MachO.h @@ -1,9 +1,8 @@ //===-- llvm/BinaryFormat/MachO.h - The MachO file format -------*- C++/-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -335,6 +334,7 @@ enum { N_WEAK_DEF = 0x0080u, N_SYMBOL_RESOLVER = 0x0100u, N_ALT_ENTRY = 0x0200u, + N_COLD_FUNC = 0x0400u, // For undefined symbols coming from libraries, see GET_LIBRARY_ORDINAL() // as these are in the top 8 bits. SELF_LIBRARY_ORDINAL = 0x0, @@ -487,6 +487,7 @@ enum PlatformType { PLATFORM_TVOS = 3, PLATFORM_WATCHOS = 4, PLATFORM_BRIDGEOS = 5, + PLATFORM_MACCATALYST = 6, PLATFORM_IOSSIMULATOR = 7, PLATFORM_TVOSSIMULATOR = 8, PLATFORM_WATCHOSSIMULATOR = 9 @@ -942,8 +943,13 @@ struct fat_arch_64 { // Structs from struct relocation_info { int32_t r_address; +#if defined(BYTE_ORDER) && defined(BIG_ENDIAN) && (BYTE_ORDER == BIG_ENDIAN) + uint32_t r_type : 4, r_extern : 1, r_length : 2, r_pcrel : 1, + r_symbolnum : 24; +#else uint32_t r_symbolnum : 24, r_pcrel : 1, r_length : 2, r_extern : 1, r_type : 4; +#endif }; struct scattered_relocation_info { @@ -1396,7 +1402,8 @@ inline void SET_COMM_ALIGN(uint16_t &n_desc, uint8_t align) { enum : uint32_t { // Capability bits used in the definition of cpu_type. CPU_ARCH_MASK = 0xff000000, // Mask for architecture bits - CPU_ARCH_ABI64 = 0x01000000 // 64 bit ABI + CPU_ARCH_ABI64 = 0x01000000, // 64 bit ABI + CPU_ARCH_ABI64_32 = 0x02000000, // ILP32 ABI on 64-bit hardware }; // Constants for the cputype field. @@ -1409,6 +1416,7 @@ enum CPUType { CPU_TYPE_MC98000 = 10, // Old Motorola PowerPC CPU_TYPE_ARM = 12, CPU_TYPE_ARM64 = CPU_TYPE_ARM | CPU_ARCH_ABI64, + CPU_TYPE_ARM64_32 = CPU_TYPE_ARM | CPU_ARCH_ABI64_32, CPU_TYPE_SPARC = 14, CPU_TYPE_POWERPC = 18, CPU_TYPE_POWERPC64 = CPU_TYPE_POWERPC | CPU_ARCH_ABI64 @@ -1477,7 +1485,12 @@ enum CPUSubTypeARM { CPU_SUBTYPE_ARM_V7EM = 16 }; -enum CPUSubTypeARM64 { CPU_SUBTYPE_ARM64_ALL = 0 }; +enum CPUSubTypeARM64 { + CPU_SUBTYPE_ARM64_ALL = 0, + CPU_SUBTYPE_ARM64E = 2, +}; + +enum CPUSubTypeARM64_32 { CPU_SUBTYPE_ARM64_32_V8 = 1 }; enum CPUSubTypeSPARC { CPU_SUBTYPE_SPARC_ALL = 0 }; diff --git a/include/llvm/BinaryFormat/Magic.h b/include/llvm/BinaryFormat/Magic.h index 04801f810be3..cd9833ec4d22 100644 --- a/include/llvm/BinaryFormat/Magic.h +++ b/include/llvm/BinaryFormat/Magic.h @@ -1,9 +1,8 @@ //===- llvm/BinaryFormat/Magic.h - File magic identification ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -40,11 +39,14 @@ struct file_magic { macho_dsym_companion, ///< Mach-O dSYM companion file macho_kext_bundle, ///< Mach-O kext bundle file macho_universal_binary, ///< Mach-O universal binary + minidump, ///< Windows minidump file coff_cl_gl_object, ///< Microsoft cl.exe's intermediate code file coff_object, ///< COFF object file coff_import_library, ///< COFF import library pecoff_executable, ///< PECOFF executable file windows_resource, ///< Windows compiled resource file (.res) + xcoff_object_32, ///< 32-bit XCOFF object file + xcoff_object_64, ///< 64-bit XCOFF object file wasm_object, ///< WebAssembly Object file pdb, ///< Windows PDB debug info file }; diff --git a/include/llvm/BinaryFormat/Minidump.h b/include/llvm/BinaryFormat/Minidump.h new file mode 100644 index 000000000000..65c17d1eb00c --- /dev/null +++ b/include/llvm/BinaryFormat/Minidump.h @@ -0,0 +1,203 @@ +//===- Minidump.h - Minidump constants and structures -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This header constants and data structures pertaining to the Windows Minidump +// core file format. +// +// Reference: +// https://msdn.microsoft.com/en-us/library/windows/desktop/ms679293(v=vs.85).aspx +// https://chromium.googlesource.com/breakpad/breakpad/ +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_BINARYFORMAT_MINIDUMP_H +#define LLVM_BINARYFORMAT_MINIDUMP_H + +#include "llvm/ADT/DenseMapInfo.h" +#include "llvm/Support/Endian.h" + +namespace llvm { +namespace minidump { + +/// The minidump header is the first part of a minidump file. It identifies the +/// file as a minidump file, and gives the location of the stream directory. +struct Header { + static constexpr uint32_t MagicSignature = 0x504d444d; // PMDM + static constexpr uint16_t MagicVersion = 0xa793; + + support::ulittle32_t Signature; + // The high 16 bits of version field are implementation specific. The low 16 + // bits should be MagicVersion. + support::ulittle32_t Version; + support::ulittle32_t NumberOfStreams; + support::ulittle32_t StreamDirectoryRVA; + support::ulittle32_t Checksum; + support::ulittle32_t TimeDateStamp; + support::ulittle64_t Flags; +}; +static_assert(sizeof(Header) == 32, ""); + +/// The type of a minidump stream identifies its contents. Streams numbers after +/// LastReserved are for application-defined data streams. +enum class StreamType : uint32_t { +#define HANDLE_MDMP_STREAM_TYPE(CODE, NAME) NAME = CODE, +#include "llvm/BinaryFormat/MinidumpConstants.def" + Unused = 0, + LastReserved = 0x0000ffff, +}; + +/// Specifies the location (and size) of various objects in the minidump file. +/// The location is relative to the start of the file. +struct LocationDescriptor { + support::ulittle32_t DataSize; + support::ulittle32_t RVA; +}; +static_assert(sizeof(LocationDescriptor) == 8, ""); + +/// Describes a single memory range (both its VM address and where to find it in +/// the file) of the process from which this minidump file was generated. +struct MemoryDescriptor { + support::ulittle64_t StartOfMemoryRange; + LocationDescriptor Memory; +}; +static_assert(sizeof(MemoryDescriptor) == 16, ""); + +/// Specifies the location and type of a single stream in the minidump file. The +/// minidump stream directory is an array of entries of this type, with its size +/// given by Header.NumberOfStreams. +struct Directory { + support::little_t Type; + LocationDescriptor Location; +}; +static_assert(sizeof(Directory) == 12, ""); + +/// The processor architecture of the system that generated this minidump. Used +/// in the ProcessorArch field of the SystemInfo stream. +enum class ProcessorArchitecture : uint16_t { +#define HANDLE_MDMP_ARCH(CODE, NAME) NAME = CODE, +#include "llvm/BinaryFormat/MinidumpConstants.def" +}; + +/// The OS Platform of the system that generated this minidump. Used in the +/// PlatformId field of the SystemInfo stream. +enum class OSPlatform : uint32_t { +#define HANDLE_MDMP_PLATFORM(CODE, NAME) NAME = CODE, +#include "llvm/BinaryFormat/MinidumpConstants.def" +}; + +/// Detailed information about the processor of the system that generated this +/// minidump. Its interpretation depends on the ProcessorArchitecture enum. +union CPUInfo { + struct X86Info { + char VendorID[12]; // cpuid 0: ebx, edx, ecx + support::ulittle32_t VersionInfo; // cpuid 1: eax + support::ulittle32_t FeatureInfo; // cpuid 1: edx + support::ulittle32_t AMDExtendedFeatures; // cpuid 0x80000001, ebx + } X86; + struct ArmInfo { + support::ulittle32_t CPUID; + support::ulittle32_t ElfHWCaps; // linux specific, 0 otherwise + } Arm; + struct OtherInfo { + uint8_t ProcessorFeatures[16]; + } Other; +}; +static_assert(sizeof(CPUInfo) == 24, ""); + +/// The SystemInfo stream, containing various information about the system where +/// this minidump was generated. +struct SystemInfo { + support::little_t ProcessorArch; + support::ulittle16_t ProcessorLevel; + support::ulittle16_t ProcessorRevision; + + uint8_t NumberOfProcessors; + uint8_t ProductType; + + support::ulittle32_t MajorVersion; + support::ulittle32_t MinorVersion; + support::ulittle32_t BuildNumber; + support::little_t PlatformId; + support::ulittle32_t CSDVersionRVA; + + support::ulittle16_t SuiteMask; + support::ulittle16_t Reserved; + + CPUInfo CPU; +}; +static_assert(sizeof(SystemInfo) == 56, ""); + +struct VSFixedFileInfo { + support::ulittle32_t Signature; + support::ulittle32_t StructVersion; + support::ulittle32_t FileVersionHigh; + support::ulittle32_t FileVersionLow; + support::ulittle32_t ProductVersionHigh; + support::ulittle32_t ProductVersionLow; + support::ulittle32_t FileFlagsMask; + support::ulittle32_t FileFlags; + support::ulittle32_t FileOS; + support::ulittle32_t FileType; + support::ulittle32_t FileSubtype; + support::ulittle32_t FileDateHigh; + support::ulittle32_t FileDateLow; +}; +static_assert(sizeof(VSFixedFileInfo) == 52, ""); + +inline bool operator==(const VSFixedFileInfo &LHS, const VSFixedFileInfo &RHS) { + return memcmp(&LHS, &RHS, sizeof(VSFixedFileInfo)) == 0; +} + +struct Module { + support::ulittle64_t BaseOfImage; + support::ulittle32_t SizeOfImage; + support::ulittle32_t Checksum; + support::ulittle32_t TimeDateStamp; + support::ulittle32_t ModuleNameRVA; + VSFixedFileInfo VersionInfo; + LocationDescriptor CvRecord; + LocationDescriptor MiscRecord; + support::ulittle64_t Reserved0; + support::ulittle64_t Reserved1; +}; +static_assert(sizeof(Module) == 108, ""); + +/// Describes a single thread in the minidump file. Part of the ThreadList +/// stream. +struct Thread { + support::ulittle32_t ThreadId; + support::ulittle32_t SuspendCount; + support::ulittle32_t PriorityClass; + support::ulittle32_t Priority; + support::ulittle64_t EnvironmentBlock; + MemoryDescriptor Stack; + LocationDescriptor Context; +}; +static_assert(sizeof(Thread) == 48, ""); + +} // namespace minidump + +template <> struct DenseMapInfo { + static minidump::StreamType getEmptyKey() { return minidump::StreamType(-1); } + + static minidump::StreamType getTombstoneKey() { + return minidump::StreamType(-2); + } + + static unsigned getHashValue(minidump::StreamType Val) { + return DenseMapInfo::getHashValue(static_cast(Val)); + } + + static bool isEqual(minidump::StreamType LHS, minidump::StreamType RHS) { + return LHS == RHS; + } +}; + +} // namespace llvm + +#endif // LLVM_BINARYFORMAT_MINIDUMP_H diff --git a/include/llvm/BinaryFormat/MinidumpConstants.def b/include/llvm/BinaryFormat/MinidumpConstants.def new file mode 100644 index 000000000000..d4f13dd99217 --- /dev/null +++ b/include/llvm/BinaryFormat/MinidumpConstants.def @@ -0,0 +1,107 @@ +//===- MinidumpConstants.def - Iteration over minidump constants-*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#if !(defined HANDLE_MDMP_STREAM_TYPE || defined HANDLE_MDMP_ARCH || \ + defined HANDLE_MDMP_PLATFORM) +#error "Missing HANDLE_MDMP definition" +#endif + +#ifndef HANDLE_MDMP_STREAM_TYPE +#define HANDLE_MDMP_STREAM_TYPE(CODE, NAME) +#endif + +#ifndef HANDLE_MDMP_ARCH +#define HANDLE_MDMP_ARCH(CODE, NAME) +#endif + +#ifndef HANDLE_MDMP_PLATFORM +#define HANDLE_MDMP_PLATFORM(CODE, NAME) +#endif + +HANDLE_MDMP_STREAM_TYPE(0x0003, ThreadList) +HANDLE_MDMP_STREAM_TYPE(0x0004, ModuleList) +HANDLE_MDMP_STREAM_TYPE(0x0005, MemoryList) +HANDLE_MDMP_STREAM_TYPE(0x0006, Exception) +HANDLE_MDMP_STREAM_TYPE(0x0007, SystemInfo) +HANDLE_MDMP_STREAM_TYPE(0x0008, ThreadExList) +HANDLE_MDMP_STREAM_TYPE(0x0009, Memory64List) +HANDLE_MDMP_STREAM_TYPE(0x000a, CommentA) +HANDLE_MDMP_STREAM_TYPE(0x000b, CommentW) +HANDLE_MDMP_STREAM_TYPE(0x000c, HandleData) +HANDLE_MDMP_STREAM_TYPE(0x000d, FunctionTable) +HANDLE_MDMP_STREAM_TYPE(0x000e, UnloadedModuleList) +HANDLE_MDMP_STREAM_TYPE(0x000f, MiscInfo) +HANDLE_MDMP_STREAM_TYPE(0x0010, MemoryInfoList) +HANDLE_MDMP_STREAM_TYPE(0x0011, ThreadInfoList) +HANDLE_MDMP_STREAM_TYPE(0x0012, HandleOperationList) +HANDLE_MDMP_STREAM_TYPE(0x0013, Token) +HANDLE_MDMP_STREAM_TYPE(0x0014, JavascriptData) +HANDLE_MDMP_STREAM_TYPE(0x0015, SystemMemoryInfo) +HANDLE_MDMP_STREAM_TYPE(0x0016, ProcessVMCounters) +// Breakpad extension types. 0x4767 = "Gg" +HANDLE_MDMP_STREAM_TYPE(0x47670001, BreakpadInfo) +HANDLE_MDMP_STREAM_TYPE(0x47670002, AssertionInfo) +// These are additional minidump stream values which are specific to the linux +// breakpad implementation. +HANDLE_MDMP_STREAM_TYPE(0x47670003, LinuxCPUInfo) // /proc/cpuinfo +HANDLE_MDMP_STREAM_TYPE(0x47670004, LinuxProcStatus) // /proc/$x/status +HANDLE_MDMP_STREAM_TYPE(0x47670005, LinuxLSBRelease) // /etc/lsb-release +HANDLE_MDMP_STREAM_TYPE(0x47670006, LinuxCMDLine) // /proc/$x/cmdline +HANDLE_MDMP_STREAM_TYPE(0x47670007, LinuxEnviron) // /proc/$x/environ +HANDLE_MDMP_STREAM_TYPE(0x47670008, LinuxAuxv) // /proc/$x/auxv +HANDLE_MDMP_STREAM_TYPE(0x47670009, LinuxMaps) // /proc/$x/maps +HANDLE_MDMP_STREAM_TYPE(0x4767000A, LinuxDSODebug) +HANDLE_MDMP_STREAM_TYPE(0x4767000B, LinuxProcStat) // /proc/$x/stat +HANDLE_MDMP_STREAM_TYPE(0x4767000C, LinuxProcUptime) // uptime +HANDLE_MDMP_STREAM_TYPE(0x4767000D, LinuxProcFD) // /proc/$x/fd +// Facebook-defined stream types +HANDLE_MDMP_STREAM_TYPE(0xFACE1CA7, FacebookLogcat) +HANDLE_MDMP_STREAM_TYPE(0xFACECAFA, FacebookAppCustomData) +HANDLE_MDMP_STREAM_TYPE(0xFACECAFB, FacebookBuildID) +HANDLE_MDMP_STREAM_TYPE(0xFACECAFC, FacebookAppVersionName) +HANDLE_MDMP_STREAM_TYPE(0xFACECAFD, FacebookJavaStack) +HANDLE_MDMP_STREAM_TYPE(0xFACECAFE, FacebookDalvikInfo) +HANDLE_MDMP_STREAM_TYPE(0xFACECAFF, FacebookUnwindSymbols) +HANDLE_MDMP_STREAM_TYPE(0xFACECB00, FacebookDumpErrorLog) +HANDLE_MDMP_STREAM_TYPE(0xFACECCCC, FacebookAppStateLog) +HANDLE_MDMP_STREAM_TYPE(0xFACEDEAD, FacebookAbortReason) +HANDLE_MDMP_STREAM_TYPE(0xFACEE000, FacebookThreadName) + +HANDLE_MDMP_ARCH(0x0000, X86) // PROCESSOR_ARCHITECTURE_INTEL +HANDLE_MDMP_ARCH(0x0001, MIPS) // PROCESSOR_ARCHITECTURE_MIPS +HANDLE_MDMP_ARCH(0x0002, Alpha) // PROCESSOR_ARCHITECTURE_ALPHA +HANDLE_MDMP_ARCH(0x0003, PPC) // PROCESSOR_ARCHITECTURE_PPC +HANDLE_MDMP_ARCH(0x0004, SHX) // PROCESSOR_ARCHITECTURE_SHX (Super-H) +HANDLE_MDMP_ARCH(0x0005, ARM) // PROCESSOR_ARCHITECTURE_ARM +HANDLE_MDMP_ARCH(0x0006, IA64) // PROCESSOR_ARCHITECTURE_IA64 +HANDLE_MDMP_ARCH(0x0007, Alpha64) // PROCESSOR_ARCHITECTURE_ALPHA64 +HANDLE_MDMP_ARCH(0x0008, MSIL) // PROCESSOR_ARCHITECTURE_MSIL +HANDLE_MDMP_ARCH(0x0009, AMD64) // PROCESSOR_ARCHITECTURE_AMD64 +HANDLE_MDMP_ARCH(0x000a, X86Win64) // PROCESSOR_ARCHITECTURE_IA32_ON_WIN64 +HANDLE_MDMP_ARCH(0x8001, SPARC) // Breakpad-defined value for SPARC +HANDLE_MDMP_ARCH(0x8002, PPC64) // Breakpad-defined value for PPC64 +HANDLE_MDMP_ARCH(0x8003, ARM64) // Breakpad-defined value for ARM64 +HANDLE_MDMP_ARCH(0x8004, MIPS64) // Breakpad-defined value for MIPS64 + +HANDLE_MDMP_PLATFORM(0x0000, Win32S) // Win32 on Windows 3.1 +HANDLE_MDMP_PLATFORM(0x0001, Win32Windows) // Windows 95-98-Me +HANDLE_MDMP_PLATFORM(0x0002, Win32NT) // Windows NT, 2000+ +HANDLE_MDMP_PLATFORM(0x0003, Win32CE) // Windows CE, Windows Mobile, "Handheld" +// Breakpad-defined values. +HANDLE_MDMP_PLATFORM(0x8000, Unix) // Generic Unix-ish +HANDLE_MDMP_PLATFORM(0x8101, MacOSX) // Mac OS X/Darwin +HANDLE_MDMP_PLATFORM(0x8102, IOS) // iOS +HANDLE_MDMP_PLATFORM(0x8201, Linux) // Linux +HANDLE_MDMP_PLATFORM(0x8202, Solaris) // Solaris +HANDLE_MDMP_PLATFORM(0x8203, Android) // Android +HANDLE_MDMP_PLATFORM(0x8204, PS3) // PS3 +HANDLE_MDMP_PLATFORM(0x8205, NaCl) // Native Client (NaCl) + +#undef HANDLE_MDMP_STREAM_TYPE +#undef HANDLE_MDMP_ARCH +#undef HANDLE_MDMP_PLATFORM diff --git a/include/llvm/BinaryFormat/MsgPack.def b/include/llvm/BinaryFormat/MsgPack.def index 781b49f46aeb..7ad83ff21c42 100644 --- a/include/llvm/BinaryFormat/MsgPack.def +++ b/include/llvm/BinaryFormat/MsgPack.def @@ -1,9 +1,8 @@ //===- MsgPack.def - MessagePack definitions --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/include/llvm/BinaryFormat/MsgPack.h b/include/llvm/BinaryFormat/MsgPack.h index d431912a53e5..9fda14b21c71 100644 --- a/include/llvm/BinaryFormat/MsgPack.h +++ b/include/llvm/BinaryFormat/MsgPack.h @@ -1,9 +1,8 @@ //===-- MsgPack.h - MessagePack Constants -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/include/llvm/BinaryFormat/MsgPackDocument.h b/include/llvm/BinaryFormat/MsgPackDocument.h new file mode 100644 index 000000000000..824ecc353207 --- /dev/null +++ b/include/llvm/BinaryFormat/MsgPackDocument.h @@ -0,0 +1,385 @@ +//===-- MsgPackDocument.h - MsgPack Document --------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// This file declares a class that exposes a simple in-memory representation +/// of a document of MsgPack objects, that can be read from MsgPack, written to +/// MsgPack, and inspected and modified in memory. This is intended to be a +/// lighter-weight (in terms of memory allocations) replacement for +/// MsgPackTypes. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_BINARYFORMAT_MSGPACKDOCUMENT_H +#define LLVM_BINARYFORMAT_MSGPACKDOCUMENT_H + +#include "llvm/BinaryFormat/MsgPackReader.h" +#include + +namespace llvm { +namespace msgpack { + +class ArrayDocNode; +class Document; +class MapDocNode; + +/// The kind of a DocNode and its owning Document. +struct KindAndDocument { + Document *Doc; + Type Kind; +}; + +/// A node in a MsgPack Document. This is a simple copyable and +/// passable-by-value type that does not own any memory. +class DocNode { + friend Document; + +public: + typedef std::map MapTy; + typedef std::vector ArrayTy; + +private: + // Using KindAndDocument allows us to squeeze Kind and a pointer to the + // owning Document into the same word. Having a pointer to the owning + // Document makes the API of DocNode more convenient, and allows its use in + // YAMLIO. + const KindAndDocument *KindAndDoc; + +protected: + // The union of different values. + union { + int64_t Int; + uint64_t UInt; + bool Bool; + double Float; + StringRef Raw; + ArrayTy *Array; + MapTy *Map; + }; + +public: + DocNode() : KindAndDoc(nullptr) {} + + // Type methods + bool isMap() const { return getKind() == Type::Map; } + bool isArray() const { return getKind() == Type::Array; } + bool isScalar() const { return !isMap() && !isArray(); } + bool isString() const { return getKind() == Type::String; } + + // Accessors + bool isEmpty() const { return !KindAndDoc; } + Type getKind() const { return KindAndDoc->Kind; } + Document *getDocument() const { return KindAndDoc->Doc; } + + int64_t &getInt() { + assert(getKind() == Type::Int); + return Int; + } + + uint64_t &getUInt() { + assert(getKind() == Type::UInt); + return UInt; + } + + bool &getBool() { + assert(getKind() == Type::Boolean); + return Bool; + } + + double &getFloat() { + assert(getKind() == Type::Float); + return Float; + } + + int64_t getInt() const { + assert(getKind() == Type::Int); + return Int; + } + + uint64_t getUInt() const { + assert(getKind() == Type::UInt); + return UInt; + } + + bool getBool() const { + assert(getKind() == Type::Boolean); + return Bool; + } + + double getFloat() const { + assert(getKind() == Type::Float); + return Float; + } + + StringRef getString() const { + assert(getKind() == Type::String); + return Raw; + } + + /// Get an ArrayDocNode for an array node. If Convert, convert the node to an + /// array node if necessary. + ArrayDocNode &getArray(bool Convert = false) { + if (getKind() != Type::Array) { + assert(Convert); + convertToArray(); + } + // This could be a static_cast, except ArrayDocNode is a forward reference. + return *reinterpret_cast(this); + } + + /// Get a MapDocNode for a map node. If Convert, convert the node to a map + /// node if necessary. + MapDocNode &getMap(bool Convert = false) { + if (getKind() != Type::Map) { + assert(Convert); + convertToMap(); + } + // This could be a static_cast, except MapDocNode is a forward reference. + return *reinterpret_cast(this); + } + + /// Comparison operator, used for map keys. + friend bool operator<(const DocNode &Lhs, const DocNode &Rhs) { + // This has to cope with one or both of the nodes being default-constructed, + // such that KindAndDoc is not set. + if (Lhs.KindAndDoc != Rhs.KindAndDoc) { + if (!Rhs.KindAndDoc) + return false; + if (!Lhs.KindAndDoc) + return true; + return (unsigned)Lhs.getKind() < (unsigned)Rhs.getKind(); + } + switch (Lhs.getKind()) { + case Type::Int: + return Lhs.Int < Rhs.Int; + case Type::UInt: + return Lhs.UInt < Rhs.UInt; + case Type::Nil: + return false; + case Type::Boolean: + return Lhs.Bool < Rhs.Bool; + case Type::Float: + return Lhs.Float < Rhs.Float; + case Type::String: + case Type::Binary: + return Lhs.Raw < Rhs.Raw; + default: + llvm_unreachable("bad map key type"); + } + } + + /// Equality operator + friend bool operator==(const DocNode &Lhs, const DocNode &Rhs) { + return !(Lhs < Rhs) && !(Rhs < Lhs); + } + + /// Convert this node to a string, assuming it is scalar. + std::string toString() const; + + /// Convert the StringRef and use it to set this DocNode (assuming scalar). If + /// it is a string, copy the string into the Document's strings list so we do + /// not rely on S having a lifetime beyond this call. Tag is "" or a YAML tag. + StringRef fromString(StringRef S, StringRef Tag = ""); + +private: + // Private constructor setting KindAndDoc, used by methods in Document. + DocNode(const KindAndDocument *KindAndDoc) : KindAndDoc(KindAndDoc) {} + + void convertToArray(); + void convertToMap(); +}; + +/// A DocNode that is a map. +class MapDocNode : public DocNode { +public: + MapDocNode() {} + MapDocNode(DocNode &N) : DocNode(N) { assert(getKind() == Type::Map); } + + // Map access methods. + size_t size() const { return Map->size(); } + bool empty() const { return !size(); } + MapTy::iterator begin() { return Map->begin(); } + MapTy::iterator end() { return Map->end(); } + MapTy::iterator find(DocNode Key) { return Map->find(Key); } + MapTy::iterator find(StringRef Key); + /// Member access. The string data must remain valid for the lifetime of the + /// Document. + DocNode &operator[](StringRef S); + /// Member access. + DocNode &operator[](DocNode Key); +}; + +/// A DocNode that is an array. +class ArrayDocNode : public DocNode { +public: + ArrayDocNode() {} + ArrayDocNode(DocNode &N) : DocNode(N) { assert(getKind() == Type::Array); } + + // Array access methods. + size_t size() const { return Array->size(); } + bool empty() const { return !size(); } + ArrayTy::iterator begin() { return Array->begin(); } + ArrayTy::iterator end() { return Array->end(); } + void push_back(DocNode N) { + assert(N.getDocument() == getDocument()); + Array->push_back(N); + } + + /// Element access. This extends the array if necessary. + DocNode &operator[](size_t Index); +}; + +/// Simple in-memory representation of a document of msgpack objects with +/// ability to find and create array and map elements. Does not currently cope +/// with any extension types. +class Document { + // Maps, arrays and strings used by nodes in the document. No attempt is made + // to free unused ones. + std::vector> Maps; + std::vector> Arrays; + std::vector> Strings; + + // The root node of the document. + DocNode Root; + + // The KindAndDocument structs pointed to by nodes in the document. + KindAndDocument KindAndDocs[size_t(Type::Extension) + 1]; + + // Whether YAML output uses hex for UInt. + bool HexMode = false; + +public: + Document() { + clear(); + for (unsigned T = 0; T != size_t(Type::Extension) + 1; ++T) + KindAndDocs[T] = {this, Type(T)}; + } + + /// Get ref to the document's root element. + DocNode &getRoot() { return Root; } + + /// Restore the Document to an empty state. + void clear() { getRoot() = getNode(); } + + /// Create a nil node associated with this Document. + DocNode getNode() { + auto N = DocNode(&KindAndDocs[size_t(Type::Nil)]); + return N; + } + + /// Create an Int node associated with this Document. + DocNode getNode(int64_t V) { + auto N = DocNode(&KindAndDocs[size_t(Type::Int)]); + N.Int = V; + return N; + } + + /// Create an Int node associated with this Document. + DocNode getNode(int V) { + auto N = DocNode(&KindAndDocs[size_t(Type::Int)]); + N.Int = V; + return N; + } + + /// Create a UInt node associated with this Document. + DocNode getNode(uint64_t V) { + auto N = DocNode(&KindAndDocs[size_t(Type::UInt)]); + N.UInt = V; + return N; + } + + /// Create a UInt node associated with this Document. + DocNode getNode(unsigned V) { + auto N = DocNode(&KindAndDocs[size_t(Type::UInt)]); + N.UInt = V; + return N; + } + + /// Create a Boolean node associated with this Document. + DocNode getNode(bool V) { + auto N = DocNode(&KindAndDocs[size_t(Type::Boolean)]); + N.Bool = V; + return N; + } + + /// Create a Float node associated with this Document. + DocNode getNode(double V) { + auto N = DocNode(&KindAndDocs[size_t(Type::Float)]); + N.Float = V; + return N; + } + + /// Create a String node associated with this Document. If !Copy, the passed + /// string must remain valid for the lifetime of the Document. + DocNode getNode(StringRef V, bool Copy = false) { + if (Copy) + V = addString(V); + auto N = DocNode(&KindAndDocs[size_t(Type::String)]); + N.Raw = V; + return N; + } + + /// Create a String node associated with this Document. If !Copy, the passed + /// string must remain valid for the lifetime of the Document. + DocNode getNode(const char *V, bool Copy = false) { + return getNode(StringRef(V), Copy); + } + + /// Create an empty Map node associated with this Document. + MapDocNode getMapNode() { + auto N = DocNode(&KindAndDocs[size_t(Type::Map)]); + Maps.push_back(std::unique_ptr(new DocNode::MapTy)); + N.Map = Maps.back().get(); + return N.getMap(); + } + + /// Create an empty Array node associated with this Document. + ArrayDocNode getArrayNode() { + auto N = DocNode(&KindAndDocs[size_t(Type::Array)]); + Arrays.push_back(std::unique_ptr(new DocNode::ArrayTy)); + N.Array = Arrays.back().get(); + return N.getArray(); + } + + /// Read a MsgPack document from a binary MsgPack blob. + /// The blob data must remain valid for the lifetime of this Document (because + /// a string object in the document contains a StringRef into the original + /// blob). + /// If Multi, then this sets root to an array and adds top-level objects to + /// it. If !Multi, then it only reads a single top-level object, even if there + /// are more, and sets root to that. + /// Returns false if failed due to illegal format. + bool readFromBlob(StringRef Blob, bool Multi); + + /// Write a MsgPack document to a binary MsgPack blob. + void writeToBlob(std::string &Blob); + + /// Copy a string into the Document's strings list, and return the copy that + /// is owned by the Document. + StringRef addString(StringRef S) { + Strings.push_back(std::unique_ptr(new char[S.size()])); + memcpy(&Strings.back()[0], S.data(), S.size()); + return StringRef(&Strings.back()[0], S.size()); + } + + /// Set whether YAML output uses hex for UInt. Default off. + void setHexMode(bool Val = true) { HexMode = Val; } + + /// Get Hexmode flag. + bool getHexMode() const { return HexMode; } + + /// Convert MsgPack Document to YAML text. + void toYAML(raw_ostream &OS); + + /// Read YAML text into the MsgPack document. Returns false on failure. + bool fromYAML(StringRef S); +}; + +} // namespace msgpack +} // namespace llvm + +#endif // LLVM_BINARYFORMAT_MSGPACKDOCUMENT_H diff --git a/include/llvm/BinaryFormat/MsgPackReader.h b/include/llvm/BinaryFormat/MsgPackReader.h index 511c31407455..2d332f531b23 100644 --- a/include/llvm/BinaryFormat/MsgPackReader.h +++ b/include/llvm/BinaryFormat/MsgPackReader.h @@ -1,9 +1,8 @@ //===- MsgPackReader.h - Simple MsgPack reader ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/include/llvm/BinaryFormat/MsgPackTypes.h b/include/llvm/BinaryFormat/MsgPackTypes.h deleted file mode 100644 index f96cd4c338fd..000000000000 --- a/include/llvm/BinaryFormat/MsgPackTypes.h +++ /dev/null @@ -1,372 +0,0 @@ -//===- MsgPackTypes.h - MsgPack Types ---------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -/// \file -/// This is a data structure for representing MessagePack "documents", with -/// methods to go to and from MessagePack. The types also specialize YAMLIO -/// traits in order to go to and from YAML. -// -//===----------------------------------------------------------------------===// - -#include "llvm/ADT/Optional.h" -#include "llvm/BinaryFormat/MsgPackReader.h" -#include "llvm/BinaryFormat/MsgPackWriter.h" -#include "llvm/Support/Casting.h" -#include "llvm/Support/YAMLTraits.h" -#include - -#ifndef LLVM_BINARYFORMAT_MSGPACKTYPES_H -#define LLVM_BINARYFORMAT_MSGPACKTYPES_H - -namespace llvm { -namespace msgpack { - -class Node; - -/// Short-hand for a Node pointer. -using NodePtr = std::shared_ptr; - -/// Short-hand for an Optional Node pointer. -using OptNodePtr = Optional; - -/// Abstract base-class which can be any MessagePack type. -class Node { -public: - enum NodeKind { - NK_Scalar, - NK_Array, - NK_Map, - }; - -private: - virtual void anchor() = 0; - const NodeKind Kind; - - static Expected readArray(Reader &MPReader, size_t Length); - static Expected readMap(Reader &MPReader, size_t Length); - -public: - NodeKind getKind() const { return Kind; } - - /// Construct a Node. Used by derived classes to track kind information. - Node(NodeKind Kind) : Kind(Kind) {} - - virtual ~Node() = default; - - /// Read from a MessagePack reader \p MPReader, returning an error if one is - /// encountered, or None if \p MPReader is at the end of stream, or some Node - /// pointer if some type is read. - static Expected read(Reader &MPReader); - - /// Write to a MessagePack writer \p MPWriter. - virtual void write(Writer &MPWriter) = 0; -}; - -/// A MessagePack scalar. -class ScalarNode : public Node { -public: - enum ScalarKind { - SK_Int, - SK_UInt, - SK_Nil, - SK_Boolean, - SK_Float, - SK_String, - SK_Binary, - }; - -private: - void anchor() override; - - void destroy(); - - ScalarKind SKind; - - union { - int64_t IntValue; - uint64_t UIntValue; - bool BoolValue; - double FloatValue; - std::string StringValue; - }; - -public: - /// Construct an Int ScalarNode. - ScalarNode(int64_t IntValue); - /// Construct an Int ScalarNode. - ScalarNode(int32_t IntValue); - /// Construct an UInt ScalarNode. - ScalarNode(uint64_t UIntValue); - /// Construct an UInt ScalarNode. - ScalarNode(uint32_t UIntValue); - /// Construct a Nil ScalarNode. - ScalarNode(); - /// Construct a Boolean ScalarNode. - ScalarNode(bool BoolValue); - /// Construct a Float ScalarNode. - ScalarNode(double FloatValue); - /// Construct a String ScalarNode. - ScalarNode(StringRef StringValue); - /// Construct a String ScalarNode. - ScalarNode(const char *StringValue); - /// Construct a String ScalarNode. - ScalarNode(std::string &&StringValue); - /// Construct a Binary ScalarNode. - ScalarNode(MemoryBufferRef BinaryValue); - - ~ScalarNode(); - - ScalarNode &operator=(const ScalarNode &RHS) = delete; - /// A ScalarNode can only be move assigned. - ScalarNode &operator=(ScalarNode &&RHS); - - /// Change the kind of this ScalarNode, zero initializing it to the new type. - void setScalarKind(ScalarKind SKind) { - switch (SKind) { - case SK_Int: - *this = int64_t(0); - break; - case SK_UInt: - *this = uint64_t(0); - break; - case SK_Boolean: - *this = false; - break; - case SK_Float: - *this = 0.0; - break; - case SK_String: - *this = StringRef(); - break; - case SK_Binary: - *this = MemoryBufferRef("", ""); - break; - case SK_Nil: - *this = ScalarNode(); - break; - } - } - - /// Get the current kind of ScalarNode. - ScalarKind getScalarKind() { return SKind; } - - /// Get the value of an Int scalar. - /// - /// \warning Assumes getScalarKind() == SK_Int - int64_t getInt() { - assert(SKind == SK_Int); - return IntValue; - } - - /// Get the value of a UInt scalar. - /// - /// \warning Assumes getScalarKind() == SK_UInt - uint64_t getUInt() { - assert(SKind == SK_UInt); - return UIntValue; - } - - /// Get the value of an Boolean scalar. - /// - /// \warning Assumes getScalarKind() == SK_Boolean - bool getBool() { - assert(SKind == SK_Boolean); - return BoolValue; - } - - /// Get the value of an Float scalar. - /// - /// \warning Assumes getScalarKind() == SK_Float - double getFloat() { - assert(SKind == SK_Float); - return FloatValue; - } - - /// Get the value of a String scalar. - /// - /// \warning Assumes getScalarKind() == SK_String - StringRef getString() { - assert(SKind == SK_String); - return StringValue; - } - - /// Get the value of a Binary scalar. - /// - /// \warning Assumes getScalarKind() == SK_Binary - StringRef getBinary() { - assert(SKind == SK_Binary); - return StringValue; - } - - static bool classof(const Node *N) { return N->getKind() == NK_Scalar; } - - void write(Writer &MPWriter) override; - - /// Parse a YAML scalar of the current ScalarKind from \p ScalarStr. - /// - /// \returns An empty string on success, otherwise an error message. - StringRef inputYAML(StringRef ScalarStr); - - /// Output a YAML scalar of the current ScalarKind into \p OS. - void outputYAML(raw_ostream &OS) const; - - /// Determine which YAML quoting type the current value would need when - /// output. - yaml::QuotingType mustQuoteYAML(StringRef ScalarStr) const; - - /// Get the YAML tag for the current ScalarKind. - StringRef getYAMLTag() const; - - /// Flag which affects how the type handles YAML tags when reading and - /// writing. - /// - /// When false, tags are used when reading and writing. When reading, the tag - /// is used to decide the ScalarKind before parsing. When writing, the tag is - /// output along with the value. - /// - /// When true, tags are ignored when reading and writing. When reading, the - /// ScalarKind is always assumed to be String. When writing, the tag is not - /// output. - bool IgnoreTag = false; - - static const char *IntTag; - static const char *NilTag; - static const char *BooleanTag; - static const char *FloatTag; - static const char *StringTag; - static const char *BinaryTag; -}; - -class ArrayNode : public Node, public std::vector { - void anchor() override; - -public: - ArrayNode() : Node(NK_Array) {} - static bool classof(const Node *N) { return N->getKind() == NK_Array; } - - void write(Writer &MPWriter) override { - MPWriter.writeArraySize(this->size()); - for (auto &N : *this) - N->write(MPWriter); - } -}; - -class MapNode : public Node, public StringMap { - void anchor() override; - -public: - MapNode() : Node(NK_Map) {} - static bool classof(const Node *N) { return N->getKind() == NK_Map; } - - void write(Writer &MPWriter) override { - MPWriter.writeMapSize(this->size()); - for (auto &N : *this) { - MPWriter.write(N.first()); - N.second->write(MPWriter); - } - } -}; - -} // end namespace msgpack - -namespace yaml { - -template <> struct PolymorphicTraits { - static NodeKind getKind(const msgpack::NodePtr &N) { - if (isa(*N)) - return NodeKind::Scalar; - if (isa(*N)) - return NodeKind::Map; - if (isa(*N)) - return NodeKind::Sequence; - llvm_unreachable("NodeKind not supported"); - } - static msgpack::ScalarNode &getAsScalar(msgpack::NodePtr &N) { - if (!N || !isa(*N)) - N.reset(new msgpack::ScalarNode()); - return *cast(N.get()); - } - static msgpack::MapNode &getAsMap(msgpack::NodePtr &N) { - if (!N || !isa(*N)) - N.reset(new msgpack::MapNode()); - return *cast(N.get()); - } - static msgpack::ArrayNode &getAsSequence(msgpack::NodePtr &N) { - if (!N || !isa(*N)) - N.reset(new msgpack::ArrayNode()); - return *cast(N.get()); - } -}; - -template <> struct TaggedScalarTraits { - static void output(const msgpack::ScalarNode &S, void *Ctxt, - raw_ostream &ScalarOS, raw_ostream &TagOS) { - if (!S.IgnoreTag) - TagOS << S.getYAMLTag(); - S.outputYAML(ScalarOS); - } - - static StringRef input(StringRef ScalarStr, StringRef Tag, void *Ctxt, - msgpack::ScalarNode &S) { - if (Tag == msgpack::ScalarNode::IntTag) { - S.setScalarKind(msgpack::ScalarNode::SK_UInt); - if (S.inputYAML(ScalarStr) == StringRef()) - return StringRef(); - S.setScalarKind(msgpack::ScalarNode::SK_Int); - return S.inputYAML(ScalarStr); - } - - if (S.IgnoreTag || Tag == msgpack::ScalarNode::StringTag || - Tag == "tag:yaml.org,2002:str") - S.setScalarKind(msgpack::ScalarNode::SK_String); - else if (Tag == msgpack::ScalarNode::NilTag) - S.setScalarKind(msgpack::ScalarNode::SK_Nil); - else if (Tag == msgpack::ScalarNode::BooleanTag) - S.setScalarKind(msgpack::ScalarNode::SK_Boolean); - else if (Tag == msgpack::ScalarNode::FloatTag) - S.setScalarKind(msgpack::ScalarNode::SK_Float); - else if (Tag == msgpack::ScalarNode::StringTag) - S.setScalarKind(msgpack::ScalarNode::SK_String); - else if (Tag == msgpack::ScalarNode::BinaryTag) - S.setScalarKind(msgpack::ScalarNode::SK_Binary); - else - return "Unsupported messagepack tag"; - - return S.inputYAML(ScalarStr); - } - - static QuotingType mustQuote(const msgpack::ScalarNode &S, StringRef Str) { - return S.mustQuoteYAML(Str); - } -}; - -template <> struct CustomMappingTraits { - static void inputOne(IO &IO, StringRef Key, msgpack::MapNode &M) { - IO.mapRequired(Key.str().c_str(), M[Key]); - } - static void output(IO &IO, msgpack::MapNode &M) { - for (auto &N : M) - IO.mapRequired(N.getKey().str().c_str(), N.getValue()); - } -}; - -template <> struct SequenceTraits { - static size_t size(IO &IO, msgpack::ArrayNode &A) { return A.size(); } - static msgpack::NodePtr &element(IO &IO, msgpack::ArrayNode &A, - size_t Index) { - if (Index >= A.size()) - A.resize(Index + 1); - return A[Index]; - } -}; - -} // end namespace yaml -} // end namespace llvm - -#endif // LLVM_BINARYFORMAT_MSGPACKTYPES_H diff --git a/include/llvm/BinaryFormat/MsgPackWriter.h b/include/llvm/BinaryFormat/MsgPackWriter.h index 98af422c9f19..3b610b774f77 100644 --- a/include/llvm/BinaryFormat/MsgPackWriter.h +++ b/include/llvm/BinaryFormat/MsgPackWriter.h @@ -1,9 +1,8 @@ //===- MsgPackWriter.h - Simple MsgPack writer ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/include/llvm/BinaryFormat/Wasm.h b/include/llvm/BinaryFormat/Wasm.h index d9f0f94b298d..0f22bfe610c6 100644 --- a/include/llvm/BinaryFormat/Wasm.h +++ b/include/llvm/BinaryFormat/Wasm.h @@ -1,9 +1,8 @@ //===- Wasm.h - Wasm object file format -------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -43,6 +42,17 @@ struct WasmDylinkInfo { std::vector Needed; // Shared library depenedencies }; +struct WasmProducerInfo { + std::vector> Languages; + std::vector> Tools; + std::vector> SDKs; +}; + +struct WasmFeatureEntry { + uint8_t Prefix; + std::string Name; +}; + struct WasmExport { StringRef Name; uint8_t Kind; @@ -126,12 +136,13 @@ struct WasmFunction { }; struct WasmDataSegment { - uint32_t MemoryIndex; - WasmInitExpr Offset; + uint32_t InitFlags; + uint32_t MemoryIndex; // present if InitFlags & WASM_SEGMENT_HAS_MEMINDEX + WasmInitExpr Offset; // present if InitFlags & WASM_SEGMENT_IS_PASSIVE == 0 ArrayRef Content; StringRef Name; // from the "segment info" section uint32_t Alignment; - uint32_t Flags; + uint32_t LinkerFlags; uint32_t Comdat; // from the "comdat info" section }; @@ -165,7 +176,8 @@ struct WasmSymbolInfo { StringRef Name; uint8_t Kind; uint32_t Flags; - StringRef Module; // For undefined symbols the module name of the import + StringRef ImportModule; // For undefined symbols the module of the import + StringRef ImportName; // For undefined symbols the name of the import union { // For function or global symbols, the index in function or global index // space. @@ -212,7 +224,7 @@ enum : unsigned { WASM_TYPE_F64 = 0x7C, WASM_TYPE_V128 = 0x7B, WASM_TYPE_FUNCREF = 0x70, - WASM_TYPE_EXCEPT_REF = 0x68, + WASM_TYPE_EXNREF = 0x68, WASM_TYPE_FUNC = 0x60, WASM_TYPE_NORESULT = 0x40, // for blocks with no result values }; @@ -229,11 +241,19 @@ enum : unsigned { // Opcodes used in initializer expressions. enum : unsigned { WASM_OPCODE_END = 0x0b, + WASM_OPCODE_CALL = 0x10, + WASM_OPCODE_LOCAL_GET = 0x20, WASM_OPCODE_GLOBAL_GET = 0x23, + WASM_OPCODE_GLOBAL_SET = 0x24, + WASM_OPCODE_I32_STORE = 0x36, WASM_OPCODE_I32_CONST = 0x41, WASM_OPCODE_I64_CONST = 0x42, WASM_OPCODE_F32_CONST = 0x43, WASM_OPCODE_F64_CONST = 0x44, + WASM_OPCODE_I32_ADD = 0x6a, + WASM_OPCODE_MISC_PREFIX = 0xfc, + WASM_OPCODE_MEMORY_INIT = 0x08, + WASM_OPCODE_DATA_DROP = 0x09, }; enum : unsigned { @@ -241,6 +261,18 @@ enum : unsigned { WASM_LIMITS_FLAG_IS_SHARED = 0x2, }; +enum : unsigned { + WASM_SEGMENT_IS_PASSIVE = 0x01, + WASM_SEGMENT_HAS_MEMINDEX = 0x02, +}; + +// Feature policy prefixes used in the custom "target_features" section +enum : uint8_t { + WASM_FEATURE_PREFIX_USED = '+', + WASM_FEATURE_PREFIX_REQUIRED = '=', + WASM_FEATURE_PREFIX_DISALLOWED = '-', +}; + // Kind codes used in the custom "name" section enum : unsigned { WASM_NAMES_FUNCTION = 0x1, @@ -284,6 +316,8 @@ const unsigned WASM_SYMBOL_BINDING_LOCAL = 0x2; const unsigned WASM_SYMBOL_VISIBILITY_DEFAULT = 0x0; const unsigned WASM_SYMBOL_VISIBILITY_HIDDEN = 0x4; const unsigned WASM_SYMBOL_UNDEFINED = 0x10; +const unsigned WASM_SYMBOL_EXPORTED = 0x20; +const unsigned WASM_SYMBOL_EXPLICIT_NAME = 0x40; #define WASM_RELOC(name, value) name = value, @@ -300,17 +334,17 @@ enum class ValType { F32 = WASM_TYPE_F32, F64 = WASM_TYPE_F64, V128 = WASM_TYPE_V128, - EXCEPT_REF = WASM_TYPE_EXCEPT_REF, + EXNREF = WASM_TYPE_EXNREF, }; struct WasmSignature { - SmallVector Returns; - SmallVector Params; + SmallVector Returns; + SmallVector Params; // Support empty and tombstone instances, needed by DenseMap. enum { Plain, Empty, Tombstone } State = Plain; - WasmSignature(SmallVector &&InReturns, - SmallVector &&InParams) + WasmSignature(SmallVector &&InReturns, + SmallVector &&InParams) : Returns(InReturns), Params(InParams) {} WasmSignature() = default; }; @@ -333,8 +367,9 @@ inline bool operator!=(const WasmGlobalType &LHS, const WasmGlobalType &RHS) { return !(LHS == RHS); } -std::string toString(wasm::WasmSymbolType type); +std::string toString(WasmSymbolType type); std::string relocTypetoString(uint32_t type); +bool relocTypeHasAddend(uint32_t type); } // end namespace wasm } // end namespace llvm diff --git a/include/llvm/BinaryFormat/WasmRelocs.def b/include/llvm/BinaryFormat/WasmRelocs.def index b3a08e70c1d5..00dacf72abb0 100644 --- a/include/llvm/BinaryFormat/WasmRelocs.def +++ b/include/llvm/BinaryFormat/WasmRelocs.def @@ -2,14 +2,16 @@ #error "WASM_RELOC must be defined" #endif -WASM_RELOC(R_WEBASSEMBLY_FUNCTION_INDEX_LEB, 0) -WASM_RELOC(R_WEBASSEMBLY_TABLE_INDEX_SLEB, 1) -WASM_RELOC(R_WEBASSEMBLY_TABLE_INDEX_I32, 2) -WASM_RELOC(R_WEBASSEMBLY_MEMORY_ADDR_LEB, 3) -WASM_RELOC(R_WEBASSEMBLY_MEMORY_ADDR_SLEB, 4) -WASM_RELOC(R_WEBASSEMBLY_MEMORY_ADDR_I32, 5) -WASM_RELOC(R_WEBASSEMBLY_TYPE_INDEX_LEB, 6) -WASM_RELOC(R_WEBASSEMBLY_GLOBAL_INDEX_LEB, 7) -WASM_RELOC(R_WEBASSEMBLY_FUNCTION_OFFSET_I32, 8) -WASM_RELOC(R_WEBASSEMBLY_SECTION_OFFSET_I32, 9) -WASM_RELOC(R_WEBASSEMBLY_EVENT_INDEX_LEB, 10) +WASM_RELOC(R_WASM_FUNCTION_INDEX_LEB, 0) +WASM_RELOC(R_WASM_TABLE_INDEX_SLEB, 1) +WASM_RELOC(R_WASM_TABLE_INDEX_I32, 2) +WASM_RELOC(R_WASM_MEMORY_ADDR_LEB, 3) +WASM_RELOC(R_WASM_MEMORY_ADDR_SLEB, 4) +WASM_RELOC(R_WASM_MEMORY_ADDR_I32, 5) +WASM_RELOC(R_WASM_TYPE_INDEX_LEB, 6) +WASM_RELOC(R_WASM_GLOBAL_INDEX_LEB, 7) +WASM_RELOC(R_WASM_FUNCTION_OFFSET_I32, 8) +WASM_RELOC(R_WASM_SECTION_OFFSET_I32, 9) +WASM_RELOC(R_WASM_EVENT_INDEX_LEB, 10) +WASM_RELOC(R_WASM_MEMORY_ADDR_REL_SLEB, 11) +WASM_RELOC(R_WASM_TABLE_INDEX_REL_SLEB, 12) diff --git a/include/llvm/BinaryFormat/XCOFF.h b/include/llvm/BinaryFormat/XCOFF.h new file mode 100644 index 000000000000..7774ab3ed24a --- /dev/null +++ b/include/llvm/BinaryFormat/XCOFF.h @@ -0,0 +1,145 @@ +//===-- llvm/BinaryFormat/XCOFF.h - The XCOFF file format -------*- C++/-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines manifest constants for the XCOFF object file format. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_BINARYFORMAT_XCOFF_H +#define LLVM_BINARYFORMAT_XCOFF_H + +#include + +namespace llvm { +namespace XCOFF { + +// Constants used in the XCOFF definition. +enum { SectionNameSize = 8, SymbolNameSize = 8 }; +enum ReservedSectionNum { N_DEBUG = -2, N_ABS = -1, N_UNDEF = 0 }; + +// x_smclas field of x_csect from system header: /usr/include/syms.h +/// Storage Mapping Class definitions. +enum StorageMappingClass { + // READ ONLY CLASSES + XMC_PR = 0, ///< Program Code + XMC_RO = 1, ///< Read Only Constant + XMC_DB = 2, ///< Debug Dictionary Table + XMC_GL = 6, ///< Global Linkage (Interfile Interface Code) + XMC_XO = 7, ///< Extended Operation (Pseudo Machine Instruction) + XMC_SV = 8, ///< Supervisor Call (32-bit process only) + XMC_SV64 = 17, ///< Supervisor Call for 64-bit process + XMC_SV3264 = 18, ///< Supervisor Call for both 32- and 64-bit processes + XMC_TI = 12, ///< Traceback Index csect + XMC_TB = 13, ///< Traceback Table csect + + // READ WRITE CLASSES + XMC_RW = 5, ///< Read Write Data + XMC_TC0 = 15, ///< TOC Anchor for TOC Addressability + XMC_TC = 3, ///< General TOC item + XMC_TD = 16, ///< Scalar data item in the TOC + XMC_DS = 10, ///< Descriptor csect + XMC_UA = 4, ///< Unclassified - Treated as Read Write + XMC_BS = 9, ///< BSS class (uninitialized static internal) + XMC_UC = 11, ///< Un-named Fortran Common + + XMC_TL = 20, ///< Initialized thread-local variable + XMC_UL = 21, ///< Uninitialized thread-local variable + XMC_TE = 22 ///< Symbol mapped at the end of TOC +}; + +// Flags for defining the section type. Used for the s_flags field of +// the section header structure. Defined in the system header `scnhdr.h`. +enum SectionTypeFlags { + STYP_PAD = 0x0008, + STYP_DWARF = 0x0010, + STYP_TEXT = 0x0020, + STYP_DATA = 0x0040, + STYP_BSS = 0x0080, + STYP_EXCEPT = 0x0100, + STYP_INFO = 0x0200, + STYP_TDATA = 0x0400, + STYP_TBSS = 0x0800, + STYP_LOADER = 0x1000, + STYP_DEBUG = 0x2000, + STYP_TYPCHK = 0x4000, + STYP_OVRFLO = 0x8000 +}; + +// STORAGE CLASSES, n_sclass field of syment. +// The values come from `storclass.h` and `dbxstclass.h`. +enum StorageClass : uint8_t { + // Storage classes used for symbolic debugging symbols. + C_FILE = 103, // File name + C_BINCL = 108, // Beginning of include file + C_EINCL = 109, // Ending of include file + C_GSYM = 128, // Global variable + C_STSYM = 133, // Statically allocated symbol + C_BCOMM = 135, // Beginning of common block + C_ECOMM = 137, // End of common block + C_ENTRY = 141, // Alternate entry + C_BSTAT = 143, // Beginning of static block + C_ESTAT = 144, // End of static block + C_GTLS = 145, // Global thread-local variable + C_STTLS = 146, // Static thread-local variable + + // Storage classes used for DWARF symbols. + C_DWARF = 112, // DWARF section symbol + + // Storage classes used for absolute symbols. + C_LSYM = 129, // Automatic variable allocated on stack + C_PSYM = 130, // Argument to subroutine allocated on stack + C_RSYM = 131, // Register variable + C_RPSYM = 132, // Argument to function or procedure stored in register + C_ECOML = 136, // Local member of common block + C_FUN = 142, // Function or procedure + + // Storage classes used for undefined external symbols or + // symbols of general sections. + C_EXT = 2, // External symbol + C_WEAKEXT = 111, // Weak external symbol + + // Storage classes used for symbols of general sections. + C_NULL = 0, + C_STAT = 3, // Static + C_BLOCK = 100, // ".bb" or ".eb" + C_FCN = 101, // ".bf" or ".ef" + C_HIDEXT = 107, // Un-named external symbol + C_INFO = 110, // Comment string in .info section + C_DECL = 140, // Declaration of object (type) + + // Storage classes - Obsolete/Undocumented. + C_AUTO = 1, // Automatic variable + C_REG = 4, // Register variable + C_EXTDEF = 5, // External definition + C_LABEL = 6, // Label + C_ULABEL = 7, // Undefined label + C_MOS = 8, // Member of structure + C_ARG = 9, // Function argument + C_STRTAG = 10, // Structure tag + C_MOU = 11, // Member of union + C_UNTAG = 12, // Union tag + C_TPDEF = 13, // Type definition + C_USTATIC = 14, // Undefined static + C_ENTAG = 15, // Enumeration tag + C_MOE = 16, // Member of enumeration + C_REGPARM = 17, // Register parameter + C_FIELD = 18, // Bit field + C_EOS = 102, // End of structure + C_LINE = 104, + C_ALIAS = 105, // Duplicate tag + C_HIDDEN = 106, // Special storage class for external + C_EFCN = 255, // Physical end of function + + // Storage classes - reserved + C_TCSYM = 134 // Reserved +}; + +} // end namespace XCOFF +} // end namespace llvm + +#endif diff --git a/include/llvm/Bitcode/BitCodes.h b/include/llvm/Bitcode/BitCodes.h deleted file mode 100644 index bf21e146e771..000000000000 --- a/include/llvm/Bitcode/BitCodes.h +++ /dev/null @@ -1,185 +0,0 @@ -//===- BitCodes.h - Enum values for the bitcode format ----------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This header Bitcode enum values. -// -// The enum values defined in this file should be considered permanent. If -// new features are added, they should have values added at the end of the -// respective lists. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_BITCODE_BITCODES_H -#define LLVM_BITCODE_BITCODES_H - -#include "llvm/ADT/SmallVector.h" -#include "llvm/Support/DataTypes.h" -#include "llvm/Support/ErrorHandling.h" -#include - -namespace llvm { -/// Offsets of the 32-bit fields of bitcode wrapper header. -static const unsigned BWH_MagicField = 0 * 4; -static const unsigned BWH_VersionField = 1 * 4; -static const unsigned BWH_OffsetField = 2 * 4; -static const unsigned BWH_SizeField = 3 * 4; -static const unsigned BWH_CPUTypeField = 4 * 4; -static const unsigned BWH_HeaderSize = 5 * 4; - -namespace bitc { - enum StandardWidths { - BlockIDWidth = 8, // We use VBR-8 for block IDs. - CodeLenWidth = 4, // Codelen are VBR-4. - BlockSizeWidth = 32 // BlockSize up to 2^32 32-bit words = 16GB per block. - }; - - // The standard abbrev namespace always has a way to exit a block, enter a - // nested block, define abbrevs, and define an unabbreviated record. - enum FixedAbbrevIDs { - END_BLOCK = 0, // Must be zero to guarantee termination for broken bitcode. - ENTER_SUBBLOCK = 1, - - /// DEFINE_ABBREV - Defines an abbrev for the current block. It consists - /// of a vbr5 for # operand infos. Each operand info is emitted with a - /// single bit to indicate if it is a literal encoding. If so, the value is - /// emitted with a vbr8. If not, the encoding is emitted as 3 bits followed - /// by the info value as a vbr5 if needed. - DEFINE_ABBREV = 2, - - // UNABBREV_RECORDs are emitted with a vbr6 for the record code, followed by - // a vbr6 for the # operands, followed by vbr6's for each operand. - UNABBREV_RECORD = 3, - - // This is not a code, this is a marker for the first abbrev assignment. - FIRST_APPLICATION_ABBREV = 4 - }; - - /// StandardBlockIDs - All bitcode files can optionally include a BLOCKINFO - /// block, which contains metadata about other blocks in the file. - enum StandardBlockIDs { - /// BLOCKINFO_BLOCK is used to define metadata about blocks, for example, - /// standard abbrevs that should be available to all blocks of a specified - /// ID. - BLOCKINFO_BLOCK_ID = 0, - - // Block IDs 1-7 are reserved for future expansion. - FIRST_APPLICATION_BLOCKID = 8 - }; - - /// BlockInfoCodes - The blockinfo block contains metadata about user-defined - /// blocks. - enum BlockInfoCodes { - // DEFINE_ABBREV has magic semantics here, applying to the current SETBID'd - // block, instead of the BlockInfo block. - - BLOCKINFO_CODE_SETBID = 1, // SETBID: [blockid#] - BLOCKINFO_CODE_BLOCKNAME = 2, // BLOCKNAME: [name] - BLOCKINFO_CODE_SETRECORDNAME = 3 // BLOCKINFO_CODE_SETRECORDNAME: - // [id, name] - }; - -} // End bitc namespace - -/// BitCodeAbbrevOp - This describes one or more operands in an abbreviation. -/// This is actually a union of two different things: -/// 1. It could be a literal integer value ("the operand is always 17"). -/// 2. It could be an encoding specification ("this operand encoded like so"). -/// -class BitCodeAbbrevOp { - uint64_t Val; // A literal value or data for an encoding. - bool IsLiteral : 1; // Indicate whether this is a literal value or not. - unsigned Enc : 3; // The encoding to use. -public: - enum Encoding { - Fixed = 1, // A fixed width field, Val specifies number of bits. - VBR = 2, // A VBR field where Val specifies the width of each chunk. - Array = 3, // A sequence of fields, next field species elt encoding. - Char6 = 4, // A 6-bit fixed field which maps to [a-zA-Z0-9._]. - Blob = 5 // 32-bit aligned array of 8-bit characters. - }; - - explicit BitCodeAbbrevOp(uint64_t V) : Val(V), IsLiteral(true) {} - explicit BitCodeAbbrevOp(Encoding E, uint64_t Data = 0) - : Val(Data), IsLiteral(false), Enc(E) {} - - bool isLiteral() const { return IsLiteral; } - bool isEncoding() const { return !IsLiteral; } - - // Accessors for literals. - uint64_t getLiteralValue() const { assert(isLiteral()); return Val; } - - // Accessors for encoding info. - Encoding getEncoding() const { assert(isEncoding()); return (Encoding)Enc; } - uint64_t getEncodingData() const { - assert(isEncoding() && hasEncodingData()); - return Val; - } - - bool hasEncodingData() const { return hasEncodingData(getEncoding()); } - static bool hasEncodingData(Encoding E) { - switch (E) { - case Fixed: - case VBR: - return true; - case Array: - case Char6: - case Blob: - return false; - } - report_fatal_error("Invalid encoding"); - } - - /// isChar6 - Return true if this character is legal in the Char6 encoding. - static bool isChar6(char C) { - if (C >= 'a' && C <= 'z') return true; - if (C >= 'A' && C <= 'Z') return true; - if (C >= '0' && C <= '9') return true; - if (C == '.' || C == '_') return true; - return false; - } - static unsigned EncodeChar6(char C) { - if (C >= 'a' && C <= 'z') return C-'a'; - if (C >= 'A' && C <= 'Z') return C-'A'+26; - if (C >= '0' && C <= '9') return C-'0'+26+26; - if (C == '.') return 62; - if (C == '_') return 63; - llvm_unreachable("Not a value Char6 character!"); - } - - static char DecodeChar6(unsigned V) { - assert((V & ~63) == 0 && "Not a Char6 encoded character!"); - return "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._" - [V]; - } - -}; - -template <> struct isPodLike { static const bool value=true; }; - -/// BitCodeAbbrev - This class represents an abbreviation record. An -/// abbreviation allows a complex record that has redundancy to be stored in a -/// specialized format instead of the fully-general, fully-vbr, format. -class BitCodeAbbrev { - SmallVector OperandList; - -public: - unsigned getNumOperandInfos() const { - return static_cast(OperandList.size()); - } - const BitCodeAbbrevOp &getOperandInfo(unsigned N) const { - return OperandList[N]; - } - - void Add(const BitCodeAbbrevOp &OpInfo) { - OperandList.push_back(OpInfo); - } -}; -} // End llvm namespace - -#endif diff --git a/include/llvm/Bitcode/BitcodeAnalyzer.h b/include/llvm/Bitcode/BitcodeAnalyzer.h new file mode 100644 index 000000000000..cfdebd6fe6cb --- /dev/null +++ b/include/llvm/Bitcode/BitcodeAnalyzer.h @@ -0,0 +1,103 @@ +//===- llvm/Bitcode/BitcodeAnalyzer.h - Bitcode analyzer --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This header defines interfaces to analyze LLVM bitcode files/streams. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_BITCODE_BITCODE_ANALYZER_H +#define LLVM_BITCODE_BITCODE_ANALYZER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Bitstream/BitstreamReader.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/raw_ostream.h" +#include +#include + +namespace llvm { + +/// CurStreamTypeType - A type for CurStreamType +enum CurStreamTypeType { + UnknownBitstream, + LLVMIRBitstream, + ClangSerializedASTBitstream, + ClangSerializedDiagnosticsBitstream, +}; + +struct BCDumpOptions { + /// The stream. + raw_ostream &OS; + /// Print per-code histogram. + bool Histogram = false; + /// Don't emit numeric info in dump if symbolic info is available. + bool Symbolic = false; + /// Print binary blobs using hex escapes. + bool ShowBinaryBlobs = false; + + BCDumpOptions(raw_ostream &OS) : OS(OS) {} +}; + +class BitcodeAnalyzer { + BitstreamCursor Stream; + BitstreamBlockInfo BlockInfo; + CurStreamTypeType CurStreamType; + Optional BlockInfoStream; + unsigned NumTopBlocks = 0; + + struct PerRecordStats { + unsigned NumInstances; + unsigned NumAbbrev; + uint64_t TotalBits; + PerRecordStats() : NumInstances(0), NumAbbrev(0), TotalBits(0) {} + }; + + struct PerBlockIDStats { + /// NumInstances - This the number of times this block ID has been seen. + unsigned NumInstances; + /// NumBits - The total size in bits of all of these blocks. + uint64_t NumBits; + /// NumSubBlocks - The total number of blocks these blocks contain. + unsigned NumSubBlocks; + /// NumAbbrevs - The total number of abbreviations. + unsigned NumAbbrevs; + /// NumRecords - The total number of records these blocks contain, and the + /// number that are abbreviated. + unsigned NumRecords, NumAbbreviatedRecords; + /// CodeFreq - Keep track of the number of times we see each code. + std::vector CodeFreq; + PerBlockIDStats() + : NumInstances(0), NumBits(0), NumSubBlocks(0), NumAbbrevs(0), + NumRecords(0), NumAbbreviatedRecords(0) {} + }; + + std::map BlockIDStats; + +public: + BitcodeAnalyzer(StringRef Buffer, Optional BlockInfoBuffer = None); + /// Analyze the bitcode file. + Error analyze(Optional O = None, + Optional CheckHash = None); + /// Print stats about the bitcode file. + void printStats(BCDumpOptions O, Optional Filename = None); + +private: + /// Read a block, updating statistics, etc. + Error parseBlock(unsigned BlockID, unsigned IndentLevel, + Optional O = None, + Optional CheckHash = None); + + Error decodeMetadataStringsBlob(StringRef Indent, ArrayRef Record, + StringRef Blob, raw_ostream &OS); +}; + +} // end namespace llvm + +#endif // LLVM_BITCODE_BITCODE_ANALYZER_H diff --git a/include/llvm/Bitcode/BitcodeReader.h b/include/llvm/Bitcode/BitcodeReader.h index 0d7cc141f2ce..ba61da733bea 100644 --- a/include/llvm/Bitcode/BitcodeReader.h +++ b/include/llvm/Bitcode/BitcodeReader.h @@ -1,9 +1,8 @@ //===- llvm/Bitcode/BitcodeReader.h - Bitcode reader ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -16,7 +15,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/StringRef.h" -#include "llvm/Bitcode/BitCodes.h" +#include "llvm/Bitstream/BitCodes.h" #include "llvm/IR/ModuleSummaryIndex.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" diff --git a/include/llvm/Bitcode/BitcodeWriter.h b/include/llvm/Bitcode/BitcodeWriter.h index 0010cf6c0544..39061e09cda5 100644 --- a/include/llvm/Bitcode/BitcodeWriter.h +++ b/include/llvm/Bitcode/BitcodeWriter.h @@ -1,9 +1,8 @@ //===- llvm/Bitcode/BitcodeWriter.h - Bitcode writers -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Bitcode/BitcodeWriterPass.h b/include/llvm/Bitcode/BitcodeWriterPass.h index 05044c9ae11c..1773d1b9f11b 100644 --- a/include/llvm/Bitcode/BitcodeWriterPass.h +++ b/include/llvm/Bitcode/BitcodeWriterPass.h @@ -1,9 +1,8 @@ //===-- BitcodeWriterPass.h - Bitcode writing pass --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/Bitcode/BitstreamReader.h b/include/llvm/Bitcode/BitstreamReader.h deleted file mode 100644 index 72e7619d9e1c..000000000000 --- a/include/llvm/Bitcode/BitstreamReader.h +++ /dev/null @@ -1,506 +0,0 @@ -//===- BitstreamReader.h - Low-level bitstream reader interface -*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This header defines the BitstreamReader class. This class can be used to -// read an arbitrary bitstream, regardless of its contents. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_BITCODE_BITSTREAMREADER_H -#define LLVM_BITCODE_BITSTREAMREADER_H - -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/Bitcode/BitCodes.h" -#include "llvm/Support/Endian.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/MathExtras.h" -#include "llvm/Support/MemoryBuffer.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace llvm { - -/// This class maintains the abbreviations read from a block info block. -class BitstreamBlockInfo { -public: - /// This contains information emitted to BLOCKINFO_BLOCK blocks. These - /// describe abbreviations that all blocks of the specified ID inherit. - struct BlockInfo { - unsigned BlockID; - std::vector> Abbrevs; - std::string Name; - std::vector> RecordNames; - }; - -private: - std::vector BlockInfoRecords; - -public: - /// If there is block info for the specified ID, return it, otherwise return - /// null. - const BlockInfo *getBlockInfo(unsigned BlockID) const { - // Common case, the most recent entry matches BlockID. - if (!BlockInfoRecords.empty() && BlockInfoRecords.back().BlockID == BlockID) - return &BlockInfoRecords.back(); - - for (unsigned i = 0, e = static_cast(BlockInfoRecords.size()); - i != e; ++i) - if (BlockInfoRecords[i].BlockID == BlockID) - return &BlockInfoRecords[i]; - return nullptr; - } - - BlockInfo &getOrCreateBlockInfo(unsigned BlockID) { - if (const BlockInfo *BI = getBlockInfo(BlockID)) - return *const_cast(BI); - - // Otherwise, add a new record. - BlockInfoRecords.emplace_back(); - BlockInfoRecords.back().BlockID = BlockID; - return BlockInfoRecords.back(); - } -}; - -/// This represents a position within a bitstream. There may be multiple -/// independent cursors reading within one bitstream, each maintaining their -/// own local state. -class SimpleBitstreamCursor { - ArrayRef BitcodeBytes; - size_t NextChar = 0; - -public: - /// This is the current data we have pulled from the stream but have not - /// returned to the client. This is specifically and intentionally defined to - /// follow the word size of the host machine for efficiency. We use word_t in - /// places that are aware of this to make it perfectly explicit what is going - /// on. - using word_t = size_t; - -private: - word_t CurWord = 0; - - /// This is the number of bits in CurWord that are valid. This is always from - /// [0...bits_of(size_t)-1] inclusive. - unsigned BitsInCurWord = 0; - -public: - static const size_t MaxChunkSize = sizeof(word_t) * 8; - - SimpleBitstreamCursor() = default; - explicit SimpleBitstreamCursor(ArrayRef BitcodeBytes) - : BitcodeBytes(BitcodeBytes) {} - explicit SimpleBitstreamCursor(StringRef BitcodeBytes) - : BitcodeBytes(reinterpret_cast(BitcodeBytes.data()), - BitcodeBytes.size()) {} - explicit SimpleBitstreamCursor(MemoryBufferRef BitcodeBytes) - : SimpleBitstreamCursor(BitcodeBytes.getBuffer()) {} - - bool canSkipToPos(size_t pos) const { - // pos can be skipped to if it is a valid address or one byte past the end. - return pos <= BitcodeBytes.size(); - } - - bool AtEndOfStream() { - return BitsInCurWord == 0 && BitcodeBytes.size() <= NextChar; - } - - /// Return the bit # of the bit we are reading. - uint64_t GetCurrentBitNo() const { - return NextChar*CHAR_BIT - BitsInCurWord; - } - - // Return the byte # of the current bit. - uint64_t getCurrentByteNo() const { return GetCurrentBitNo() / 8; } - - ArrayRef getBitcodeBytes() const { return BitcodeBytes; } - - /// Reset the stream to the specified bit number. - void JumpToBit(uint64_t BitNo) { - size_t ByteNo = size_t(BitNo/8) & ~(sizeof(word_t)-1); - unsigned WordBitNo = unsigned(BitNo & (sizeof(word_t)*8-1)); - assert(canSkipToPos(ByteNo) && "Invalid location"); - - // Move the cursor to the right word. - NextChar = ByteNo; - BitsInCurWord = 0; - - // Skip over any bits that are already consumed. - if (WordBitNo) - Read(WordBitNo); - } - - /// Get a pointer into the bitstream at the specified byte offset. - const uint8_t *getPointerToByte(uint64_t ByteNo, uint64_t NumBytes) { - return BitcodeBytes.data() + ByteNo; - } - - /// Get a pointer into the bitstream at the specified bit offset. - /// - /// The bit offset must be on a byte boundary. - const uint8_t *getPointerToBit(uint64_t BitNo, uint64_t NumBytes) { - assert(!(BitNo % 8) && "Expected bit on byte boundary"); - return getPointerToByte(BitNo / 8, NumBytes); - } - - void fillCurWord() { - if (NextChar >= BitcodeBytes.size()) - report_fatal_error("Unexpected end of file"); - - // Read the next word from the stream. - const uint8_t *NextCharPtr = BitcodeBytes.data() + NextChar; - unsigned BytesRead; - if (BitcodeBytes.size() >= NextChar + sizeof(word_t)) { - BytesRead = sizeof(word_t); - CurWord = - support::endian::read( - NextCharPtr); - } else { - // Short read. - BytesRead = BitcodeBytes.size() - NextChar; - CurWord = 0; - for (unsigned B = 0; B != BytesRead; ++B) - CurWord |= uint64_t(NextCharPtr[B]) << (B * 8); - } - NextChar += BytesRead; - BitsInCurWord = BytesRead * 8; - } - - word_t Read(unsigned NumBits) { - static const unsigned BitsInWord = MaxChunkSize; - - assert(NumBits && NumBits <= BitsInWord && - "Cannot return zero or more than BitsInWord bits!"); - - static const unsigned Mask = sizeof(word_t) > 4 ? 0x3f : 0x1f; - - // If the field is fully contained by CurWord, return it quickly. - if (BitsInCurWord >= NumBits) { - word_t R = CurWord & (~word_t(0) >> (BitsInWord - NumBits)); - - // Use a mask to avoid undefined behavior. - CurWord >>= (NumBits & Mask); - - BitsInCurWord -= NumBits; - return R; - } - - word_t R = BitsInCurWord ? CurWord : 0; - unsigned BitsLeft = NumBits - BitsInCurWord; - - fillCurWord(); - - // If we run out of data, abort. - if (BitsLeft > BitsInCurWord) - report_fatal_error("Unexpected end of file"); - - word_t R2 = CurWord & (~word_t(0) >> (BitsInWord - BitsLeft)); - - // Use a mask to avoid undefined behavior. - CurWord >>= (BitsLeft & Mask); - - BitsInCurWord -= BitsLeft; - - R |= R2 << (NumBits - BitsLeft); - - return R; - } - - uint32_t ReadVBR(unsigned NumBits) { - uint32_t Piece = Read(NumBits); - if ((Piece & (1U << (NumBits-1))) == 0) - return Piece; - - uint32_t Result = 0; - unsigned NextBit = 0; - while (true) { - Result |= (Piece & ((1U << (NumBits-1))-1)) << NextBit; - - if ((Piece & (1U << (NumBits-1))) == 0) - return Result; - - NextBit += NumBits-1; - Piece = Read(NumBits); - } - } - - // Read a VBR that may have a value up to 64-bits in size. The chunk size of - // the VBR must still be <= 32 bits though. - uint64_t ReadVBR64(unsigned NumBits) { - uint32_t Piece = Read(NumBits); - if ((Piece & (1U << (NumBits-1))) == 0) - return uint64_t(Piece); - - uint64_t Result = 0; - unsigned NextBit = 0; - while (true) { - Result |= uint64_t(Piece & ((1U << (NumBits-1))-1)) << NextBit; - - if ((Piece & (1U << (NumBits-1))) == 0) - return Result; - - NextBit += NumBits-1; - Piece = Read(NumBits); - } - } - - void SkipToFourByteBoundary() { - // If word_t is 64-bits and if we've read less than 32 bits, just dump - // the bits we have up to the next 32-bit boundary. - if (sizeof(word_t) > 4 && - BitsInCurWord >= 32) { - CurWord >>= BitsInCurWord-32; - BitsInCurWord = 32; - return; - } - - BitsInCurWord = 0; - } - - /// Skip to the end of the file. - void skipToEnd() { NextChar = BitcodeBytes.size(); } -}; - -/// When advancing through a bitstream cursor, each advance can discover a few -/// different kinds of entries: -struct BitstreamEntry { - enum { - Error, // Malformed bitcode was found. - EndBlock, // We've reached the end of the current block, (or the end of the - // file, which is treated like a series of EndBlock records. - SubBlock, // This is the start of a new subblock of a specific ID. - Record // This is a record with a specific AbbrevID. - } Kind; - - unsigned ID; - - static BitstreamEntry getError() { - BitstreamEntry E; E.Kind = Error; return E; - } - - static BitstreamEntry getEndBlock() { - BitstreamEntry E; E.Kind = EndBlock; return E; - } - - static BitstreamEntry getSubBlock(unsigned ID) { - BitstreamEntry E; E.Kind = SubBlock; E.ID = ID; return E; - } - - static BitstreamEntry getRecord(unsigned AbbrevID) { - BitstreamEntry E; E.Kind = Record; E.ID = AbbrevID; return E; - } -}; - -/// This represents a position within a bitcode file, implemented on top of a -/// SimpleBitstreamCursor. -/// -/// Unlike iterators, BitstreamCursors are heavy-weight objects that should not -/// be passed by value. -class BitstreamCursor : SimpleBitstreamCursor { - // This is the declared size of code values used for the current block, in - // bits. - unsigned CurCodeSize = 2; - - /// Abbrevs installed at in this block. - std::vector> CurAbbrevs; - - struct Block { - unsigned PrevCodeSize; - std::vector> PrevAbbrevs; - - explicit Block(unsigned PCS) : PrevCodeSize(PCS) {} - }; - - /// This tracks the codesize of parent blocks. - SmallVector BlockScope; - - BitstreamBlockInfo *BlockInfo = nullptr; - -public: - static const size_t MaxChunkSize = sizeof(word_t) * 8; - - BitstreamCursor() = default; - explicit BitstreamCursor(ArrayRef BitcodeBytes) - : SimpleBitstreamCursor(BitcodeBytes) {} - explicit BitstreamCursor(StringRef BitcodeBytes) - : SimpleBitstreamCursor(BitcodeBytes) {} - explicit BitstreamCursor(MemoryBufferRef BitcodeBytes) - : SimpleBitstreamCursor(BitcodeBytes) {} - - using SimpleBitstreamCursor::canSkipToPos; - using SimpleBitstreamCursor::AtEndOfStream; - using SimpleBitstreamCursor::getBitcodeBytes; - using SimpleBitstreamCursor::GetCurrentBitNo; - using SimpleBitstreamCursor::getCurrentByteNo; - using SimpleBitstreamCursor::getPointerToByte; - using SimpleBitstreamCursor::JumpToBit; - using SimpleBitstreamCursor::fillCurWord; - using SimpleBitstreamCursor::Read; - using SimpleBitstreamCursor::ReadVBR; - using SimpleBitstreamCursor::ReadVBR64; - - /// Return the number of bits used to encode an abbrev #. - unsigned getAbbrevIDWidth() const { return CurCodeSize; } - - /// Flags that modify the behavior of advance(). - enum { - /// If this flag is used, the advance() method does not automatically pop - /// the block scope when the end of a block is reached. - AF_DontPopBlockAtEnd = 1, - - /// If this flag is used, abbrev entries are returned just like normal - /// records. - AF_DontAutoprocessAbbrevs = 2 - }; - - /// Advance the current bitstream, returning the next entry in the stream. - BitstreamEntry advance(unsigned Flags = 0) { - while (true) { - if (AtEndOfStream()) - return BitstreamEntry::getError(); - - unsigned Code = ReadCode(); - if (Code == bitc::END_BLOCK) { - // Pop the end of the block unless Flags tells us not to. - if (!(Flags & AF_DontPopBlockAtEnd) && ReadBlockEnd()) - return BitstreamEntry::getError(); - return BitstreamEntry::getEndBlock(); - } - - if (Code == bitc::ENTER_SUBBLOCK) - return BitstreamEntry::getSubBlock(ReadSubBlockID()); - - if (Code == bitc::DEFINE_ABBREV && - !(Flags & AF_DontAutoprocessAbbrevs)) { - // We read and accumulate abbrev's, the client can't do anything with - // them anyway. - ReadAbbrevRecord(); - continue; - } - - return BitstreamEntry::getRecord(Code); - } - } - - /// This is a convenience function for clients that don't expect any - /// subblocks. This just skips over them automatically. - BitstreamEntry advanceSkippingSubblocks(unsigned Flags = 0) { - while (true) { - // If we found a normal entry, return it. - BitstreamEntry Entry = advance(Flags); - if (Entry.Kind != BitstreamEntry::SubBlock) - return Entry; - - // If we found a sub-block, just skip over it and check the next entry. - if (SkipBlock()) - return BitstreamEntry::getError(); - } - } - - unsigned ReadCode() { - return Read(CurCodeSize); - } - - // Block header: - // [ENTER_SUBBLOCK, blockid, newcodelen, , blocklen] - - /// Having read the ENTER_SUBBLOCK code, read the BlockID for the block. - unsigned ReadSubBlockID() { - return ReadVBR(bitc::BlockIDWidth); - } - - /// Having read the ENTER_SUBBLOCK abbrevid and a BlockID, skip over the body - /// of this block. If the block record is malformed, return true. - bool SkipBlock() { - // Read and ignore the codelen value. Since we are skipping this block, we - // don't care what code widths are used inside of it. - ReadVBR(bitc::CodeLenWidth); - SkipToFourByteBoundary(); - size_t NumFourBytes = Read(bitc::BlockSizeWidth); - - // Check that the block wasn't partially defined, and that the offset isn't - // bogus. - size_t SkipTo = GetCurrentBitNo() + NumFourBytes*4*8; - if (AtEndOfStream() || !canSkipToPos(SkipTo/8)) - return true; - - JumpToBit(SkipTo); - return false; - } - - /// Having read the ENTER_SUBBLOCK abbrevid, enter the block, and return true - /// if the block has an error. - bool EnterSubBlock(unsigned BlockID, unsigned *NumWordsP = nullptr); - - bool ReadBlockEnd() { - if (BlockScope.empty()) return true; - - // Block tail: - // [END_BLOCK, ] - SkipToFourByteBoundary(); - - popBlockScope(); - return false; - } - -private: - void popBlockScope() { - CurCodeSize = BlockScope.back().PrevCodeSize; - - CurAbbrevs = std::move(BlockScope.back().PrevAbbrevs); - BlockScope.pop_back(); - } - - //===--------------------------------------------------------------------===// - // Record Processing - //===--------------------------------------------------------------------===// - -public: - /// Return the abbreviation for the specified AbbrevId. - const BitCodeAbbrev *getAbbrev(unsigned AbbrevID) { - unsigned AbbrevNo = AbbrevID - bitc::FIRST_APPLICATION_ABBREV; - if (AbbrevNo >= CurAbbrevs.size()) - report_fatal_error("Invalid abbrev number"); - return CurAbbrevs[AbbrevNo].get(); - } - - /// Read the current record and discard it, returning the code for the record. - unsigned skipRecord(unsigned AbbrevID); - - unsigned readRecord(unsigned AbbrevID, SmallVectorImpl &Vals, - StringRef *Blob = nullptr); - - //===--------------------------------------------------------------------===// - // Abbrev Processing - //===--------------------------------------------------------------------===// - void ReadAbbrevRecord(); - - /// Read and return a block info block from the bitstream. If an error was - /// encountered, return None. - /// - /// \param ReadBlockInfoNames Whether to read block/record name information in - /// the BlockInfo block. Only llvm-bcanalyzer uses this. - Optional - ReadBlockInfoBlock(bool ReadBlockInfoNames = false); - - /// Set the block info to be used by this BitstreamCursor to interpret - /// abbreviated records. - void setBlockInfo(BitstreamBlockInfo *BI) { BlockInfo = BI; } -}; - -} // end llvm namespace - -#endif // LLVM_BITCODE_BITSTREAMREADER_H diff --git a/include/llvm/Bitcode/BitstreamWriter.h b/include/llvm/Bitcode/BitstreamWriter.h deleted file mode 100644 index c854769e0622..000000000000 --- a/include/llvm/Bitcode/BitstreamWriter.h +++ /dev/null @@ -1,550 +0,0 @@ -//===- BitstreamWriter.h - Low-level bitstream writer interface -*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This header defines the BitstreamWriter class. This class can be used to -// write an arbitrary bitstream, regardless of its contents. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_BITCODE_BITSTREAMWRITER_H -#define LLVM_BITCODE_BITSTREAMWRITER_H - -#include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/Optional.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringRef.h" -#include "llvm/Bitcode/BitCodes.h" -#include "llvm/Support/Endian.h" -#include - -namespace llvm { - -class BitstreamWriter { - SmallVectorImpl &Out; - - /// CurBit - Always between 0 and 31 inclusive, specifies the next bit to use. - unsigned CurBit; - - /// CurValue - The current value. Only bits < CurBit are valid. - uint32_t CurValue; - - /// CurCodeSize - This is the declared size of code values used for the - /// current block, in bits. - unsigned CurCodeSize; - - /// BlockInfoCurBID - When emitting a BLOCKINFO_BLOCK, this is the currently - /// selected BLOCK ID. - unsigned BlockInfoCurBID; - - /// CurAbbrevs - Abbrevs installed at in this block. - std::vector> CurAbbrevs; - - struct Block { - unsigned PrevCodeSize; - size_t StartSizeWord; - std::vector> PrevAbbrevs; - Block(unsigned PCS, size_t SSW) : PrevCodeSize(PCS), StartSizeWord(SSW) {} - }; - - /// BlockScope - This tracks the current blocks that we have entered. - std::vector BlockScope; - - /// BlockInfo - This contains information emitted to BLOCKINFO_BLOCK blocks. - /// These describe abbreviations that all blocks of the specified ID inherit. - struct BlockInfo { - unsigned BlockID; - std::vector> Abbrevs; - }; - std::vector BlockInfoRecords; - - void WriteByte(unsigned char Value) { - Out.push_back(Value); - } - - void WriteWord(unsigned Value) { - Value = support::endian::byte_swap(Value); - Out.append(reinterpret_cast(&Value), - reinterpret_cast(&Value + 1)); - } - - size_t GetBufferOffset() const { return Out.size(); } - - size_t GetWordIndex() const { - size_t Offset = GetBufferOffset(); - assert((Offset & 3) == 0 && "Not 32-bit aligned"); - return Offset / 4; - } - -public: - explicit BitstreamWriter(SmallVectorImpl &O) - : Out(O), CurBit(0), CurValue(0), CurCodeSize(2) {} - - ~BitstreamWriter() { - assert(CurBit == 0 && "Unflushed data remaining"); - assert(BlockScope.empty() && CurAbbrevs.empty() && "Block imbalance"); - } - - /// Retrieve the current position in the stream, in bits. - uint64_t GetCurrentBitNo() const { return GetBufferOffset() * 8 + CurBit; } - - /// Retrieve the number of bits currently used to encode an abbrev ID. - unsigned GetAbbrevIDWidth() const { return CurCodeSize; } - - //===--------------------------------------------------------------------===// - // Basic Primitives for emitting bits to the stream. - //===--------------------------------------------------------------------===// - - /// Backpatch a 32-bit word in the output at the given bit offset - /// with the specified value. - void BackpatchWord(uint64_t BitNo, unsigned NewWord) { - using namespace llvm::support; - unsigned ByteNo = BitNo / 8; - assert((!endian::readAtBitAlignment( - &Out[ByteNo], BitNo & 7)) && - "Expected to be patching over 0-value placeholders"); - endian::writeAtBitAlignment( - &Out[ByteNo], NewWord, BitNo & 7); - } - - void BackpatchWord64(uint64_t BitNo, uint64_t Val) { - BackpatchWord(BitNo, (uint32_t)Val); - BackpatchWord(BitNo + 32, (uint32_t)(Val >> 32)); - } - - void Emit(uint32_t Val, unsigned NumBits) { - assert(NumBits && NumBits <= 32 && "Invalid value size!"); - assert((Val & ~(~0U >> (32-NumBits))) == 0 && "High bits set!"); - CurValue |= Val << CurBit; - if (CurBit + NumBits < 32) { - CurBit += NumBits; - return; - } - - // Add the current word. - WriteWord(CurValue); - - if (CurBit) - CurValue = Val >> (32-CurBit); - else - CurValue = 0; - CurBit = (CurBit+NumBits) & 31; - } - - void FlushToWord() { - if (CurBit) { - WriteWord(CurValue); - CurBit = 0; - CurValue = 0; - } - } - - void EmitVBR(uint32_t Val, unsigned NumBits) { - assert(NumBits <= 32 && "Too many bits to emit!"); - uint32_t Threshold = 1U << (NumBits-1); - - // Emit the bits with VBR encoding, NumBits-1 bits at a time. - while (Val >= Threshold) { - Emit((Val & ((1 << (NumBits-1))-1)) | (1 << (NumBits-1)), NumBits); - Val >>= NumBits-1; - } - - Emit(Val, NumBits); - } - - void EmitVBR64(uint64_t Val, unsigned NumBits) { - assert(NumBits <= 32 && "Too many bits to emit!"); - if ((uint32_t)Val == Val) - return EmitVBR((uint32_t)Val, NumBits); - - uint32_t Threshold = 1U << (NumBits-1); - - // Emit the bits with VBR encoding, NumBits-1 bits at a time. - while (Val >= Threshold) { - Emit(((uint32_t)Val & ((1 << (NumBits-1))-1)) | - (1 << (NumBits-1)), NumBits); - Val >>= NumBits-1; - } - - Emit((uint32_t)Val, NumBits); - } - - /// EmitCode - Emit the specified code. - void EmitCode(unsigned Val) { - Emit(Val, CurCodeSize); - } - - //===--------------------------------------------------------------------===// - // Block Manipulation - //===--------------------------------------------------------------------===// - - /// getBlockInfo - If there is block info for the specified ID, return it, - /// otherwise return null. - BlockInfo *getBlockInfo(unsigned BlockID) { - // Common case, the most recent entry matches BlockID. - if (!BlockInfoRecords.empty() && BlockInfoRecords.back().BlockID == BlockID) - return &BlockInfoRecords.back(); - - for (unsigned i = 0, e = static_cast(BlockInfoRecords.size()); - i != e; ++i) - if (BlockInfoRecords[i].BlockID == BlockID) - return &BlockInfoRecords[i]; - return nullptr; - } - - void EnterSubblock(unsigned BlockID, unsigned CodeLen) { - // Block header: - // [ENTER_SUBBLOCK, blockid, newcodelen, , blocklen] - EmitCode(bitc::ENTER_SUBBLOCK); - EmitVBR(BlockID, bitc::BlockIDWidth); - EmitVBR(CodeLen, bitc::CodeLenWidth); - FlushToWord(); - - size_t BlockSizeWordIndex = GetWordIndex(); - unsigned OldCodeSize = CurCodeSize; - - // Emit a placeholder, which will be replaced when the block is popped. - Emit(0, bitc::BlockSizeWidth); - - CurCodeSize = CodeLen; - - // Push the outer block's abbrev set onto the stack, start out with an - // empty abbrev set. - BlockScope.emplace_back(OldCodeSize, BlockSizeWordIndex); - BlockScope.back().PrevAbbrevs.swap(CurAbbrevs); - - // If there is a blockinfo for this BlockID, add all the predefined abbrevs - // to the abbrev list. - if (BlockInfo *Info = getBlockInfo(BlockID)) { - CurAbbrevs.insert(CurAbbrevs.end(), Info->Abbrevs.begin(), - Info->Abbrevs.end()); - } - } - - void ExitBlock() { - assert(!BlockScope.empty() && "Block scope imbalance!"); - const Block &B = BlockScope.back(); - - // Block tail: - // [END_BLOCK, ] - EmitCode(bitc::END_BLOCK); - FlushToWord(); - - // Compute the size of the block, in words, not counting the size field. - size_t SizeInWords = GetWordIndex() - B.StartSizeWord - 1; - uint64_t BitNo = uint64_t(B.StartSizeWord) * 32; - - // Update the block size field in the header of this sub-block. - BackpatchWord(BitNo, SizeInWords); - - // Restore the inner block's code size and abbrev table. - CurCodeSize = B.PrevCodeSize; - CurAbbrevs = std::move(B.PrevAbbrevs); - BlockScope.pop_back(); - } - - //===--------------------------------------------------------------------===// - // Record Emission - //===--------------------------------------------------------------------===// - -private: - /// EmitAbbreviatedLiteral - Emit a literal value according to its abbrev - /// record. This is a no-op, since the abbrev specifies the literal to use. - template - void EmitAbbreviatedLiteral(const BitCodeAbbrevOp &Op, uintty V) { - assert(Op.isLiteral() && "Not a literal"); - // If the abbrev specifies the literal value to use, don't emit - // anything. - assert(V == Op.getLiteralValue() && - "Invalid abbrev for record!"); - } - - /// EmitAbbreviatedField - Emit a single scalar field value with the specified - /// encoding. - template - void EmitAbbreviatedField(const BitCodeAbbrevOp &Op, uintty V) { - assert(!Op.isLiteral() && "Literals should use EmitAbbreviatedLiteral!"); - - // Encode the value as we are commanded. - switch (Op.getEncoding()) { - default: llvm_unreachable("Unknown encoding!"); - case BitCodeAbbrevOp::Fixed: - if (Op.getEncodingData()) - Emit((unsigned)V, (unsigned)Op.getEncodingData()); - break; - case BitCodeAbbrevOp::VBR: - if (Op.getEncodingData()) - EmitVBR64(V, (unsigned)Op.getEncodingData()); - break; - case BitCodeAbbrevOp::Char6: - Emit(BitCodeAbbrevOp::EncodeChar6((char)V), 6); - break; - } - } - - /// EmitRecordWithAbbrevImpl - This is the core implementation of the record - /// emission code. If BlobData is non-null, then it specifies an array of - /// data that should be emitted as part of the Blob or Array operand that is - /// known to exist at the end of the record. If Code is specified, then - /// it is the record code to emit before the Vals, which must not contain - /// the code. - template - void EmitRecordWithAbbrevImpl(unsigned Abbrev, ArrayRef Vals, - StringRef Blob, Optional Code) { - const char *BlobData = Blob.data(); - unsigned BlobLen = (unsigned) Blob.size(); - unsigned AbbrevNo = Abbrev-bitc::FIRST_APPLICATION_ABBREV; - assert(AbbrevNo < CurAbbrevs.size() && "Invalid abbrev #!"); - const BitCodeAbbrev *Abbv = CurAbbrevs[AbbrevNo].get(); - - EmitCode(Abbrev); - - unsigned i = 0, e = static_cast(Abbv->getNumOperandInfos()); - if (Code) { - assert(e && "Expected non-empty abbreviation"); - const BitCodeAbbrevOp &Op = Abbv->getOperandInfo(i++); - - if (Op.isLiteral()) - EmitAbbreviatedLiteral(Op, Code.getValue()); - else { - assert(Op.getEncoding() != BitCodeAbbrevOp::Array && - Op.getEncoding() != BitCodeAbbrevOp::Blob && - "Expected literal or scalar"); - EmitAbbreviatedField(Op, Code.getValue()); - } - } - - unsigned RecordIdx = 0; - for (; i != e; ++i) { - const BitCodeAbbrevOp &Op = Abbv->getOperandInfo(i); - if (Op.isLiteral()) { - assert(RecordIdx < Vals.size() && "Invalid abbrev/record"); - EmitAbbreviatedLiteral(Op, Vals[RecordIdx]); - ++RecordIdx; - } else if (Op.getEncoding() == BitCodeAbbrevOp::Array) { - // Array case. - assert(i + 2 == e && "array op not second to last?"); - const BitCodeAbbrevOp &EltEnc = Abbv->getOperandInfo(++i); - - // If this record has blob data, emit it, otherwise we must have record - // entries to encode this way. - if (BlobData) { - assert(RecordIdx == Vals.size() && - "Blob data and record entries specified for array!"); - // Emit a vbr6 to indicate the number of elements present. - EmitVBR(static_cast(BlobLen), 6); - - // Emit each field. - for (unsigned i = 0; i != BlobLen; ++i) - EmitAbbreviatedField(EltEnc, (unsigned char)BlobData[i]); - - // Know that blob data is consumed for assertion below. - BlobData = nullptr; - } else { - // Emit a vbr6 to indicate the number of elements present. - EmitVBR(static_cast(Vals.size()-RecordIdx), 6); - - // Emit each field. - for (unsigned e = Vals.size(); RecordIdx != e; ++RecordIdx) - EmitAbbreviatedField(EltEnc, Vals[RecordIdx]); - } - } else if (Op.getEncoding() == BitCodeAbbrevOp::Blob) { - // If this record has blob data, emit it, otherwise we must have record - // entries to encode this way. - - if (BlobData) { - assert(RecordIdx == Vals.size() && - "Blob data and record entries specified for blob operand!"); - - assert(Blob.data() == BlobData && "BlobData got moved"); - assert(Blob.size() == BlobLen && "BlobLen got changed"); - emitBlob(Blob); - BlobData = nullptr; - } else { - emitBlob(Vals.slice(RecordIdx)); - } - } else { // Single scalar field. - assert(RecordIdx < Vals.size() && "Invalid abbrev/record"); - EmitAbbreviatedField(Op, Vals[RecordIdx]); - ++RecordIdx; - } - } - assert(RecordIdx == Vals.size() && "Not all record operands emitted!"); - assert(BlobData == nullptr && - "Blob data specified for record that doesn't use it!"); - } - -public: - /// Emit a blob, including flushing before and tail-padding. - template - void emitBlob(ArrayRef Bytes, bool ShouldEmitSize = true) { - // Emit a vbr6 to indicate the number of elements present. - if (ShouldEmitSize) - EmitVBR(static_cast(Bytes.size()), 6); - - // Flush to a 32-bit alignment boundary. - FlushToWord(); - - // Emit literal bytes. - for (const auto &B : Bytes) { - assert(isUInt<8>(B) && "Value too large to emit as byte"); - WriteByte((unsigned char)B); - } - - // Align end to 32-bits. - while (GetBufferOffset() & 3) - WriteByte(0); - } - void emitBlob(StringRef Bytes, bool ShouldEmitSize = true) { - emitBlob(makeArrayRef((const uint8_t *)Bytes.data(), Bytes.size()), - ShouldEmitSize); - } - - /// EmitRecord - Emit the specified record to the stream, using an abbrev if - /// we have one to compress the output. - template - void EmitRecord(unsigned Code, const Container &Vals, unsigned Abbrev = 0) { - if (!Abbrev) { - // If we don't have an abbrev to use, emit this in its fully unabbreviated - // form. - auto Count = static_cast(makeArrayRef(Vals).size()); - EmitCode(bitc::UNABBREV_RECORD); - EmitVBR(Code, 6); - EmitVBR(Count, 6); - for (unsigned i = 0, e = Count; i != e; ++i) - EmitVBR64(Vals[i], 6); - return; - } - - EmitRecordWithAbbrevImpl(Abbrev, makeArrayRef(Vals), StringRef(), Code); - } - - /// EmitRecordWithAbbrev - Emit a record with the specified abbreviation. - /// Unlike EmitRecord, the code for the record should be included in Vals as - /// the first entry. - template - void EmitRecordWithAbbrev(unsigned Abbrev, const Container &Vals) { - EmitRecordWithAbbrevImpl(Abbrev, makeArrayRef(Vals), StringRef(), None); - } - - /// EmitRecordWithBlob - Emit the specified record to the stream, using an - /// abbrev that includes a blob at the end. The blob data to emit is - /// specified by the pointer and length specified at the end. In contrast to - /// EmitRecord, this routine expects that the first entry in Vals is the code - /// of the record. - template - void EmitRecordWithBlob(unsigned Abbrev, const Container &Vals, - StringRef Blob) { - EmitRecordWithAbbrevImpl(Abbrev, makeArrayRef(Vals), Blob, None); - } - template - void EmitRecordWithBlob(unsigned Abbrev, const Container &Vals, - const char *BlobData, unsigned BlobLen) { - return EmitRecordWithAbbrevImpl(Abbrev, makeArrayRef(Vals), - StringRef(BlobData, BlobLen), None); - } - - /// EmitRecordWithArray - Just like EmitRecordWithBlob, works with records - /// that end with an array. - template - void EmitRecordWithArray(unsigned Abbrev, const Container &Vals, - StringRef Array) { - EmitRecordWithAbbrevImpl(Abbrev, makeArrayRef(Vals), Array, None); - } - template - void EmitRecordWithArray(unsigned Abbrev, const Container &Vals, - const char *ArrayData, unsigned ArrayLen) { - return EmitRecordWithAbbrevImpl(Abbrev, makeArrayRef(Vals), - StringRef(ArrayData, ArrayLen), None); - } - - //===--------------------------------------------------------------------===// - // Abbrev Emission - //===--------------------------------------------------------------------===// - -private: - // Emit the abbreviation as a DEFINE_ABBREV record. - void EncodeAbbrev(const BitCodeAbbrev &Abbv) { - EmitCode(bitc::DEFINE_ABBREV); - EmitVBR(Abbv.getNumOperandInfos(), 5); - for (unsigned i = 0, e = static_cast(Abbv.getNumOperandInfos()); - i != e; ++i) { - const BitCodeAbbrevOp &Op = Abbv.getOperandInfo(i); - Emit(Op.isLiteral(), 1); - if (Op.isLiteral()) { - EmitVBR64(Op.getLiteralValue(), 8); - } else { - Emit(Op.getEncoding(), 3); - if (Op.hasEncodingData()) - EmitVBR64(Op.getEncodingData(), 5); - } - } - } -public: - - /// EmitAbbrev - This emits an abbreviation to the stream. Note that this - /// method takes ownership of the specified abbrev. - unsigned EmitAbbrev(std::shared_ptr Abbv) { - // Emit the abbreviation as a record. - EncodeAbbrev(*Abbv); - CurAbbrevs.push_back(std::move(Abbv)); - return static_cast(CurAbbrevs.size())-1 + - bitc::FIRST_APPLICATION_ABBREV; - } - - //===--------------------------------------------------------------------===// - // BlockInfo Block Emission - //===--------------------------------------------------------------------===// - - /// EnterBlockInfoBlock - Start emitting the BLOCKINFO_BLOCK. - void EnterBlockInfoBlock() { - EnterSubblock(bitc::BLOCKINFO_BLOCK_ID, 2); - BlockInfoCurBID = ~0U; - BlockInfoRecords.clear(); - } -private: - /// SwitchToBlockID - If we aren't already talking about the specified block - /// ID, emit a BLOCKINFO_CODE_SETBID record. - void SwitchToBlockID(unsigned BlockID) { - if (BlockInfoCurBID == BlockID) return; - SmallVector V; - V.push_back(BlockID); - EmitRecord(bitc::BLOCKINFO_CODE_SETBID, V); - BlockInfoCurBID = BlockID; - } - - BlockInfo &getOrCreateBlockInfo(unsigned BlockID) { - if (BlockInfo *BI = getBlockInfo(BlockID)) - return *BI; - - // Otherwise, add a new record. - BlockInfoRecords.emplace_back(); - BlockInfoRecords.back().BlockID = BlockID; - return BlockInfoRecords.back(); - } - -public: - - /// EmitBlockInfoAbbrev - Emit a DEFINE_ABBREV record for the specified - /// BlockID. - unsigned EmitBlockInfoAbbrev(unsigned BlockID, std::shared_ptr Abbv) { - SwitchToBlockID(BlockID); - EncodeAbbrev(*Abbv); - - // Add the abbrev to the specified block record. - BlockInfo &Info = getOrCreateBlockInfo(BlockID); - Info.Abbrevs.push_back(std::move(Abbv)); - - return Info.Abbrevs.size()-1+bitc::FIRST_APPLICATION_ABBREV; - } -}; - - -} // End llvm namespace - -#endif diff --git a/include/llvm/Bitcode/LLVMBitCodes.h b/include/llvm/Bitcode/LLVMBitCodes.h index f0d11e9c1689..decd4dd3a965 100644 --- a/include/llvm/Bitcode/LLVMBitCodes.h +++ b/include/llvm/Bitcode/LLVMBitCodes.h @@ -1,9 +1,8 @@ //===- LLVMBitCodes.h - Enum values for the LLVM bitcode format -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,7 +17,7 @@ #ifndef LLVM_BITCODE_LLVMBITCODES_H #define LLVM_BITCODE_LLVMBITCODES_H -#include "llvm/Bitcode/BitCodes.h" +#include "llvm/Bitstream/BitCodes.h" namespace llvm { namespace bitc { @@ -264,10 +263,31 @@ enum GlobalValueSummarySymtabCodes { // Index-wide flags FS_FLAGS = 20, // Maps type identifier to summary information for that type identifier. + // Produced by the thin link (only lives in combined index). // TYPE_ID: [typeid, kind, bitwidth, align, size, bitmask, inlinebits, // n x (typeid, kind, name, numrba, // numrba x (numarg, numarg x arg, kind, info, byte, bit))] FS_TYPE_ID = 21, + // For background see overview at https://llvm.org/docs/TypeMetadata.html. + // The type metadata includes both the type identifier and the offset of + // the address point of the type (the address held by objects of that type + // which may not be the beginning of the virtual table). Vtable definitions + // are decorated with type metadata for the types they are compatible with. + // + // Maps type identifier to summary information for that type identifier + // computed from type metadata: the valueid of each vtable definition + // decorated with a type metadata for that identifier, and the offset from + // the corresponding type metadata. + // Exists in the per-module summary to provide information to thin link + // for index-based whole program devirtualization. + // TYPE_ID_METADATA: [typeid, n x (valueid, offset)] + FS_TYPE_ID_METADATA = 22, + // Summarizes vtable definition for use in index-based whole program + // devirtualization during the thin link. + // PERMODULE_VTABLE_GLOBALVAR_INIT_REFS: [valueid, flags, varflags, + // numrefs, numrefs x valueid, + // n x (valueid, offset)] + FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS = 23, }; enum MetadataCodes { @@ -311,6 +331,7 @@ enum MetadataCodes { METADATA_INDEX_OFFSET = 38, // [offset] METADATA_INDEX = 39, // [bitpos] METADATA_LABEL = 40, // [distinct, scope, name, file, line] + METADATA_COMMON_BLOCK = 44, // [distinct, scope, name, variable,...] }; // The constants block (CONSTANTS_BLOCK_ID) describes emission for each @@ -407,7 +428,9 @@ enum RMWOperations { RMW_MAX = 7, RMW_MIN = 8, RMW_UMAX = 9, - RMW_UMIN = 10 + RMW_UMIN = 10, + RMW_FADD = 11, + RMW_FSUB = 12 }; /// OverflowingBinaryOperatorOptionalFlags - Flags for serializing @@ -534,6 +557,8 @@ enum FunctionCodes { // 54 is unused. FUNC_CODE_OPERAND_BUNDLE = 55, // OPERAND_BUNDLE: [tag#, value...] FUNC_CODE_INST_UNOP = 56, // UNOP: [opcode, ty, opval] + FUNC_CODE_INST_CALLBR = 57, // CALLBR: [attr, cc, norm, transfs, + // fnty, fnid, args...] }; enum UseListCodes { @@ -602,6 +627,11 @@ enum AttributeKindCodes { ATTR_KIND_OPT_FOR_FUZZING = 57, ATTR_KIND_SHADOWCALLSTACK = 58, ATTR_KIND_SPECULATIVE_LOAD_HARDENING = 59, + ATTR_KIND_IMMARG = 60, + ATTR_KIND_WILLRETURN = 61, + ATTR_KIND_NOFREE = 62, + ATTR_KIND_NOSYNC = 63, + ATTR_KIND_SANITIZE_MEMTAG = 64, }; enum ComdatSelectionKindCodes { diff --git a/include/llvm/Bitstream/BitCodes.h b/include/llvm/Bitstream/BitCodes.h new file mode 100644 index 000000000000..adf54ba96396 --- /dev/null +++ b/include/llvm/Bitstream/BitCodes.h @@ -0,0 +1,184 @@ +//===- BitCodes.h - Enum values for the bitstream format --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This header defines bitstream enum values. +// +// The enum values defined in this file should be considered permanent. If +// new features are added, they should have values added at the end of the +// respective lists. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_BITSTREAM_BITCODES_H +#define LLVM_BITSTREAM_BITCODES_H + +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/DataTypes.h" +#include "llvm/Support/ErrorHandling.h" +#include + +namespace llvm { +/// Offsets of the 32-bit fields of bitstream wrapper header. +enum BitstreamWrapperHeader : unsigned { + BWH_MagicField = 0 * 4, + BWH_VersionField = 1 * 4, + BWH_OffsetField = 2 * 4, + BWH_SizeField = 3 * 4, + BWH_CPUTypeField = 4 * 4, + BWH_HeaderSize = 5 * 4 +}; + +namespace bitc { + enum StandardWidths { + BlockIDWidth = 8, // We use VBR-8 for block IDs. + CodeLenWidth = 4, // Codelen are VBR-4. + BlockSizeWidth = 32 // BlockSize up to 2^32 32-bit words = 16GB per block. + }; + + // The standard abbrev namespace always has a way to exit a block, enter a + // nested block, define abbrevs, and define an unabbreviated record. + enum FixedAbbrevIDs { + END_BLOCK = 0, // Must be zero to guarantee termination for broken bitcode. + ENTER_SUBBLOCK = 1, + + /// DEFINE_ABBREV - Defines an abbrev for the current block. It consists + /// of a vbr5 for # operand infos. Each operand info is emitted with a + /// single bit to indicate if it is a literal encoding. If so, the value is + /// emitted with a vbr8. If not, the encoding is emitted as 3 bits followed + /// by the info value as a vbr5 if needed. + DEFINE_ABBREV = 2, + + // UNABBREV_RECORDs are emitted with a vbr6 for the record code, followed by + // a vbr6 for the # operands, followed by vbr6's for each operand. + UNABBREV_RECORD = 3, + + // This is not a code, this is a marker for the first abbrev assignment. + FIRST_APPLICATION_ABBREV = 4 + }; + + /// StandardBlockIDs - All bitcode files can optionally include a BLOCKINFO + /// block, which contains metadata about other blocks in the file. + enum StandardBlockIDs { + /// BLOCKINFO_BLOCK is used to define metadata about blocks, for example, + /// standard abbrevs that should be available to all blocks of a specified + /// ID. + BLOCKINFO_BLOCK_ID = 0, + + // Block IDs 1-7 are reserved for future expansion. + FIRST_APPLICATION_BLOCKID = 8 + }; + + /// BlockInfoCodes - The blockinfo block contains metadata about user-defined + /// blocks. + enum BlockInfoCodes { + // DEFINE_ABBREV has magic semantics here, applying to the current SETBID'd + // block, instead of the BlockInfo block. + + BLOCKINFO_CODE_SETBID = 1, // SETBID: [blockid#] + BLOCKINFO_CODE_BLOCKNAME = 2, // BLOCKNAME: [name] + BLOCKINFO_CODE_SETRECORDNAME = 3 // BLOCKINFO_CODE_SETRECORDNAME: + // [id, name] + }; + +} // End bitc namespace + +/// BitCodeAbbrevOp - This describes one or more operands in an abbreviation. +/// This is actually a union of two different things: +/// 1. It could be a literal integer value ("the operand is always 17"). +/// 2. It could be an encoding specification ("this operand encoded like so"). +/// +class BitCodeAbbrevOp { + uint64_t Val; // A literal value or data for an encoding. + bool IsLiteral : 1; // Indicate whether this is a literal value or not. + unsigned Enc : 3; // The encoding to use. +public: + enum Encoding { + Fixed = 1, // A fixed width field, Val specifies number of bits. + VBR = 2, // A VBR field where Val specifies the width of each chunk. + Array = 3, // A sequence of fields, next field species elt encoding. + Char6 = 4, // A 6-bit fixed field which maps to [a-zA-Z0-9._]. + Blob = 5 // 32-bit aligned array of 8-bit characters. + }; + + explicit BitCodeAbbrevOp(uint64_t V) : Val(V), IsLiteral(true) {} + explicit BitCodeAbbrevOp(Encoding E, uint64_t Data = 0) + : Val(Data), IsLiteral(false), Enc(E) {} + + bool isLiteral() const { return IsLiteral; } + bool isEncoding() const { return !IsLiteral; } + + // Accessors for literals. + uint64_t getLiteralValue() const { assert(isLiteral()); return Val; } + + // Accessors for encoding info. + Encoding getEncoding() const { assert(isEncoding()); return (Encoding)Enc; } + uint64_t getEncodingData() const { + assert(isEncoding() && hasEncodingData()); + return Val; + } + + bool hasEncodingData() const { return hasEncodingData(getEncoding()); } + static bool hasEncodingData(Encoding E) { + switch (E) { + case Fixed: + case VBR: + return true; + case Array: + case Char6: + case Blob: + return false; + } + report_fatal_error("Invalid encoding"); + } + + /// isChar6 - Return true if this character is legal in the Char6 encoding. + static bool isChar6(char C) { + if (C >= 'a' && C <= 'z') return true; + if (C >= 'A' && C <= 'Z') return true; + if (C >= '0' && C <= '9') return true; + if (C == '.' || C == '_') return true; + return false; + } + static unsigned EncodeChar6(char C) { + if (C >= 'a' && C <= 'z') return C-'a'; + if (C >= 'A' && C <= 'Z') return C-'A'+26; + if (C >= '0' && C <= '9') return C-'0'+26+26; + if (C == '.') return 62; + if (C == '_') return 63; + llvm_unreachable("Not a value Char6 character!"); + } + + static char DecodeChar6(unsigned V) { + assert((V & ~63) == 0 && "Not a Char6 encoded character!"); + return "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._" + [V]; + } + +}; + +/// BitCodeAbbrev - This class represents an abbreviation record. An +/// abbreviation allows a complex record that has redundancy to be stored in a +/// specialized format instead of the fully-general, fully-vbr, format. +class BitCodeAbbrev { + SmallVector OperandList; + +public: + unsigned getNumOperandInfos() const { + return static_cast(OperandList.size()); + } + const BitCodeAbbrevOp &getOperandInfo(unsigned N) const { + return OperandList[N]; + } + + void Add(const BitCodeAbbrevOp &OpInfo) { + OperandList.push_back(OpInfo); + } +}; +} // End llvm namespace + +#endif diff --git a/include/llvm/Bitstream/BitstreamReader.h b/include/llvm/Bitstream/BitstreamReader.h new file mode 100644 index 000000000000..ee82e7ec1ba2 --- /dev/null +++ b/include/llvm/Bitstream/BitstreamReader.h @@ -0,0 +1,557 @@ +//===- BitstreamReader.h - Low-level bitstream reader interface -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This header defines the BitstreamReader class. This class can be used to +// read an arbitrary bitstream, regardless of its contents. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_BITSTREAM_BITSTREAMREADER_H +#define LLVM_BITSTREAM_BITSTREAMREADER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Bitstream/BitCodes.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/MathExtras.h" +#include "llvm/Support/MemoryBuffer.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace llvm { + +/// This class maintains the abbreviations read from a block info block. +class BitstreamBlockInfo { +public: + /// This contains information emitted to BLOCKINFO_BLOCK blocks. These + /// describe abbreviations that all blocks of the specified ID inherit. + struct BlockInfo { + unsigned BlockID; + std::vector> Abbrevs; + std::string Name; + std::vector> RecordNames; + }; + +private: + std::vector BlockInfoRecords; + +public: + /// If there is block info for the specified ID, return it, otherwise return + /// null. + const BlockInfo *getBlockInfo(unsigned BlockID) const { + // Common case, the most recent entry matches BlockID. + if (!BlockInfoRecords.empty() && BlockInfoRecords.back().BlockID == BlockID) + return &BlockInfoRecords.back(); + + for (unsigned i = 0, e = static_cast(BlockInfoRecords.size()); + i != e; ++i) + if (BlockInfoRecords[i].BlockID == BlockID) + return &BlockInfoRecords[i]; + return nullptr; + } + + BlockInfo &getOrCreateBlockInfo(unsigned BlockID) { + if (const BlockInfo *BI = getBlockInfo(BlockID)) + return *const_cast(BI); + + // Otherwise, add a new record. + BlockInfoRecords.emplace_back(); + BlockInfoRecords.back().BlockID = BlockID; + return BlockInfoRecords.back(); + } +}; + +/// This represents a position within a bitstream. There may be multiple +/// independent cursors reading within one bitstream, each maintaining their +/// own local state. +class SimpleBitstreamCursor { + ArrayRef BitcodeBytes; + size_t NextChar = 0; + +public: + /// This is the current data we have pulled from the stream but have not + /// returned to the client. This is specifically and intentionally defined to + /// follow the word size of the host machine for efficiency. We use word_t in + /// places that are aware of this to make it perfectly explicit what is going + /// on. + using word_t = size_t; + +private: + word_t CurWord = 0; + + /// This is the number of bits in CurWord that are valid. This is always from + /// [0...bits_of(size_t)-1] inclusive. + unsigned BitsInCurWord = 0; + +public: + static const constexpr size_t MaxChunkSize = sizeof(word_t) * 8; + + SimpleBitstreamCursor() = default; + explicit SimpleBitstreamCursor(ArrayRef BitcodeBytes) + : BitcodeBytes(BitcodeBytes) {} + explicit SimpleBitstreamCursor(StringRef BitcodeBytes) + : BitcodeBytes(arrayRefFromStringRef(BitcodeBytes)) {} + explicit SimpleBitstreamCursor(MemoryBufferRef BitcodeBytes) + : SimpleBitstreamCursor(BitcodeBytes.getBuffer()) {} + + bool canSkipToPos(size_t pos) const { + // pos can be skipped to if it is a valid address or one byte past the end. + return pos <= BitcodeBytes.size(); + } + + bool AtEndOfStream() { + return BitsInCurWord == 0 && BitcodeBytes.size() <= NextChar; + } + + /// Return the bit # of the bit we are reading. + uint64_t GetCurrentBitNo() const { + return NextChar*CHAR_BIT - BitsInCurWord; + } + + // Return the byte # of the current bit. + uint64_t getCurrentByteNo() const { return GetCurrentBitNo() / 8; } + + ArrayRef getBitcodeBytes() const { return BitcodeBytes; } + + /// Reset the stream to the specified bit number. + Error JumpToBit(uint64_t BitNo) { + size_t ByteNo = size_t(BitNo/8) & ~(sizeof(word_t)-1); + unsigned WordBitNo = unsigned(BitNo & (sizeof(word_t)*8-1)); + assert(canSkipToPos(ByteNo) && "Invalid location"); + + // Move the cursor to the right word. + NextChar = ByteNo; + BitsInCurWord = 0; + + // Skip over any bits that are already consumed. + if (WordBitNo) { + if (Expected Res = Read(WordBitNo)) + return Error::success(); + else + return Res.takeError(); + } + + return Error::success(); + } + + /// Get a pointer into the bitstream at the specified byte offset. + const uint8_t *getPointerToByte(uint64_t ByteNo, uint64_t NumBytes) { + return BitcodeBytes.data() + ByteNo; + } + + /// Get a pointer into the bitstream at the specified bit offset. + /// + /// The bit offset must be on a byte boundary. + const uint8_t *getPointerToBit(uint64_t BitNo, uint64_t NumBytes) { + assert(!(BitNo % 8) && "Expected bit on byte boundary"); + return getPointerToByte(BitNo / 8, NumBytes); + } + + Error fillCurWord() { + if (NextChar >= BitcodeBytes.size()) + return createStringError(std::errc::io_error, + "Unexpected end of file reading %u of %u bytes", + NextChar, BitcodeBytes.size()); + + // Read the next word from the stream. + const uint8_t *NextCharPtr = BitcodeBytes.data() + NextChar; + unsigned BytesRead; + if (BitcodeBytes.size() >= NextChar + sizeof(word_t)) { + BytesRead = sizeof(word_t); + CurWord = + support::endian::read( + NextCharPtr); + } else { + // Short read. + BytesRead = BitcodeBytes.size() - NextChar; + CurWord = 0; + for (unsigned B = 0; B != BytesRead; ++B) + CurWord |= uint64_t(NextCharPtr[B]) << (B * 8); + } + NextChar += BytesRead; + BitsInCurWord = BytesRead * 8; + return Error::success(); + } + + Expected Read(unsigned NumBits) { + static const unsigned BitsInWord = MaxChunkSize; + + assert(NumBits && NumBits <= BitsInWord && + "Cannot return zero or more than BitsInWord bits!"); + + static const unsigned Mask = sizeof(word_t) > 4 ? 0x3f : 0x1f; + + // If the field is fully contained by CurWord, return it quickly. + if (BitsInCurWord >= NumBits) { + word_t R = CurWord & (~word_t(0) >> (BitsInWord - NumBits)); + + // Use a mask to avoid undefined behavior. + CurWord >>= (NumBits & Mask); + + BitsInCurWord -= NumBits; + return R; + } + + word_t R = BitsInCurWord ? CurWord : 0; + unsigned BitsLeft = NumBits - BitsInCurWord; + + if (Error fillResult = fillCurWord()) + return std::move(fillResult); + + // If we run out of data, abort. + if (BitsLeft > BitsInCurWord) + return createStringError(std::errc::io_error, + "Unexpected end of file reading %u of %u bits", + BitsInCurWord, BitsLeft); + + word_t R2 = CurWord & (~word_t(0) >> (BitsInWord - BitsLeft)); + + // Use a mask to avoid undefined behavior. + CurWord >>= (BitsLeft & Mask); + + BitsInCurWord -= BitsLeft; + + R |= R2 << (NumBits - BitsLeft); + + return R; + } + + Expected ReadVBR(unsigned NumBits) { + Expected MaybeRead = Read(NumBits); + if (!MaybeRead) + return MaybeRead; + uint32_t Piece = MaybeRead.get(); + + if ((Piece & (1U << (NumBits-1))) == 0) + return Piece; + + uint32_t Result = 0; + unsigned NextBit = 0; + while (true) { + Result |= (Piece & ((1U << (NumBits-1))-1)) << NextBit; + + if ((Piece & (1U << (NumBits-1))) == 0) + return Result; + + NextBit += NumBits-1; + MaybeRead = Read(NumBits); + if (!MaybeRead) + return MaybeRead; + Piece = MaybeRead.get(); + } + } + + // Read a VBR that may have a value up to 64-bits in size. The chunk size of + // the VBR must still be <= 32 bits though. + Expected ReadVBR64(unsigned NumBits) { + Expected MaybeRead = Read(NumBits); + if (!MaybeRead) + return MaybeRead; + uint32_t Piece = MaybeRead.get(); + + if ((Piece & (1U << (NumBits-1))) == 0) + return uint64_t(Piece); + + uint64_t Result = 0; + unsigned NextBit = 0; + while (true) { + Result |= uint64_t(Piece & ((1U << (NumBits-1))-1)) << NextBit; + + if ((Piece & (1U << (NumBits-1))) == 0) + return Result; + + NextBit += NumBits-1; + MaybeRead = Read(NumBits); + if (!MaybeRead) + return MaybeRead; + Piece = MaybeRead.get(); + } + } + + void SkipToFourByteBoundary() { + // If word_t is 64-bits and if we've read less than 32 bits, just dump + // the bits we have up to the next 32-bit boundary. + if (sizeof(word_t) > 4 && + BitsInCurWord >= 32) { + CurWord >>= BitsInCurWord-32; + BitsInCurWord = 32; + return; + } + + BitsInCurWord = 0; + } + + /// Return the size of the stream in bytes. + size_t SizeInBytes() const { return BitcodeBytes.size(); } + + /// Skip to the end of the file. + void skipToEnd() { NextChar = BitcodeBytes.size(); } +}; + +/// When advancing through a bitstream cursor, each advance can discover a few +/// different kinds of entries: +struct BitstreamEntry { + enum { + Error, // Malformed bitcode was found. + EndBlock, // We've reached the end of the current block, (or the end of the + // file, which is treated like a series of EndBlock records. + SubBlock, // This is the start of a new subblock of a specific ID. + Record // This is a record with a specific AbbrevID. + } Kind; + + unsigned ID; + + static BitstreamEntry getError() { + BitstreamEntry E; E.Kind = Error; return E; + } + + static BitstreamEntry getEndBlock() { + BitstreamEntry E; E.Kind = EndBlock; return E; + } + + static BitstreamEntry getSubBlock(unsigned ID) { + BitstreamEntry E; E.Kind = SubBlock; E.ID = ID; return E; + } + + static BitstreamEntry getRecord(unsigned AbbrevID) { + BitstreamEntry E; E.Kind = Record; E.ID = AbbrevID; return E; + } +}; + +/// This represents a position within a bitcode file, implemented on top of a +/// SimpleBitstreamCursor. +/// +/// Unlike iterators, BitstreamCursors are heavy-weight objects that should not +/// be passed by value. +class BitstreamCursor : SimpleBitstreamCursor { + // This is the declared size of code values used for the current block, in + // bits. + unsigned CurCodeSize = 2; + + /// Abbrevs installed at in this block. + std::vector> CurAbbrevs; + + struct Block { + unsigned PrevCodeSize; + std::vector> PrevAbbrevs; + + explicit Block(unsigned PCS) : PrevCodeSize(PCS) {} + }; + + /// This tracks the codesize of parent blocks. + SmallVector BlockScope; + + BitstreamBlockInfo *BlockInfo = nullptr; + +public: + static const size_t MaxChunkSize = sizeof(word_t) * 8; + + BitstreamCursor() = default; + explicit BitstreamCursor(ArrayRef BitcodeBytes) + : SimpleBitstreamCursor(BitcodeBytes) {} + explicit BitstreamCursor(StringRef BitcodeBytes) + : SimpleBitstreamCursor(BitcodeBytes) {} + explicit BitstreamCursor(MemoryBufferRef BitcodeBytes) + : SimpleBitstreamCursor(BitcodeBytes) {} + + using SimpleBitstreamCursor::AtEndOfStream; + using SimpleBitstreamCursor::canSkipToPos; + using SimpleBitstreamCursor::fillCurWord; + using SimpleBitstreamCursor::getBitcodeBytes; + using SimpleBitstreamCursor::GetCurrentBitNo; + using SimpleBitstreamCursor::getCurrentByteNo; + using SimpleBitstreamCursor::getPointerToByte; + using SimpleBitstreamCursor::JumpToBit; + using SimpleBitstreamCursor::Read; + using SimpleBitstreamCursor::ReadVBR; + using SimpleBitstreamCursor::ReadVBR64; + using SimpleBitstreamCursor::SizeInBytes; + + /// Return the number of bits used to encode an abbrev #. + unsigned getAbbrevIDWidth() const { return CurCodeSize; } + + /// Flags that modify the behavior of advance(). + enum { + /// If this flag is used, the advance() method does not automatically pop + /// the block scope when the end of a block is reached. + AF_DontPopBlockAtEnd = 1, + + /// If this flag is used, abbrev entries are returned just like normal + /// records. + AF_DontAutoprocessAbbrevs = 2 + }; + + /// Advance the current bitstream, returning the next entry in the stream. + Expected advance(unsigned Flags = 0) { + while (true) { + if (AtEndOfStream()) + return BitstreamEntry::getError(); + + Expected MaybeCode = ReadCode(); + if (!MaybeCode) + return MaybeCode.takeError(); + unsigned Code = MaybeCode.get(); + + if (Code == bitc::END_BLOCK) { + // Pop the end of the block unless Flags tells us not to. + if (!(Flags & AF_DontPopBlockAtEnd) && ReadBlockEnd()) + return BitstreamEntry::getError(); + return BitstreamEntry::getEndBlock(); + } + + if (Code == bitc::ENTER_SUBBLOCK) { + if (Expected MaybeSubBlock = ReadSubBlockID()) + return BitstreamEntry::getSubBlock(MaybeSubBlock.get()); + else + return MaybeSubBlock.takeError(); + } + + if (Code == bitc::DEFINE_ABBREV && + !(Flags & AF_DontAutoprocessAbbrevs)) { + // We read and accumulate abbrev's, the client can't do anything with + // them anyway. + if (Error Err = ReadAbbrevRecord()) + return std::move(Err); + continue; + } + + return BitstreamEntry::getRecord(Code); + } + } + + /// This is a convenience function for clients that don't expect any + /// subblocks. This just skips over them automatically. + Expected advanceSkippingSubblocks(unsigned Flags = 0) { + while (true) { + // If we found a normal entry, return it. + Expected MaybeEntry = advance(Flags); + if (!MaybeEntry) + return MaybeEntry; + BitstreamEntry Entry = MaybeEntry.get(); + + if (Entry.Kind != BitstreamEntry::SubBlock) + return Entry; + + // If we found a sub-block, just skip over it and check the next entry. + if (Error Err = SkipBlock()) + return std::move(Err); + } + } + + Expected ReadCode() { return Read(CurCodeSize); } + + // Block header: + // [ENTER_SUBBLOCK, blockid, newcodelen, , blocklen] + + /// Having read the ENTER_SUBBLOCK code, read the BlockID for the block. + Expected ReadSubBlockID() { return ReadVBR(bitc::BlockIDWidth); } + + /// Having read the ENTER_SUBBLOCK abbrevid and a BlockID, skip over the body + /// of this block. + Error SkipBlock() { + // Read and ignore the codelen value. + if (Expected Res = ReadVBR(bitc::CodeLenWidth)) + ; // Since we are skipping this block, we don't care what code widths are + // used inside of it. + else + return Res.takeError(); + + SkipToFourByteBoundary(); + Expected MaybeNum = Read(bitc::BlockSizeWidth); + if (!MaybeNum) + return MaybeNum.takeError(); + size_t NumFourBytes = MaybeNum.get(); + + // Check that the block wasn't partially defined, and that the offset isn't + // bogus. + size_t SkipTo = GetCurrentBitNo() + NumFourBytes * 4 * 8; + if (AtEndOfStream()) + return createStringError(std::errc::illegal_byte_sequence, + "can't skip block: already at end of stream"); + if (!canSkipToPos(SkipTo / 8)) + return createStringError(std::errc::illegal_byte_sequence, + "can't skip to bit %zu from %" PRIu64, SkipTo, + GetCurrentBitNo()); + + if (Error Res = JumpToBit(SkipTo)) + return Res; + + return Error::success(); + } + + /// Having read the ENTER_SUBBLOCK abbrevid, and enter the block. + Error EnterSubBlock(unsigned BlockID, unsigned *NumWordsP = nullptr); + + bool ReadBlockEnd() { + if (BlockScope.empty()) return true; + + // Block tail: + // [END_BLOCK, ] + SkipToFourByteBoundary(); + + popBlockScope(); + return false; + } + +private: + void popBlockScope() { + CurCodeSize = BlockScope.back().PrevCodeSize; + + CurAbbrevs = std::move(BlockScope.back().PrevAbbrevs); + BlockScope.pop_back(); + } + + //===--------------------------------------------------------------------===// + // Record Processing + //===--------------------------------------------------------------------===// + +public: + /// Return the abbreviation for the specified AbbrevId. + const BitCodeAbbrev *getAbbrev(unsigned AbbrevID) { + unsigned AbbrevNo = AbbrevID - bitc::FIRST_APPLICATION_ABBREV; + if (AbbrevNo >= CurAbbrevs.size()) + report_fatal_error("Invalid abbrev number"); + return CurAbbrevs[AbbrevNo].get(); + } + + /// Read the current record and discard it, returning the code for the record. + Expected skipRecord(unsigned AbbrevID); + + Expected readRecord(unsigned AbbrevID, + SmallVectorImpl &Vals, + StringRef *Blob = nullptr); + + //===--------------------------------------------------------------------===// + // Abbrev Processing + //===--------------------------------------------------------------------===// + Error ReadAbbrevRecord(); + + /// Read and return a block info block from the bitstream. If an error was + /// encountered, return None. + /// + /// \param ReadBlockInfoNames Whether to read block/record name information in + /// the BlockInfo block. Only llvm-bcanalyzer uses this. + Expected> + ReadBlockInfoBlock(bool ReadBlockInfoNames = false); + + /// Set the block info to be used by this BitstreamCursor to interpret + /// abbreviated records. + void setBlockInfo(BitstreamBlockInfo *BI) { BlockInfo = BI; } +}; + +} // end llvm namespace + +#endif // LLVM_BITSTREAM_BITSTREAMREADER_H diff --git a/include/llvm/Bitstream/BitstreamWriter.h b/include/llvm/Bitstream/BitstreamWriter.h new file mode 100644 index 000000000000..c0ead19dc71d --- /dev/null +++ b/include/llvm/Bitstream/BitstreamWriter.h @@ -0,0 +1,547 @@ +//===- BitstreamWriter.h - Low-level bitstream writer interface -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This header defines the BitstreamWriter class. This class can be used to +// write an arbitrary bitstream, regardless of its contents. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_BITSTREAM_BITSTREAMWRITER_H +#define LLVM_BITSTREAM_BITSTREAMWRITER_H + +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Bitstream/BitCodes.h" +#include "llvm/Support/Endian.h" +#include + +namespace llvm { + +class BitstreamWriter { + SmallVectorImpl &Out; + + /// CurBit - Always between 0 and 31 inclusive, specifies the next bit to use. + unsigned CurBit; + + /// CurValue - The current value. Only bits < CurBit are valid. + uint32_t CurValue; + + /// CurCodeSize - This is the declared size of code values used for the + /// current block, in bits. + unsigned CurCodeSize; + + /// BlockInfoCurBID - When emitting a BLOCKINFO_BLOCK, this is the currently + /// selected BLOCK ID. + unsigned BlockInfoCurBID; + + /// CurAbbrevs - Abbrevs installed at in this block. + std::vector> CurAbbrevs; + + struct Block { + unsigned PrevCodeSize; + size_t StartSizeWord; + std::vector> PrevAbbrevs; + Block(unsigned PCS, size_t SSW) : PrevCodeSize(PCS), StartSizeWord(SSW) {} + }; + + /// BlockScope - This tracks the current blocks that we have entered. + std::vector BlockScope; + + /// BlockInfo - This contains information emitted to BLOCKINFO_BLOCK blocks. + /// These describe abbreviations that all blocks of the specified ID inherit. + struct BlockInfo { + unsigned BlockID; + std::vector> Abbrevs; + }; + std::vector BlockInfoRecords; + + void WriteByte(unsigned char Value) { + Out.push_back(Value); + } + + void WriteWord(unsigned Value) { + Value = support::endian::byte_swap(Value); + Out.append(reinterpret_cast(&Value), + reinterpret_cast(&Value + 1)); + } + + size_t GetBufferOffset() const { return Out.size(); } + + size_t GetWordIndex() const { + size_t Offset = GetBufferOffset(); + assert((Offset & 3) == 0 && "Not 32-bit aligned"); + return Offset / 4; + } + +public: + explicit BitstreamWriter(SmallVectorImpl &O) + : Out(O), CurBit(0), CurValue(0), CurCodeSize(2) {} + + ~BitstreamWriter() { + assert(CurBit == 0 && "Unflushed data remaining"); + assert(BlockScope.empty() && CurAbbrevs.empty() && "Block imbalance"); + } + + /// Retrieve the current position in the stream, in bits. + uint64_t GetCurrentBitNo() const { return GetBufferOffset() * 8 + CurBit; } + + /// Retrieve the number of bits currently used to encode an abbrev ID. + unsigned GetAbbrevIDWidth() const { return CurCodeSize; } + + //===--------------------------------------------------------------------===// + // Basic Primitives for emitting bits to the stream. + //===--------------------------------------------------------------------===// + + /// Backpatch a 32-bit word in the output at the given bit offset + /// with the specified value. + void BackpatchWord(uint64_t BitNo, unsigned NewWord) { + using namespace llvm::support; + unsigned ByteNo = BitNo / 8; + assert((!endian::readAtBitAlignment( + &Out[ByteNo], BitNo & 7)) && + "Expected to be patching over 0-value placeholders"); + endian::writeAtBitAlignment( + &Out[ByteNo], NewWord, BitNo & 7); + } + + void BackpatchWord64(uint64_t BitNo, uint64_t Val) { + BackpatchWord(BitNo, (uint32_t)Val); + BackpatchWord(BitNo + 32, (uint32_t)(Val >> 32)); + } + + void Emit(uint32_t Val, unsigned NumBits) { + assert(NumBits && NumBits <= 32 && "Invalid value size!"); + assert((Val & ~(~0U >> (32-NumBits))) == 0 && "High bits set!"); + CurValue |= Val << CurBit; + if (CurBit + NumBits < 32) { + CurBit += NumBits; + return; + } + + // Add the current word. + WriteWord(CurValue); + + if (CurBit) + CurValue = Val >> (32-CurBit); + else + CurValue = 0; + CurBit = (CurBit+NumBits) & 31; + } + + void FlushToWord() { + if (CurBit) { + WriteWord(CurValue); + CurBit = 0; + CurValue = 0; + } + } + + void EmitVBR(uint32_t Val, unsigned NumBits) { + assert(NumBits <= 32 && "Too many bits to emit!"); + uint32_t Threshold = 1U << (NumBits-1); + + // Emit the bits with VBR encoding, NumBits-1 bits at a time. + while (Val >= Threshold) { + Emit((Val & ((1 << (NumBits-1))-1)) | (1 << (NumBits-1)), NumBits); + Val >>= NumBits-1; + } + + Emit(Val, NumBits); + } + + void EmitVBR64(uint64_t Val, unsigned NumBits) { + assert(NumBits <= 32 && "Too many bits to emit!"); + if ((uint32_t)Val == Val) + return EmitVBR((uint32_t)Val, NumBits); + + uint32_t Threshold = 1U << (NumBits-1); + + // Emit the bits with VBR encoding, NumBits-1 bits at a time. + while (Val >= Threshold) { + Emit(((uint32_t)Val & ((1 << (NumBits-1))-1)) | + (1 << (NumBits-1)), NumBits); + Val >>= NumBits-1; + } + + Emit((uint32_t)Val, NumBits); + } + + /// EmitCode - Emit the specified code. + void EmitCode(unsigned Val) { + Emit(Val, CurCodeSize); + } + + //===--------------------------------------------------------------------===// + // Block Manipulation + //===--------------------------------------------------------------------===// + + /// getBlockInfo - If there is block info for the specified ID, return it, + /// otherwise return null. + BlockInfo *getBlockInfo(unsigned BlockID) { + // Common case, the most recent entry matches BlockID. + if (!BlockInfoRecords.empty() && BlockInfoRecords.back().BlockID == BlockID) + return &BlockInfoRecords.back(); + + for (unsigned i = 0, e = static_cast(BlockInfoRecords.size()); + i != e; ++i) + if (BlockInfoRecords[i].BlockID == BlockID) + return &BlockInfoRecords[i]; + return nullptr; + } + + void EnterSubblock(unsigned BlockID, unsigned CodeLen) { + // Block header: + // [ENTER_SUBBLOCK, blockid, newcodelen, , blocklen] + EmitCode(bitc::ENTER_SUBBLOCK); + EmitVBR(BlockID, bitc::BlockIDWidth); + EmitVBR(CodeLen, bitc::CodeLenWidth); + FlushToWord(); + + size_t BlockSizeWordIndex = GetWordIndex(); + unsigned OldCodeSize = CurCodeSize; + + // Emit a placeholder, which will be replaced when the block is popped. + Emit(0, bitc::BlockSizeWidth); + + CurCodeSize = CodeLen; + + // Push the outer block's abbrev set onto the stack, start out with an + // empty abbrev set. + BlockScope.emplace_back(OldCodeSize, BlockSizeWordIndex); + BlockScope.back().PrevAbbrevs.swap(CurAbbrevs); + + // If there is a blockinfo for this BlockID, add all the predefined abbrevs + // to the abbrev list. + if (BlockInfo *Info = getBlockInfo(BlockID)) { + CurAbbrevs.insert(CurAbbrevs.end(), Info->Abbrevs.begin(), + Info->Abbrevs.end()); + } + } + + void ExitBlock() { + assert(!BlockScope.empty() && "Block scope imbalance!"); + const Block &B = BlockScope.back(); + + // Block tail: + // [END_BLOCK, ] + EmitCode(bitc::END_BLOCK); + FlushToWord(); + + // Compute the size of the block, in words, not counting the size field. + size_t SizeInWords = GetWordIndex() - B.StartSizeWord - 1; + uint64_t BitNo = uint64_t(B.StartSizeWord) * 32; + + // Update the block size field in the header of this sub-block. + BackpatchWord(BitNo, SizeInWords); + + // Restore the inner block's code size and abbrev table. + CurCodeSize = B.PrevCodeSize; + CurAbbrevs = std::move(B.PrevAbbrevs); + BlockScope.pop_back(); + } + + //===--------------------------------------------------------------------===// + // Record Emission + //===--------------------------------------------------------------------===// + +private: + /// EmitAbbreviatedLiteral - Emit a literal value according to its abbrev + /// record. This is a no-op, since the abbrev specifies the literal to use. + template + void EmitAbbreviatedLiteral(const BitCodeAbbrevOp &Op, uintty V) { + assert(Op.isLiteral() && "Not a literal"); + // If the abbrev specifies the literal value to use, don't emit + // anything. + assert(V == Op.getLiteralValue() && + "Invalid abbrev for record!"); + } + + /// EmitAbbreviatedField - Emit a single scalar field value with the specified + /// encoding. + template + void EmitAbbreviatedField(const BitCodeAbbrevOp &Op, uintty V) { + assert(!Op.isLiteral() && "Literals should use EmitAbbreviatedLiteral!"); + + // Encode the value as we are commanded. + switch (Op.getEncoding()) { + default: llvm_unreachable("Unknown encoding!"); + case BitCodeAbbrevOp::Fixed: + if (Op.getEncodingData()) + Emit((unsigned)V, (unsigned)Op.getEncodingData()); + break; + case BitCodeAbbrevOp::VBR: + if (Op.getEncodingData()) + EmitVBR64(V, (unsigned)Op.getEncodingData()); + break; + case BitCodeAbbrevOp::Char6: + Emit(BitCodeAbbrevOp::EncodeChar6((char)V), 6); + break; + } + } + + /// EmitRecordWithAbbrevImpl - This is the core implementation of the record + /// emission code. If BlobData is non-null, then it specifies an array of + /// data that should be emitted as part of the Blob or Array operand that is + /// known to exist at the end of the record. If Code is specified, then + /// it is the record code to emit before the Vals, which must not contain + /// the code. + template + void EmitRecordWithAbbrevImpl(unsigned Abbrev, ArrayRef Vals, + StringRef Blob, Optional Code) { + const char *BlobData = Blob.data(); + unsigned BlobLen = (unsigned) Blob.size(); + unsigned AbbrevNo = Abbrev-bitc::FIRST_APPLICATION_ABBREV; + assert(AbbrevNo < CurAbbrevs.size() && "Invalid abbrev #!"); + const BitCodeAbbrev *Abbv = CurAbbrevs[AbbrevNo].get(); + + EmitCode(Abbrev); + + unsigned i = 0, e = static_cast(Abbv->getNumOperandInfos()); + if (Code) { + assert(e && "Expected non-empty abbreviation"); + const BitCodeAbbrevOp &Op = Abbv->getOperandInfo(i++); + + if (Op.isLiteral()) + EmitAbbreviatedLiteral(Op, Code.getValue()); + else { + assert(Op.getEncoding() != BitCodeAbbrevOp::Array && + Op.getEncoding() != BitCodeAbbrevOp::Blob && + "Expected literal or scalar"); + EmitAbbreviatedField(Op, Code.getValue()); + } + } + + unsigned RecordIdx = 0; + for (; i != e; ++i) { + const BitCodeAbbrevOp &Op = Abbv->getOperandInfo(i); + if (Op.isLiteral()) { + assert(RecordIdx < Vals.size() && "Invalid abbrev/record"); + EmitAbbreviatedLiteral(Op, Vals[RecordIdx]); + ++RecordIdx; + } else if (Op.getEncoding() == BitCodeAbbrevOp::Array) { + // Array case. + assert(i + 2 == e && "array op not second to last?"); + const BitCodeAbbrevOp &EltEnc = Abbv->getOperandInfo(++i); + + // If this record has blob data, emit it, otherwise we must have record + // entries to encode this way. + if (BlobData) { + assert(RecordIdx == Vals.size() && + "Blob data and record entries specified for array!"); + // Emit a vbr6 to indicate the number of elements present. + EmitVBR(static_cast(BlobLen), 6); + + // Emit each field. + for (unsigned i = 0; i != BlobLen; ++i) + EmitAbbreviatedField(EltEnc, (unsigned char)BlobData[i]); + + // Know that blob data is consumed for assertion below. + BlobData = nullptr; + } else { + // Emit a vbr6 to indicate the number of elements present. + EmitVBR(static_cast(Vals.size()-RecordIdx), 6); + + // Emit each field. + for (unsigned e = Vals.size(); RecordIdx != e; ++RecordIdx) + EmitAbbreviatedField(EltEnc, Vals[RecordIdx]); + } + } else if (Op.getEncoding() == BitCodeAbbrevOp::Blob) { + // If this record has blob data, emit it, otherwise we must have record + // entries to encode this way. + + if (BlobData) { + assert(RecordIdx == Vals.size() && + "Blob data and record entries specified for blob operand!"); + + assert(Blob.data() == BlobData && "BlobData got moved"); + assert(Blob.size() == BlobLen && "BlobLen got changed"); + emitBlob(Blob); + BlobData = nullptr; + } else { + emitBlob(Vals.slice(RecordIdx)); + } + } else { // Single scalar field. + assert(RecordIdx < Vals.size() && "Invalid abbrev/record"); + EmitAbbreviatedField(Op, Vals[RecordIdx]); + ++RecordIdx; + } + } + assert(RecordIdx == Vals.size() && "Not all record operands emitted!"); + assert(BlobData == nullptr && + "Blob data specified for record that doesn't use it!"); + } + +public: + /// Emit a blob, including flushing before and tail-padding. + template + void emitBlob(ArrayRef Bytes, bool ShouldEmitSize = true) { + // Emit a vbr6 to indicate the number of elements present. + if (ShouldEmitSize) + EmitVBR(static_cast(Bytes.size()), 6); + + // Flush to a 32-bit alignment boundary. + FlushToWord(); + + // Emit literal bytes. + for (const auto &B : Bytes) { + assert(isUInt<8>(B) && "Value too large to emit as byte"); + WriteByte((unsigned char)B); + } + + // Align end to 32-bits. + while (GetBufferOffset() & 3) + WriteByte(0); + } + void emitBlob(StringRef Bytes, bool ShouldEmitSize = true) { + emitBlob(makeArrayRef((const uint8_t *)Bytes.data(), Bytes.size()), + ShouldEmitSize); + } + + /// EmitRecord - Emit the specified record to the stream, using an abbrev if + /// we have one to compress the output. + template + void EmitRecord(unsigned Code, const Container &Vals, unsigned Abbrev = 0) { + if (!Abbrev) { + // If we don't have an abbrev to use, emit this in its fully unabbreviated + // form. + auto Count = static_cast(makeArrayRef(Vals).size()); + EmitCode(bitc::UNABBREV_RECORD); + EmitVBR(Code, 6); + EmitVBR(Count, 6); + for (unsigned i = 0, e = Count; i != e; ++i) + EmitVBR64(Vals[i], 6); + return; + } + + EmitRecordWithAbbrevImpl(Abbrev, makeArrayRef(Vals), StringRef(), Code); + } + + /// EmitRecordWithAbbrev - Emit a record with the specified abbreviation. + /// Unlike EmitRecord, the code for the record should be included in Vals as + /// the first entry. + template + void EmitRecordWithAbbrev(unsigned Abbrev, const Container &Vals) { + EmitRecordWithAbbrevImpl(Abbrev, makeArrayRef(Vals), StringRef(), None); + } + + /// EmitRecordWithBlob - Emit the specified record to the stream, using an + /// abbrev that includes a blob at the end. The blob data to emit is + /// specified by the pointer and length specified at the end. In contrast to + /// EmitRecord, this routine expects that the first entry in Vals is the code + /// of the record. + template + void EmitRecordWithBlob(unsigned Abbrev, const Container &Vals, + StringRef Blob) { + EmitRecordWithAbbrevImpl(Abbrev, makeArrayRef(Vals), Blob, None); + } + template + void EmitRecordWithBlob(unsigned Abbrev, const Container &Vals, + const char *BlobData, unsigned BlobLen) { + return EmitRecordWithAbbrevImpl(Abbrev, makeArrayRef(Vals), + StringRef(BlobData, BlobLen), None); + } + + /// EmitRecordWithArray - Just like EmitRecordWithBlob, works with records + /// that end with an array. + template + void EmitRecordWithArray(unsigned Abbrev, const Container &Vals, + StringRef Array) { + EmitRecordWithAbbrevImpl(Abbrev, makeArrayRef(Vals), Array, None); + } + template + void EmitRecordWithArray(unsigned Abbrev, const Container &Vals, + const char *ArrayData, unsigned ArrayLen) { + return EmitRecordWithAbbrevImpl(Abbrev, makeArrayRef(Vals), + StringRef(ArrayData, ArrayLen), None); + } + + //===--------------------------------------------------------------------===// + // Abbrev Emission + //===--------------------------------------------------------------------===// + +private: + // Emit the abbreviation as a DEFINE_ABBREV record. + void EncodeAbbrev(const BitCodeAbbrev &Abbv) { + EmitCode(bitc::DEFINE_ABBREV); + EmitVBR(Abbv.getNumOperandInfos(), 5); + for (unsigned i = 0, e = static_cast(Abbv.getNumOperandInfos()); + i != e; ++i) { + const BitCodeAbbrevOp &Op = Abbv.getOperandInfo(i); + Emit(Op.isLiteral(), 1); + if (Op.isLiteral()) { + EmitVBR64(Op.getLiteralValue(), 8); + } else { + Emit(Op.getEncoding(), 3); + if (Op.hasEncodingData()) + EmitVBR64(Op.getEncodingData(), 5); + } + } + } +public: + + /// Emits the abbreviation \p Abbv to the stream. + unsigned EmitAbbrev(std::shared_ptr Abbv) { + EncodeAbbrev(*Abbv); + CurAbbrevs.push_back(std::move(Abbv)); + return static_cast(CurAbbrevs.size())-1 + + bitc::FIRST_APPLICATION_ABBREV; + } + + //===--------------------------------------------------------------------===// + // BlockInfo Block Emission + //===--------------------------------------------------------------------===// + + /// EnterBlockInfoBlock - Start emitting the BLOCKINFO_BLOCK. + void EnterBlockInfoBlock() { + EnterSubblock(bitc::BLOCKINFO_BLOCK_ID, 2); + BlockInfoCurBID = ~0U; + BlockInfoRecords.clear(); + } +private: + /// SwitchToBlockID - If we aren't already talking about the specified block + /// ID, emit a BLOCKINFO_CODE_SETBID record. + void SwitchToBlockID(unsigned BlockID) { + if (BlockInfoCurBID == BlockID) return; + SmallVector V; + V.push_back(BlockID); + EmitRecord(bitc::BLOCKINFO_CODE_SETBID, V); + BlockInfoCurBID = BlockID; + } + + BlockInfo &getOrCreateBlockInfo(unsigned BlockID) { + if (BlockInfo *BI = getBlockInfo(BlockID)) + return *BI; + + // Otherwise, add a new record. + BlockInfoRecords.emplace_back(); + BlockInfoRecords.back().BlockID = BlockID; + return BlockInfoRecords.back(); + } + +public: + + /// EmitBlockInfoAbbrev - Emit a DEFINE_ABBREV record for the specified + /// BlockID. + unsigned EmitBlockInfoAbbrev(unsigned BlockID, std::shared_ptr Abbv) { + SwitchToBlockID(BlockID); + EncodeAbbrev(*Abbv); + + // Add the abbrev to the specified block record. + BlockInfo &Info = getOrCreateBlockInfo(BlockID); + Info.Abbrevs.push_back(std::move(Abbv)); + + return Info.Abbrevs.size()-1+bitc::FIRST_APPLICATION_ABBREV; + } +}; + + +} // End llvm namespace + +#endif diff --git a/include/llvm/CodeGen/AccelTable.h b/include/llvm/CodeGen/AccelTable.h index 13928582f2dd..734531a65d50 100644 --- a/include/llvm/CodeGen/AccelTable.h +++ b/include/llvm/CodeGen/AccelTable.h @@ -1,9 +1,8 @@ //==- include/llvm/CodeGen/AccelTable.h - Accelerator Tables -----*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -327,14 +326,8 @@ public: void emit(AsmPrinter *Asm) const override; -#ifndef _MSC_VER - // The line below is rejected by older versions (TBD) of MSVC. static constexpr Atom Atoms[] = { Atom(dwarf::DW_ATOM_die_offset, dwarf::DW_FORM_data4)}; -#else - // FIXME: Erase this path once the minimum MSCV version has been bumped. - static const SmallVector Atoms; -#endif #ifndef NDEBUG void print(raw_ostream &OS) const override; @@ -352,16 +345,10 @@ public: void emit(AsmPrinter *Asm) const override; -#ifndef _MSC_VER - // The line below is rejected by older versions (TBD) of MSVC. static constexpr Atom Atoms[] = { Atom(dwarf::DW_ATOM_die_offset, dwarf::DW_FORM_data4), Atom(dwarf::DW_ATOM_die_tag, dwarf::DW_FORM_data2), Atom(dwarf::DW_ATOM_type_flags, dwarf::DW_FORM_data1)}; -#else - // FIXME: Erase this path once the minimum MSCV version has been bumped. - static const SmallVector Atoms; -#endif #ifndef NDEBUG void print(raw_ostream &OS) const override; @@ -376,14 +363,8 @@ public: void emit(AsmPrinter *Asm) const override; -#ifndef _MSC_VER - // The line below is rejected by older versions (TBD) of MSVC. static constexpr Atom Atoms[] = { Atom(dwarf::DW_ATOM_die_offset, dwarf::DW_FORM_data4)}; -#else - // FIXME: Erase this path once the minimum MSCV version has been bumped. - static const SmallVector Atoms; -#endif #ifndef NDEBUG void print(raw_ostream &OS) const override; @@ -407,16 +388,10 @@ public: void emit(AsmPrinter *Asm) const override; -#ifndef _MSC_VER - // The line below is rejected by older versions (TBD) of MSVC. static constexpr Atom Atoms[] = { Atom(dwarf::DW_ATOM_die_offset, dwarf::DW_FORM_data4), Atom(dwarf::DW_ATOM_die_tag, dwarf::DW_FORM_data2), Atom(5, dwarf::DW_FORM_data1), Atom(6, dwarf::DW_FORM_data4)}; -#else - // FIXME: Erase this path once the minimum MSCV version has been bumped. - static const SmallVector Atoms; -#endif #ifndef NDEBUG void print(raw_ostream &OS) const override; diff --git a/include/llvm/CodeGen/Analysis.h b/include/llvm/CodeGen/Analysis.h index d77aee66ed76..0be0ac22a74d 100644 --- a/include/llvm/CodeGen/Analysis.h +++ b/include/llvm/CodeGen/Analysis.h @@ -1,9 +1,8 @@ //===- CodeGen/Analysis.h - CodeGen LLVM IR Analysis Utilities --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -26,6 +25,7 @@ namespace llvm { class GlobalValue; +class LLT; class MachineBasicBlock; class MachineFunction; class TargetLoweringBase; @@ -74,6 +74,25 @@ void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl *Offsets = nullptr, uint64_t StartingOffset = 0); +/// Variant of ComputeValueVTs that also produces the memory VTs. +void ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, + SmallVectorImpl &ValueVTs, + SmallVectorImpl *MemVTs, + SmallVectorImpl *Offsets = nullptr, + uint64_t StartingOffset = 0); + +/// computeValueLLTs - Given an LLVM IR type, compute a sequence of +/// LLTs that represent all the individual underlying +/// non-aggregate types that comprise it. +/// +/// If Offsets is non-null, it points to a vector to be filled in +/// with the in-memory offsets of each of the individual values. +/// +void computeValueLLTs(const DataLayout &DL, Type &Ty, + SmallVectorImpl &ValueTys, + SmallVectorImpl *Offsets = nullptr, + uint64_t StartingOffset = 0); + /// ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V. GlobalValue *ExtractTypeInfo(Value *V); diff --git a/include/llvm/CodeGen/AsmPrinter.h b/include/llvm/CodeGen/AsmPrinter.h index 413901d218f9..d110f8b01cb5 100644 --- a/include/llvm/CodeGen/AsmPrinter.h +++ b/include/llvm/CodeGen/AsmPrinter.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/AsmPrinter.h - AsmPrinter Framework ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -20,6 +19,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" +#include "llvm/CodeGen/AsmPrinterHandler.h" #include "llvm/CodeGen/DwarfStringPoolEntry.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/IR/InlineAsm.h" @@ -33,7 +33,6 @@ namespace llvm { -class AsmPrinterHandler; class BasicBlock; class BlockAddress; class Constant; @@ -122,9 +121,6 @@ public: using GOTEquivUsePair = std::pair; MapVector GlobalGOTEquivs; - /// Enable print [latency:throughput] in output. - bool EnablePrintSchedInfo = false; - private: MCSymbol *CurrentFnBegin = nullptr; MCSymbol *CurrentFnEnd = nullptr; @@ -142,16 +138,16 @@ protected: /// Protected struct HandlerInfo and Handlers permit target extended /// AsmPrinter adds their own handlers. struct HandlerInfo { - AsmPrinterHandler *Handler; + std::unique_ptr Handler; const char *TimerName; const char *TimerDescription; const char *TimerGroupName; const char *TimerGroupDescription; - HandlerInfo(AsmPrinterHandler *Handler, const char *TimerName, - const char *TimerDescription, const char *TimerGroupName, - const char *TimerGroupDescription) - : Handler(Handler), TimerName(TimerName), + HandlerInfo(std::unique_ptr Handler, + const char *TimerName, const char *TimerDescription, + const char *TimerGroupName, const char *TimerGroupDescription) + : Handler(std::move(Handler)), TimerName(TimerName), TimerDescription(TimerDescription), TimerGroupName(TimerGroupName), TimerGroupDescription(TimerGroupDescription) {} }; @@ -227,6 +223,9 @@ public: void EmitToStreamer(MCStreamer &S, const MCInst &Inst); + /// Emits inital debug location directive. + void emitInitialRawDwarfLocDirective(const MachineFunction &MF); + /// Return the current section we are emitting to. const MCSection *getCurrentSection() const; @@ -316,6 +315,8 @@ public: void emitStackSizeSection(const MachineFunction &MF); + void emitRemarksSection(Module &M); + enum CFIMoveType { CFI_M_None, CFI_M_EH, CFI_M_Debug }; CFIMoveType needsCFIMoves() const; @@ -511,7 +512,7 @@ public: void EmitSLEB128(int64_t Value, const char *Desc = nullptr) const; /// Emit the specified unsigned leb128 value. - void EmitULEB128(uint64_t Value, const char *Desc = nullptr) const; + void EmitULEB128(uint64_t Value, const char *Desc = nullptr, unsigned PadTo = 0) const; /// Emit a .byte 42 directive that corresponds to an encoding. If verbose /// assembly output is enabled, we output comments describing the encoding. @@ -542,6 +543,12 @@ public: emitDwarfStringOffset(S.getEntry()); } + /// Emit reference to a call site with a specified encoding + void EmitCallSiteOffset(const MCSymbol *Hi, const MCSymbol *Lo, + unsigned Encoding) const; + /// Emit an integer value corresponding to the call site encoding + void EmitCallSiteValue(uint64_t Value, unsigned Encoding) const; + /// Get the value for DW_AT_APPLE_isa. Zero if no isa encoding specified. virtual unsigned getISAEncoding() { return 0; } @@ -589,20 +596,22 @@ public: virtual void PrintSpecial(const MachineInstr *MI, raw_ostream &OS, const char *Code) const; + /// Print the MachineOperand as a symbol. Targets with complex handling of + /// symbol references should override the base implementation. + virtual void PrintSymbolOperand(const MachineOperand &MO, raw_ostream &OS); + /// Print the specified operand of MI, an INLINEASM instruction, using the /// specified assembler variant. Targets should override this to format as /// appropriate. This method can return true if the operand is erroneous. virtual bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, - unsigned AsmVariant, const char *ExtraCode, - raw_ostream &OS); + const char *ExtraCode, raw_ostream &OS); /// Print the specified operand of MI, an INLINEASM instruction, using the /// specified assembler variant as an address. Targets should override this to /// format as appropriate. This method can return true if the operand is /// erroneous. virtual bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo, - unsigned AsmVariant, const char *ExtraCode, - raw_ostream &OS); + const char *ExtraCode, raw_ostream &OS); /// Let the target do anything it needs to do before emitting inlineasm. /// \p StartInfo - the subtarget info before parsing inline asm @@ -617,6 +626,15 @@ public: virtual void emitInlineAsmEnd(const MCSubtargetInfo &StartInfo, const MCSubtargetInfo *EndInfo) const; + /// This emits visibility information about symbol, if this is supported by + /// the target. + void EmitVisibility(MCSymbol *Sym, unsigned Visibility, + bool IsDefinition = true) const; + + /// This emits linkage information about \p GVSym based on \p GV, if this is + /// supported by the target. + void EmitLinkage(const GlobalValue *GV, MCSymbol *GVSym) const; + private: /// Private state for PrintSpecial() // Assign a unique ID to this machine instruction. @@ -647,13 +665,6 @@ private: // Internal Implementation Details //===------------------------------------------------------------------===// - /// This emits visibility information about symbol, if this is supported by - /// the target. - void EmitVisibility(MCSymbol *Sym, unsigned Visibility, - bool IsDefinition = true) const; - - void EmitLinkage(const GlobalValue *GV, MCSymbol *GVSym) const; - void EmitJumpTableEntry(const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB, unsigned uid) const; void EmitLLVMUsedList(const ConstantArray *InitList); diff --git a/include/llvm/CodeGen/AsmPrinterHandler.h b/include/llvm/CodeGen/AsmPrinterHandler.h index a8b13200dd4e..affb558f2fa6 100644 --- a/include/llvm/CodeGen/AsmPrinterHandler.h +++ b/include/llvm/CodeGen/AsmPrinterHandler.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/AsmPrinterHandler.h -----------------------*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/AtomicExpandUtils.h b/include/llvm/CodeGen/AtomicExpandUtils.h index b1adf66e7ff4..8a46c6e00d22 100644 --- a/include/llvm/CodeGen/AtomicExpandUtils.h +++ b/include/llvm/CodeGen/AtomicExpandUtils.h @@ -1,9 +1,8 @@ //===- AtomicExpandUtils.h - Utilities for expanding atomic instructions --===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/CodeGen/BasicTTIImpl.h b/include/llvm/CodeGen/BasicTTIImpl.h index f105d887c397..70bf670fdf0b 100644 --- a/include/llvm/CodeGen/BasicTTIImpl.h +++ b/include/llvm/CodeGen/BasicTTIImpl.h @@ -1,9 +1,8 @@ //===- BasicTTIImpl.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -197,11 +196,12 @@ protected: public: /// \name Scalar TTI Implementations /// @{ - bool allowsMisalignedMemoryAccesses(LLVMContext &Context, - unsigned BitWidth, unsigned AddressSpace, - unsigned Alignment, bool *Fast) const { + bool allowsMisalignedMemoryAccesses(LLVMContext &Context, unsigned BitWidth, + unsigned AddressSpace, unsigned Alignment, + bool *Fast) const { EVT E = EVT::getIntegerVT(Context, BitWidth); - return getTLI()->allowsMisalignedMemoryAccesses(E, AddressSpace, Alignment, Fast); + return getTLI()->allowsMisalignedMemoryAccesses( + E, AddressSpace, Alignment, MachineMemOperand::MONone, Fast); } bool hasBranchDivergence() { return false; } @@ -293,12 +293,12 @@ public: } unsigned getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, - ArrayRef Arguments) { - return BaseT::getIntrinsicCost(IID, RetTy, Arguments); + ArrayRef Arguments, const User *U) { + return BaseT::getIntrinsicCost(IID, RetTy, Arguments, U); } unsigned getIntrinsicCost(Intrinsic::ID IID, Type *RetTy, - ArrayRef ParamTys) { + ArrayRef ParamTys, const User *U) { if (IID == Intrinsic::cttz) { if (getTLI()->isCheapToSpeculateCttz()) return TargetTransformInfo::TCC_Basic; @@ -311,7 +311,7 @@ public: return TargetTransformInfo::TCC_Expensive; } - return BaseT::getIntrinsicCost(IID, RetTy, ParamTys); + return BaseT::getIntrinsicCost(IID, RetTy, ParamTys, U); } unsigned getEstimatedNumberOfCaseClusters(const SwitchInst &SI, @@ -414,6 +414,12 @@ public: if (TLI->isZExtFree(OpTy, Ty)) return TargetTransformInfo::TCC_Free; return TargetTransformInfo::TCC_Basic; + + case Instruction::AddrSpaceCast: + if (TLI->isFreeAddrSpaceCast(OpTy->getPointerAddressSpace(), + Ty->getPointerAddressSpace())) + return TargetTransformInfo::TCC_Free; + return TargetTransformInfo::TCC_Basic; } return BaseT::getOperationCost(Opcode, Ty, OpTy); @@ -421,6 +427,8 @@ public: unsigned getInliningThresholdMultiplier() { return 1; } + int getInlinerVectorBonusPercent() { return 150; } + void getUnrollingPreferences(Loop *L, ScalarEvolution &SE, TTI::UnrollingPreferences &UP) { // This unrolling functionality is target independent, but to provide some @@ -486,6 +494,13 @@ public: UP.BEInsns = 2; } + bool isHardwareLoopProfitable(Loop *L, ScalarEvolution &SE, + AssumptionCache &AC, + TargetLibraryInfo *LibInfo, + HardwareLoopInfo &HWLoopInfo) { + return BaseT::isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo); + } + int getInstructionLatency(const Instruction *I) { if (isa(I)) return getST()->getSchedModel().DefaultLoadLatency; @@ -657,7 +672,7 @@ public: return 0; if (Opcode == Instruction::AddrSpaceCast && - TLI->isNoopAddrSpaceCast(Src->getPointerAddressSpace(), + TLI->isFreeAddrSpaceCast(Src->getPointerAddressSpace(), Dst->getPointerAddressSpace())) return 0; @@ -997,7 +1012,7 @@ public: // inside the loop. if (UseMaskForGaps) Cost += static_cast(this)->getArithmeticInstrCost( - BinaryOperator::And, MaskVT); + BinaryOperator::And, MaskVT); return Cost; } @@ -1058,8 +1073,8 @@ public: case Intrinsic::experimental_vector_reduce_and: case Intrinsic::experimental_vector_reduce_or: case Intrinsic::experimental_vector_reduce_xor: - case Intrinsic::experimental_vector_reduce_fadd: - case Intrinsic::experimental_vector_reduce_fmul: + case Intrinsic::experimental_vector_reduce_v2_fadd: + case Intrinsic::experimental_vector_reduce_v2_fmul: case Intrinsic::experimental_vector_reduce_smax: case Intrinsic::experimental_vector_reduce_smin: case Intrinsic::experimental_vector_reduce_fmax: @@ -1116,6 +1131,9 @@ public: unsigned getIntrinsicInstrCost( Intrinsic::ID IID, Type *RetTy, ArrayRef Tys, FastMathFlags FMF, unsigned ScalarizationCostPassed = std::numeric_limits::max()) { + unsigned RetVF = (RetTy->isVectorTy() ? RetTy->getVectorNumElements() : 1); + auto *ConcreteTTI = static_cast(this); + SmallVector ISDs; unsigned SingleCallCost = 10; // Library call cost. Make it expensive. switch (IID) { @@ -1144,8 +1162,8 @@ public: if (ScalarCalls == 1) return 1; // Return cost of a scalar intrinsic. Assume it to be cheap. - unsigned ScalarCost = static_cast(this)->getIntrinsicInstrCost( - IID, ScalarRetTy, ScalarTys, FMF); + unsigned ScalarCost = + ConcreteTTI->getIntrinsicInstrCost(IID, ScalarRetTy, ScalarTys, FMF); return ScalarCalls * ScalarCost + ScalarizationCost; } @@ -1227,44 +1245,181 @@ public: case Intrinsic::sideeffect: return 0; case Intrinsic::masked_store: - return static_cast(this) - ->getMaskedMemoryOpCost(Instruction::Store, Tys[0], 0, 0); + return ConcreteTTI->getMaskedMemoryOpCost(Instruction::Store, Tys[0], 0, + 0); case Intrinsic::masked_load: - return static_cast(this) - ->getMaskedMemoryOpCost(Instruction::Load, RetTy, 0, 0); + return ConcreteTTI->getMaskedMemoryOpCost(Instruction::Load, RetTy, 0, 0); case Intrinsic::experimental_vector_reduce_add: - return static_cast(this)->getArithmeticReductionCost( - Instruction::Add, Tys[0], /*IsPairwiseForm=*/false); + return ConcreteTTI->getArithmeticReductionCost(Instruction::Add, Tys[0], + /*IsPairwiseForm=*/false); case Intrinsic::experimental_vector_reduce_mul: - return static_cast(this)->getArithmeticReductionCost( - Instruction::Mul, Tys[0], /*IsPairwiseForm=*/false); + return ConcreteTTI->getArithmeticReductionCost(Instruction::Mul, Tys[0], + /*IsPairwiseForm=*/false); case Intrinsic::experimental_vector_reduce_and: - return static_cast(this)->getArithmeticReductionCost( - Instruction::And, Tys[0], /*IsPairwiseForm=*/false); + return ConcreteTTI->getArithmeticReductionCost(Instruction::And, Tys[0], + /*IsPairwiseForm=*/false); case Intrinsic::experimental_vector_reduce_or: - return static_cast(this)->getArithmeticReductionCost( - Instruction::Or, Tys[0], /*IsPairwiseForm=*/false); + return ConcreteTTI->getArithmeticReductionCost(Instruction::Or, Tys[0], + /*IsPairwiseForm=*/false); case Intrinsic::experimental_vector_reduce_xor: - return static_cast(this)->getArithmeticReductionCost( - Instruction::Xor, Tys[0], /*IsPairwiseForm=*/false); - case Intrinsic::experimental_vector_reduce_fadd: - return static_cast(this)->getArithmeticReductionCost( - Instruction::FAdd, Tys[0], /*IsPairwiseForm=*/false); - case Intrinsic::experimental_vector_reduce_fmul: - return static_cast(this)->getArithmeticReductionCost( - Instruction::FMul, Tys[0], /*IsPairwiseForm=*/false); + return ConcreteTTI->getArithmeticReductionCost(Instruction::Xor, Tys[0], + /*IsPairwiseForm=*/false); + case Intrinsic::experimental_vector_reduce_v2_fadd: + return ConcreteTTI->getArithmeticReductionCost( + Instruction::FAdd, Tys[0], + /*IsPairwiseForm=*/false); // FIXME: Add new flag for cost of strict + // reductions. + case Intrinsic::experimental_vector_reduce_v2_fmul: + return ConcreteTTI->getArithmeticReductionCost( + Instruction::FMul, Tys[0], + /*IsPairwiseForm=*/false); // FIXME: Add new flag for cost of strict + // reductions. case Intrinsic::experimental_vector_reduce_smax: case Intrinsic::experimental_vector_reduce_smin: case Intrinsic::experimental_vector_reduce_fmax: case Intrinsic::experimental_vector_reduce_fmin: - return static_cast(this)->getMinMaxReductionCost( + return ConcreteTTI->getMinMaxReductionCost( Tys[0], CmpInst::makeCmpResultType(Tys[0]), /*IsPairwiseForm=*/false, - /*IsSigned=*/true); + /*IsUnsigned=*/true); case Intrinsic::experimental_vector_reduce_umax: case Intrinsic::experimental_vector_reduce_umin: - return static_cast(this)->getMinMaxReductionCost( + return ConcreteTTI->getMinMaxReductionCost( Tys[0], CmpInst::makeCmpResultType(Tys[0]), /*IsPairwiseForm=*/false, - /*IsSigned=*/false); + /*IsUnsigned=*/false); + case Intrinsic::sadd_sat: + case Intrinsic::ssub_sat: { + Type *CondTy = Type::getInt1Ty(RetTy->getContext()); + if (RetVF > 1) + CondTy = VectorType::get(CondTy, RetVF); + + Type *OpTy = StructType::create({RetTy, CondTy}); + Intrinsic::ID OverflowOp = IID == Intrinsic::sadd_sat + ? Intrinsic::sadd_with_overflow + : Intrinsic::ssub_with_overflow; + + // SatMax -> Overflow && SumDiff < 0 + // SatMin -> Overflow && SumDiff >= 0 + unsigned Cost = 0; + Cost += ConcreteTTI->getIntrinsicInstrCost( + OverflowOp, OpTy, {RetTy, RetTy}, FMF, ScalarizationCostPassed); + Cost += ConcreteTTI->getCmpSelInstrCost(BinaryOperator::ICmp, RetTy, + CondTy, nullptr); + Cost += 2 * ConcreteTTI->getCmpSelInstrCost(BinaryOperator::Select, RetTy, + CondTy, nullptr); + return Cost; + } + case Intrinsic::uadd_sat: + case Intrinsic::usub_sat: { + Type *CondTy = Type::getInt1Ty(RetTy->getContext()); + if (RetVF > 1) + CondTy = VectorType::get(CondTy, RetVF); + + Type *OpTy = StructType::create({RetTy, CondTy}); + Intrinsic::ID OverflowOp = IID == Intrinsic::uadd_sat + ? Intrinsic::uadd_with_overflow + : Intrinsic::usub_with_overflow; + + unsigned Cost = 0; + Cost += ConcreteTTI->getIntrinsicInstrCost( + OverflowOp, OpTy, {RetTy, RetTy}, FMF, ScalarizationCostPassed); + Cost += ConcreteTTI->getCmpSelInstrCost(BinaryOperator::Select, RetTy, + CondTy, nullptr); + return Cost; + } + case Intrinsic::smul_fix: + case Intrinsic::umul_fix: { + unsigned ExtSize = RetTy->getScalarSizeInBits() * 2; + Type *ExtTy = Type::getIntNTy(RetTy->getContext(), ExtSize); + if (RetVF > 1) + ExtTy = VectorType::get(ExtTy, RetVF); + + unsigned ExtOp = + IID == Intrinsic::smul_fix ? Instruction::SExt : Instruction::ZExt; + + unsigned Cost = 0; + Cost += 2 * ConcreteTTI->getCastInstrCost(ExtOp, ExtTy, RetTy); + Cost += ConcreteTTI->getArithmeticInstrCost(Instruction::Mul, ExtTy); + Cost += + 2 * ConcreteTTI->getCastInstrCost(Instruction::Trunc, RetTy, ExtTy); + Cost += ConcreteTTI->getArithmeticInstrCost(Instruction::LShr, RetTy, + TTI::OK_AnyValue, + TTI::OK_UniformConstantValue); + Cost += ConcreteTTI->getArithmeticInstrCost(Instruction::Shl, RetTy, + TTI::OK_AnyValue, + TTI::OK_UniformConstantValue); + Cost += ConcreteTTI->getArithmeticInstrCost(Instruction::Or, RetTy); + return Cost; + } + case Intrinsic::sadd_with_overflow: + case Intrinsic::ssub_with_overflow: { + Type *SumTy = RetTy->getContainedType(0); + Type *OverflowTy = RetTy->getContainedType(1); + unsigned Opcode = IID == Intrinsic::sadd_with_overflow + ? BinaryOperator::Add + : BinaryOperator::Sub; + + // LHSSign -> LHS >= 0 + // RHSSign -> RHS >= 0 + // SumSign -> Sum >= 0 + // + // Add: + // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) + // Sub: + // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) + unsigned Cost = 0; + Cost += ConcreteTTI->getArithmeticInstrCost(Opcode, SumTy); + Cost += 3 * ConcreteTTI->getCmpSelInstrCost(BinaryOperator::ICmp, SumTy, + OverflowTy, nullptr); + Cost += 2 * ConcreteTTI->getCmpSelInstrCost( + BinaryOperator::ICmp, OverflowTy, OverflowTy, nullptr); + Cost += + ConcreteTTI->getArithmeticInstrCost(BinaryOperator::And, OverflowTy); + return Cost; + } + case Intrinsic::uadd_with_overflow: + case Intrinsic::usub_with_overflow: { + Type *SumTy = RetTy->getContainedType(0); + Type *OverflowTy = RetTy->getContainedType(1); + unsigned Opcode = IID == Intrinsic::uadd_with_overflow + ? BinaryOperator::Add + : BinaryOperator::Sub; + + unsigned Cost = 0; + Cost += ConcreteTTI->getArithmeticInstrCost(Opcode, SumTy); + Cost += ConcreteTTI->getCmpSelInstrCost(BinaryOperator::ICmp, SumTy, + OverflowTy, nullptr); + return Cost; + } + case Intrinsic::smul_with_overflow: + case Intrinsic::umul_with_overflow: { + Type *MulTy = RetTy->getContainedType(0); + Type *OverflowTy = RetTy->getContainedType(1); + unsigned ExtSize = MulTy->getScalarSizeInBits() * 2; + Type *ExtTy = Type::getIntNTy(RetTy->getContext(), ExtSize); + if (MulTy->isVectorTy()) + ExtTy = VectorType::get(ExtTy, MulTy->getVectorNumElements() ); + + unsigned ExtOp = + IID == Intrinsic::smul_fix ? Instruction::SExt : Instruction::ZExt; + + unsigned Cost = 0; + Cost += 2 * ConcreteTTI->getCastInstrCost(ExtOp, ExtTy, MulTy); + Cost += ConcreteTTI->getArithmeticInstrCost(Instruction::Mul, ExtTy); + Cost += + 2 * ConcreteTTI->getCastInstrCost(Instruction::Trunc, MulTy, ExtTy); + Cost += ConcreteTTI->getArithmeticInstrCost(Instruction::LShr, MulTy, + TTI::OK_AnyValue, + TTI::OK_UniformConstantValue); + + if (IID == Intrinsic::smul_with_overflow) + Cost += ConcreteTTI->getArithmeticInstrCost( + Instruction::AShr, MulTy, TTI::OK_AnyValue, + TTI::OK_UniformConstantValue); + + Cost += ConcreteTTI->getCmpSelInstrCost(BinaryOperator::ICmp, MulTy, + OverflowTy, nullptr); + return Cost; + } case Intrinsic::ctpop: ISDs.push_back(ISD::CTPOP); // In case of legalization use TCC_Expensive. This is cheaper than a @@ -1305,17 +1460,16 @@ public: if (MinLegalCostI != LegalCost.end()) return *MinLegalCostI; - auto MinCustomCostI = std::min_element(CustomCost.begin(), CustomCost.end()); + auto MinCustomCostI = + std::min_element(CustomCost.begin(), CustomCost.end()); if (MinCustomCostI != CustomCost.end()) return *MinCustomCostI; // If we can't lower fmuladd into an FMA estimate the cost as a floating // point mul followed by an add. if (IID == Intrinsic::fmuladd) - return static_cast(this) - ->getArithmeticInstrCost(BinaryOperator::FMul, RetTy) + - static_cast(this) - ->getArithmeticInstrCost(BinaryOperator::FAdd, RetTy); + return ConcreteTTI->getArithmeticInstrCost(BinaryOperator::FMul, RetTy) + + ConcreteTTI->getArithmeticInstrCost(BinaryOperator::FAdd, RetTy); // Else, assume that we need to scalarize this intrinsic. For math builtins // this will emit a costly libcall, adding call overhead and spills. Make it @@ -1333,7 +1487,7 @@ public: Ty = Ty->getScalarType(); ScalarTys.push_back(Ty); } - unsigned ScalarCost = static_cast(this)->getIntrinsicInstrCost( + unsigned ScalarCost = ConcreteTTI->getIntrinsicInstrCost( IID, RetTy->getScalarType(), ScalarTys, FMF); for (unsigned i = 0, ie = Tys.size(); i != ie; ++i) { if (Tys[i]->isVectorTy()) { diff --git a/include/llvm/CodeGen/BuiltinGCs.h b/include/llvm/CodeGen/BuiltinGCs.h index 1767922fb5ac..d44183dab0f7 100644 --- a/include/llvm/CodeGen/BuiltinGCs.h +++ b/include/llvm/CodeGen/BuiltinGCs.h @@ -1,9 +1,8 @@ //===-- BuiltinGCs.h - Garbage collector linkage hacks --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/CSEConfigBase.h b/include/llvm/CodeGen/CSEConfigBase.h new file mode 100644 index 000000000000..70b5e5c17eb1 --- /dev/null +++ b/include/llvm/CodeGen/CSEConfigBase.h @@ -0,0 +1,28 @@ +//===- CSEConfigBase.h - A CSEConfig interface ------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_CSECONFIG_BASE_H +#define LLVM_CODEGEN_CSECONFIG_BASE_H + +namespace llvm { +// Class representing some configuration that can be done during GlobalISel's +// CSEInfo analysis. We define it here because TargetPassConfig can't depend on +// the GlobalISel library, and so we use this in the interface between them +// so that the derived classes in GISel can reference generic opcodes. +class CSEConfigBase { +public: + virtual ~CSEConfigBase() = default; + // Hook for defining which Generic instructions should be CSEd. + // GISelCSEInfo currently only calls this hook when dealing with generic + // opcodes. + virtual bool shouldCSEOpc(unsigned Opc) { return false; } +}; + +} // namespace llvm + +#endif // LLVM_CODEGEN_CSECONFIG_BASE_H diff --git a/include/llvm/CodeGen/CalcSpillWeights.h b/include/llvm/CodeGen/CalcSpillWeights.h index f85767f1fc11..9b8b7324f30a 100644 --- a/include/llvm/CodeGen/CalcSpillWeights.h +++ b/include/llvm/CodeGen/CalcSpillWeights.h @@ -1,9 +1,8 @@ //===- lib/CodeGen/CalcSpillWeights.h ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/CodeGen/CallingConvLower.h b/include/llvm/CodeGen/CallingConvLower.h index efcf80ba0b4e..aa339e1cc913 100644 --- a/include/llvm/CodeGen/CallingConvLower.h +++ b/include/llvm/CodeGen/CallingConvLower.h @@ -1,9 +1,8 @@ //===- llvm/CallingConvLower.h - Calling Conventions ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -146,7 +145,7 @@ public: bool needsCustom() const { return isCustom; } - unsigned getLocReg() const { assert(isRegLoc()); return Loc; } + Register getLocReg() const { assert(isRegLoc()); return Loc; } unsigned getLocMemOffset() const { assert(isMemLoc()); return Loc; } unsigned getExtraInfo() const { return Loc; } MVT getLocVT() const { return LocVT; } @@ -557,7 +556,7 @@ public: // Sort the locations of the arguments according to their original position. SmallVector TmpArgLocs; - std::swap(TmpArgLocs, Locs); + TmpArgLocs.swap(Locs); auto B = TmpArgLocs.begin(), E = TmpArgLocs.end(); std::merge(B, B + NumFirstPassLocs, B + NumFirstPassLocs, E, std::back_inserter(Locs), diff --git a/include/llvm/CodeGen/CommandFlags.inc b/include/llvm/CodeGen/CommandFlags.inc index 568d329a5e8c..cb69e9f61405 100644 --- a/include/llvm/CodeGen/CommandFlags.inc +++ b/include/llvm/CodeGen/CommandFlags.inc @@ -1,9 +1,8 @@ //===-- CommandFlags.h - Command Line Flags Interface -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -272,6 +271,11 @@ static cl::opt EnableAddrsig("addrsig", cl::desc("Emit an address-significance table"), cl::init(false)); +static cl::opt + EnableDebugEntryValues("debug-entry-values", + cl::desc("Emit debug info about parameter's entry values"), + cl::init(false)); + // Common utility function tightly tied to the options listed here. Initializes // a TargetOptions object with CodeGen flags and returns it. static TargetOptions InitTargetOptionsFromCodeGenFlags() { @@ -301,6 +305,7 @@ static TargetOptions InitTargetOptionsFromCodeGenFlags() { Options.ExceptionModel = ExceptionModel; Options.EmitStackSizeSection = EnableStackSizeSection; Options.EmitAddrsig = EnableAddrsig; + Options.EnableDebugEntryValues = EnableDebugEntryValues; Options.MCOptions = InitMCTargetOptionsFromFlags(); diff --git a/include/llvm/CodeGen/CostTable.h b/include/llvm/CodeGen/CostTable.h index 48ad76971520..52f3bfaea180 100644 --- a/include/llvm/CodeGen/CostTable.h +++ b/include/llvm/CodeGen/CostTable.h @@ -1,9 +1,8 @@ //===-- CostTable.h - Instruction Cost Table handling -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/include/llvm/CodeGen/DAGCombine.h b/include/llvm/CodeGen/DAGCombine.h index 8b5919005451..944187341455 100644 --- a/include/llvm/CodeGen/DAGCombine.h +++ b/include/llvm/CodeGen/DAGCombine.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/DAGCombine.h ------- SelectionDAG Nodes ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/DFAPacketizer.h b/include/llvm/CodeGen/DFAPacketizer.h index d3aabe22f216..cf58ee0cabea 100644 --- a/include/llvm/CodeGen/DFAPacketizer.h +++ b/include/llvm/CodeGen/DFAPacketizer.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/DFAPacketizer.h - DFA Packetizer for VLIW ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This class implements a deterministic finite automaton (DFA) based diff --git a/include/llvm/CodeGen/DIE.h b/include/llvm/CodeGen/DIE.h index 7d486b1df56d..684f9e40ca5a 100644 --- a/include/llvm/CodeGen/DIE.h +++ b/include/llvm/CodeGen/DIE.h @@ -1,9 +1,8 @@ //===- lib/CodeGen/DIE.h - DWARF Info Entries -------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -39,6 +38,7 @@ namespace llvm { class AsmPrinter; class DIE; class DIEUnit; +class DwarfCompileUnit; class MCExpr; class MCSection; class MCSymbol; @@ -230,6 +230,25 @@ public: void print(raw_ostream &O) const; }; +//===--------------------------------------------------------------------===// +/// A BaseTypeRef DIE. +class DIEBaseTypeRef { + const DwarfCompileUnit *CU; + const uint64_t Index; + static constexpr unsigned ULEB128PadSize = 4; + +public: + explicit DIEBaseTypeRef(const DwarfCompileUnit *TheCU, uint64_t Idx) + : CU(TheCU), Index(Idx) {} + + /// EmitValue - Emit base type reference. + void EmitValue(const AsmPrinter *AP, dwarf::Form Form) const; + /// SizeOf - Determine size of the base type reference in bytes. + unsigned SizeOf(const AsmPrinter *AP, dwarf::Form Form) const; + + void print(raw_ostream &O) const; +}; + //===--------------------------------------------------------------------===// /// A simple label difference DIE. /// @@ -350,7 +369,7 @@ private: /// should be stored by reference instead of by value. using ValTy = AlignedCharArrayUnion; + DIELoc *, DIELocList, DIEBaseTypeRef *>; static_assert(sizeof(ValTy) <= sizeof(uint64_t) || sizeof(ValTy) <= sizeof(void *), @@ -502,6 +521,18 @@ struct IntrusiveBackListBase { } Last = &N; } + + void push_front(Node &N) { + assert(N.Next.getPointer() == &N && "Expected unlinked node"); + assert(N.Next.getInt() == true && "Expected unlinked node"); + + if (Last) { + N.Next.setPointerAndInt(Last->Next.getPointer(), false); + Last->Next.setPointerAndInt(&N, true); + } else { + Last = &N; + } + } }; template class IntrusiveBackList : IntrusiveBackListBase { @@ -509,8 +540,15 @@ public: using IntrusiveBackListBase::empty; void push_back(T &N) { IntrusiveBackListBase::push_back(N); } + void push_front(T &N) { IntrusiveBackListBase::push_front(N); } T &back() { return *static_cast(Last); } const T &back() const { return *static_cast(Last); } + T &front() { + return *static_cast(Last ? Last->Next.getPointer() : nullptr); + } + const T &front() const { + return *static_cast(Last ? Last->Next.getPointer() : nullptr); + } class const_iterator; class iterator @@ -760,7 +798,7 @@ public: /// /// \returns the DIEUnit that represents the compile or type unit that owns /// this DIE, or NULL if this DIE hasn't been added to a unit DIE. - const DIEUnit *getUnit() const; + DIEUnit *getUnit() const; void setOffset(unsigned O) { Offset = O; } void setSize(unsigned S) { Size = S; } @@ -773,6 +811,13 @@ public: return Children.back(); } + DIE &addChildFront(DIE *Child) { + assert(!Child->getParent() && "Child should be orphaned"); + Child->Owner = this; + Children.push_front(*Child); + return Children.front(); + } + /// Find a value in the DIE with the attribute given. /// /// Returns a default-constructed DIEValue (where \a DIEValue::getType() @@ -800,7 +845,7 @@ class DIEUnit { const uint16_t Version; /// The Dwarf version number for this unit. const uint8_t AddrSize; /// The size in bytes of an address for this unit. protected: - ~DIEUnit() = default; + virtual ~DIEUnit() = default; public: DIEUnit(uint16_t Version, uint8_t AddrSize, dwarf::Tag UnitTag); diff --git a/include/llvm/CodeGen/DIEValue.def b/include/llvm/CodeGen/DIEValue.def index a3fce9b1d20c..92afeb3868b4 100644 --- a/include/llvm/CodeGen/DIEValue.def +++ b/include/llvm/CodeGen/DIEValue.def @@ -1,9 +1,8 @@ //===- llvm/CodeGen/DIEValue.def - DIEValue types ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -35,6 +34,7 @@ HANDLE_DIEVALUE_SMALL(Integer) HANDLE_DIEVALUE_SMALL(String) HANDLE_DIEVALUE_SMALL(Expr) HANDLE_DIEVALUE_SMALL(Label) +HANDLE_DIEVALUE_LARGE(BaseTypeRef) HANDLE_DIEVALUE_LARGE(Delta) HANDLE_DIEVALUE_SMALL(Entry) HANDLE_DIEVALUE_LARGE(Block) diff --git a/include/llvm/CodeGen/DbgEntityHistoryCalculator.h b/include/llvm/CodeGen/DbgEntityHistoryCalculator.h index befc28f084e7..7eec75bc81bf 100644 --- a/include/llvm/CodeGen/DbgEntityHistoryCalculator.h +++ b/include/llvm/CodeGen/DbgEntityHistoryCalculator.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/DbgEntityHistoryCalculator.h ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -11,6 +10,7 @@ #define LLVM_CODEGEN_DBGVALUEHISTORYCALCULATOR_H #include "llvm/ADT/MapVector.h" +#include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/DebugInfoMetadata.h" #include @@ -22,35 +22,76 @@ class MachineFunction; class MachineInstr; class TargetRegisterInfo; -// For each user variable, keep a list of instruction ranges where this variable -// is accessible. The variables are listed in order of appearance. +/// For each user variable, keep a list of instruction ranges where this +/// variable is accessible. The variables are listed in order of appearance. class DbgValueHistoryMap { - // Each instruction range starts with a DBG_VALUE instruction, specifying the - // location of a variable, which is assumed to be valid until the end of the - // range. If end is not specified, location is valid until the start - // instruction of the next instruction range, or until the end of the - // function. public: - using InstrRange = std::pair; - using InstrRanges = SmallVector; + /// Index in the entry vector. + typedef size_t EntryIndex; + + /// Special value to indicate that an entry is valid until the end of the + /// function. + static const EntryIndex NoEntry = std::numeric_limits::max(); + + /// Specifies a change in a variable's debug value history. + /// + /// There exist two types of entries: + /// + /// * Debug value entry: + /// + /// A new debug value becomes live. If the entry's \p EndIndex is \p NoEntry, + /// the value is valid until the end of the function. For other values, the + /// index points to the entry in the entry vector that ends this debug + /// value. The ending entry can either be an overlapping debug value, or + /// an instruction that clobbers the value. + /// + /// * Clobbering entry: + /// + /// This entry's instruction clobbers one or more preceding + /// register-described debug values that have their end index + /// set to this entry's position in the entry vector. + class Entry { + public: + enum EntryKind { DbgValue, Clobber }; + + Entry(const MachineInstr *Instr, EntryKind Kind) + : Instr(Instr, Kind), EndIndex(NoEntry) {} + + const MachineInstr *getInstr() const { return Instr.getPointer(); } + EntryIndex getEndIndex() const { return EndIndex; } + EntryKind getEntryKind() const { return Instr.getInt(); } + + bool isClobber() const { return getEntryKind() == Clobber; } + bool isDbgValue() const { return getEntryKind() == DbgValue; } + bool isClosed() const { return EndIndex != NoEntry; } + + void endEntry(EntryIndex EndIndex); + + private: + PointerIntPair Instr; + EntryIndex EndIndex; + }; + using Entries = SmallVector; using InlinedEntity = std::pair; - using InstrRangesMap = MapVector; + using EntriesMap = MapVector; private: - InstrRangesMap VarInstrRanges; + EntriesMap VarEntries; public: - void startInstrRange(InlinedEntity Var, const MachineInstr &MI); - void endInstrRange(InlinedEntity Var, const MachineInstr &MI); - - // Returns register currently describing @Var. If @Var is currently - // unaccessible or is not described by a register, returns 0. - unsigned getRegisterForVar(InlinedEntity Var) const; - - bool empty() const { return VarInstrRanges.empty(); } - void clear() { VarInstrRanges.clear(); } - InstrRangesMap::const_iterator begin() const { return VarInstrRanges.begin(); } - InstrRangesMap::const_iterator end() const { return VarInstrRanges.end(); } + bool startDbgValue(InlinedEntity Var, const MachineInstr &MI, + EntryIndex &NewIndex); + EntryIndex startClobber(InlinedEntity Var, const MachineInstr &MI); + + Entry &getEntry(InlinedEntity Var, EntryIndex Index) { + auto &Entries = VarEntries[Var]; + return Entries[Index]; + } + + bool empty() const { return VarEntries.empty(); } + void clear() { VarEntries.clear(); } + EntriesMap::const_iterator begin() const { return VarEntries.begin(); } + EntriesMap::const_iterator end() const { return VarEntries.end(); } #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) LLVM_DUMP_METHOD void dump() const; diff --git a/include/llvm/CodeGen/DebugHandlerBase.h b/include/llvm/CodeGen/DebugHandlerBase.h index 4f0d14d317f2..4008d597395e 100644 --- a/include/llvm/CodeGen/DebugHandlerBase.h +++ b/include/llvm/CodeGen/DebugHandlerBase.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/DebugHandlerBase.h -----------------------*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -130,7 +129,7 @@ public: const MCExpr *getFunctionLocalOffsetAfterInsn(const MachineInstr *MI); /// If this type is derived from a base type then return base type size. - static uint64_t getBaseTypeSize(const DITypeRef TyRef); + static uint64_t getBaseTypeSize(const DIType *Ty); }; } diff --git a/include/llvm/CodeGen/DwarfStringPoolEntry.h b/include/llvm/CodeGen/DwarfStringPoolEntry.h index 8b1a7af17bbf..e189352a7b2d 100644 --- a/include/llvm/CodeGen/DwarfStringPoolEntry.h +++ b/include/llvm/CodeGen/DwarfStringPoolEntry.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/DwarfStringPoolEntry.h - String pool entry --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/CodeGen/EdgeBundles.h b/include/llvm/CodeGen/EdgeBundles.h index c31fad246c96..28cdf54e0575 100644 --- a/include/llvm/CodeGen/EdgeBundles.h +++ b/include/llvm/CodeGen/EdgeBundles.h @@ -1,9 +1,8 @@ //===-------- EdgeBundles.h - Bundles of CFG edges --------------*- c++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/ExecutionDomainFix.h b/include/llvm/CodeGen/ExecutionDomainFix.h index 338c214dd073..6836678e2101 100644 --- a/include/llvm/CodeGen/ExecutionDomainFix.h +++ b/include/llvm/CodeGen/ExecutionDomainFix.h @@ -1,9 +1,8 @@ //==-- llvm/CodeGen/ExecutionDomainFix.h - Execution Domain Fix -*- C++ -*--==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/ExpandReductions.h b/include/llvm/CodeGen/ExpandReductions.h index c6aaaad967b3..5dbed07873c1 100644 --- a/include/llvm/CodeGen/ExpandReductions.h +++ b/include/llvm/CodeGen/ExpandReductions.h @@ -1,9 +1,8 @@ //===----- ExpandReductions.h - Expand experimental reduction intrinsics --===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/CodeGen/FastISel.h b/include/llvm/CodeGen/FastISel.h index 865d8a88b8cc..f09b59daf4dd 100644 --- a/include/llvm/CodeGen/FastISel.h +++ b/include/llvm/CodeGen/FastISel.h @@ -1,9 +1,8 @@ //===- FastISel.h - Definition of the FastISel class ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -528,7 +527,7 @@ protected: /// Select and emit code for a binary operator instruction, which has /// an opcode which directly corresponds to the given ISD opcode. bool selectBinaryOp(const User *I, unsigned ISDOpcode); - bool selectFNeg(const User *I); + bool selectFNeg(const User *I, const Value *In); bool selectGetElementPtr(const User *I); bool selectStackmap(const CallInst *I); bool selectPatchpoint(const CallInst *I); diff --git a/include/llvm/CodeGen/FaultMaps.h b/include/llvm/CodeGen/FaultMaps.h index 55e25c9823b1..a1e2349c413e 100644 --- a/include/llvm/CodeGen/FaultMaps.h +++ b/include/llvm/CodeGen/FaultMaps.h @@ -1,9 +1,8 @@ //===- FaultMaps.h - The "FaultMaps" section --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/CodeGen/FunctionLoweringInfo.h b/include/llvm/CodeGen/FunctionLoweringInfo.h index 7c658515de09..fb60191abd3a 100644 --- a/include/llvm/CodeGen/FunctionLoweringInfo.h +++ b/include/llvm/CodeGen/FunctionLoweringInfo.h @@ -1,9 +1,8 @@ //===- FunctionLoweringInfo.h - Lower functions from LLVM IR ---*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,13 +13,14 @@ #ifndef LLVM_CODEGEN_FUNCTIONLOWERINGINFO_H #define LLVM_CODEGEN_FUNCTIONLOWERINGINFO_H - #include "llvm/ADT/APInt.h" +#include "llvm/ADT/BitVector.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/IndexedMap.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/LegacyDivergenceAnalysis.h" #include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/TargetRegisterInfo.h" @@ -57,6 +57,7 @@ public: const TargetLowering *TLI; MachineRegisterInfo *RegInfo; BranchProbabilityInfo *BPI; + const LegacyDivergenceAnalysis *DA; /// CanLowerReturn - true iff the function's return value can be lowered to /// registers. bool CanLowerReturn; @@ -71,48 +72,6 @@ public: /// MBBMap - A mapping from LLVM basic blocks to their machine code entry. DenseMap MBBMap; - /// A map from swifterror value in a basic block to the virtual register it is - /// currently represented by. - DenseMap, unsigned> - SwiftErrorVRegDefMap; - - /// A list of upward exposed vreg uses that need to be satisfied by either a - /// copy def or a phi node at the beginning of the basic block representing - /// the predecessor(s) swifterror value. - DenseMap, unsigned> - SwiftErrorVRegUpwardsUse; - - /// A map from instructions that define/use a swifterror value to the virtual - /// register that represents that def/use. - llvm::DenseMap, unsigned> - SwiftErrorVRegDefUses; - - /// The swifterror argument of the current function. - const Value *SwiftErrorArg; - - using SwiftErrorValues = SmallVector; - /// A function can only have a single swifterror argument. And if it does - /// have a swifterror argument, it must be the first entry in - /// SwiftErrorVals. - SwiftErrorValues SwiftErrorVals; - - /// Get or create the swifterror value virtual register in - /// SwiftErrorVRegDefMap for this basic block. - unsigned getOrCreateSwiftErrorVReg(const MachineBasicBlock *, - const Value *); - - /// Set the swifterror virtual register in the SwiftErrorVRegDefMap for this - /// basic block. - void setCurrentSwiftErrorVReg(const MachineBasicBlock *MBB, const Value *, - unsigned); - - /// Get or create the swifterror value virtual register for a def of a - /// swifterror by an instruction. - std::pair getOrCreateSwiftErrorVRegDefAt(const Instruction *); - std::pair - getOrCreateSwiftErrorVRegUseAt(const Instruction *, const MachineBasicBlock *, - const Value *); - /// ValueMap - Since we emit code for the function a basic block at a time, /// we must remember which virtual registers hold the values for /// cross-basic-block values. @@ -175,6 +134,10 @@ public: /// function arguments that are inserted after scheduling is completed. SmallVector ArgDbgValues; + /// Bitvector with a bit set if corresponding argument is described in + /// ArgDbgValues. Using arg numbers according to Argument numbering. + BitVector DescribedArgs; + /// RegFixups - Registers which need to be replaced after isel is done. DenseMap RegFixups; @@ -236,9 +199,11 @@ public: return ValueMap.count(V); } - unsigned CreateReg(MVT VT); + unsigned CreateReg(MVT VT, bool isDivergent = false); + + unsigned CreateRegs(const Value *V); - unsigned CreateRegs(Type *Ty); + unsigned CreateRegs(Type *Ty, bool isDivergent = false); unsigned InitializeRegForValue(const Value *V) { // Tokens never live in vregs. @@ -247,7 +212,7 @@ public: unsigned &R = ValueMap[V]; assert(R == 0 && "Already initialized this value register!"); assert(VirtReg2Value.empty()); - return R = CreateRegs(V->getType()); + return R = CreateRegs(V); } /// GetLiveOutRegInfo - Gets LiveOutInfo for a register, returning NULL if the diff --git a/include/llvm/CodeGen/GCMetadata.h b/include/llvm/CodeGen/GCMetadata.h index 7fb27202c122..77cd356c49dd 100644 --- a/include/llvm/CodeGen/GCMetadata.h +++ b/include/llvm/CodeGen/GCMetadata.h @@ -1,9 +1,8 @@ //===- GCMetadata.h - Garbage collector metadata ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/GCMetadataPrinter.h b/include/llvm/CodeGen/GCMetadataPrinter.h index 5f1efb2ce02c..f9527c9f8752 100644 --- a/include/llvm/CodeGen/GCMetadataPrinter.h +++ b/include/llvm/CodeGen/GCMetadataPrinter.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GCMetadataPrinter.h - Prints asm GC tables --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/GCStrategy.h b/include/llvm/CodeGen/GCStrategy.h index 5a60cd7cb823..c5731528da4e 100644 --- a/include/llvm/CodeGen/GCStrategy.h +++ b/include/llvm/CodeGen/GCStrategy.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GCStrategy.h - Garbage collection -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/GlobalISel/CSEInfo.h b/include/llvm/CodeGen/GlobalISel/CSEInfo.h index ce2d285a99e5..5a44e67992ad 100644 --- a/include/llvm/CodeGen/GlobalISel/CSEInfo.h +++ b/include/llvm/CodeGen/GlobalISel/CSEInfo.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/CSEInfo.h ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,6 +13,7 @@ #define LLVM_CODEGEN_GLOBALISEL_CSEINFO_H #include "llvm/ADT/FoldingSet.h" +#include "llvm/CodeGen/CSEConfigBase.h" #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" #include "llvm/CodeGen/GlobalISel/GISelWorkList.h" #include "llvm/CodeGen/GlobalISel/Utils.h" @@ -37,25 +37,27 @@ public: void Profile(FoldingSetNodeID &ID); }; -// Class representing some configuration that can be done during CSE analysis. -// Currently it only supports shouldCSE method that each pass can set. -class CSEConfig { +// A CSE config for fully optimized builds. +class CSEConfigFull : public CSEConfigBase { public: - virtual ~CSEConfig() = default; - // Hook for defining which Generic instructions should be CSEd. - // GISelCSEInfo currently only calls this hook when dealing with generic - // opcodes. - virtual bool shouldCSEOpc(unsigned Opc); + virtual ~CSEConfigFull() = default; + virtual bool shouldCSEOpc(unsigned Opc) override; }; -// TODO: Find a better place for this. // Commonly used for O0 config. -class CSEConfigConstantOnly : public CSEConfig { +class CSEConfigConstantOnly : public CSEConfigBase { public: virtual ~CSEConfigConstantOnly() = default; virtual bool shouldCSEOpc(unsigned Opc) override; }; +// Returns the standard expected CSEConfig for the given optimization level. +// We have this logic here so targets can make use of it from their derived +// TargetPassConfig, but can't put this logic into TargetPassConfig directly +// because the CodeGen library can't depend on GlobalISel. +std::unique_ptr +getStandardCSEConfigForOpt(CodeGenOpt::Level Level); + /// The CSE Analysis object. /// This installs itself as a delegate to the MachineFunction to track /// new instructions as well as deletions. It however will not be able to @@ -74,7 +76,7 @@ class GISelCSEInfo : public GISelChangeObserver { FoldingSet CSEMap; MachineRegisterInfo *MRI = nullptr; MachineFunction *MF = nullptr; - std::unique_ptr CSEOpt; + std::unique_ptr CSEOpt; /// Keep a cache of UniqueInstrs for each MachineInstr. In GISel, /// often instructions are mutated (while their ID has completely changed). /// Whenever mutation happens, invalidate the UniqueMachineInstr for the @@ -139,7 +141,9 @@ public: void releaseMemory(); - void setCSEConfig(std::unique_ptr Opt) { CSEOpt = std::move(Opt); } + void setCSEConfig(std::unique_ptr Opt) { + CSEOpt = std::move(Opt); + } bool shouldCSE(unsigned Opc) const; @@ -199,11 +203,12 @@ class GISelCSEAnalysisWrapper { bool AlreadyComputed = false; public: - /// Takes a CSEConfig object that defines what opcodes get CSEd. + /// Takes a CSEConfigBase object that defines what opcodes get CSEd. /// If CSEConfig is already set, and the CSE Analysis has been preserved, /// it will not use the new CSEOpt(use Recompute to force using the new /// CSEOpt). - GISelCSEInfo &get(std::unique_ptr CSEOpt, bool ReCompute = false); + GISelCSEInfo &get(std::unique_ptr CSEOpt, + bool ReCompute = false); void setMF(MachineFunction &MFunc) { MF = &MFunc; } void setComputed(bool Computed) { AlreadyComputed = Computed; } void releaseMemory() { Info.releaseMemory(); } diff --git a/include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h b/include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h index a8fb736ebbb5..4f95335db74b 100644 --- a/include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h +++ b/include/llvm/CodeGen/GlobalISel/CSEMIRBuilder.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/GlobalISel/CSEMIRBuilder.h --*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/CodeGen/GlobalISel/CallLowering.h b/include/llvm/CodeGen/GlobalISel/CallLowering.h index ab498e8f070b..d717121ad78e 100644 --- a/include/llvm/CodeGen/GlobalISel/CallLowering.h +++ b/include/llvm/CodeGen/GlobalISel/CallLowering.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/CallLowering.h - Call lowering ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -16,6 +15,7 @@ #define LLVM_CODEGEN_GLOBALISEL_CALLLOWERING_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/CallingConvLower.h" #include "llvm/CodeGen/TargetCallingConv.h" #include "llvm/IR/CallSite.h" @@ -27,6 +27,7 @@ namespace llvm { +class CCState; class DataLayout; class Function; class MachineIRBuilder; @@ -43,14 +44,19 @@ class CallLowering { virtual void anchor(); public: struct ArgInfo { - unsigned Reg; + SmallVector Regs; Type *Ty; ISD::ArgFlagsTy Flags; bool IsFixed; - ArgInfo(unsigned Reg, Type *Ty, ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy{}, - bool IsFixed = true) - : Reg(Reg), Ty(Ty), Flags(Flags), IsFixed(IsFixed) {} + ArgInfo(ArrayRef Regs, Type *Ty, + ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy{}, bool IsFixed = true) + : Regs(Regs.begin(), Regs.end()), Ty(Ty), Flags(Flags), + IsFixed(IsFixed) { + // FIXME: We should have just one way of saying "no register". + assert((Ty->isVoidTy() == (Regs.empty() || Regs[0] == 0)) && + "only void types should have no register"); + } }; /// Argument handling is mostly uniform between the four places that @@ -66,24 +72,28 @@ public: virtual ~ValueHandler() = default; + /// Returns true if the handler is dealing with formal arguments, + /// not with return values etc. + virtual bool isArgumentHandler() const { return false; } + /// Materialize a VReg containing the address of the specified /// stack-based object. This is either based on a FrameIndex or /// direct SP manipulation, depending on the context. \p MPO /// should be initialized to an appropriate description of the /// address created. - virtual unsigned getStackAddress(uint64_t Size, int64_t Offset, + virtual Register getStackAddress(uint64_t Size, int64_t Offset, MachinePointerInfo &MPO) = 0; /// The specified value has been assigned to a physical register, /// handle the appropriate COPY (either to or from) and mark any /// relevant uses/defines as needed. - virtual void assignValueToReg(unsigned ValVReg, unsigned PhysReg, + virtual void assignValueToReg(Register ValVReg, Register PhysReg, CCValAssign &VA) = 0; /// The specified value has been assigned to a stack /// location. Load or store it there, with appropriate extension /// if necessary. - virtual void assignValueToAddress(unsigned ValVReg, unsigned Addr, + virtual void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size, MachinePointerInfo &MPO, CCValAssign &VA) = 0; @@ -98,7 +108,7 @@ public: llvm_unreachable("Custom values not supported"); } - unsigned extendRegister(unsigned ValReg, CCValAssign &VA); + Register extendRegister(Register ValReg, CCValAssign &VA); virtual bool assignArg(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, const ArgInfo &Info, @@ -130,39 +140,83 @@ protected: void setArgFlags(ArgInfo &Arg, unsigned OpIdx, const DataLayout &DL, const FuncInfoTy &FuncInfo) const; + /// Generate instructions for packing \p SrcRegs into one big register + /// corresponding to the aggregate type \p PackedTy. + /// + /// \param SrcRegs should contain one virtual register for each base type in + /// \p PackedTy, as returned by computeValueLLTs. + /// + /// \return The packed register. + Register packRegs(ArrayRef SrcRegs, Type *PackedTy, + MachineIRBuilder &MIRBuilder) const; + + /// Generate instructions for unpacking \p SrcReg into the \p DstRegs + /// corresponding to the aggregate type \p PackedTy. + /// + /// \param DstRegs should contain one virtual register for each base type in + /// \p PackedTy, as returned by computeValueLLTs. + void unpackRegs(ArrayRef DstRegs, Register SrcReg, Type *PackedTy, + MachineIRBuilder &MIRBuilder) const; + /// Invoke Handler::assignArg on each of the given \p Args and then use /// \p Callback to move them to the assigned locations. /// /// \return True if everything has succeeded, false otherwise. bool handleAssignments(MachineIRBuilder &MIRBuilder, ArrayRef Args, ValueHandler &Handler) const; - + bool handleAssignments(CCState &CCState, + SmallVectorImpl &ArgLocs, + MachineIRBuilder &MIRBuilder, ArrayRef Args, + ValueHandler &Handler) const; public: CallLowering(const TargetLowering *TLI) : TLI(TLI) {} virtual ~CallLowering() = default; + /// \return true if the target is capable of handling swifterror values that + /// have been promoted to a specified register. The extended versions of + /// lowerReturn and lowerCall should be implemented. + virtual bool supportSwiftError() const { + return false; + } + /// This hook must be implemented to lower outgoing return values, described /// by \p Val, into the specified virtual registers \p VRegs. /// This hook is used by GlobalISel. /// + /// \p SwiftErrorVReg is non-zero if the function has a swifterror parameter + /// that needs to be implicitly returned. + /// /// \return True if the lowering succeeds, false otherwise. virtual bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, - ArrayRef VRegs) const { + ArrayRef VRegs, + Register SwiftErrorVReg) const { + if (!supportSwiftError()) { + assert(SwiftErrorVReg == 0 && "attempt to use unsupported swifterror"); + return lowerReturn(MIRBuilder, Val, VRegs); + } + return false; + } + + /// This hook behaves as the extended lowerReturn function, but for targets + /// that do not support swifterror value promotion. + virtual bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, + ArrayRef VRegs) const { return false; } /// This hook must be implemented to lower the incoming (formal) - /// arguments, described by \p Args, for GlobalISel. Each argument - /// must end up in the related virtual register described by VRegs. - /// In other words, the first argument should end up in VRegs[0], - /// the second in VRegs[1], and so on. + /// arguments, described by \p VRegs, for GlobalISel. Each argument + /// must end up in the related virtual registers described by \p VRegs. + /// In other words, the first argument should end up in \c VRegs[0], + /// the second in \c VRegs[1], and so on. For each argument, there will be one + /// register for each non-aggregate type, as returned by \c computeValueLLTs. /// \p MIRBuilder is set to the proper insertion for the argument /// lowering. /// /// \return True if the lowering succeeded, false otherwise. virtual bool lowerFormalArguments(MachineIRBuilder &MIRBuilder, const Function &F, - ArrayRef VRegs) const { + ArrayRef> VRegs) const { return false; } @@ -174,18 +228,29 @@ public: /// \p Callee is the destination of the call. It should be either a register, /// globaladdress, or externalsymbol. /// - /// \p ResTy is the type returned by the function - /// - /// \p ResReg is the generic virtual register that the returned - /// value should be lowered into. + /// \p OrigRet is a descriptor for the return type of the function. /// - /// \p ArgTys is a list of the types each member of \p ArgRegs has; used by - /// the target to decide which register/stack slot should be allocated. + /// \p OrigArgs is a list of descriptors of the arguments passed to the + /// function. /// - /// \p ArgRegs is a list of virtual registers containing each argument that - /// needs to be passed. + /// \p SwiftErrorVReg is non-zero if the call has a swifterror inout + /// parameter, and contains the vreg that the swifterror should be copied into + /// after the call. /// /// \return true if the lowering succeeded, false otherwise. + virtual bool lowerCall(MachineIRBuilder &MIRBuilder, CallingConv::ID CallConv, + const MachineOperand &Callee, const ArgInfo &OrigRet, + ArrayRef OrigArgs, + Register SwiftErrorVReg) const { + if (!supportSwiftError()) { + assert(SwiftErrorVReg == 0 && "trying to use unsupported swifterror"); + return lowerCall(MIRBuilder, CallConv, Callee, OrigRet, OrigArgs); + } + return false; + } + + /// This hook behaves as the extended lowerCall function, but for targets that + /// do not support swifterror value promotion. virtual bool lowerCall(MachineIRBuilder &MIRBuilder, CallingConv::ID CallConv, const MachineOperand &Callee, const ArgInfo &OrigRet, ArrayRef OrigArgs) const { @@ -197,11 +262,18 @@ public: /// /// \p CI is the call/invoke instruction. /// - /// \p ResReg is a register where the call's return value should be stored (or - /// 0 if there is no return value). + /// \p ResRegs are the registers where the call's return value should be + /// stored (or 0 if there is no return value). There will be one register for + /// each non-aggregate type, as returned by \c computeValueLLTs. + /// + /// \p ArgRegs is a list of lists of virtual registers containing each + /// argument that needs to be passed (argument \c i should be placed in \c + /// ArgRegs[i]). For each argument, there will be one register for each + /// non-aggregate type, as returned by \c computeValueLLTs. /// - /// \p ArgRegs is a list of virtual registers containing each argument that - /// needs to be passed. + /// \p SwiftErrorVReg is non-zero if the call has a swifterror inout + /// parameter, and contains the vreg that the swifterror should be copied into + /// after the call. /// /// \p GetCalleeReg is a callback to materialize a register for the callee if /// the target determines it cannot jump to the destination based purely on \p @@ -210,7 +282,8 @@ public: /// /// \return true if the lowering succeeded, false otherwise. bool lowerCall(MachineIRBuilder &MIRBuilder, ImmutableCallSite CS, - unsigned ResReg, ArrayRef ArgRegs, + ArrayRef ResRegs, + ArrayRef> ArgRegs, Register SwiftErrorVReg, std::function GetCalleeReg) const; }; diff --git a/include/llvm/CodeGen/GlobalISel/Combiner.h b/include/llvm/CodeGen/GlobalISel/Combiner.h index b097c7817762..efe8bdf93664 100644 --- a/include/llvm/CodeGen/GlobalISel/Combiner.h +++ b/include/llvm/CodeGen/GlobalISel/Combiner.h @@ -1,9 +1,8 @@ -//== ----- llvm/CodeGen/GlobalISel/Combiner.h --------------------- == // +//== ----- llvm/CodeGen/GlobalISel/Combiner.h -------------------*- C++ -*-== // // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/include/llvm/CodeGen/GlobalISel/CombinerHelper.h index 6e9ac01c1ee2..0c50c9c5e0cf 100644 --- a/include/llvm/CodeGen/GlobalISel/CombinerHelper.h +++ b/include/llvm/CodeGen/GlobalISel/CombinerHelper.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/GlobalISel/CombinerHelper.h --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===--------------------------------------------------------------------===// // @@ -18,6 +17,9 @@ #ifndef LLVM_CODEGEN_GLOBALISEL_COMBINER_HELPER_H #define LLVM_CODEGEN_GLOBALISEL_COMBINER_HELPER_H +#include "llvm/CodeGen/LowLevelType.h" +#include "llvm/CodeGen/Register.h" + namespace llvm { class GISelChangeObserver; @@ -26,6 +28,12 @@ class MachineRegisterInfo; class MachineInstr; class MachineOperand; +struct PreferredTuple { + LLT Ty; // The result type of the extend. + unsigned ExtendOpcode; // G_ANYEXT/G_SEXT/G_ZEXT + MachineInstr *MI; +}; + class CombinerHelper { MachineIRBuilder &Builder; MachineRegisterInfo &MRI; @@ -35,20 +43,27 @@ public: CombinerHelper(GISelChangeObserver &Observer, MachineIRBuilder &B); /// MachineRegisterInfo::replaceRegWith() and inform the observer of the changes - void replaceRegWith(MachineRegisterInfo &MRI, unsigned FromReg, unsigned ToReg) const; + void replaceRegWith(MachineRegisterInfo &MRI, Register FromReg, Register ToReg) const; /// Replace a single register operand with a new register and inform the /// observer of the changes. void replaceRegOpWith(MachineRegisterInfo &MRI, MachineOperand &FromRegOp, - unsigned ToReg) const; + Register ToReg) const; /// If \p MI is COPY, try to combine it. /// Returns true if MI changed. bool tryCombineCopy(MachineInstr &MI); + bool matchCombineCopy(MachineInstr &MI); + void applyCombineCopy(MachineInstr &MI); /// If \p MI is extend that consumes the result of a load, try to combine it. /// Returns true if MI changed. bool tryCombineExtendingLoads(MachineInstr &MI); + bool matchCombineExtendingLoads(MachineInstr &MI, PreferredTuple &MatchInfo); + void applyCombineExtendingLoads(MachineInstr &MI, PreferredTuple &MatchInfo); + + bool matchCombineBr(MachineInstr &MI); + bool tryCombineBr(MachineInstr &MI); /// Try to transform \p MI by using all of the above /// combine functions. Returns true if changed. diff --git a/include/llvm/CodeGen/GlobalISel/CombinerInfo.h b/include/llvm/CodeGen/GlobalISel/CombinerInfo.h index d21aa3f725d9..3b09a8e2b479 100644 --- a/include/llvm/CodeGen/GlobalISel/CombinerInfo.h +++ b/include/llvm/CodeGen/GlobalISel/CombinerInfo.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/CombinerInfo.h ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h b/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h index 220a571b21db..e817d9b4550e 100644 --- a/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h +++ b/include/llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/GlobalISel/ConstantFoldingMIRBuilder.h --*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h b/include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h index c8e8a7a5a7cb..e5691cb35174 100644 --- a/include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h +++ b/include/llvm/CodeGen/GlobalISel/GISelChangeObserver.h @@ -1,9 +1,8 @@ -//===----- llvm/CodeGen/GlobalISel/GISelChangeObserver.h ------------------===// +//===----- llvm/CodeGen/GlobalISel/GISelChangeObserver.h --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -34,10 +33,17 @@ public: /// An instruction is about to be erased. virtual void erasingInstr(MachineInstr &MI) = 0; - /// An instruction was created and inserted into the function. + + /// An instruction has been created and inserted into the function. + /// Note that the instruction might not be a fully fledged instruction at this + /// point and won't be if the MachineFunction::Delegate is calling it. This is + /// because the delegate only sees the construction of the MachineInstr before + /// operands have been added. virtual void createdInstr(MachineInstr &MI) = 0; + /// This instruction is about to be mutated in some way. virtual void changingInstr(MachineInstr &MI) = 0; + /// This instruction was mutated in some way. virtual void changedInstr(MachineInstr &MI) = 0; diff --git a/include/llvm/CodeGen/GlobalISel/GISelWorkList.h b/include/llvm/CodeGen/GlobalISel/GISelWorkList.h index 1571841a208d..b0bb519283b1 100644 --- a/include/llvm/CodeGen/GlobalISel/GISelWorkList.h +++ b/include/llvm/CodeGen/GlobalISel/GISelWorkList.h @@ -1,9 +1,8 @@ //===- GISelWorkList.h - Worklist for GISel passes ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -33,23 +32,61 @@ class GISelWorkList { SmallVector Worklist; DenseMap WorklistMap; +#ifndef NDEBUG + bool Finalized = true; +#endif + public: - GISelWorkList() {} + GISelWorkList() : WorklistMap(N) {} bool empty() const { return WorklistMap.empty(); } unsigned size() const { return WorklistMap.size(); } + // Since we don't know ahead of time how many instructions we're going to add + // to the worklist, and migrating densemap's elements is quite expensive + // everytime we resize, only insert to the smallvector (typically during the + // initial phase of populating lists). Before the worklist can be used, + // finalize should be called. Also assert with NDEBUG if list is ever used + // without finalizing. Note that unlike insert, we won't check for duplicates + // - so the ideal place to use this is during the initial prepopulating phase + // of most passes. + void deferred_insert(MachineInstr *I) { + Worklist.push_back(I); +#ifndef NDEBUG + Finalized = false; +#endif + } + + // This should only be called when using deferred_insert. + // This asserts that the WorklistMap is empty, and then + // inserts all the elements in the Worklist into the map. + // It also asserts if there are any duplicate elements found. + void finalize() { + assert(WorklistMap.empty() && "Expecting empty worklistmap"); + if (Worklist.size() > N) + WorklistMap.reserve(Worklist.size()); + for (unsigned i = 0; i < Worklist.size(); ++i) + if (!WorklistMap.try_emplace(Worklist[i], i).second) + llvm_unreachable("Duplicate elements in the list"); +#ifndef NDEBUG + Finalized = true; +#endif + } + /// Add the specified instruction to the worklist if it isn't already in it. void insert(MachineInstr *I) { + assert(Finalized && "GISelWorkList used without finalizing"); if (WorklistMap.try_emplace(I, Worklist.size()).second) Worklist.push_back(I); } /// Remove I from the worklist if it exists. void remove(const MachineInstr *I) { + assert((Finalized || WorklistMap.empty()) && "Neither finalized nor empty"); auto It = WorklistMap.find(I); - if (It == WorklistMap.end()) return; // Not in worklist. + if (It == WorklistMap.end()) + return; // Not in worklist. // Don't bother moving everything down, just null out the slot. Worklist[It->second] = nullptr; @@ -63,6 +100,7 @@ public: } MachineInstr *pop_back_val() { + assert(Finalized && "GISelWorkList used without finalizing"); MachineInstr *I; do { I = Worklist.pop_back_val(); diff --git a/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/include/llvm/CodeGen/GlobalISel/IRTranslator.h index d1770bf6e4ce..8654ba83f08d 100644 --- a/include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ b/include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/IRTranslator.h - IRTranslator ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -23,7 +22,9 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/GlobalISel/CSEMIRBuilder.h" #include "llvm/CodeGen/GlobalISel/Types.h" +#include "llvm/CodeGen/SwiftErrorValueTracking.h" #include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/SwitchLoweringUtils.h" #include "llvm/IR/Intrinsics.h" #include "llvm/Support/Allocator.h" #include @@ -37,6 +38,7 @@ class CallInst; class CallLowering; class Constant; class DataLayout; +class FunctionLoweringInfo; class Instruction; class MachineBasicBlock; class MachineFunction; @@ -69,7 +71,7 @@ private: public: ValueToVRegInfo() = default; - using VRegListT = SmallVector; + using VRegListT = SmallVector; using OffsetListT = SmallVector; using const_vreg_iterator = @@ -164,6 +166,8 @@ private: /// this function. DenseMap FrameIndices; + SwiftErrorValueTracking SwiftError; + /// \name Methods for translating form LLVM IR to MachineInstr. /// \see ::translate for general information on the translate methods. /// @{ @@ -196,7 +200,7 @@ private: /// the function. /// /// \return true if the materialization succeeded. - bool translate(const Constant &C, unsigned Reg); + bool translate(const Constant &C, Register Reg); /// Translate an LLVM bitcast into generic IR. Either a COPY or a G_BITCAST is /// emitted. @@ -212,24 +216,27 @@ private: bool translateMemfunc(const CallInst &CI, MachineIRBuilder &MIRBuilder, unsigned ID); - void getStackGuard(unsigned DstReg, MachineIRBuilder &MIRBuilder); + void getStackGuard(Register DstReg, MachineIRBuilder &MIRBuilder); bool translateOverflowIntrinsic(const CallInst &CI, unsigned Op, MachineIRBuilder &MIRBuilder); + /// Helper function for translateSimpleIntrinsic. + /// \return The generic opcode for \p IntrinsicID if \p IntrinsicID is a + /// simple intrinsic (ceil, fabs, etc.). Otherwise, returns + /// Intrinsic::not_intrinsic. + unsigned getSimpleIntrinsicOpcode(Intrinsic::ID ID); + + /// Translates the intrinsics defined in getSimpleIntrinsicOpcode. + /// \return true if the translation succeeded. + bool translateSimpleIntrinsic(const CallInst &CI, Intrinsic::ID ID, + MachineIRBuilder &MIRBuilder); + bool translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder); bool translateInlineAsm(const CallInst &CI, MachineIRBuilder &MIRBuilder); - // FIXME: temporary function to expose previous interface to call lowering - // until it is refactored. - /// Combines all component registers of \p V into a single scalar with size - /// "max(Offsets) + last size". - unsigned packRegs(const Value &V, MachineIRBuilder &MIRBuilder); - - void unpackRegs(const Value &V, unsigned Src, MachineIRBuilder &MIRBuilder); - /// Returns true if the value should be split into multiple LLTs. /// If \p Offsets is given then the split type's offsets will be stored in it. /// If \p Offsets is not empty it will be cleared first. @@ -242,6 +249,8 @@ private: bool translateInvoke(const User &U, MachineIRBuilder &MIRBuilder); + bool translateCallBr(const User &U, MachineIRBuilder &MIRBuilder); + bool translateLandingPad(const User &U, MachineIRBuilder &MIRBuilder); /// Translate one of LLVM's cast instructions into MachineInstrs, with the @@ -278,7 +287,42 @@ private: /// \pre \p U is a branch instruction. bool translateBr(const User &U, MachineIRBuilder &MIRBuilder); + // Begin switch lowering functions. + bool emitJumpTableHeader(SwitchCG::JumpTable &JT, + SwitchCG::JumpTableHeader &JTH, + MachineBasicBlock *HeaderBB); + void emitJumpTable(SwitchCG::JumpTable &JT, MachineBasicBlock *MBB); + + void emitSwitchCase(SwitchCG::CaseBlock &CB, MachineBasicBlock *SwitchBB, + MachineIRBuilder &MIB); + + bool lowerJumpTableWorkItem(SwitchCG::SwitchWorkListItem W, + MachineBasicBlock *SwitchMBB, + MachineBasicBlock *CurMBB, + MachineBasicBlock *DefaultMBB, + MachineIRBuilder &MIB, + MachineFunction::iterator BBI, + BranchProbability UnhandledProbs, + SwitchCG::CaseClusterIt I, + MachineBasicBlock *Fallthrough, + bool FallthroughUnreachable); + + bool lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I, + Value *Cond, + MachineBasicBlock *Fallthrough, + bool FallthroughUnreachable, + BranchProbability UnhandledProbs, + MachineBasicBlock *CurMBB, + MachineIRBuilder &MIB, + MachineBasicBlock *SwitchMBB); + + bool lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W, Value *Cond, + MachineBasicBlock *SwitchMBB, + MachineBasicBlock *DefaultMBB, + MachineIRBuilder &MIB); + bool translateSwitch(const User &U, MachineIRBuilder &MIRBuilder); + // End switch lowering section. bool translateIndirectBr(const User &U, MachineIRBuilder &MIRBuilder); @@ -404,6 +448,7 @@ private: bool translateAtomicCmpXchg(const User &U, MachineIRBuilder &MIRBuilder); bool translateAtomicRMW(const User &U, MachineIRBuilder &MIRBuilder); + bool translateFence(const User &U, MachineIRBuilder &MIRBuilder); // Stubs to keep the compiler happy while we implement the rest of the // translation. @@ -419,9 +464,6 @@ private: bool translateCatchSwitch(const User &U, MachineIRBuilder &MIRBuilder) { return false; } - bool translateFence(const User &U, MachineIRBuilder &MIRBuilder) { - return false; - } bool translateAddrSpaceCast(const User &U, MachineIRBuilder &MIRBuilder) { return translateCast(TargetOpcode::G_ADDRSPACE_CAST, U, MIRBuilder); } @@ -466,19 +508,50 @@ private: /// Current optimization remark emitter. Used to report failures. std::unique_ptr ORE; + FunctionLoweringInfo FuncInfo; + + // True when either the Target Machine specifies no optimizations or the + // function has the optnone attribute. + bool EnableOpts = false; + + /// Switch analysis and optimization. + class GISelSwitchLowering : public SwitchCG::SwitchLowering { + public: + GISelSwitchLowering(IRTranslator *irt, FunctionLoweringInfo &funcinfo) + : SwitchLowering(funcinfo), IRT(irt) { + assert(irt && "irt is null!"); + } + + virtual void addSuccessorWithProb( + MachineBasicBlock *Src, MachineBasicBlock *Dst, + BranchProbability Prob = BranchProbability::getUnknown()) override { + IRT->addSuccessorWithProb(Src, Dst, Prob); + } + + virtual ~GISelSwitchLowering() = default; + + private: + IRTranslator *IRT; + }; + + std::unique_ptr SL; + // * Insert all the code needed to materialize the constants // at the proper place. E.g., Entry block or dominator block // of each constant depending on how fancy we want to be. // * Clear the different maps. void finalizeFunction(); + // Handle emitting jump tables for each basic block. + void finalizeBasicBlock(); + /// Get the VRegs that represent \p Val. /// Non-aggregate types have just one corresponding VReg and the list can be /// used as a single "unsigned". Aggregates get flattened. If such VRegs do /// not exist, they are created. - ArrayRef getOrCreateVRegs(const Value &Val); + ArrayRef getOrCreateVRegs(const Value &Val); - unsigned getOrCreateVReg(const Value &Val) { + Register getOrCreateVReg(const Value &Val) { auto Regs = getOrCreateVRegs(Val); if (Regs.empty()) return 0; @@ -522,6 +595,14 @@ private: return SmallVector(1, &getMBB(*Edge.first)); } + /// Return branch probability calculated by BranchProbabilityInfo for IR + /// blocks. + BranchProbability getEdgeProbability(const MachineBasicBlock *Src, + const MachineBasicBlock *Dst) const; + + void addSuccessorWithProb(MachineBasicBlock *Src, MachineBasicBlock *Dst, + BranchProbability Prob); + public: // Ctor, nothing fancy. IRTranslator(); diff --git a/include/llvm/CodeGen/GlobalISel/InstructionSelect.h b/include/llvm/CodeGen/GlobalISel/InstructionSelect.h index 01521c46ab6a..1af46e0a9e76 100644 --- a/include/llvm/CodeGen/GlobalISel/InstructionSelect.h +++ b/include/llvm/CodeGen/GlobalISel/InstructionSelect.h @@ -1,9 +1,8 @@ //== llvm/CodeGen/GlobalISel/InstructionSelect.h -----------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file This file describes the interface of the MachineFunctionPass diff --git a/include/llvm/CodeGen/GlobalISel/InstructionSelector.h b/include/llvm/CodeGen/GlobalISel/InstructionSelector.h index 471def7f45a3..e9b93be76754 100644 --- a/include/llvm/CodeGen/GlobalISel/InstructionSelector.h +++ b/include/llvm/CodeGen/GlobalISel/InstructionSelector.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/InstructionSelector.h ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -139,6 +138,16 @@ enum { /// - MMOIdx - MMO index /// - Size - The size in bytes of the memory access GIM_CheckMemorySizeEqualTo, + + /// Check the address space of the memory access for the given machine memory + /// operand. + /// - InsnID - Instruction ID + /// - MMOIdx - MMO index + /// - NumAddrSpace - Number of valid address spaces + /// - AddrSpaceN - An allowed space of the memory access + /// - AddrSpaceN+1 ... + GIM_CheckMemoryAddressSpace, + /// Check the size of the memory access for the given machine memory operand /// against the size of an operand. /// - InsnID - Instruction ID diff --git a/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h b/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h index 2003a79f6b20..e8ee4af0cb0b 100644 --- a/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h +++ b/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -371,6 +370,45 @@ bool InstructionSelector::executeMatchTable( return false; break; } + case GIM_CheckMemoryAddressSpace: { + int64_t InsnID = MatchTable[CurrentIdx++]; + int64_t MMOIdx = MatchTable[CurrentIdx++]; + // This accepts a list of possible address spaces. + const int NumAddrSpace = MatchTable[CurrentIdx++]; + + if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) { + if (handleReject() == RejectAndGiveUp) + return false; + break; + } + + // Need to still jump to the end of the list of address spaces if we find + // a match earlier. + const uint64_t LastIdx = CurrentIdx + NumAddrSpace; + + const MachineMemOperand *MMO + = *(State.MIs[InsnID]->memoperands_begin() + MMOIdx); + const unsigned MMOAddrSpace = MMO->getAddrSpace(); + + bool Success = false; + for (int I = 0; I != NumAddrSpace; ++I) { + unsigned AddrSpace = MatchTable[CurrentIdx++]; + DEBUG_WITH_TYPE( + TgtInstructionSelector::getName(), + dbgs() << "addrspace(" << MMOAddrSpace << ") vs " + << AddrSpace << '\n'); + + if (AddrSpace == MMOAddrSpace) { + Success = true; + break; + } + } + + CurrentIdx = LastIdx; + if (!Success && handleReject() == RejectAndGiveUp) + return false; + break; + } case GIM_CheckMemorySizeEqualTo: { int64_t InsnID = MatchTable[CurrentIdx++]; int64_t MMOIdx = MatchTable[CurrentIdx++]; @@ -438,15 +476,15 @@ bool InstructionSelector::executeMatchTable( unsigned Size = MRI.getType(MO.getReg()).getSizeInBits(); if (MatcherOpcode == GIM_CheckMemorySizeEqualToLLT && - MMO->getSize() * 8 != Size) { + MMO->getSizeInBits() != Size) { if (handleReject() == RejectAndGiveUp) return false; } else if (MatcherOpcode == GIM_CheckMemorySizeLessThanLLT && - MMO->getSize() * 8 >= Size) { + MMO->getSizeInBits() >= Size) { if (handleReject() == RejectAndGiveUp) return false; } else if (MatcherOpcode == GIM_CheckMemorySizeGreaterThanLLT && - MMO->getSize() * 8 <= Size) + MMO->getSizeInBits() <= Size) if (handleReject() == RejectAndGiveUp) return false; @@ -479,17 +517,19 @@ bool InstructionSelector::executeMatchTable( << InsnID << "]->getOperand(" << OpIdx << "), SizeInBits=" << SizeInBits << ")\n"); assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); + const LLT Ty = MRI.getType(MO.getReg()); + // iPTR must be looked up in the target. if (SizeInBits == 0) { MachineFunction *MF = State.MIs[InsnID]->getParent()->getParent(); - SizeInBits = MF->getDataLayout().getPointerSizeInBits(0); + const unsigned AddrSpace = Ty.getAddressSpace(); + SizeInBits = MF->getDataLayout().getPointerSizeInBits(AddrSpace); } assert(SizeInBits != 0 && "Pointer size must be known"); - MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx); if (MO.isReg()) { - const LLT &Ty = MRI.getType(MO.getReg()); if (!Ty.isPointer() || Ty.getSizeInBits() != SizeInBits) if (handleReject() == RejectAndGiveUp) return false; diff --git a/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h b/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h index 20bec7650179..a22778b8848c 100644 --- a/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h +++ b/include/llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/GlobalISel/LegalizationArtifactCombiner.h -----*- C++ -*-// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This file contains some helper functions which try to cleanup artifacts @@ -29,6 +28,18 @@ class LegalizationArtifactCombiner { MachineRegisterInfo &MRI; const LegalizerInfo &LI; + static bool isArtifactCast(unsigned Opc) { + switch (Opc) { + case TargetOpcode::G_TRUNC: + case TargetOpcode::G_SEXT: + case TargetOpcode::G_ZEXT: + case TargetOpcode::G_ANYEXT: + return true; + default: + return false; + } + } + public: LegalizationArtifactCombiner(MachineIRBuilder &B, MachineRegisterInfo &MRI, const LegalizerInfo &LI) @@ -40,11 +51,11 @@ public: return false; Builder.setInstr(MI); - unsigned DstReg = MI.getOperand(0).getReg(); - unsigned SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg()); + Register DstReg = MI.getOperand(0).getReg(); + Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg()); // aext(trunc x) - > aext/copy/trunc x - unsigned TruncSrc; + Register TruncSrc; if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) { LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;); Builder.buildAnyExtOrTrunc(DstReg, TruncSrc); @@ -53,7 +64,7 @@ public: } // aext([asz]ext x) -> [asz]ext x - unsigned ExtSrc; + Register ExtSrc; MachineInstr *ExtMI; if (mi_match(SrcReg, MRI, m_all_of(m_MInstr(ExtMI), m_any_of(m_GAnyExt(m_Reg(ExtSrc)), @@ -63,6 +74,20 @@ public: markInstAndDefDead(MI, *ExtMI, DeadInsts); return true; } + + // Try to fold aext(g_constant) when the larger constant type is legal. + // Can't use MIPattern because we don't have a specific constant in mind. + auto *SrcMI = MRI.getVRegDef(SrcReg); + if (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT) { + const LLT &DstTy = MRI.getType(DstReg); + if (isInstLegal({TargetOpcode::G_CONSTANT, {DstTy}})) { + auto &CstVal = SrcMI->getOperand(1); + Builder.buildConstant( + DstReg, CstVal.getCImm()->getValue().sext(DstTy.getSizeInBits())); + markInstAndDefDead(MI, *SrcMI, DeadInsts); + return true; + } + } return tryFoldImplicitDef(MI, DeadInsts); } @@ -73,25 +98,39 @@ public: return false; Builder.setInstr(MI); - unsigned DstReg = MI.getOperand(0).getReg(); - unsigned SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg()); + Register DstReg = MI.getOperand(0).getReg(); + Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg()); // zext(trunc x) - > and (aext/copy/trunc x), mask - unsigned TruncSrc; + Register TruncSrc; if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) { LLT DstTy = MRI.getType(DstReg); if (isInstUnsupported({TargetOpcode::G_AND, {DstTy}}) || - isInstUnsupported({TargetOpcode::G_CONSTANT, {DstTy}})) + isConstantUnsupported(DstTy)) return false; LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;); LLT SrcTy = MRI.getType(SrcReg); - APInt Mask = APInt::getAllOnesValue(SrcTy.getSizeInBits()); + APInt Mask = APInt::getAllOnesValue(SrcTy.getScalarSizeInBits()); auto MIBMask = Builder.buildConstant(DstTy, Mask.getZExtValue()); Builder.buildAnd(DstReg, Builder.buildAnyExtOrTrunc(DstTy, TruncSrc), MIBMask); markInstAndDefDead(MI, *MRI.getVRegDef(SrcReg), DeadInsts); return true; } + + // Try to fold zext(g_constant) when the larger constant type is legal. + // Can't use MIPattern because we don't have a specific constant in mind. + auto *SrcMI = MRI.getVRegDef(SrcReg); + if (SrcMI->getOpcode() == TargetOpcode::G_CONSTANT) { + const LLT &DstTy = MRI.getType(DstReg); + if (isInstLegal({TargetOpcode::G_CONSTANT, {DstTy}})) { + auto &CstVal = SrcMI->getOperand(1); + Builder.buildConstant( + DstReg, CstVal.getCImm()->getValue().zext(DstTy.getSizeInBits())); + markInstAndDefDead(MI, *SrcMI, DeadInsts); + return true; + } + } return tryFoldImplicitDef(MI, DeadInsts); } @@ -102,20 +141,22 @@ public: return false; Builder.setInstr(MI); - unsigned DstReg = MI.getOperand(0).getReg(); - unsigned SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg()); + Register DstReg = MI.getOperand(0).getReg(); + Register SrcReg = lookThroughCopyInstrs(MI.getOperand(1).getReg()); // sext(trunc x) - > ashr (shl (aext/copy/trunc x), c), c - unsigned TruncSrc; + Register TruncSrc; if (mi_match(SrcReg, MRI, m_GTrunc(m_Reg(TruncSrc)))) { LLT DstTy = MRI.getType(DstReg); - if (isInstUnsupported({TargetOpcode::G_SHL, {DstTy}}) || - isInstUnsupported({TargetOpcode::G_ASHR, {DstTy}}) || - isInstUnsupported({TargetOpcode::G_CONSTANT, {DstTy}})) + // Guess on the RHS shift amount type, which should be re-legalized if + // applicable. + if (isInstUnsupported({TargetOpcode::G_SHL, {DstTy, DstTy}}) || + isInstUnsupported({TargetOpcode::G_ASHR, {DstTy, DstTy}}) || + isConstantUnsupported(DstTy)) return false; LLVM_DEBUG(dbgs() << ".. Combine MI: " << MI;); LLT SrcTy = MRI.getType(SrcReg); - unsigned ShAmt = DstTy.getSizeInBits() - SrcTy.getSizeInBits(); + unsigned ShAmt = DstTy.getScalarSizeInBits() - SrcTy.getScalarSizeInBits(); auto MIBShAmt = Builder.buildConstant(DstTy, ShAmt); auto MIBShl = Builder.buildInstr( TargetOpcode::G_SHL, {DstTy}, @@ -138,7 +179,7 @@ public: if (MachineInstr *DefMI = getOpcodeDef(TargetOpcode::G_IMPLICIT_DEF, MI.getOperand(1).getReg(), MRI)) { Builder.setInstr(MI); - unsigned DstReg = MI.getOperand(0).getReg(); + Register DstReg = MI.getOperand(0).getReg(); LLT DstTy = MRI.getType(DstReg); if (Opcode == TargetOpcode::G_ANYEXT) { @@ -150,7 +191,7 @@ public: } else { // G_[SZ]EXT (G_IMPLICIT_DEF) -> G_CONSTANT 0 because the top // bits will be 0 for G_ZEXT and 0/1 for the G_SEXT. - if (isInstUnsupported({TargetOpcode::G_CONSTANT, {DstTy}})) + if (isConstantUnsupported(DstTy)) return false; LLVM_DEBUG(dbgs() << ".. Combine G_[SZ]EXT(G_IMPLICIT_DEF): " << MI;); Builder.buildConstant(DstReg, 0); @@ -162,6 +203,16 @@ public: return false; } + static unsigned getMergeOpcode(LLT OpTy, LLT DestTy) { + if (OpTy.isVector() && DestTy.isVector()) + return TargetOpcode::G_CONCAT_VECTORS; + + if (OpTy.isVector() && !DestTy.isVector()) + return TargetOpcode::G_BUILD_VECTOR; + + return TargetOpcode::G_MERGE_VALUES; + } + bool tryCombineMerges(MachineInstr &MI, SmallVectorImpl &DeadInsts) { @@ -169,27 +220,33 @@ public: return false; unsigned NumDefs = MI.getNumOperands() - 1; + MachineInstr *SrcDef = + getDefIgnoringCopies(MI.getOperand(NumDefs).getReg(), MRI); + if (!SrcDef) + return false; - unsigned MergingOpcode; LLT OpTy = MRI.getType(MI.getOperand(NumDefs).getReg()); LLT DestTy = MRI.getType(MI.getOperand(0).getReg()); - if (OpTy.isVector() && DestTy.isVector()) - MergingOpcode = TargetOpcode::G_CONCAT_VECTORS; - else if (OpTy.isVector() && !DestTy.isVector()) - MergingOpcode = TargetOpcode::G_BUILD_VECTOR; - else - MergingOpcode = TargetOpcode::G_MERGE_VALUES; - - MachineInstr *MergeI = - getOpcodeDef(MergingOpcode, MI.getOperand(NumDefs).getReg(), MRI); + MachineInstr *MergeI = SrcDef; + unsigned ConvertOp = 0; + + // Handle intermediate conversions + unsigned SrcOp = SrcDef->getOpcode(); + if (isArtifactCast(SrcOp)) { + ConvertOp = SrcOp; + MergeI = getDefIgnoringCopies(SrcDef->getOperand(1).getReg(), MRI); + } - if (!MergeI) + // FIXME: Handle scalarizing concat_vectors (scalar result type with vector + // source) + unsigned MergingOpcode = getMergeOpcode(OpTy, DestTy); + if (!MergeI || MergeI->getOpcode() != MergingOpcode) return false; const unsigned NumMergeRegs = MergeI->getNumOperands() - 1; if (NumMergeRegs < NumDefs) { - if (NumDefs % NumMergeRegs != 0) + if (ConvertOp != 0 || NumDefs % NumMergeRegs != 0) return false; Builder.setInstr(MI); @@ -202,7 +259,7 @@ public: const unsigned NewNumDefs = NumDefs / NumMergeRegs; for (unsigned Idx = 0; Idx < NumMergeRegs; ++Idx) { - SmallVector DstRegs; + SmallVector DstRegs; for (unsigned j = 0, DefIdx = Idx * NewNumDefs; j < NewNumDefs; ++j, ++DefIdx) DstRegs.push_back(MI.getOperand(DefIdx).getReg()); @@ -211,7 +268,7 @@ public: } } else if (NumMergeRegs > NumDefs) { - if (NumMergeRegs % NumDefs != 0) + if (ConvertOp != 0 || NumMergeRegs % NumDefs != 0) return false; Builder.setInstr(MI); @@ -224,7 +281,7 @@ public: const unsigned NumRegs = NumMergeRegs / NumDefs; for (unsigned DefIdx = 0; DefIdx < NumDefs; ++DefIdx) { - SmallVector Regs; + SmallVector Regs; for (unsigned j = 0, Idx = NumRegs * DefIdx + 1; j < NumRegs; ++j, ++Idx) Regs.push_back(MergeI->getOperand(Idx).getReg()); @@ -233,10 +290,22 @@ public: } } else { + LLT MergeSrcTy = MRI.getType(MergeI->getOperand(1).getReg()); + if (ConvertOp) { + Builder.setInstr(MI); + + for (unsigned Idx = 0; Idx < NumDefs; ++Idx) { + Register MergeSrc = MergeI->getOperand(Idx + 1).getReg(); + Builder.buildInstr(ConvertOp, {MI.getOperand(Idx).getReg()}, + {MergeSrc}); + } + + markInstAndDefDead(MI, *MergeI, DeadInsts); + return true; + } // FIXME: is a COPY appropriate if the types mismatch? We know both // registers are allocatable by now. - if (MRI.getType(MI.getOperand(0).getReg()) != - MRI.getType(MergeI->getOperand(1).getReg())) + if (DestTy != MergeSrcTy) return false; for (unsigned Idx = 0; Idx < NumDefs; ++Idx) @@ -248,12 +317,77 @@ public: return true; } + static bool isMergeLikeOpcode(unsigned Opc) { + switch (Opc) { + case TargetOpcode::G_MERGE_VALUES: + case TargetOpcode::G_BUILD_VECTOR: + case TargetOpcode::G_CONCAT_VECTORS: + return true; + default: + return false; + } + } + + bool tryCombineExtract(MachineInstr &MI, + SmallVectorImpl &DeadInsts) { + assert(MI.getOpcode() == TargetOpcode::G_EXTRACT); + + // Try to use the source registers from a G_MERGE_VALUES + // + // %2 = G_MERGE_VALUES %0, %1 + // %3 = G_EXTRACT %2, N + // => + // + // for N < %2.getSizeInBits() / 2 + // %3 = G_EXTRACT %0, N + // + // for N >= %2.getSizeInBits() / 2 + // %3 = G_EXTRACT %1, (N - %0.getSizeInBits() + + unsigned Src = lookThroughCopyInstrs(MI.getOperand(1).getReg()); + MachineInstr *MergeI = MRI.getVRegDef(Src); + if (!MergeI || !isMergeLikeOpcode(MergeI->getOpcode())) + return false; + + LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); + LLT SrcTy = MRI.getType(Src); + + // TODO: Do we need to check if the resulting extract is supported? + unsigned ExtractDstSize = DstTy.getSizeInBits(); + unsigned Offset = MI.getOperand(2).getImm(); + unsigned NumMergeSrcs = MergeI->getNumOperands() - 1; + unsigned MergeSrcSize = SrcTy.getSizeInBits() / NumMergeSrcs; + unsigned MergeSrcIdx = Offset / MergeSrcSize; + + // Compute the offset of the last bit the extract needs. + unsigned EndMergeSrcIdx = (Offset + ExtractDstSize - 1) / MergeSrcSize; + + // Can't handle the case where the extract spans multiple inputs. + if (MergeSrcIdx != EndMergeSrcIdx) + return false; + + // TODO: We could modify MI in place in most cases. + Builder.setInstr(MI); + Builder.buildExtract( + MI.getOperand(0).getReg(), + MergeI->getOperand(MergeSrcIdx + 1).getReg(), + Offset - MergeSrcIdx * MergeSrcSize); + markInstAndDefDead(MI, *MergeI, DeadInsts); + return true; + } + /// Try to combine away MI. /// Returns true if it combined away the MI. /// Adds instructions that are dead as a result of the combine /// into DeadInsts, which can include MI. bool tryCombineInstruction(MachineInstr &MI, - SmallVectorImpl &DeadInsts) { + SmallVectorImpl &DeadInsts, + GISelObserverWrapper &WrapperObserver) { + // This might be a recursive call, and we might have DeadInsts already + // populated. To avoid bad things happening later with multiple vreg defs + // etc, process the dead instructions now if any. + if (!DeadInsts.empty()) + deleteMarkedDeadInsts(DeadInsts, WrapperObserver); switch (MI.getOpcode()) { default: return false; @@ -265,16 +399,35 @@ public: return tryCombineSExt(MI, DeadInsts); case TargetOpcode::G_UNMERGE_VALUES: return tryCombineMerges(MI, DeadInsts); + case TargetOpcode::G_EXTRACT: + return tryCombineExtract(MI, DeadInsts); case TargetOpcode::G_TRUNC: { bool Changed = false; for (auto &Use : MRI.use_instructions(MI.getOperand(0).getReg())) - Changed |= tryCombineInstruction(Use, DeadInsts); + Changed |= tryCombineInstruction(Use, DeadInsts, WrapperObserver); return Changed; } } } private: + + static unsigned getArtifactSrcReg(const MachineInstr &MI) { + switch (MI.getOpcode()) { + case TargetOpcode::COPY: + case TargetOpcode::G_TRUNC: + case TargetOpcode::G_ZEXT: + case TargetOpcode::G_ANYEXT: + case TargetOpcode::G_SEXT: + case TargetOpcode::G_UNMERGE_VALUES: + return MI.getOperand(MI.getNumOperands() - 1).getReg(); + case TargetOpcode::G_EXTRACT: + return MI.getOperand(1).getReg(); + default: + llvm_unreachable("Not a legalization artifact happen"); + } + } + /// Mark MI as dead. If a def of one of MI's operands, DefMI, would also be /// dead due to MI being killed, then mark DefMI as dead too. /// Some of the combines (extends(trunc)), try to walk through redundant @@ -295,13 +448,15 @@ private: // and as a result, %3, %2, %1 are dead. MachineInstr *PrevMI = &MI; while (PrevMI != &DefMI) { - unsigned PrevRegSrc = - PrevMI->getOperand(PrevMI->getNumOperands() - 1).getReg(); + unsigned PrevRegSrc = getArtifactSrcReg(*PrevMI); + MachineInstr *TmpDef = MRI.getVRegDef(PrevRegSrc); if (MRI.hasOneUse(PrevRegSrc)) { if (TmpDef != &DefMI) { - assert(TmpDef->getOpcode() == TargetOpcode::COPY && - "Expecting copy here"); + assert((TmpDef->getOpcode() == TargetOpcode::COPY || + isArtifactCast(TmpDef->getOpcode())) && + "Expecting copy or artifact cast here"); + DeadInsts.push_back(TmpDef); } } else @@ -312,6 +467,22 @@ private: DeadInsts.push_back(&DefMI); } + /// Erase the dead instructions in the list and call the observer hooks. + /// Normally the Legalizer will deal with erasing instructions that have been + /// marked dead. However, for the trunc(ext(x)) cases we can end up trying to + /// process instructions which have been marked dead, but otherwise break the + /// MIR by introducing multiple vreg defs. For those cases, allow the combines + /// to explicitly delete the instructions before we run into trouble. + void deleteMarkedDeadInsts(SmallVectorImpl &DeadInsts, + GISelObserverWrapper &WrapperObserver) { + for (auto *DeadMI : DeadInsts) { + LLVM_DEBUG(dbgs() << *DeadMI << "Is dead, eagerly deleting\n"); + WrapperObserver.erasingInstr(*DeadMI); + DeadMI->eraseFromParentAndMarkDBGValuesForRemoval(); + } + DeadInsts.clear(); + } + /// Checks if the target legalizer info has specified anything about the /// instruction, or if unsupported. bool isInstUnsupported(const LegalityQuery &Query) const { @@ -320,10 +491,23 @@ private: return Step.Action == Unsupported || Step.Action == NotFound; } + bool isInstLegal(const LegalityQuery &Query) const { + return LI.getAction(Query).Action == LegalizeActions::Legal; + } + + bool isConstantUnsupported(LLT Ty) const { + if (!Ty.isVector()) + return isInstUnsupported({TargetOpcode::G_CONSTANT, {Ty}}); + + LLT EltTy = Ty.getElementType(); + return isInstUnsupported({TargetOpcode::G_CONSTANT, {EltTy}}) || + isInstUnsupported({TargetOpcode::G_BUILD_VECTOR, {Ty, EltTy}}); + } + /// Looks through copy instructions and returns the actual /// source register. - unsigned lookThroughCopyInstrs(unsigned Reg) { - unsigned TmpReg; + unsigned lookThroughCopyInstrs(Register Reg) { + Register TmpReg; while (mi_match(Reg, MRI, m_Copy(m_Reg(TmpReg)))) { if (MRI.getType(TmpReg).isValid()) Reg = TmpReg; diff --git a/include/llvm/CodeGen/GlobalISel/Legalizer.h b/include/llvm/CodeGen/GlobalISel/Legalizer.h index 8284ab6dac65..13cf3f7e694d 100644 --- a/include/llvm/CodeGen/GlobalISel/Legalizer.h +++ b/include/llvm/CodeGen/GlobalISel/Legalizer.h @@ -1,9 +1,8 @@ -//== llvm/CodeGen/GlobalISel/LegalizePass.h ------------- -*- C++ -*-==// +//== llvm/CodeGen/GlobalISel/Legalizer.h ---------------- -*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -55,6 +54,11 @@ public: MachineFunctionProperties::Property::Legalized); } + MachineFunctionProperties getClearedProperties() const override { + return MachineFunctionProperties() + .set(MachineFunctionProperties::Property::NoPHIs); + } + bool combineExtracts(MachineInstr &MI, MachineRegisterInfo &MRI, const TargetInstrInfo &TII); diff --git a/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h index 9b4ecf9284e3..a0f21e8b19d7 100644 --- a/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ b/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -1,9 +1,8 @@ //== llvm/CodeGen/GlobalISel/LegalizerHelper.h ---------------- -*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -87,7 +86,7 @@ public: /// Legalize a vector instruction by increasing the number of vector elements /// involved and ignoring the added elements later. LegalizeResult moreElementsVector(MachineInstr &MI, unsigned TypeIdx, - LLT WideTy); + LLT MoreTy); /// Expose MIRBuilder so clients can set their own RecordInsertInstruction /// functions @@ -104,20 +103,127 @@ private: void widenScalarSrc(MachineInstr &MI, LLT WideTy, unsigned OpIdx, unsigned ExtOpcode); + /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a + /// Use by truncating the operand's type to \p NarrowTy using G_TRUNC, and + /// replacing the vreg of the operand in place. + void narrowScalarSrc(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx); + /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a /// Def by extending the operand's type to \p WideTy and truncating it back /// with the \p TruncOpcode, and replacing the vreg of the operand in place. void widenScalarDst(MachineInstr &MI, LLT WideTy, unsigned OpIdx = 0, unsigned TruncOpcode = TargetOpcode::G_TRUNC); + // Legalize a single operand \p OpIdx of the machine instruction \p MI as a + // Def by truncating the operand's type to \p NarrowTy, replacing in place and + // extending back with \p ExtOpcode. + void narrowScalarDst(MachineInstr &MI, LLT NarrowTy, unsigned OpIdx, + unsigned ExtOpcode); + /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a + /// Def by performing it with additional vector elements and extracting the + /// result elements, and replacing the vreg of the operand in place. + void moreElementsVectorDst(MachineInstr &MI, LLT MoreTy, unsigned OpIdx); + + /// Legalize a single operand \p OpIdx of the machine instruction \p MI as a + /// Use by producing a vector with undefined high elements, extracting the + /// original vector type, and replacing the vreg of the operand in place. + void moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy, unsigned OpIdx); + + LegalizeResult + widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); + LegalizeResult + widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); + LegalizeResult + widenScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); + LegalizeResult + widenScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT WideTy); + /// Helper function to split a wide generic register into bitwise blocks with /// the given Type (which implies the number of blocks needed). The generic /// registers created are appended to Ops, starting at bit 0 of Reg. - void extractParts(unsigned Reg, LLT Ty, int NumParts, - SmallVectorImpl &VRegs); + void extractParts(Register Reg, LLT Ty, int NumParts, + SmallVectorImpl &VRegs); + + /// Version which handles irregular splits. + bool extractParts(Register Reg, LLT RegTy, LLT MainTy, + LLT &LeftoverTy, + SmallVectorImpl &VRegs, + SmallVectorImpl &LeftoverVRegs); + + /// Helper function to build a wide generic register \p DstReg of type \p + /// RegTy from smaller parts. This will produce a G_MERGE_VALUES, + /// G_BUILD_VECTOR, G_CONCAT_VECTORS, or sequence of G_INSERT as appropriate + /// for the types. + /// + /// \p PartRegs must be registers of type \p PartTy. + /// + /// If \p ResultTy does not evenly break into \p PartTy sized pieces, the + /// remainder must be specified with \p LeftoverRegs of type \p LeftoverTy. + void insertParts(Register DstReg, LLT ResultTy, + LLT PartTy, ArrayRef PartRegs, + LLT LeftoverTy = LLT(), ArrayRef LeftoverRegs = {}); + + /// Perform generic multiplication of values held in multiple registers. + /// Generated instructions use only types NarrowTy and i1. + /// Destination can be same or two times size of the source. + void multiplyRegisters(SmallVectorImpl &DstRegs, + ArrayRef Src1Regs, + ArrayRef Src2Regs, LLT NarrowTy); + +public: + LegalizeResult fewerElementsVectorImplicitDef(MachineInstr &MI, + unsigned TypeIdx, LLT NarrowTy); + + /// Legalize a simple vector instruction where all operands are the same type + /// by splitting into multiple components. + LegalizeResult fewerElementsVectorBasic(MachineInstr &MI, unsigned TypeIdx, + LLT NarrowTy); + + /// Legalize a instruction with a vector type where each operand may have a + /// different element type. All type indexes must have the same number of + /// elements. + LegalizeResult fewerElementsVectorMultiEltType(MachineInstr &MI, + unsigned TypeIdx, LLT NarrowTy); + + LegalizeResult fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx, + LLT NarrowTy); + + LegalizeResult + fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); + + LegalizeResult + fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); + + LegalizeResult fewerElementsVectorPhi(MachineInstr &MI, + unsigned TypeIdx, LLT NarrowTy); + + LegalizeResult moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, + LLT MoreTy); + + LegalizeResult + reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy); + + LegalizeResult narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt, + LLT HalfTy, LLT ShiftAmtTy); + + LegalizeResult narrowScalarShift(MachineInstr &MI, unsigned TypeIdx, LLT Ty); + LegalizeResult narrowScalarMul(MachineInstr &MI, LLT Ty); + LegalizeResult narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx, LLT Ty); + LegalizeResult narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx, LLT Ty); + + LegalizeResult narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx, LLT Ty); + LegalizeResult narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx, LLT Ty); LegalizeResult lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty); + LegalizeResult lowerU64ToF32BitOps(MachineInstr &MI); + LegalizeResult lowerUITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty); + LegalizeResult lowerSITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty); + LegalizeResult lowerMinMax(MachineInstr &MI, unsigned TypeIdx, LLT Ty); + LegalizeResult lowerFCopySign(MachineInstr &MI, unsigned TypeIdx, LLT Ty); + LegalizeResult lowerFMinNumMaxNum(MachineInstr &MI); + +private: MachineRegisterInfo &MRI; const LegalizerInfo &LI; /// To keep track of changes made by the LegalizerHelper. diff --git a/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h index 13776dd3e87d..513c98f2d23f 100644 --- a/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h +++ b/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/LegalizerInfo.h ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -93,6 +92,7 @@ enum LegalizeAction : std::uint8_t { UseLegacyRules, }; } // end namespace LegalizeActions +raw_ostream &operator<<(raw_ostream &OS, LegalizeActions::LegalizeAction Action); using LegalizeActions::LegalizeAction; @@ -123,6 +123,7 @@ struct LegalityQuery { struct MemDesc { uint64_t SizeInBits; + uint64_t AlignInBits; AtomicOrdering Ordering; }; @@ -165,13 +166,23 @@ using LegalizeMutation = std::function(const LegalityQuery &)>; namespace LegalityPredicates { -struct TypePairAndMemSize { +struct TypePairAndMemDesc { LLT Type0; LLT Type1; uint64_t MemSize; + uint64_t Align; + + bool operator==(const TypePairAndMemDesc &Other) const { + return Type0 == Other.Type0 && Type1 == Other.Type1 && + Align == Other.Align && + MemSize == Other.MemSize; + } - bool operator==(const TypePairAndMemSize &Other) const { + /// \returns true if this memory access is legal with for the acecss described + /// by \p Other (The alignment is sufficient for the size and result type). + bool isCompatible(const TypePairAndMemDesc &Other) const { return Type0 == Other.Type0 && Type1 == Other.Type1 && + Align >= Other.Align && MemSize == Other.MemSize; } }; @@ -200,20 +211,45 @@ typePairInSet(unsigned TypeIdx0, unsigned TypeIdx1, std::initializer_list> TypesInit); /// True iff the given types for the given pair of type indexes is one of the /// specified type pairs. -LegalityPredicate typePairAndMemSizeInSet( +LegalityPredicate typePairAndMemDescInSet( unsigned TypeIdx0, unsigned TypeIdx1, unsigned MMOIdx, - std::initializer_list TypesAndMemSizeInit); + std::initializer_list TypesAndMemDescInit); /// True iff the specified type index is a scalar. LegalityPredicate isScalar(unsigned TypeIdx); +/// True iff the specified type index is a vector. +LegalityPredicate isVector(unsigned TypeIdx); +/// True iff the specified type index is a pointer (with any address space). +LegalityPredicate isPointer(unsigned TypeIdx); +/// True iff the specified type index is a pointer with the specified address +/// space. +LegalityPredicate isPointer(unsigned TypeIdx, unsigned AddrSpace); + /// True iff the specified type index is a scalar that's narrower than the given /// size. LegalityPredicate narrowerThan(unsigned TypeIdx, unsigned Size); + /// True iff the specified type index is a scalar that's wider than the given /// size. LegalityPredicate widerThan(unsigned TypeIdx, unsigned Size); + +/// True iff the specified type index is a scalar or vector with an element type +/// that's narrower than the given size. +LegalityPredicate scalarOrEltNarrowerThan(unsigned TypeIdx, unsigned Size); + +/// True iff the specified type index is a scalar or a vector with an element +/// type that's wider than the given size. +LegalityPredicate scalarOrEltWiderThan(unsigned TypeIdx, unsigned Size); + /// True iff the specified type index is a scalar whose size is not a power of /// 2. LegalityPredicate sizeNotPow2(unsigned TypeIdx); + +/// True iff the specified type index is a scalar or vector whose element size +/// is not a power of 2. +LegalityPredicate scalarOrEltSizeNotPow2(unsigned TypeIdx); + +/// True iff the specified type indices are both the same bit size. +LegalityPredicate sameSize(unsigned TypeIdx0, unsigned TypeIdx1); /// True iff the specified MMO index has a size that is not a power of 2 LegalityPredicate memSizeInBytesNotPow2(unsigned MMOIdx); /// True iff the specified type index is a vector whose element count is not a @@ -228,13 +264,25 @@ LegalityPredicate atomicOrderingAtLeastOrStrongerThan(unsigned MMOIdx, namespace LegalizeMutations { /// Select this specific type for the given type index. LegalizeMutation changeTo(unsigned TypeIdx, LLT Ty); + /// Keep the same type as the given type index. LegalizeMutation changeTo(unsigned TypeIdx, unsigned FromTypeIdx); -/// Widen the type for the given type index to the next power of 2. -LegalizeMutation widenScalarToNextPow2(unsigned TypeIdx, unsigned Min = 0); + +/// Keep the same scalar or element type as the given type index. +LegalizeMutation changeElementTo(unsigned TypeIdx, unsigned FromTypeIdx); + +/// Keep the same scalar or element type as the given type. +LegalizeMutation changeElementTo(unsigned TypeIdx, LLT Ty); + +/// Widen the scalar type or vector element type for the given type index to the +/// next power of 2. +LegalizeMutation widenScalarOrEltToNextPow2(unsigned TypeIdx, unsigned Min = 0); + /// Add more elements to the type for the given type index to the next power of /// 2. LegalizeMutation moreElementsToNextPow2(unsigned TypeIdx, unsigned Min = 0); +/// Break up the vector type for the given type index into the element type. +LegalizeMutation scalarize(unsigned TypeIdx); } // end namespace LegalizeMutations /// A single rule in a legalizer info ruleset. @@ -419,13 +467,13 @@ public: return actionFor(LegalizeAction::Legal, Types); } /// The instruction is legal when type indexes 0 and 1 along with the memory - /// size is any type and size tuple in the given list. - LegalizeRuleSet &legalForTypesWithMemSize( - std::initializer_list - TypesAndMemSize) { + /// size and minimum alignment is any type and size tuple in the given list. + LegalizeRuleSet &legalForTypesWithMemDesc( + std::initializer_list + TypesAndMemDesc) { return actionIf(LegalizeAction::Legal, - LegalityPredicates::typePairAndMemSizeInSet( - typeIdx(0), typeIdx(1), /*MMOIdx*/ 0, TypesAndMemSize)); + LegalityPredicates::typePairAndMemDescInSet( + typeIdx(0), typeIdx(1), /*MMOIdx*/ 0, TypesAndMemDesc)); } /// The instruction is legal when type indexes 0 and 1 are both in the given /// list. That is, the type pair is in the cartesian product of the list. @@ -438,6 +486,20 @@ public: std::initializer_list Types1) { return actionForCartesianProduct(LegalizeAction::Legal, Types0, Types1); } + /// The instruction is legal when type indexes 0, 1, and 2 are both their + /// respective lists. + LegalizeRuleSet &legalForCartesianProduct(std::initializer_list Types0, + std::initializer_list Types1, + std::initializer_list Types2) { + return actionForCartesianProduct(LegalizeAction::Legal, Types0, Types1, + Types2); + } + + LegalizeRuleSet &alwaysLegal() { + using namespace LegalizeMutations; + markAllTypeIdxsAsCovered(); + return actionIf(LegalizeAction::Legal, always); + } /// The instruction is lowered. LegalizeRuleSet &lower() { @@ -588,6 +650,13 @@ public: LegalizeRuleSet &customFor(std::initializer_list Types) { return actionFor(LegalizeAction::Custom, Types); } + + /// The instruction is custom when type indexes 0 and 1 is any type pair in the + /// given list. + LegalizeRuleSet &customFor(std::initializer_list> Types) { + return actionFor(LegalizeAction::Custom, Types); + } + LegalizeRuleSet &customForCartesianProduct(std::initializer_list Types) { return actionForCartesianProduct(LegalizeAction::Custom, Types); } @@ -597,13 +666,29 @@ public: return actionForCartesianProduct(LegalizeAction::Custom, Types0, Types1); } + /// Unconditionally custom lower. + LegalizeRuleSet &custom() { + return customIf(always); + } + /// Widen the scalar to the next power of two that is at least MinSize. /// No effect if the type is not a scalar or is a power of two. LegalizeRuleSet &widenScalarToNextPow2(unsigned TypeIdx, unsigned MinSize = 0) { using namespace LegalityPredicates; - return actionIf(LegalizeAction::WidenScalar, sizeNotPow2(typeIdx(TypeIdx)), - LegalizeMutations::widenScalarToNextPow2(TypeIdx, MinSize)); + return actionIf( + LegalizeAction::WidenScalar, sizeNotPow2(typeIdx(TypeIdx)), + LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, MinSize)); + } + + /// Widen the scalar or vector element type to the next power of two that is + /// at least MinSize. No effect if the scalar size is a power of two. + LegalizeRuleSet &widenScalarOrEltToNextPow2(unsigned TypeIdx, + unsigned MinSize = 0) { + using namespace LegalityPredicates; + return actionIf( + LegalizeAction::WidenScalar, scalarOrEltSizeNotPow2(typeIdx(TypeIdx)), + LegalizeMutations::widenScalarOrEltToNextPow2(TypeIdx, MinSize)); } LegalizeRuleSet &narrowScalar(unsigned TypeIdx, LegalizeMutation Mutation) { @@ -612,6 +697,32 @@ public: Mutation); } + LegalizeRuleSet &scalarize(unsigned TypeIdx) { + using namespace LegalityPredicates; + return actionIf(LegalizeAction::FewerElements, isVector(typeIdx(TypeIdx)), + LegalizeMutations::scalarize(TypeIdx)); + } + + /// Ensure the scalar or element is at least as wide as Ty. + LegalizeRuleSet &minScalarOrElt(unsigned TypeIdx, const LLT &Ty) { + using namespace LegalityPredicates; + using namespace LegalizeMutations; + return actionIf(LegalizeAction::WidenScalar, + scalarOrEltNarrowerThan(TypeIdx, Ty.getScalarSizeInBits()), + changeElementTo(typeIdx(TypeIdx), Ty)); + } + + /// Ensure the scalar or element is at least as wide as Ty. + LegalizeRuleSet &minScalarOrEltIf(LegalityPredicate Predicate, + unsigned TypeIdx, const LLT &Ty) { + using namespace LegalityPredicates; + using namespace LegalizeMutations; + return actionIf(LegalizeAction::WidenScalar, + all(Predicate, scalarOrEltNarrowerThan( + TypeIdx, Ty.getScalarSizeInBits())), + changeElementTo(typeIdx(TypeIdx), Ty)); + } + /// Ensure the scalar is at least as wide as Ty. LegalizeRuleSet &minScalar(unsigned TypeIdx, const LLT &Ty) { using namespace LegalityPredicates; @@ -621,6 +732,15 @@ public: changeTo(typeIdx(TypeIdx), Ty)); } + /// Ensure the scalar is at most as wide as Ty. + LegalizeRuleSet &maxScalarOrElt(unsigned TypeIdx, const LLT &Ty) { + using namespace LegalityPredicates; + using namespace LegalizeMutations; + return actionIf(LegalizeAction::NarrowScalar, + scalarOrEltWiderThan(TypeIdx, Ty.getScalarSizeInBits()), + changeElementTo(typeIdx(TypeIdx), Ty)); + } + /// Ensure the scalar is at most as wide as Ty. LegalizeRuleSet &maxScalar(unsigned TypeIdx, const LLT &Ty) { using namespace LegalityPredicates; @@ -637,12 +757,12 @@ public: const LLT &Ty) { using namespace LegalityPredicates; using namespace LegalizeMutations; - return actionIf(LegalizeAction::NarrowScalar, - [=](const LegalityQuery &Query) { - return widerThan(TypeIdx, Ty.getSizeInBits()) && - Predicate(Query); - }, - changeTo(typeIdx(TypeIdx), Ty)); + return actionIf( + LegalizeAction::NarrowScalar, + [=](const LegalityQuery &Query) { + return widerThan(TypeIdx, Ty.getSizeInBits()) && Predicate(Query); + }, + changeElementTo(typeIdx(TypeIdx), Ty)); } /// Limit the range of scalar sizes to MinTy and MaxTy. @@ -652,6 +772,12 @@ public: return minScalar(TypeIdx, MinTy).maxScalar(TypeIdx, MaxTy); } + /// Limit the range of scalar sizes to MinTy and MaxTy. + LegalizeRuleSet &clampScalarOrElt(unsigned TypeIdx, const LLT &MinTy, + const LLT &MaxTy) { + return minScalarOrElt(TypeIdx, MinTy).maxScalarOrElt(TypeIdx, MaxTy); + } + /// Widen the scalar to match the size of another. LegalizeRuleSet &minScalarSameAs(unsigned TypeIdx, unsigned LargeTypeIdx) { typeIdx(TypeIdx); @@ -661,8 +787,25 @@ public: Query.Types[TypeIdx].getSizeInBits(); }, [=](const LegalityQuery &Query) { + LLT T = Query.Types[LargeTypeIdx]; return std::make_pair(TypeIdx, - Query.Types[LargeTypeIdx].getElementType()); + T.isVector() ? T.getElementType() : T); + }); + } + + /// Conditionally widen the scalar or elt to match the size of another. + LegalizeRuleSet &minScalarEltSameAsIf(LegalityPredicate Predicate, + unsigned TypeIdx, unsigned LargeTypeIdx) { + typeIdx(TypeIdx); + return widenScalarIf( + [=](const LegalityQuery &Query) { + return Query.Types[LargeTypeIdx].getScalarSizeInBits() > + Query.Types[TypeIdx].getScalarSizeInBits() && + Predicate(Query); + }, + [=](const LegalityQuery &Query) { + LLT T = Query.Types[LargeTypeIdx]; + return std::make_pair(TypeIdx, T); }); } @@ -691,7 +834,7 @@ public: [=](const LegalityQuery &Query) { LLT VecTy = Query.Types[TypeIdx]; return std::make_pair( - TypeIdx, LLT::vector(MinElements, VecTy.getScalarSizeInBits())); + TypeIdx, LLT::vector(MinElements, VecTy.getElementType())); }); } /// Limit the number of elements in EltTy vectors to at most MaxElements. @@ -708,10 +851,8 @@ public: }, [=](const LegalityQuery &Query) { LLT VecTy = Query.Types[TypeIdx]; - if (MaxElements == 1) - return std::make_pair(TypeIdx, VecTy.getElementType()); - return std::make_pair( - TypeIdx, LLT::vector(MaxElements, VecTy.getScalarSizeInBits())); + LLT NewTy = LLT::scalarOrVector(MaxElements, VecTy.getElementType()); + return std::make_pair(TypeIdx, NewTy); }); } /// Limit the number of elements for the given vectors to at least MinTy's @@ -962,12 +1103,22 @@ public: LegalizeActionStep getAction(const MachineInstr &MI, const MachineRegisterInfo &MRI) const; + bool isLegal(const LegalityQuery &Query) const { + return getAction(Query).Action == LegalizeAction::Legal; + } bool isLegal(const MachineInstr &MI, const MachineRegisterInfo &MRI) const; + bool isLegalOrCustom(const MachineInstr &MI, + const MachineRegisterInfo &MRI) const; virtual bool legalizeCustom(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder, GISelChangeObserver &Observer) const; + /// Return true if MI is either legal or has been legalized and false + /// if not legal. + virtual bool legalizeIntrinsic(MachineInstr &MI, MachineRegisterInfo &MRI, + MachineIRBuilder &MIRBuilder) const; + private: /// Determine what action should be taken to legalize the given generic /// instruction opcode, type-index and type. Requires computeTables to have diff --git a/include/llvm/CodeGen/GlobalISel/Localizer.h b/include/llvm/CodeGen/GlobalISel/Localizer.h index 1e2d4763e5e1..06de5800b8b7 100644 --- a/include/llvm/CodeGen/GlobalISel/Localizer.h +++ b/include/llvm/CodeGen/GlobalISel/Localizer.h @@ -1,9 +1,8 @@ //== llvm/CodeGen/GlobalISel/Localizer.h - Localizer -------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -22,12 +21,14 @@ #ifndef LLVM_CODEGEN_GLOBALISEL_LOCALIZER_H #define LLVM_CODEGEN_GLOBALISEL_LOCALIZER_H +#include "llvm/ADT/SetVector.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" #include "llvm/CodeGen/MachineFunctionPass.h" namespace llvm { // Forward declarations. class MachineRegisterInfo; +class TargetTransformInfo; /// This pass implements the localization mechanism described at the /// top of this file. One specificity of the implementation is that @@ -44,9 +45,11 @@ private: /// MRI contains all the register class/bank information that this /// pass uses and updates. MachineRegisterInfo *MRI; + /// TTI used for getting remat costs for instructions. + TargetTransformInfo *TTI; /// Check whether or not \p MI needs to be moved close to its uses. - static bool shouldLocalize(const MachineInstr &MI); + bool shouldLocalize(const MachineInstr &MI); /// Check if \p MOUse is used in the same basic block as \p Def. /// If the use is in the same block, we say it is local. @@ -58,6 +61,15 @@ private: /// Initialize the field members using \p MF. void init(MachineFunction &MF); + typedef SmallSetVector LocalizedSetVecT; + + /// Do inter-block localization from the entry block. + bool localizeInterBlock(MachineFunction &MF, + LocalizedSetVecT &LocalizedInstrs); + + /// Do intra-block localization of already localized instructions. + bool localizeIntraBlock(LocalizedSetVecT &LocalizedInstrs); + public: Localizer(); diff --git a/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h b/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h index f77f9a8df7ee..13eddd9539fa 100644 --- a/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h +++ b/include/llvm/CodeGen/GlobalISel/MIPatternMatch.h @@ -1,9 +1,8 @@ -//== ----- llvm/CodeGen/GlobalISel/MIPatternMatch.h --------------------- == // +//==------ llvm/CodeGen/GlobalISel/MIPatternMatch.h -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -31,8 +30,7 @@ template struct OneUse_match { SubPatternT SubPat; OneUse_match(const SubPatternT &SP) : SubPat(SP) {} - template - bool match(const MachineRegisterInfo &MRI, unsigned Reg) { + bool match(MachineRegisterInfo &MRI, unsigned Reg) { return MRI.hasOneUse(Reg) && SubPat.match(MRI, Reg); } }; @@ -162,7 +160,7 @@ template struct bind_ty { } }; -inline bind_ty m_Reg(unsigned &R) { return R; } +inline bind_ty m_Reg(Register &R) { return R; } inline bind_ty m_MInstr(MachineInstr *&MI) { return MI; } inline bind_ty m_Type(LLT &Ty) { return Ty; } diff --git a/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index 37de8f030410..10d712176b1b 100644 --- a/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/GlobalISel/MachineIRBuilder.h - MIBuilder --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -60,13 +59,15 @@ struct MachineIRBuilderState { class DstOp { union { LLT LLTTy; - unsigned Reg; + Register Reg; const TargetRegisterClass *RC; }; public: enum class DstType { Ty_LLT, Ty_Reg, Ty_RC }; DstOp(unsigned R) : Reg(R), Ty(DstType::Ty_Reg) {} + DstOp(Register R) : Reg(R), Ty(DstType::Ty_Reg) {} + DstOp(const MachineOperand &Op) : Reg(Op.getReg()), Ty(DstType::Ty_Reg) {} DstOp(const LLT &T) : LLTTy(T), Ty(DstType::Ty_LLT) {} DstOp(const TargetRegisterClass *TRC) : RC(TRC), Ty(DstType::Ty_RC) {} @@ -96,7 +97,7 @@ public: llvm_unreachable("Unrecognised DstOp::DstType enum"); } - unsigned getReg() const { + Register getReg() const { assert(Ty == DstType::Ty_Reg && "Not a register"); return Reg; } @@ -119,13 +120,14 @@ private: class SrcOp { union { MachineInstrBuilder SrcMIB; - unsigned Reg; + Register Reg; CmpInst::Predicate Pred; }; public: enum class SrcType { Ty_Reg, Ty_MIB, Ty_Predicate }; - SrcOp(unsigned R) : Reg(R), Ty(SrcType::Ty_Reg) {} + SrcOp(Register R) : Reg(R), Ty(SrcType::Ty_Reg) {} + SrcOp(const MachineOperand &Op) : Reg(Op.getReg()), Ty(SrcType::Ty_Reg) {} SrcOp(const MachineInstrBuilder &MIB) : SrcMIB(MIB), Ty(SrcType::Ty_MIB) {} SrcOp(const CmpInst::Predicate P) : Pred(P), Ty(SrcType::Ty_Predicate) {} @@ -155,7 +157,7 @@ public: llvm_unreachable("Unrecognised SrcOp::SrcType enum"); } - unsigned getReg() const { + Register getReg() const { switch (Ty) { case SrcType::Ty_Predicate: llvm_unreachable("Not a register operand"); @@ -202,6 +204,7 @@ protected: void validateTruncExt(const LLT &Dst, const LLT &Src, bool IsExtend); void validateBinaryOp(const LLT &Res, const LLT &Op0, const LLT &Op1); + void validateShiftOp(const LLT &Res, const LLT &Op0, const LLT &Op1); void validateSelectOp(const LLT &ResTy, const LLT &TstTy, const LLT &Op0Ty, const LLT &Op1Ty); @@ -230,6 +233,15 @@ public: return *State.MF; } + const MachineFunction &getMF() const { + assert(State.MF && "MachineFunction is not set"); + return *State.MF; + } + + const DataLayout &getDataLayout() const { + return getMF().getFunction().getParent()->getDataLayout(); + } + /// Getter for DebugLoc const DebugLoc &getDL() { return State.DL; } @@ -310,13 +322,13 @@ public: /// Build and insert a DBG_VALUE instruction expressing the fact that the /// associated \p Variable lives in \p Reg (suitably modified by \p Expr). - MachineInstrBuilder buildDirectDbgValue(unsigned Reg, const MDNode *Variable, + MachineInstrBuilder buildDirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr); /// Build and insert a DBG_VALUE instruction expressing the fact that the /// associated \p Variable lives in memory at \p Reg (suitably modified by \p /// Expr). - MachineInstrBuilder buildIndirectDbgValue(unsigned Reg, + MachineInstrBuilder buildIndirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr); @@ -345,7 +357,7 @@ public: /// \pre \p Res must be a generic virtual register with pointer type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildFrameIndex(unsigned Res, int Idx); + MachineInstrBuilder buildFrameIndex(const DstOp &Res, int Idx); /// Build and insert \p Res = G_GLOBAL_VALUE \p GV /// @@ -357,8 +369,7 @@ public: /// in the same address space as \p GV. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildGlobalValue(unsigned Res, const GlobalValue *GV); - + MachineInstrBuilder buildGlobalValue(const DstOp &Res, const GlobalValue *GV); /// Build and insert \p Res = G_GEP \p Op0, \p Op1 /// @@ -371,8 +382,8 @@ public: /// \pre \p Op1 must be a generic virtual register with scalar type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildGEP(unsigned Res, unsigned Op0, - unsigned Op1); + MachineInstrBuilder buildGEP(const DstOp &Res, const SrcOp &Op0, + const SrcOp &Op1); /// Materialize and insert \p Res = G_GEP \p Op0, (G_CONSTANT \p Value) /// @@ -390,7 +401,7 @@ public: /// type as \p Op0 or \p Op0 itself. /// /// \return a MachineInstrBuilder for the newly created instruction. - Optional materializeGEP(unsigned &Res, unsigned Op0, + Optional materializeGEP(Register &Res, Register Op0, const LLT &ValueTy, uint64_t Value); @@ -407,9 +418,24 @@ public: /// be cleared in \p Op0. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildPtrMask(unsigned Res, unsigned Op0, + MachineInstrBuilder buildPtrMask(const DstOp &Res, const SrcOp &Op0, uint32_t NumBits); + /// Build and insert \p Res, \p CarryOut = G_UADDO \p Op0, \p Op1 + /// + /// G_UADDO sets \p Res to \p Op0 + \p Op1 (truncated to the bit width) and + /// sets \p CarryOut to 1 if the result overflowed in unsigned arithmetic. + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p Res, \p Op0 and \p Op1 must be generic virtual registers with the + /// same scalar type. + ////\pre \p CarryOut must be generic virtual register with scalar type + ///(typically s1) + /// + /// \return The newly created instruction. + MachineInstrBuilder buildUAddo(const DstOp &Res, const DstOp &CarryOut, + const SrcOp &Op0, const SrcOp &Op1); + /// Build and insert \p Res, \p CarryOut = G_UADDE \p Op0, /// \p Op1, \p CarryIn /// @@ -458,6 +484,25 @@ public: /// \return The newly created instruction. MachineInstrBuilder buildSExt(const DstOp &Res, const SrcOp &Op); + /// Build and insert a G_PTRTOINT instruction. + MachineInstrBuilder buildPtrToInt(const DstOp &Dst, const SrcOp &Src) { + return buildInstr(TargetOpcode::G_PTRTOINT, {Dst}, {Src}); + } + + /// Build and insert \p Dst = G_BITCAST \p Src + MachineInstrBuilder buildBitcast(const DstOp &Dst, const SrcOp &Src) { + return buildInstr(TargetOpcode::G_BITCAST, {Dst}, {Src}); + } + + /// \return The opcode of the extension the target wants to use for boolean + /// values. + unsigned getBoolExtOp(bool IsVec, bool IsFP) const; + + // Build and insert \p Res = G_ANYEXT \p Op, \p Res = G_SEXT \p Op, or \p Res + // = G_ZEXT \p Op depending on how the target wants to extend boolean values. + MachineInstrBuilder buildBoolExt(const DstOp &Res, const SrcOp &Op, + bool IsFP); + /// Build and insert \p Res = G_ZEXT \p Op /// /// G_ZEXT produces a register of the specified width, with bits 0 to @@ -538,7 +583,7 @@ public: /// depend on bit 0 (for now). /// /// \return The newly created instruction. - MachineInstrBuilder buildBrCond(unsigned Tst, MachineBasicBlock &Dest); + MachineInstrBuilder buildBrCond(Register Tst, MachineBasicBlock &Dest); /// Build and insert G_BRINDIRECT \p Tgt /// @@ -548,7 +593,21 @@ public: /// \pre \p Tgt must be a generic virtual register with pointer type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildBrIndirect(unsigned Tgt); + MachineInstrBuilder buildBrIndirect(Register Tgt); + + /// Build and insert G_BRJT \p TablePtr, \p JTI, \p IndexReg + /// + /// G_BRJT is a jump table branch using a table base pointer \p TablePtr, + /// jump table index \p JTI and index \p IndexReg + /// + /// \pre setBasicBlock or setMI must have been called. + /// \pre \p TablePtr must be a generic virtual register with pointer type. + /// \pre \p JTI must be be a jump table index. + /// \pre \p IndexReg must be a generic virtual register with pointer type. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildBrJT(Register TablePtr, unsigned JTI, + Register IndexReg); /// Build and insert \p Res = G_CONSTANT \p Val /// @@ -572,6 +631,7 @@ public: /// /// \return The newly created instruction. MachineInstrBuilder buildConstant(const DstOp &Res, int64_t Val); + MachineInstrBuilder buildConstant(const DstOp &Res, const APInt &Val); /// Build and insert \p Res = G_FCONSTANT \p Val /// @@ -586,6 +646,7 @@ public: const ConstantFP &Val); MachineInstrBuilder buildFConstant(const DstOp &Res, double Val); + MachineInstrBuilder buildFConstant(const DstOp &Res, const APFloat &Val); /// Build and insert \p Res = COPY Op /// @@ -605,7 +666,7 @@ public: /// \pre \p Addr must be a generic virtual register with pointer type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildLoad(unsigned Res, unsigned Addr, + MachineInstrBuilder buildLoad(const DstOp &Res, const SrcOp &Addr, MachineMemOperand &MMO); /// Build and insert `Res = Addr, MMO`. @@ -617,8 +678,8 @@ public: /// \pre \p Addr must be a generic virtual register with pointer type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildLoadInstr(unsigned Opcode, unsigned Res, - unsigned Addr, MachineMemOperand &MMO); + MachineInstrBuilder buildLoadInstr(unsigned Opcode, const DstOp &Res, + const SrcOp &Addr, MachineMemOperand &MMO); /// Build and insert `G_STORE Val, Addr, MMO`. /// @@ -629,7 +690,7 @@ public: /// \pre \p Addr must be a generic virtual register with pointer type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildStore(unsigned Val, unsigned Addr, + MachineInstrBuilder buildStore(const SrcOp &Val, const SrcOp &Addr, MachineMemOperand &MMO); /// Build and insert `Res0, ... = G_EXTRACT Src, Idx0`. @@ -638,7 +699,7 @@ public: /// \pre \p Res and \p Src must be generic virtual registers. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildExtract(unsigned Res, unsigned Src, uint64_t Index); + MachineInstrBuilder buildExtract(const DstOp &Res, const SrcOp &Src, uint64_t Index); /// Build and insert \p Res = IMPLICIT_DEF. MachineInstrBuilder buildUndef(const DstOp &Res); @@ -656,7 +717,7 @@ public: /// \pre The bits defined by each Op (derived from index and scalar size) must /// not overlap. /// \pre \p Indices must be in ascending order of bit position. - void buildSequence(unsigned Res, ArrayRef Ops, + void buildSequence(Register Res, ArrayRef Ops, ArrayRef Indices); /// Build and insert \p Res = G_MERGE_VALUES \p Op0, ... @@ -670,7 +731,7 @@ public: /// \pre The type of all \p Ops registers must be identical. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildMerge(const DstOp &Res, ArrayRef Ops); + MachineInstrBuilder buildMerge(const DstOp &Res, ArrayRef Ops); /// Build and insert \p Res0, ... = G_UNMERGE_VALUES \p Op /// @@ -683,7 +744,10 @@ public: /// /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildUnmerge(ArrayRef Res, const SrcOp &Op); - MachineInstrBuilder buildUnmerge(ArrayRef Res, const SrcOp &Op); + MachineInstrBuilder buildUnmerge(ArrayRef Res, const SrcOp &Op); + + /// Build and insert an unmerge of \p Res sized pieces to cover \p Op + MachineInstrBuilder buildUnmerge(LLT Res, const SrcOp &Op); /// Build and insert \p Res = G_BUILD_VECTOR \p Op0, ... /// @@ -695,7 +759,12 @@ public: /// /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildBuildVector(const DstOp &Res, - ArrayRef Ops); + ArrayRef Ops); + + /// Build and insert \p Res = G_BUILD_VECTOR with \p Src replicated to fill + /// the number of elements + MachineInstrBuilder buildSplatVector(const DstOp &Res, + const SrcOp &Src); /// Build and insert \p Res = G_BUILD_VECTOR_TRUNC \p Op0, ... /// @@ -711,7 +780,7 @@ public: /// /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildBuildVectorTrunc(const DstOp &Res, - ArrayRef Ops); + ArrayRef Ops); /// Build and insert \p Res = G_CONCAT_VECTORS \p Op0, ... /// @@ -725,10 +794,10 @@ public: /// /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder buildConcatVectors(const DstOp &Res, - ArrayRef Ops); + ArrayRef Ops); - MachineInstrBuilder buildInsert(unsigned Res, unsigned Src, - unsigned Op, unsigned Index); + MachineInstrBuilder buildInsert(Register Res, Register Src, + Register Op, unsigned Index); /// Build and insert either a G_INTRINSIC (if \p HasSideEffects is false) or /// G_INTRINSIC_W_SIDE_EFFECTS instruction. Its first operand will be the @@ -740,7 +809,9 @@ public: /// \pre setBasicBlock or setMI must have been called. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, unsigned Res, + MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef Res, + bool HasSideEffects); + MachineInstrBuilder buildIntrinsic(Intrinsic::ID ID, ArrayRef Res, bool HasSideEffects); /// Build and insert \p Res = G_FPTRUNC \p Op @@ -855,8 +926,8 @@ public: /// /// \return a MachineInstrBuilder for the newly created instruction. MachineInstrBuilder - buildAtomicCmpXchgWithSuccess(unsigned OldValRes, unsigned SuccessRes, - unsigned Addr, unsigned CmpVal, unsigned NewVal, + buildAtomicCmpXchgWithSuccess(Register OldValRes, Register SuccessRes, + Register Addr, Register CmpVal, Register NewVal, MachineMemOperand &MMO); /// Build and insert `OldValRes = G_ATOMIC_CMPXCHG Addr, CmpVal, NewVal, @@ -873,8 +944,8 @@ public: /// registers of the same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicCmpXchg(unsigned OldValRes, unsigned Addr, - unsigned CmpVal, unsigned NewVal, + MachineInstrBuilder buildAtomicCmpXchg(Register OldValRes, Register Addr, + Register CmpVal, Register NewVal, MachineMemOperand &MMO); /// Build and insert `OldValRes = G_ATOMICRMW_ Addr, Val, MMO`. @@ -890,8 +961,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMW(unsigned Opcode, unsigned OldValRes, - unsigned Addr, unsigned Val, + MachineInstrBuilder buildAtomicRMW(unsigned Opcode, Register OldValRes, + Register Addr, Register Val, MachineMemOperand &MMO); /// Build and insert `OldValRes = G_ATOMICRMW_XCHG Addr, Val, MMO`. @@ -906,8 +977,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMWXchg(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO); + MachineInstrBuilder buildAtomicRMWXchg(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO); /// Build and insert `OldValRes = G_ATOMICRMW_ADD Addr, Val, MMO`. /// @@ -921,8 +992,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMWAdd(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO); + MachineInstrBuilder buildAtomicRMWAdd(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO); /// Build and insert `OldValRes = G_ATOMICRMW_SUB Addr, Val, MMO`. /// @@ -936,8 +1007,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMWSub(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO); + MachineInstrBuilder buildAtomicRMWSub(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO); /// Build and insert `OldValRes = G_ATOMICRMW_AND Addr, Val, MMO`. /// @@ -951,8 +1022,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMWAnd(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO); + MachineInstrBuilder buildAtomicRMWAnd(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO); /// Build and insert `OldValRes = G_ATOMICRMW_NAND Addr, Val, MMO`. /// @@ -967,8 +1038,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMWNand(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO); + MachineInstrBuilder buildAtomicRMWNand(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO); /// Build and insert `OldValRes = G_ATOMICRMW_OR Addr, Val, MMO`. /// @@ -982,8 +1053,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMWOr(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO); + MachineInstrBuilder buildAtomicRMWOr(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO); /// Build and insert `OldValRes = G_ATOMICRMW_XOR Addr, Val, MMO`. /// @@ -997,8 +1068,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMWXor(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO); + MachineInstrBuilder buildAtomicRMWXor(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO); /// Build and insert `OldValRes = G_ATOMICRMW_MAX Addr, Val, MMO`. /// @@ -1013,8 +1084,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMWMax(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO); + MachineInstrBuilder buildAtomicRMWMax(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO); /// Build and insert `OldValRes = G_ATOMICRMW_MIN Addr, Val, MMO`. /// @@ -1029,8 +1100,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMWMin(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO); + MachineInstrBuilder buildAtomicRMWMin(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO); /// Build and insert `OldValRes = G_ATOMICRMW_UMAX Addr, Val, MMO`. /// @@ -1045,8 +1116,8 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMWUmax(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO); + MachineInstrBuilder buildAtomicRMWUmax(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO); /// Build and insert `OldValRes = G_ATOMICRMW_UMIN Addr, Val, MMO`. /// @@ -1061,8 +1132,11 @@ public: /// same type. /// /// \return a MachineInstrBuilder for the newly created instruction. - MachineInstrBuilder buildAtomicRMWUmin(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO); + MachineInstrBuilder buildAtomicRMWUmin(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO); + + /// Build and insert `G_FENCE Ordering, Scope`. + MachineInstrBuilder buildFence(unsigned Ordering, unsigned Scope); /// Build and insert \p Res = G_BLOCK_ADDR \p BA /// @@ -1072,7 +1146,7 @@ public: /// \pre \p Res must be a generic virtual register of a pointer type. /// /// \return The newly created instruction. - MachineInstrBuilder buildBlockAddress(unsigned Res, const BlockAddress *BA); + MachineInstrBuilder buildBlockAddress(Register Res, const BlockAddress *BA); /// Build and insert \p Res = G_ADD \p Op0, \p Op1 /// @@ -1124,6 +1198,36 @@ public: return buildInstr(TargetOpcode::G_MUL, {Dst}, {Src0, Src1}, Flags); } + MachineInstrBuilder buildUMulH(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1, + Optional Flags = None) { + return buildInstr(TargetOpcode::G_UMULH, {Dst}, {Src0, Src1}, Flags); + } + + MachineInstrBuilder buildSMulH(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1, + Optional Flags = None) { + return buildInstr(TargetOpcode::G_SMULH, {Dst}, {Src0, Src1}, Flags); + } + + MachineInstrBuilder buildShl(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1, + Optional Flags = None) { + return buildInstr(TargetOpcode::G_SHL, {Dst}, {Src0, Src1}, Flags); + } + + MachineInstrBuilder buildLShr(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1, + Optional Flags = None) { + return buildInstr(TargetOpcode::G_LSHR, {Dst}, {Src0, Src1}, Flags); + } + + MachineInstrBuilder buildAShr(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1, + Optional Flags = None) { + return buildInstr(TargetOpcode::G_ASHR, {Dst}, {Src0, Src1}, Flags); + } + /// Build and insert \p Res = G_AND \p Op0, \p Op1 /// /// G_AND sets \p Res to the bitwise and of integer parameters \p Op0 and \p @@ -1155,6 +1259,137 @@ public: return buildInstr(TargetOpcode::G_OR, {Dst}, {Src0, Src1}); } + /// Build and insert \p Res = G_XOR \p Op0, \p Op1 + MachineInstrBuilder buildXor(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1) { + return buildInstr(TargetOpcode::G_XOR, {Dst}, {Src0, Src1}); + } + + /// Build and insert a bitwise not, + /// \p NegOne = G_CONSTANT -1 + /// \p Res = G_OR \p Op0, NegOne + MachineInstrBuilder buildNot(const DstOp &Dst, const SrcOp &Src0) { + auto NegOne = buildConstant(Dst.getLLTTy(*getMRI()), -1); + return buildInstr(TargetOpcode::G_XOR, {Dst}, {Src0, NegOne}); + } + + /// Build and insert \p Res = G_CTPOP \p Op0, \p Src0 + MachineInstrBuilder buildCTPOP(const DstOp &Dst, const SrcOp &Src0) { + return buildInstr(TargetOpcode::G_CTPOP, {Dst}, {Src0}); + } + + /// Build and insert \p Res = G_CTLZ \p Op0, \p Src0 + MachineInstrBuilder buildCTLZ(const DstOp &Dst, const SrcOp &Src0) { + return buildInstr(TargetOpcode::G_CTLZ, {Dst}, {Src0}); + } + + /// Build and insert \p Res = G_CTLZ_ZERO_UNDEF \p Op0, \p Src0 + MachineInstrBuilder buildCTLZ_ZERO_UNDEF(const DstOp &Dst, const SrcOp &Src0) { + return buildInstr(TargetOpcode::G_CTLZ_ZERO_UNDEF, {Dst}, {Src0}); + } + + /// Build and insert \p Res = G_CTTZ \p Op0, \p Src0 + MachineInstrBuilder buildCTTZ(const DstOp &Dst, const SrcOp &Src0) { + return buildInstr(TargetOpcode::G_CTTZ, {Dst}, {Src0}); + } + + /// Build and insert \p Res = G_CTTZ_ZERO_UNDEF \p Op0, \p Src0 + MachineInstrBuilder buildCTTZ_ZERO_UNDEF(const DstOp &Dst, const SrcOp &Src0) { + return buildInstr(TargetOpcode::G_CTTZ_ZERO_UNDEF, {Dst}, {Src0}); + } + + /// Build and insert \p Res = G_FADD \p Op0, \p Op1 + MachineInstrBuilder buildFAdd(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1) { + return buildInstr(TargetOpcode::G_FADD, {Dst}, {Src0, Src1}); + } + + /// Build and insert \p Res = G_FSUB \p Op0, \p Op1 + MachineInstrBuilder buildFSub(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1) { + return buildInstr(TargetOpcode::G_FSUB, {Dst}, {Src0, Src1}); + } + + /// Build and insert \p Res = G_FMA \p Op0, \p Op1, \p Op2 + MachineInstrBuilder buildFMA(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1, const SrcOp &Src2) { + return buildInstr(TargetOpcode::G_FMA, {Dst}, {Src0, Src1, Src2}); + } + + /// Build and insert \p Res = G_FNEG \p Op0 + MachineInstrBuilder buildFNeg(const DstOp &Dst, const SrcOp &Src0) { + return buildInstr(TargetOpcode::G_FNEG, {Dst}, {Src0}); + } + + /// Build and insert \p Res = G_FABS \p Op0 + MachineInstrBuilder buildFAbs(const DstOp &Dst, const SrcOp &Src0) { + return buildInstr(TargetOpcode::G_FABS, {Dst}, {Src0}); + } + + /// Build and insert \p Dst = G_FCANONICALIZE \p Src0 + MachineInstrBuilder buildFCanonicalize(const DstOp &Dst, const SrcOp &Src0, + Optional Flags = None) { + return buildInstr(TargetOpcode::G_FCANONICALIZE, {Dst}, {Src0}, Flags); + } + + /// Build and insert \p Res = G_FCOPYSIGN \p Op0, \p Op1 + MachineInstrBuilder buildFCopysign(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1) { + return buildInstr(TargetOpcode::G_FCOPYSIGN, {Dst}, {Src0, Src1}); + } + + /// Build and insert \p Res = G_UITOFP \p Src0 + MachineInstrBuilder buildUITOFP(const DstOp &Dst, const SrcOp &Src0) { + return buildInstr(TargetOpcode::G_UITOFP, {Dst}, {Src0}); + } + + /// Build and insert \p Res = G_SITOFP \p Src0 + MachineInstrBuilder buildSITOFP(const DstOp &Dst, const SrcOp &Src0) { + return buildInstr(TargetOpcode::G_SITOFP, {Dst}, {Src0}); + } + + /// Build and insert \p Res = G_FPTOUI \p Src0 + MachineInstrBuilder buildFPTOUI(const DstOp &Dst, const SrcOp &Src0) { + return buildInstr(TargetOpcode::G_FPTOUI, {Dst}, {Src0}); + } + + /// Build and insert \p Res = G_FPTOSI \p Src0 + MachineInstrBuilder buildFPTOSI(const DstOp &Dst, const SrcOp &Src0) { + return buildInstr(TargetOpcode::G_FPTOSI, {Dst}, {Src0}); + } + + /// Build and insert \p Res = G_SMIN \p Op0, \p Op1 + MachineInstrBuilder buildSMin(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1) { + return buildInstr(TargetOpcode::G_SMIN, {Dst}, {Src0, Src1}); + } + + /// Build and insert \p Res = G_SMAX \p Op0, \p Op1 + MachineInstrBuilder buildSMax(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1) { + return buildInstr(TargetOpcode::G_SMAX, {Dst}, {Src0, Src1}); + } + + /// Build and insert \p Res = G_UMIN \p Op0, \p Op1 + MachineInstrBuilder buildUMin(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1) { + return buildInstr(TargetOpcode::G_UMIN, {Dst}, {Src0, Src1}); + } + + /// Build and insert \p Res = G_UMAX \p Op0, \p Op1 + MachineInstrBuilder buildUMax(const DstOp &Dst, const SrcOp &Src0, + const SrcOp &Src1) { + return buildInstr(TargetOpcode::G_UMAX, {Dst}, {Src0, Src1}); + } + + /// Build and insert \p Res = G_JUMP_TABLE \p JTI + /// + /// G_JUMP_TABLE sets \p Res to the address of the jump table specified by + /// the jump table index \p JTI. + /// + /// \return a MachineInstrBuilder for the newly created instruction. + MachineInstrBuilder buildJumpTable(const LLT PtrTy, unsigned JTI); + virtual MachineInstrBuilder buildInstr(unsigned Opc, ArrayRef DstOps, ArrayRef SrcOps, Optional Flags = None); diff --git a/include/llvm/CodeGen/GlobalISel/RegBankSelect.h b/include/llvm/CodeGen/GlobalISel/RegBankSelect.h index c53ae416e60b..d9d076ba312c 100644 --- a/include/llvm/CodeGen/GlobalISel/RegBankSelect.h +++ b/include/llvm/CodeGen/GlobalISel/RegBankSelect.h @@ -1,9 +1,8 @@ //=- llvm/CodeGen/GlobalISel/RegBankSelect.h - Reg Bank Selector --*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -524,7 +523,7 @@ private: /// \p OnlyAssign == true means that \p Reg just needs to be assigned a /// register bank. I.e., no repairing is necessary to have the /// assignment match. - bool assignmentMatch(unsigned Reg, + bool assignmentMatch(Register Reg, const RegisterBankInfo::ValueMapping &ValMapping, bool &OnlyAssign) const; @@ -563,7 +562,7 @@ private: bool repairReg(MachineOperand &MO, const RegisterBankInfo::ValueMapping &ValMapping, RegBankSelect::RepairingPlacement &RepairPt, - const iterator_range::const_iterator> + const iterator_range::const_iterator> &NewVRegs); /// Return the cost of the instruction needed to map \p MO to \p ValMapping. @@ -634,6 +633,11 @@ public: MachineFunctionProperties::Property::RegBankSelected); } + MachineFunctionProperties getClearedProperties() const override { + return MachineFunctionProperties() + .set(MachineFunctionProperties::Property::NoPHIs); + } + /// Walk through \p MF and assign a register bank to every virtual register /// that are still mapped to nothing. /// The target needs to provide a RegisterBankInfo and in particular diff --git a/include/llvm/CodeGen/GlobalISel/RegisterBank.h b/include/llvm/CodeGen/GlobalISel/RegisterBank.h index d5612e17393c..f528d1a46012 100644 --- a/include/llvm/CodeGen/GlobalISel/RegisterBank.h +++ b/include/llvm/CodeGen/GlobalISel/RegisterBank.h @@ -1,9 +1,8 @@ //==-- llvm/CodeGen/GlobalISel/RegisterBank.h - Register Bank ----*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h b/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h index c33b32b2db40..e84b1c3ea8b1 100644 --- a/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h +++ b/include/llvm/CodeGen/GlobalISel/RegisterBankInfo.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/RegisterBankInfo.h ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,6 +18,7 @@ #include "llvm/ADT/Hashing.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/iterator_range.h" +#include "llvm/CodeGen/Register.h" #include "llvm/Support/ErrorHandling.h" #include #include @@ -161,6 +161,10 @@ public: const PartialMapping *begin() const { return BreakDown; } const PartialMapping *end() const { return BreakDown + NumBreakDowns; } + /// \return true if all partial mappings are the same size and register + /// bank. + bool partsAllUniform() const; + /// Check if this ValueMapping is valid. bool isValid() const { return BreakDown && NumBreakDowns; } @@ -190,7 +194,7 @@ public: unsigned Cost = 0; /// Mapping of all the operands. - const ValueMapping *OperandsMapping; + const ValueMapping *OperandsMapping = nullptr; /// Number of operands. unsigned NumOperands = 0; @@ -207,15 +211,11 @@ public: /// The rationale is that it is more efficient for the optimizers /// to be able to assume that the mapping of the ith operand is /// at the index i. - /// - /// \pre ID != InvalidMappingID InstructionMapping(unsigned ID, unsigned Cost, const ValueMapping *OperandsMapping, unsigned NumOperands) : ID(ID), Cost(Cost), OperandsMapping(OperandsMapping), NumOperands(NumOperands) { - assert(getID() != InvalidMappingID && - "Use the default constructor for invalid mapping"); } /// Default constructor. @@ -282,7 +282,7 @@ public: SmallVector OpToNewVRegIdx; /// Hold the registers that will be used to map MI with InstrMapping. - SmallVector NewVRegs; + SmallVector NewVRegs; /// Current MachineRegisterInfo, used to create new virtual registers. MachineRegisterInfo &MRI; @@ -303,15 +303,15 @@ public: /// \return The iterator range for the space created. // /// \pre getMI().getOperand(OpIdx).isReg() - iterator_range::iterator> + iterator_range::iterator> getVRegsMem(unsigned OpIdx); /// Get the end iterator for a range starting at \p StartIdx and /// spannig \p NumVal in NewVRegs. /// \pre StartIdx + NumVal <= NewVRegs.size() - SmallVectorImpl::const_iterator + SmallVectorImpl::const_iterator getNewVRegsEnd(unsigned StartIdx, unsigned NumVal) const; - SmallVectorImpl::iterator getNewVRegsEnd(unsigned StartIdx, + SmallVectorImpl::iterator getNewVRegsEnd(unsigned StartIdx, unsigned NumVal); public: @@ -357,7 +357,7 @@ public: /// /// \post the \p PartialMapIdx-th register of the value mapping of the \p /// OpIdx-th operand has been set. - void setVRegs(unsigned OpIdx, unsigned PartialMapIdx, unsigned NewVReg); + void setVRegs(unsigned OpIdx, unsigned PartialMapIdx, Register NewVReg); /// Get all the virtual registers required to map the \p OpIdx-th operand of /// the instruction. @@ -371,7 +371,7 @@ public: /// /// \pre getMI().getOperand(OpIdx).isReg() /// \pre ForDebug || All partial mappings have been set a register - iterator_range::const_iterator> + iterator_range::const_iterator> getVRegs(unsigned OpIdx, bool ForDebug = false) const; /// Print this operands mapper on dbgs() stream. @@ -435,7 +435,7 @@ protected: /// Get the MinimalPhysRegClass for Reg. /// \pre Reg is a physical register. const TargetRegisterClass & - getMinimalPhysRegClass(unsigned Reg, const TargetRegisterInfo &TRI) const; + getMinimalPhysRegClass(Register Reg, const TargetRegisterInfo &TRI) const; /// Try to get the mapping of \p MI. /// See getInstrMapping for more details on what a mapping represents. @@ -580,7 +580,7 @@ public: /// or a register bank, then this returns nullptr. /// /// \pre Reg != 0 (NoRegister) - const RegisterBank *getRegBank(unsigned Reg, const MachineRegisterInfo &MRI, + const RegisterBank *getRegBank(Register Reg, const MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI) const; /// Get the total number of register banks. @@ -618,6 +618,21 @@ public: return &A != &B; } + /// \returns true if emitting a copy from \p Src to \p Dst is impossible. + bool cannotCopy(const RegisterBank &Dst, const RegisterBank &Src, + unsigned Size) const { + return copyCost(Dst, Src, Size) == std::numeric_limits::max(); + } + + /// Get the cost of using \p ValMapping to decompose a register. This is + /// similar to ::copyCost, except for cases where multiple copy-like + /// operations need to be inserted. If the register is used as a source + /// operand and already has a bank assigned, \p CurBank is non-null. + virtual unsigned getBreakDownCost(const ValueMapping &ValMapping, + const RegisterBank *CurBank = nullptr) const { + return std::numeric_limits::max(); + } + /// Constrain the (possibly generic) virtual register \p Reg to \p RC. /// /// \pre \p Reg is a virtual register that either has a bank or a class. @@ -626,7 +641,7 @@ public: /// \note Use MachineRegisterInfo::constrainRegAttrs instead for any non-isel /// purpose, including non-select passes of GlobalISel static const TargetRegisterClass * - constrainGenericRegister(unsigned Reg, const TargetRegisterClass &RC, + constrainGenericRegister(Register Reg, const TargetRegisterClass &RC, MachineRegisterInfo &MRI); /// Identifier used when the related instruction mapping instance @@ -711,7 +726,7 @@ public: /// virtual register. /// /// \pre \p Reg != 0 (NoRegister). - unsigned getSizeInBits(unsigned Reg, const MachineRegisterInfo &MRI, + unsigned getSizeInBits(Register Reg, const MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI) const; /// Check that information hold by this instance make sense for the diff --git a/include/llvm/CodeGen/GlobalISel/Types.h b/include/llvm/CodeGen/GlobalISel/Types.h index 7b22e343a7f8..4fd7043ba02d 100644 --- a/include/llvm/CodeGen/GlobalISel/Types.h +++ b/include/llvm/CodeGen/GlobalISel/Types.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/Types.h - Types used by GISel ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/include/llvm/CodeGen/GlobalISel/Utils.h b/include/llvm/CodeGen/GlobalISel/Utils.h index 82b791d35b2b..4cdaa48fb689 100644 --- a/include/llvm/CodeGen/GlobalISel/Utils.h +++ b/include/llvm/CodeGen/GlobalISel/Utils.h @@ -1,9 +1,8 @@ //==-- llvm/CodeGen/GlobalISel/Utils.h ---------------------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -16,6 +15,7 @@ #define LLVM_CODEGEN_GLOBALISEL_UTILS_H #include "llvm/ADT/StringRef.h" +#include "llvm/CodeGen/Register.h" namespace llvm { @@ -37,21 +37,37 @@ class ConstantFP; class APFloat; /// Try to constrain Reg to the specified register class. If this fails, -/// create a new virtual register in the correct class and insert a COPY before -/// \p InsertPt. The debug location of \p InsertPt is used for the new copy. +/// create a new virtual register in the correct class. /// /// \return The virtual register constrained to the right register class. unsigned constrainRegToClass(MachineRegisterInfo &MRI, const TargetInstrInfo &TII, - const RegisterBankInfo &RBI, - MachineInstr &InsertPt, unsigned Reg, + const RegisterBankInfo &RBI, unsigned Reg, const TargetRegisterClass &RegClass); +/// Constrain the Register operand OpIdx, so that it is now constrained to the +/// TargetRegisterClass passed as an argument (RegClass). +/// If this fails, create a new virtual register in the correct class and +/// insert a COPY before \p InsertPt if it is a use or after if it is a +/// definition. The debug location of \p InsertPt is used for the new copy. +/// +/// \return The virtual register constrained to the right register class. +unsigned constrainOperandRegClass(const MachineFunction &MF, + const TargetRegisterInfo &TRI, + MachineRegisterInfo &MRI, + const TargetInstrInfo &TII, + const RegisterBankInfo &RBI, + MachineInstr &InsertPt, + const TargetRegisterClass &RegClass, + const MachineOperand &RegMO, unsigned OpIdx); + /// Try to constrain Reg so that it is usable by argument OpIdx of the /// provided MCInstrDesc \p II. If this fails, create a new virtual -/// register in the correct class and insert a COPY before \p InsertPt. -/// This is equivalent to constrainRegToClass() with RegClass obtained from the -/// MCInstrDesc. The debug location of \p InsertPt is used for the new copy. +/// register in the correct class and insert a COPY before \p InsertPt +/// if it is a use or after if it is a definition. +/// This is equivalent to constrainOperandRegClass(..., RegClass, ...) +/// with RegClass obtained from the MCInstrDesc. The debug location of \p +/// InsertPt is used for the new copy. /// /// \return The virtual register constrained to the right register class. unsigned constrainOperandRegClass(const MachineFunction &MF, @@ -90,17 +106,40 @@ void reportGISelFailure(MachineFunction &MF, const TargetPassConfig &TPC, const char *PassName, StringRef Msg, const MachineInstr &MI); +/// If \p VReg is defined by a G_CONSTANT fits in int64_t +/// returns it. Optional getConstantVRegVal(unsigned VReg, const MachineRegisterInfo &MRI); +/// Simple struct used to hold a constant integer value and a virtual +/// register. +struct ValueAndVReg { + int64_t Value; + unsigned VReg; +}; +/// If \p VReg is defined by a statically evaluable chain of +/// instructions rooted on a G_CONSTANT (\p LookThroughInstrs == true) +/// and that constant fits in int64_t, returns its value as well as +/// the virtual register defined by this G_CONSTANT. +/// When \p LookThroughInstrs == false, this function behaves like +/// getConstantVRegVal. +Optional +getConstantVRegValWithLookThrough(unsigned VReg, const MachineRegisterInfo &MRI, + bool LookThroughInstrs = true); const ConstantFP* getConstantFPVRegVal(unsigned VReg, const MachineRegisterInfo &MRI); /// See if Reg is defined by an single def instruction that is /// Opcode. Also try to do trivial folding if it's a COPY with /// same types. Returns null otherwise. -MachineInstr *getOpcodeDef(unsigned Opcode, unsigned Reg, +MachineInstr *getOpcodeDef(unsigned Opcode, Register Reg, const MachineRegisterInfo &MRI); +/// Find the def instruction for \p Reg, folding away any trivial copies. Note +/// it may still return a COPY, if it changes the type. May return nullptr if \p +/// Reg is not a generic virtual register. +MachineInstr *getDefIgnoringCopies(Register Reg, + const MachineRegisterInfo &MRI); + /// Returns an APFloat from Val converted to the appropriate size. APFloat getAPFloatFromSize(double Val, unsigned Size); @@ -111,5 +150,16 @@ void getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU); Optional ConstantFoldBinOp(unsigned Opcode, const unsigned Op1, const unsigned Op2, const MachineRegisterInfo &MRI); + +/// Returns true if \p Val can be assumed to never be a NaN. If \p SNaN is true, +/// this returns if \p Val can be assumed to never be a signaling NaN. +bool isKnownNeverNaN(Register Val, const MachineRegisterInfo &MRI, + bool SNaN = false); + +/// Returns true if \p Val can be assumed to never be a signaling NaN. +inline bool isKnownNeverSNaN(Register Val, const MachineRegisterInfo &MRI) { + return isKnownNeverNaN(Val, MRI, true); +} + } // End namespace llvm. #endif diff --git a/include/llvm/CodeGen/ISDOpcodes.h b/include/llvm/CodeGen/ISDOpcodes.h index 9c918ae1104f..acf27dcc5fab 100644 --- a/include/llvm/CodeGen/ISDOpcodes.h +++ b/include/llvm/CodeGen/ISDOpcodes.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/ISDOpcodes.h - CodeGen opcodes -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -272,12 +271,17 @@ namespace ISD { /// resulting value is this minimum value. SSUBSAT, USUBSAT, - /// RESULT = SMULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on + /// RESULT = [US]MULFIX(LHS, RHS, SCALE) - Perform fixed point multiplication on /// 2 integers with the same width and scale. SCALE represents the scale of /// both operands as fixed point numbers. This SCALE parameter must be a /// constant integer. A scale of zero is effectively performing /// multiplication on 2 integers. - SMULFIX, + SMULFIX, UMULFIX, + + /// Same as the corresponding unsaturated fixed point instructions, but the + /// result is clamped between the min and max values representable by the + /// bits of the first 2 operands. + SMULFIXSAT, /// Simple binary floating point operators. FADD, FSUB, FMUL, FDIV, FREM, @@ -298,6 +302,26 @@ namespace ISD { STRICT_FRINT, STRICT_FNEARBYINT, STRICT_FMAXNUM, STRICT_FMINNUM, STRICT_FCEIL, STRICT_FFLOOR, STRICT_FROUND, STRICT_FTRUNC, + /// X = STRICT_FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating + /// point type down to the precision of the destination VT. TRUNC is a + /// flag, which is always an integer that is zero or one. If TRUNC is 0, + /// this is a normal rounding, if it is 1, this FP_ROUND is known to not + /// change the value of Y. + /// + /// The TRUNC = 1 case is used in cases where we know that the value will + /// not be modified by the node, because Y is not using any of the extra + /// precision of source type. This allows certain transformations like + /// STRICT_FP_EXTEND(STRICT_FP_ROUND(X,1)) -> X which are not safe for + /// STRICT_FP_EXTEND(STRICT_FP_ROUND(X,0)) because the extra bits aren't + /// removed. + /// It is used to limit optimizations while the DAG is being optimized. + STRICT_FP_ROUND, + + /// X = STRICT_FP_EXTEND(Y) - Extend a smaller FP type into a larger FP + /// type. + /// It is used to limit optimizations while the DAG is being optimized. + STRICT_FP_EXTEND, + /// FMA - Perform a * b + c with no intermediate rounding step. FMA, @@ -580,10 +604,14 @@ namespace ISD { /// is often a storage-only type but has native conversions. FP16_TO_FP, FP_TO_FP16, - /// Perform various unary floating-point operations inspired by libm. + /// Perform various unary floating-point operations inspired by libm. For + /// FPOWI, the result is undefined if if the integer operand doesn't fit + /// into 32 bits. FNEG, FABS, FSQRT, FCBRT, FSIN, FCOS, FPOWI, FPOW, FLOG, FLOG2, FLOG10, FEXP, FEXP2, FCEIL, FTRUNC, FRINT, FNEARBYINT, FROUND, FFLOOR, + LROUND, LLROUND, LRINT, LLRINT, + /// FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two /// values. // @@ -666,6 +694,9 @@ namespace ISD { /// SDOperands. INLINEASM, + /// INLINEASM_BR - Terminator version of inline asm. Used by asm-goto. + INLINEASM_BR, + /// EH_LABEL - Represents a label in mid basic block used to track /// locations needed for debug and exception handling tables. These nodes /// take a chain as input and return a chain. @@ -819,6 +850,8 @@ namespace ISD { ATOMIC_LOAD_MAX, ATOMIC_LOAD_UMIN, ATOMIC_LOAD_UMAX, + ATOMIC_LOAD_FADD, + ATOMIC_LOAD_FSUB, // Masked load and store - consecutive vector load and store operations // with additional mask operand that prevents memory accesses to the @@ -866,11 +899,14 @@ namespace ISD { VECREDUCE_STRICT_FADD, VECREDUCE_STRICT_FMUL, /// These reductions are non-strict, and have a single vector operand. VECREDUCE_FADD, VECREDUCE_FMUL, + /// FMIN/FMAX nodes can have flags, for NaN/NoNaN variants. + VECREDUCE_FMAX, VECREDUCE_FMIN, + /// Integer reductions may have a result type larger than the vector element + /// type. However, the reduction is performed using the vector element type + /// and the value in the top bits is unspecified. VECREDUCE_ADD, VECREDUCE_MUL, VECREDUCE_AND, VECREDUCE_OR, VECREDUCE_XOR, VECREDUCE_SMAX, VECREDUCE_SMIN, VECREDUCE_UMAX, VECREDUCE_UMIN, - /// FMIN/FMAX nodes can have flags, for NaN/NoNaN variants. - VECREDUCE_FMAX, VECREDUCE_FMIN, /// BUILTIN_OP_END - This must be the last enum value in this list. /// The target-specific pre-isel opcode values start here. diff --git a/include/llvm/CodeGen/IntrinsicLowering.h b/include/llvm/CodeGen/IntrinsicLowering.h index 597d684909c1..daf2d9a47801 100644 --- a/include/llvm/CodeGen/IntrinsicLowering.h +++ b/include/llvm/CodeGen/IntrinsicLowering.h @@ -1,9 +1,8 @@ //===-- IntrinsicLowering.h - Intrinsic Function Lowering -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -31,10 +30,6 @@ class IntrinsicLowering { public: explicit IntrinsicLowering(const DataLayout &DL) : DL(DL), Warned(false) {} - /// Add all of the prototypes that might be needed by an intrinsic lowering - /// implementation to be inserted into the module specified. - void AddPrototypes(Module &M); - /// Replace a call to the specified intrinsic function. /// If an intrinsic function must be implemented by the code generator /// (such as va_start), this function should print a message and abort. diff --git a/include/llvm/CodeGen/LatencyPriorityQueue.h b/include/llvm/CodeGen/LatencyPriorityQueue.h index 9b8d83ce77ca..95f4c6473542 100644 --- a/include/llvm/CodeGen/LatencyPriorityQueue.h +++ b/include/llvm/CodeGen/LatencyPriorityQueue.h @@ -1,9 +1,8 @@ //===---- LatencyPriorityQueue.h - A latency-oriented priority queue ------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/LazyMachineBlockFrequencyInfo.h b/include/llvm/CodeGen/LazyMachineBlockFrequencyInfo.h index 221f16a03f16..ca99c6c89b19 100644 --- a/include/llvm/CodeGen/LazyMachineBlockFrequencyInfo.h +++ b/include/llvm/CodeGen/LazyMachineBlockFrequencyInfo.h @@ -1,9 +1,8 @@ ///===- LazyMachineBlockFrequencyInfo.h - Lazy Block Frequency -*- C++ -*--===// /// -/// The LLVM Compiler Infrastructure -/// -/// This file is distributed under the University of Illinois Open Source -/// License. See LICENSE.TXT for details. +/// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +/// See https://llvm.org/LICENSE.txt for license information. +/// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception /// ///===---------------------------------------------------------------------===// /// \file diff --git a/include/llvm/CodeGen/LexicalScopes.h b/include/llvm/CodeGen/LexicalScopes.h index 3ba503487823..253d4734995b 100644 --- a/include/llvm/CodeGen/LexicalScopes.h +++ b/include/llvm/CodeGen/LexicalScopes.h @@ -1,9 +1,8 @@ //===- LexicalScopes.cpp - Collecting lexical scope info --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/LinkAllAsmWriterComponents.h b/include/llvm/CodeGen/LinkAllAsmWriterComponents.h index 38fcb37b1e69..75a5c359630e 100644 --- a/include/llvm/CodeGen/LinkAllAsmWriterComponents.h +++ b/include/llvm/CodeGen/LinkAllAsmWriterComponents.h @@ -1,9 +1,8 @@ //===- llvm/Codegen/LinkAllAsmWriterComponents.h ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/LinkAllCodegenComponents.h b/include/llvm/CodeGen/LinkAllCodegenComponents.h index 18c13ca8f598..56c93b24147e 100644 --- a/include/llvm/CodeGen/LinkAllCodegenComponents.h +++ b/include/llvm/CodeGen/LinkAllCodegenComponents.h @@ -1,9 +1,8 @@ //===- llvm/Codegen/LinkAllCodegenComponents.h ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/LiveInterval.h b/include/llvm/CodeGen/LiveInterval.h index cdf9ad2588cf..8bb88165d3e1 100644 --- a/include/llvm/CodeGen/LiveInterval.h +++ b/include/llvm/CodeGen/LiveInterval.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/LiveInterval.h - Interval representation ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -606,6 +605,44 @@ namespace llvm { /// activated in the constructor of the live range. void flushSegmentSet(); + /// Stores indexes from the input index sequence R at which this LiveRange + /// is live to the output O iterator. + /// R is a range of _ascending sorted_ _random_ access iterators + /// to the input indexes. Indexes stored at O are ascending sorted so it + /// can be used directly in the subsequent search (for example for + /// subranges). Returns true if found at least one index. + template + bool findIndexesLiveAt(Range &&R, OutputIt O) const { + assert(std::is_sorted(R.begin(), R.end())); + auto Idx = R.begin(), EndIdx = R.end(); + auto Seg = segments.begin(), EndSeg = segments.end(); + bool Found = false; + while (Idx != EndIdx && Seg != EndSeg) { + // if the Seg is lower find first segment that is above Idx using binary + // search + if (Seg->end <= *Idx) { + Seg = std::upper_bound(++Seg, EndSeg, *Idx, + [=](typename std::remove_reference::type V, + const typename std::remove_reference::type &S) { + return V < S.end; + }); + if (Seg == EndSeg) + break; + } + auto NotLessStart = std::lower_bound(Idx, EndIdx, Seg->start); + if (NotLessStart == EndIdx) + break; + auto NotLessEnd = std::lower_bound(NotLessStart, EndIdx, Seg->end); + if (NotLessEnd != NotLessStart) { + Found = true; + O = std::copy(NotLessStart, NotLessEnd, O); + } + Idx = NotLessEnd; + ++Seg; + } + return Found; + } + void print(raw_ostream &OS) const; void dump() const; @@ -790,8 +827,15 @@ namespace llvm { /// L000F, refining for mask L0018. Will split the L00F0 lane into /// L00E0 and L0010 and the L000F lane into L0007 and L0008. The Mod /// function will be applied to the L0010 and L0008 subranges. + /// + /// \p Indexes and \p TRI are required to clean up the VNIs that + /// don't defne the related lane masks after they get shrunk. E.g., + /// when L000F gets split into L0007 and L0008 maybe only a subset + /// of the VNIs that defined L000F defines L0007. void refineSubRanges(BumpPtrAllocator &Allocator, LaneBitmask LaneMask, - std::function Apply); + std::function Apply, + const SlotIndexes &Indexes, + const TargetRegisterInfo &TRI); bool operator<(const LiveInterval& other) const { const SlotIndex &thisIndex = beginIndex(); diff --git a/include/llvm/CodeGen/LiveIntervalUnion.h b/include/llvm/CodeGen/LiveIntervalUnion.h index 9e2799bd4414..05506d2c3bc6 100644 --- a/include/llvm/CodeGen/LiveIntervalUnion.h +++ b/include/llvm/CodeGen/LiveIntervalUnion.h @@ -1,9 +1,8 @@ //===- LiveIntervalUnion.h - Live interval union data struct ---*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/LiveIntervals.h b/include/llvm/CodeGen/LiveIntervals.h index 16ab1dc475c4..588b0f9cf39c 100644 --- a/include/llvm/CodeGen/LiveIntervals.h +++ b/include/llvm/CodeGen/LiveIntervals.h @@ -1,9 +1,8 @@ //===- LiveIntervals.h - Live Interval Analysis -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -418,6 +417,15 @@ class VirtRegMap; RegUnitRanges[Unit] = nullptr; } + /// Remove associated live ranges for the register units associated with \p + /// Reg. Subsequent uses should rely on on-demand recomputation. \note This + /// method can result in inconsistent liveness tracking if multiple phyical + /// registers share a regunit, and should be used cautiously. + void removeAllRegUnitsForPhysReg(unsigned Reg) { + for (MCRegUnitIterator Units(Reg, TRI); Units.isValid(); ++Units) + removeRegUnit(*Units); + } + /// Remove value numbers and related live segments starting at position /// \p Pos that are part of any liverange of physical register \p Reg or one /// of its subregisters. diff --git a/include/llvm/CodeGen/LivePhysRegs.h b/include/llvm/CodeGen/LivePhysRegs.h index 7312902e21b7..50da0b3d5c48 100644 --- a/include/llvm/CodeGen/LivePhysRegs.h +++ b/include/llvm/CodeGen/LivePhysRegs.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/LivePhysRegs.h - Live Physical Register Set -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/LiveRangeEdit.h b/include/llvm/CodeGen/LiveRangeEdit.h index 53830297c525..6519937ec071 100644 --- a/include/llvm/CodeGen/LiveRangeEdit.h +++ b/include/llvm/CodeGen/LiveRangeEdit.h @@ -1,9 +1,8 @@ //===- LiveRangeEdit.h - Basic tools for split and spill --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/LiveRegMatrix.h b/include/llvm/CodeGen/LiveRegMatrix.h index f62a55c73085..ab4d44f9a611 100644 --- a/include/llvm/CodeGen/LiveRegMatrix.h +++ b/include/llvm/CodeGen/LiveRegMatrix.h @@ -1,9 +1,8 @@ //===- LiveRegMatrix.h - Track register interference ----------*- C++ -*---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/LiveRegUnits.h b/include/llvm/CodeGen/LiveRegUnits.h index 5e9dd8b3cdf6..7dbb2feab8bf 100644 --- a/include/llvm/CodeGen/LiveRegUnits.h +++ b/include/llvm/CodeGen/LiveRegUnits.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/LiveRegUnits.h - Register Unit Set ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/LiveStacks.h b/include/llvm/CodeGen/LiveStacks.h index 44ed785f7b53..7c4c64d515df 100644 --- a/include/llvm/CodeGen/LiveStacks.h +++ b/include/llvm/CodeGen/LiveStacks.h @@ -1,9 +1,8 @@ //===- LiveStacks.h - Live Stack Slot Analysis ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/LiveVariables.h b/include/llvm/CodeGen/LiveVariables.h index ed8da8662106..71de306e2942 100644 --- a/include/llvm/CodeGen/LiveVariables.h +++ b/include/llvm/CodeGen/LiveVariables.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/LiveVariables.h - Live Variable Analysis ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/LoopTraversal.h b/include/llvm/CodeGen/LoopTraversal.h index 750da0143c0d..e5810ef1ef26 100644 --- a/include/llvm/CodeGen/LoopTraversal.h +++ b/include/llvm/CodeGen/LoopTraversal.h @@ -1,9 +1,8 @@ //==------ llvm/CodeGen/LoopTraversal.h - Loop Traversal -*- C++ -*---------==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/LowLevelType.h b/include/llvm/CodeGen/LowLevelType.h index a3c5c9329f53..687233e4e168 100644 --- a/include/llvm/CodeGen/LowLevelType.h +++ b/include/llvm/CodeGen/LowLevelType.h @@ -1,9 +1,8 @@ //== llvm/CodeGen/LowLevelType.h ------------------------------- -*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/MIRParser/MIParser.h b/include/llvm/CodeGen/MIRParser/MIParser.h new file mode 100644 index 000000000000..4e32a04551c1 --- /dev/null +++ b/include/llvm/CodeGen/MIRParser/MIParser.h @@ -0,0 +1,233 @@ +//===- MIParser.h - Machine Instructions Parser -----------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file declares the function that parses the machine instructions. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_CODEGEN_MIRPARSER_MIPARSER_H +#define LLVM_LIB_CODEGEN_MIRPARSER_MIPARSER_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/CodeGen/MachineMemOperand.h" +#include "llvm/Support/Allocator.h" + +namespace llvm { + +class MachineBasicBlock; +class MachineFunction; +class MDNode; +class RegisterBank; +struct SlotMapping; +class SMDiagnostic; +class SourceMgr; +class StringRef; +class TargetRegisterClass; +class TargetSubtargetInfo; + +struct VRegInfo { + enum uint8_t { + UNKNOWN, NORMAL, GENERIC, REGBANK + } Kind = UNKNOWN; + bool Explicit = false; ///< VReg was explicitly specified in the .mir file. + union { + const TargetRegisterClass *RC; + const RegisterBank *RegBank; + } D; + unsigned VReg; + unsigned PreferredReg = 0; +}; + +using Name2RegClassMap = StringMap; +using Name2RegBankMap = StringMap; + +struct PerTargetMIParsingState { +private: + const TargetSubtargetInfo &Subtarget; + + /// Maps from instruction names to op codes. + StringMap Names2InstrOpCodes; + + /// Maps from register names to registers. + StringMap Names2Regs; + + /// Maps from register mask names to register masks. + StringMap Names2RegMasks; + + /// Maps from subregister names to subregister indices. + StringMap Names2SubRegIndices; + + /// Maps from target index names to target indices. + StringMap Names2TargetIndices; + + /// Maps from direct target flag names to the direct target flag values. + StringMap Names2DirectTargetFlags; + + /// Maps from direct target flag names to the bitmask target flag values. + StringMap Names2BitmaskTargetFlags; + + /// Maps from MMO target flag names to MMO target flag values. + StringMap Names2MMOTargetFlags; + + /// Maps from register class names to register classes. + Name2RegClassMap Names2RegClasses; + + /// Maps from register bank names to register banks. + Name2RegBankMap Names2RegBanks; + + void initNames2InstrOpCodes(); + void initNames2Regs(); + void initNames2RegMasks(); + void initNames2SubRegIndices(); + void initNames2TargetIndices(); + void initNames2DirectTargetFlags(); + void initNames2BitmaskTargetFlags(); + void initNames2MMOTargetFlags(); + + void initNames2RegClasses(); + void initNames2RegBanks(); + +public: + /// Try to convert an instruction name to an opcode. Return true if the + /// instruction name is invalid. + bool parseInstrName(StringRef InstrName, unsigned &OpCode); + + /// Try to convert a register name to a register number. Return true if the + /// register name is invalid. + bool getRegisterByName(StringRef RegName, unsigned &Reg); + + /// Check if the given identifier is a name of a register mask. + /// + /// Return null if the identifier isn't a register mask. + const uint32_t *getRegMask(StringRef Identifier); + + /// Check if the given identifier is a name of a subregister index. + /// + /// Return 0 if the name isn't a subregister index class. + unsigned getSubRegIndex(StringRef Name); + + /// Try to convert a name of target index to the corresponding target index. + /// + /// Return true if the name isn't a name of a target index. + bool getTargetIndex(StringRef Name, int &Index); + + /// Try to convert a name of a direct target flag to the corresponding + /// target flag. + /// + /// Return true if the name isn't a name of a direct flag. + bool getDirectTargetFlag(StringRef Name, unsigned &Flag); + + /// Try to convert a name of a bitmask target flag to the corresponding + /// target flag. + /// + /// Return true if the name isn't a name of a bitmask target flag. + bool getBitmaskTargetFlag(StringRef Name, unsigned &Flag); + + /// Try to convert a name of a MachineMemOperand target flag to the + /// corresponding target flag. + /// + /// Return true if the name isn't a name of a target MMO flag. + bool getMMOTargetFlag(StringRef Name, MachineMemOperand::Flags &Flag); + + /// Check if the given identifier is a name of a register class. + /// + /// Return null if the name isn't a register class. + const TargetRegisterClass *getRegClass(StringRef Name); + + /// Check if the given identifier is a name of a register bank. + /// + /// Return null if the name isn't a register bank. + const RegisterBank *getRegBank(StringRef Name); + + PerTargetMIParsingState(const TargetSubtargetInfo &STI) + : Subtarget(STI) { + initNames2RegClasses(); + initNames2RegBanks(); + } + + ~PerTargetMIParsingState() = default; + + void setTarget(const TargetSubtargetInfo &NewSubtarget); +}; + +struct PerFunctionMIParsingState { + BumpPtrAllocator Allocator; + MachineFunction &MF; + SourceMgr *SM; + const SlotMapping &IRSlots; + PerTargetMIParsingState &Target; + + DenseMap MBBSlots; + DenseMap VRegInfos; + StringMap VRegInfosNamed; + DenseMap FixedStackObjectSlots; + DenseMap StackObjectSlots; + DenseMap ConstantPoolSlots; + DenseMap JumpTableSlots; + + PerFunctionMIParsingState(MachineFunction &MF, SourceMgr &SM, + const SlotMapping &IRSlots, + PerTargetMIParsingState &Target); + + VRegInfo &getVRegInfo(unsigned Num); + VRegInfo &getVRegInfoNamed(StringRef RegName); +}; + +/// Parse the machine basic block definitions, and skip the machine +/// instructions. +/// +/// This function runs the first parsing pass on the machine function's body. +/// It parses only the machine basic block definitions and creates the machine +/// basic blocks in the given machine function. +/// +/// The machine instructions aren't parsed during the first pass because all +/// the machine basic blocks aren't defined yet - this makes it impossible to +/// resolve the machine basic block references. +/// +/// Return true if an error occurred. +bool parseMachineBasicBlockDefinitions(PerFunctionMIParsingState &PFS, + StringRef Src, SMDiagnostic &Error); + +/// Parse the machine instructions. +/// +/// This function runs the second parsing pass on the machine function's body. +/// It skips the machine basic block definitions and parses only the machine +/// instructions and basic block attributes like liveins and successors. +/// +/// The second parsing pass assumes that the first parsing pass already ran +/// on the given source string. +/// +/// Return true if an error occurred. +bool parseMachineInstructions(PerFunctionMIParsingState &PFS, StringRef Src, + SMDiagnostic &Error); + +bool parseMBBReference(PerFunctionMIParsingState &PFS, + MachineBasicBlock *&MBB, StringRef Src, + SMDiagnostic &Error); + +bool parseRegisterReference(PerFunctionMIParsingState &PFS, + unsigned &Reg, StringRef Src, + SMDiagnostic &Error); + +bool parseNamedRegisterReference(PerFunctionMIParsingState &PFS, unsigned &Reg, + StringRef Src, SMDiagnostic &Error); + +bool parseVirtualRegisterReference(PerFunctionMIParsingState &PFS, + VRegInfo *&Info, StringRef Src, + SMDiagnostic &Error); + +bool parseStackObjectReference(PerFunctionMIParsingState &PFS, int &FI, + StringRef Src, SMDiagnostic &Error); + +bool parseMDNode(PerFunctionMIParsingState &PFS, MDNode *&Node, StringRef Src, + SMDiagnostic &Error); + +} // end namespace llvm + +#endif // LLVM_LIB_CODEGEN_MIRPARSER_MIPARSER_H diff --git a/include/llvm/CodeGen/MIRParser/MIRParser.h b/include/llvm/CodeGen/MIRParser/MIRParser.h index e199a1f69ad7..6a04e48e533c 100644 --- a/include/llvm/CodeGen/MIRParser/MIRParser.h +++ b/include/llvm/CodeGen/MIRParser/MIRParser.h @@ -1,9 +1,8 @@ //===- MIRParser.h - MIR serialization format parser ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/MIRPrinter.h b/include/llvm/CodeGen/MIRPrinter.h index 078c4b2f6072..a4b03a7fb765 100644 --- a/include/llvm/CodeGen/MIRPrinter.h +++ b/include/llvm/CodeGen/MIRPrinter.h @@ -1,9 +1,8 @@ -//===- MIRPrinter.h - MIR serialization format printer --------------------===// +//===- MIRPrinter.h - MIR serialization format printer ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/MIRYamlMapping.h b/include/llvm/CodeGen/MIRYamlMapping.h index 98ac81915dc0..94e76a75e8da 100644 --- a/include/llvm/CodeGen/MIRYamlMapping.h +++ b/include/llvm/CodeGen/MIRYamlMapping.h @@ -1,9 +1,8 @@ -//===- MIRYAMLMapping.h - Describes the mapping between MIR and YAML ------===// +//===- MIRYamlMapping.h - Describe mapping between MIR and YAML--*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,6 +17,7 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/StringRef.h" #include "llvm/CodeGen/MachineJumpTableInfo.h" +#include "llvm/CodeGen/TargetFrameLowering.h" #include "llvm/Support/SMLoc.h" #include "llvm/Support/YAMLTraits.h" #include "llvm/Support/raw_ostream.h" @@ -37,6 +37,7 @@ struct StringValue { StringValue() = default; StringValue(std::string Value) : Value(std::move(Value)) {} + StringValue(const char Val[]) : Value(Val) {} bool operator==(const StringValue &Other) const { return Value == Other.Value; @@ -212,7 +213,7 @@ struct MachineStackObject { int64_t Offset = 0; uint64_t Size = 0; unsigned Alignment = 0; - uint8_t StackID = 0; + TargetStackID::Value StackID; StringValue CalleeSavedRegister; bool CalleeSavedRestored = true; Optional LocalOffset; @@ -252,7 +253,7 @@ template <> struct MappingTraits { if (Object.Type != MachineStackObject::VariableSized) YamlIO.mapRequired("size", Object.Size); YamlIO.mapOptional("alignment", Object.Alignment, (unsigned)0); - YamlIO.mapOptional("stack-id", Object.StackID); + YamlIO.mapOptional("stack-id", Object.StackID, TargetStackID::Default); YamlIO.mapOptional("callee-saved-register", Object.CalleeSavedRegister, StringValue()); // Don't print it out when it's empty. YamlIO.mapOptional("callee-saved-restored", Object.CalleeSavedRestored, @@ -278,7 +279,7 @@ struct FixedMachineStackObject { int64_t Offset = 0; uint64_t Size = 0; unsigned Alignment = 0; - uint8_t StackID = 0; + TargetStackID::Value StackID; bool IsImmutable = false; bool IsAliased = false; StringValue CalleeSavedRegister; @@ -308,6 +309,15 @@ struct ScalarEnumerationTraits { } }; +template <> +struct ScalarEnumerationTraits { + static void enumeration(yaml::IO &IO, TargetStackID::Value &ID) { + IO.enumCase(ID, "default", TargetStackID::Default); + IO.enumCase(ID, "sgpr-spill", TargetStackID::SGPRSpill); + IO.enumCase(ID, "noalloc", TargetStackID::NoAlloc); + } +}; + template <> struct MappingTraits { static void mapping(yaml::IO &YamlIO, FixedMachineStackObject &Object) { YamlIO.mapRequired("id", Object.ID); @@ -317,7 +327,7 @@ template <> struct MappingTraits { YamlIO.mapOptional("offset", Object.Offset, (int64_t)0); YamlIO.mapOptional("size", Object.Size, (uint64_t)0); YamlIO.mapOptional("alignment", Object.Alignment, (unsigned)0); - YamlIO.mapOptional("stack-id", Object.StackID); + YamlIO.mapOptional("stack-id", Object.StackID, TargetStackID::Default); if (Object.Type != FixedMachineStackObject::SpillSlot) { YamlIO.mapOptional("isImmutable", Object.IsImmutable, false); YamlIO.mapOptional("isAliased", Object.IsAliased, false); @@ -337,6 +347,66 @@ template <> struct MappingTraits { static const bool flow = true; }; + +/// Serializable representation of CallSiteInfo. +struct CallSiteInfo { + // Representation of call argument and register which is used to + // transfer it. + struct ArgRegPair { + StringValue Reg; + uint16_t ArgNo; + + bool operator==(const ArgRegPair &Other) const { + return Reg == Other.Reg && ArgNo == Other.ArgNo; + } + }; + + /// Identifies call instruction location in machine function. + struct MachineInstrLoc { + unsigned BlockNum; + unsigned Offset; + + bool operator==(const MachineInstrLoc &Other) const { + return BlockNum == Other.BlockNum && Offset == Other.Offset; + } + }; + + MachineInstrLoc CallLocation; + std::vector ArgForwardingRegs; + + bool operator==(const CallSiteInfo &Other) const { + return CallLocation.BlockNum == Other.CallLocation.BlockNum && + CallLocation.Offset == Other.CallLocation.Offset; + } +}; + +template <> struct MappingTraits { + static void mapping(IO &YamlIO, CallSiteInfo::ArgRegPair &ArgReg) { + YamlIO.mapRequired("arg", ArgReg.ArgNo); + YamlIO.mapRequired("reg", ArgReg.Reg); + } + + static const bool flow = true; +}; +} +} + +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::CallSiteInfo::ArgRegPair) + +namespace llvm { +namespace yaml { + +template <> struct MappingTraits { + static void mapping(IO &YamlIO, CallSiteInfo &CSInfo) { + YamlIO.mapRequired("bb", CSInfo.CallLocation.BlockNum); + YamlIO.mapRequired("offset", CSInfo.CallLocation.Offset); + YamlIO.mapOptional("fwdArgRegs", CSInfo.ArgForwardingRegs, + std::vector()); + } + + static const bool flow = true; +}; + struct MachineConstantPoolValue { UnsignedValue ID; StringValue Value; @@ -391,6 +461,7 @@ LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::MachineFunctionLiveIn) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::VirtualRegisterDefinition) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::MachineStackObject) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::FixedMachineStackObject) +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::CallSiteInfo) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::MachineConstantPoolValue) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::yaml::MachineJumpTable::Entry) @@ -483,6 +554,20 @@ template <> struct MappingTraits { } }; +/// Targets should override this in a way that mirrors the implementation of +/// llvm::MachineFunctionInfo. +struct MachineFunctionInfo { + virtual ~MachineFunctionInfo() {} + virtual void mappingImpl(IO &YamlIO) {} +}; + +template <> struct MappingTraits> { + static void mapping(IO &YamlIO, std::unique_ptr &MFI) { + if (MFI) + MFI->mappingImpl(YamlIO); + } +}; + struct MachineFunction { StringRef Name; unsigned Alignment = 0; @@ -504,6 +589,8 @@ struct MachineFunction { std::vector FixedStackObjects; std::vector StackObjects; std::vector Constants; /// Constant pool. + std::unique_ptr MachineFuncInfo; + std::vector CallSitesInfo; MachineJumpTable JumpTableInfo; BlockStringValue Body; }; @@ -530,8 +617,11 @@ template <> struct MappingTraits { std::vector()); YamlIO.mapOptional("stack", MF.StackObjects, std::vector()); + YamlIO.mapOptional("callSites", MF.CallSitesInfo, + std::vector()); YamlIO.mapOptional("constants", MF.Constants, std::vector()); + YamlIO.mapOptional("machineFunctionInfo", MF.MachineFuncInfo); if (!YamlIO.outputting() || !MF.JumpTableInfo.Entries.empty()) YamlIO.mapOptional("jumpTable", MF.JumpTableInfo, MachineJumpTable()); YamlIO.mapOptional("body", MF.Body, BlockStringValue()); diff --git a/include/llvm/CodeGen/MachORelocation.h b/include/llvm/CodeGen/MachORelocation.h index cbb49695af75..0185c7cbe018 100644 --- a/include/llvm/CodeGen/MachORelocation.h +++ b/include/llvm/CodeGen/MachORelocation.h @@ -1,9 +1,8 @@ //=== MachORelocation.h - Mach-O Relocation Info ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/MachineBasicBlock.h b/include/llvm/CodeGen/MachineBasicBlock.h index ec2f270fcb3f..333d0a78618c 100644 --- a/include/llvm/CodeGen/MachineBasicBlock.h +++ b/include/llvm/CodeGen/MachineBasicBlock.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/MachineBasicBlock.h -------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -115,6 +114,10 @@ private: /// branch. bool AddressTaken = false; + /// Indicate that this basic block needs its symbol be emitted regardless of + /// whether the flow just falls-through to it. + bool LabelMustBeEmitted = false; + /// Indicate that this basic block is the entry block of an EH scope, i.e., /// the block that used to have a catchpad or cleanuppad instruction in the /// LLVM IR. @@ -159,6 +162,13 @@ public: /// branch. void setHasAddressTaken() { AddressTaken = true; } + /// Test whether this block must have its label emitted. + bool hasLabelMustBeEmitted() const { return LabelMustBeEmitted; } + + /// Set this block to reflect that, regardless how we flow to it, we need + /// its label be emitted. + void setLabelMustBeEmitted() { LabelMustBeEmitted = true; } + /// Return the MachineFunction containing this basic block. const MachineFunction *getParent() const { return xParent; } MachineFunction *getParent() { return xParent; } @@ -900,11 +910,11 @@ class MachineInstrSpan { MachineBasicBlock::iterator I, B, E; public: - MachineInstrSpan(MachineBasicBlock::iterator I) - : MBB(*I->getParent()), - I(I), - B(I == MBB.begin() ? MBB.end() : std::prev(I)), - E(std::next(I)) {} + MachineInstrSpan(MachineBasicBlock::iterator I, MachineBasicBlock *BB) + : MBB(*BB), I(I), B(I == MBB.begin() ? MBB.end() : std::prev(I)), + E(std::next(I)) { + assert(I == BB->end() || I->getParent() == BB); + } MachineBasicBlock::iterator begin() { return B == MBB.end() ? MBB.begin() : std::next(B); diff --git a/include/llvm/CodeGen/MachineBlockFrequencyInfo.h b/include/llvm/CodeGen/MachineBlockFrequencyInfo.h index 5b4b99ca0a5d..a438ecfcc25e 100644 --- a/include/llvm/CodeGen/MachineBlockFrequencyInfo.h +++ b/include/llvm/CodeGen/MachineBlockFrequencyInfo.h @@ -1,9 +1,8 @@ //===- MachineBlockFrequencyInfo.h - MBB Frequency Analysis -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/MachineBranchProbabilityInfo.h b/include/llvm/CodeGen/MachineBranchProbabilityInfo.h index 81b0524cf0a4..2b9b2030eb97 100644 --- a/include/llvm/CodeGen/MachineBranchProbabilityInfo.h +++ b/include/llvm/CodeGen/MachineBranchProbabilityInfo.h @@ -1,9 +1,8 @@ //=- MachineBranchProbabilityInfo.h - Branch Probability Analysis -*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/MachineCombinerPattern.h b/include/llvm/CodeGen/MachineCombinerPattern.h index 586535f771c2..4f4034baf801 100644 --- a/include/llvm/CodeGen/MachineCombinerPattern.h +++ b/include/llvm/CodeGen/MachineCombinerPattern.h @@ -1,10 +1,9 @@ //===-- llvm/CodeGen/MachineCombinerPattern.h - Instruction pattern supported by // combiner ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/MachineConstantPool.h b/include/llvm/CodeGen/MachineConstantPool.h index b0b5420a884b..4d07b620a4b4 100644 --- a/include/llvm/CodeGen/MachineConstantPool.h +++ b/include/llvm/CodeGen/MachineConstantPool.h @@ -1,9 +1,8 @@ //===- CodeGen/MachineConstantPool.h - Abstract Constant Pool ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/MachineDominanceFrontier.h b/include/llvm/CodeGen/MachineDominanceFrontier.h index 75d75bc3669a..f7bbd07a63ab 100644 --- a/include/llvm/CodeGen/MachineDominanceFrontier.h +++ b/include/llvm/CodeGen/MachineDominanceFrontier.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/MachineDominanceFrontier.h ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/CodeGen/MachineDominators.h b/include/llvm/CodeGen/MachineDominators.h index e3d3d169db97..d2200080b897 100644 --- a/include/llvm/CodeGen/MachineDominators.h +++ b/include/llvm/CodeGen/MachineDominators.h @@ -1,9 +1,8 @@ //==- llvm/CodeGen/MachineDominators.h - Machine Dom Calculation -*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/MachineFrameInfo.h b/include/llvm/CodeGen/MachineFrameInfo.h index c2706a21a177..761735120a64 100644 --- a/include/llvm/CodeGen/MachineFrameInfo.h +++ b/include/llvm/CodeGen/MachineFrameInfo.h @@ -1,9 +1,8 @@ //===-- CodeGen/MachineFrameInfo.h - Abstract Stack Frame Rep. --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -471,7 +470,10 @@ public: assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() && "Invalid Object Idx!"); Objects[ObjectIdx+NumFixedObjects].Alignment = Align; - ensureMaxAlignment(Align); + + // Only ensure max alignment for the default stack. + if (getStackID(ObjectIdx) == 0) + ensureMaxAlignment(Align); } /// Return the underlying Alloca of the specified @@ -698,6 +700,8 @@ public: assert(unsigned(ObjectIdx+NumFixedObjects) < Objects.size() && "Invalid Object Idx!"); Objects[ObjectIdx+NumFixedObjects].StackID = ID; + // If ID > 0, MaxAlignment may now be overly conservative. + // If ID == 0, MaxAlignment will need to be updated separately. } /// Returns true if the specified index corresponds to a dead object. diff --git a/include/llvm/CodeGen/MachineFunction.h b/include/llvm/CodeGen/MachineFunction.h index 25edf5bcce51..201c126ee52e 100644 --- a/include/llvm/CodeGen/MachineFunction.h +++ b/include/llvm/CodeGen/MachineFunction.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/MachineFunction.h ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -31,11 +30,6 @@ #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineMemOperand.h" -#include "llvm/IR/DebugLoc.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/Metadata.h" -#include "llvm/MC/MCDwarf.h" -#include "llvm/MC/MCSymbol.h" #include "llvm/Support/Allocator.h" #include "llvm/Support/ArrayRecycler.h" #include "llvm/Support/AtomicOrdering.h" @@ -53,6 +47,7 @@ namespace llvm { class BasicBlock; class BlockAddress; class DataLayout; +class DebugLoc; class DIExpression; class DILocalVariable; class DILocation; @@ -67,6 +62,7 @@ class MachineModuleInfo; class MachineRegisterInfo; class MCContext; class MCInstrDesc; +class MCSymbol; class Pass; class PseudoSourceValueManager; class raw_ostream; @@ -86,7 +82,7 @@ template <> struct ilist_callback_traits { template void transferNodesFromList(ilist_callback_traits &OldList, Iterator, Iterator) { - llvm_unreachable("Never transfer between lists"); + assert(this == &OldList && "never transfer MBBs between functions"); } }; @@ -325,6 +321,10 @@ class MachineFunction { /// CodeView label annotations. std::vector> CodeViewAnnotations; + /// CodeView heapallocsites. + std::vector> + CodeViewHeapAllocSites; + bool CallsEHReturn = false; bool CallsUnwindInit = false; bool HasEHScopes = false; @@ -378,9 +378,28 @@ public: virtual void MF_HandleRemoval(MachineInstr &MI) = 0; }; + /// Structure used to represent pair of argument number after call lowering + /// and register used to transfer that argument. + /// For now we support only cases when argument is transferred through one + /// register. + struct ArgRegPair { + unsigned Reg; + uint16_t ArgNo; + ArgRegPair(unsigned R, unsigned Arg) : Reg(R), ArgNo(Arg) { + assert(Arg < (1 << 16) && "Arg out of range"); + } + }; + /// Vector of call argument and its forwarding register. + using CallSiteInfo = SmallVector; + using CallSiteInfoImpl = SmallVectorImpl; + private: Delegate *TheDelegate = nullptr; + using CallSiteInfoMap = DenseMap; + /// Map a call instruction to call site arguments forwarding info. + CallSiteInfoMap CallSitesInfo; + // Callbacks for insertion and removal. void handleInsertion(MachineInstr &MI); void handleRemoval(MachineInstr &MI); @@ -443,7 +462,6 @@ public: /// getSubtarget - Return the subtarget for which this machine code is being /// compiled. const TargetSubtargetInfo &getSubtarget() const { return *STI; } - void setSubtarget(const TargetSubtargetInfo *ST) { STI = ST; } /// getSubtarget - This method returns a pointer to the specified type of /// TargetSubtargetInfo. In debug builds, it verifies that the object being @@ -741,6 +759,12 @@ public: MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO, const AAMDNodes &AAInfo); + /// Allocate a new MachineMemOperand by copying an existing one, + /// replacing the flags. MachineMemOperands are owned + /// by the MachineFunction and need not be explicitly deallocated. + MachineMemOperand *getMachineMemOperand(const MachineMemOperand *MMO, + MachineMemOperand::Flags Flags); + using OperandCapacity = ArrayRecycler::Capacity; /// Allocate an array of MachineOperands. This is only intended for use by @@ -791,10 +815,7 @@ public: return FrameInstructions; } - LLVM_NODISCARD unsigned addFrameInst(const MCCFIInstruction &Inst) { - FrameInstructions.push_back(Inst); - return FrameInstructions.size() - 1; - } + LLVM_NODISCARD unsigned addFrameInst(const MCCFIInstruction &Inst); /// \name Exception Handling /// \{ @@ -913,6 +934,14 @@ public: return CodeViewAnnotations; } + /// Record heapallocsites + void addCodeViewHeapAllocSite(MachineInstr *I, MDNode *MD); + + ArrayRef> + getCodeViewHeapAllocSites() const { + return CodeViewHeapAllocSites; + } + /// Return a reference to the C++ typeinfo for the current function. const std::vector &getTypeInfos() const { return TypeInfos; @@ -936,6 +965,23 @@ public: const VariableDbgInfoMapTy &getVariableDbgInfo() const { return VariableDbgInfos; } + + void addCallArgsForwardingRegs(const MachineInstr *CallI, + CallSiteInfoImpl &&CallInfo) { + assert(CallI->isCall()); + CallSitesInfo[CallI] = std::move(CallInfo); + } + + const CallSiteInfoMap &getCallSitesInfo() const { + return CallSitesInfo; + } + + /// Update call sites info by deleting entry for \p Old call instruction. + /// If \p New is present then transfer \p Old call info to it. This function + /// should be called before removing call instruction or before replacing + /// call instruction with new one. + void updateCallSiteInfo(const MachineInstr *Old, + const MachineInstr *New = nullptr); }; //===--------------------------------------------------------------------===// diff --git a/include/llvm/CodeGen/MachineFunctionPass.h b/include/llvm/CodeGen/MachineFunctionPass.h index 6d978daa2018..caaf22c2139e 100644 --- a/include/llvm/CodeGen/MachineFunctionPass.h +++ b/include/llvm/CodeGen/MachineFunctionPass.h @@ -1,9 +1,8 @@ //===-- MachineFunctionPass.h - Pass for MachineFunctions --------*-C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/MachineInstr.h b/include/llvm/CodeGen/MachineInstr.h index ea1a2a536fc7..c82c5b137507 100644 --- a/include/llvm/CodeGen/MachineInstr.h +++ b/include/llvm/CodeGen/MachineInstr.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/MachineInstr.h - MachineInstr class ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,6 +24,7 @@ #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/TargetOpcodes.h" +#include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/InlineAsm.h" #include "llvm/MC/MCInstrDesc.h" @@ -103,8 +103,10 @@ public: // no unsigned wrap. NoSWrap = 1 << 12, // Instruction supports binary operator // no signed wrap. - IsExact = 1 << 13 // Instruction supports division is + IsExact = 1 << 13, // Instruction supports division is // known to be exact. + FPExcept = 1 << 14, // Instruction may raise floating-point + // exceptions. }; private: @@ -831,6 +833,17 @@ public: return mayLoad(Type) || mayStore(Type); } + /// Return true if this instruction could possibly raise a floating-point + /// exception. This is the case if the instruction is a floating-point + /// instruction that can in principle raise an exception, as indicated + /// by the MCID::MayRaiseFPException property, *and* at the same time, + /// the instruction is used in a context where we expect floating-point + /// exceptions might be enabled, as indicated by the FPExcept MI flag. + bool mayRaiseFPException() const { + return hasProperty(MCID::MayRaiseFPException) && + getFlag(MachineInstr::MIFlag::FPExcept); + } + //===--------------------------------------------------------------------===// // Flags that indicate whether an instruction can be modified by a method. //===--------------------------------------------------------------------===// @@ -1006,16 +1019,33 @@ public: && getOperand(1).isImm(); } + /// A DBG_VALUE is an entry value iff its debug expression contains the + /// DW_OP_entry_value DWARF operation. + bool isDebugEntryValue() const { + return isDebugValue() && getDebugExpression()->isEntryValue(); + } + + /// Return true if the instruction is a debug value which describes a part of + /// a variable as unavailable. + bool isUndefDebugValue() const { + return isDebugValue() && getOperand(0).isReg() && !getOperand(0).getReg(); + } + bool isPHI() const { return getOpcode() == TargetOpcode::PHI || getOpcode() == TargetOpcode::G_PHI; } bool isKill() const { return getOpcode() == TargetOpcode::KILL; } bool isImplicitDef() const { return getOpcode()==TargetOpcode::IMPLICIT_DEF; } - bool isInlineAsm() const { return getOpcode() == TargetOpcode::INLINEASM; } + bool isInlineAsm() const { + return getOpcode() == TargetOpcode::INLINEASM || + getOpcode() == TargetOpcode::INLINEASM_BR; + } + /// FIXME: Seems like a layering violation that the AsmDialect, which is X86 + /// specific, be attached to a generic MachineInstr. bool isMSInlineAsm() const { - return getOpcode() == TargetOpcode::INLINEASM && getInlineAsmDialect(); + return isInlineAsm() && getInlineAsmDialect() == InlineAsm::AD_Intel; } bool isStackAligningInlineAsm() const; @@ -1197,12 +1227,22 @@ public: /// Wrapper for findRegisterDefOperandIdx, it returns /// a pointer to the MachineOperand rather than an index. - MachineOperand *findRegisterDefOperand(unsigned Reg, bool isDead = false, - const TargetRegisterInfo *TRI = nullptr) { - int Idx = findRegisterDefOperandIdx(Reg, isDead, false, TRI); + MachineOperand * + findRegisterDefOperand(unsigned Reg, bool isDead = false, + bool Overlap = false, + const TargetRegisterInfo *TRI = nullptr) { + int Idx = findRegisterDefOperandIdx(Reg, isDead, Overlap, TRI); return (Idx == -1) ? nullptr : &getOperand(Idx); } + const MachineOperand * + findRegisterDefOperand(unsigned Reg, bool isDead = false, + bool Overlap = false, + const TargetRegisterInfo *TRI = nullptr) const { + return const_cast(this)->findRegisterDefOperand( + Reg, isDead, Overlap, TRI); + } + /// Find the index of the first operand in the /// operand list that is used to represent the predicate. It returns -1 if /// none is found. @@ -1364,7 +1404,7 @@ public: /// @param AA Optional alias analysis, used to compare memory operands. /// @param Other MachineInstr to check aliasing against. /// @param UseTBAA Whether to pass TBAA information to alias analysis. - bool mayAlias(AliasAnalysis *AA, MachineInstr &Other, bool UseTBAA); + bool mayAlias(AliasAnalysis *AA, const MachineInstr &Other, bool UseTBAA) const; /// Return true if this instruction may have an ordered /// or volatile memory reference, or if the information describing the memory @@ -1400,6 +1440,19 @@ public: /// Return true if all the defs of this instruction are dead. bool allDefsAreDead() const; + /// Return a valid size if the instruction is a spill instruction. + Optional getSpillSize(const TargetInstrInfo *TII) const; + + /// Return a valid size if the instruction is a folded spill instruction. + Optional getFoldedSpillSize(const TargetInstrInfo *TII) const; + + /// Return a valid size if the instruction is a restore instruction. + Optional getRestoreSize(const TargetInstrInfo *TII) const; + + /// Return a valid size if the instruction is a folded restore instruction. + Optional + getFoldedRestoreSize(const TargetInstrInfo *TII) const; + /// Copy implicit register operands from specified /// instruction to this instruction. void copyImplicitOps(MachineFunction &MF, const MachineInstr &MI); @@ -1521,11 +1574,17 @@ public: /// FIXME: This is not fully implemented yet. void setPostInstrSymbol(MachineFunction &MF, MCSymbol *Symbol); + /// Clone another MachineInstr's pre- and post- instruction symbols and + /// replace ours with it. + void cloneInstrSymbols(MachineFunction &MF, const MachineInstr &MI); + /// Return the MIFlags which represent both MachineInstrs. This /// should be used when merging two MachineInstrs into one. This routine does /// not modify the MIFlags of this MachineInstr. uint16_t mergeFlagsWith(const MachineInstr& Other) const; + static uint16_t copyFlagsFromInstruction(const Instruction &I); + /// Copy all flags to MachineInst MIFlags void copyIRFlags(const Instruction &I); diff --git a/include/llvm/CodeGen/MachineInstrBuilder.h b/include/llvm/CodeGen/MachineInstrBuilder.h index b5e523f655e7..6d7fb72b6bd1 100644 --- a/include/llvm/CodeGen/MachineInstrBuilder.h +++ b/include/llvm/CodeGen/MachineInstrBuilder.h @@ -1,9 +1,8 @@ //===- CodeGen/MachineInstrBuilder.h - Simplify creation of MIs --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -80,6 +79,11 @@ public: /// explicitly. MachineInstr *getInstr() const { return MI; } + /// Get the register for the operand index. + /// The operand at the index should be a register (asserted by + /// MachineOperand). + Register getReg(unsigned Idx) const { return MI->getOperand(Idx).getReg(); } + /// Add a new virtual register operand. const MachineInstrBuilder &addReg(unsigned RegNo, unsigned flags = 0, unsigned SubReg = 0) const { @@ -283,6 +287,9 @@ public: case MachineOperand::MO_GlobalAddress: return addGlobalAddress(Disp.getGlobal(), Disp.getOffset() + off, TargetFlags); + case MachineOperand::MO_BlockAddress: + return addBlockAddress(Disp.getBlockAddress(), Disp.getOffset() + off, + TargetFlags); } } diff --git a/include/llvm/CodeGen/MachineInstrBundle.h b/include/llvm/CodeGen/MachineInstrBundle.h index b5341fd1ae49..1810d23072d0 100644 --- a/include/llvm/CodeGen/MachineInstrBundle.h +++ b/include/llvm/CodeGen/MachineInstrBundle.h @@ -1,9 +1,8 @@ -//===-- CodeGen/MachineInstBundle.h - MI bundle utilities -------*- C++ -*-===// +//===- llvm/CodeGen/MachineInstrBundle.h - MI bundle utilities --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -62,7 +61,8 @@ inline MachineBasicBlock::instr_iterator getBundleEnd( MachineBasicBlock::instr_iterator I) { while (I->isBundledWithSucc()) ++I; - return ++I; + ++I; + return I; } /// Returns an iterator pointing beyond the bundle containing \p I. @@ -70,7 +70,8 @@ inline MachineBasicBlock::const_instr_iterator getBundleEnd( MachineBasicBlock::const_instr_iterator I) { while (I->isBundledWithSucc()) ++I; - return ++I; + ++I; + return I; } //===----------------------------------------------------------------------===// diff --git a/include/llvm/CodeGen/MachineInstrBundleIterator.h b/include/llvm/CodeGen/MachineInstrBundleIterator.h index 5fe4964ff116..0f59563e7e1b 100644 --- a/include/llvm/CodeGen/MachineInstrBundleIterator.h +++ b/include/llvm/CodeGen/MachineInstrBundleIterator.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/MachineInstrBundleIterator.h ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/MachineJumpTableInfo.h b/include/llvm/CodeGen/MachineJumpTableInfo.h index 25a3e6b556a3..11781145b378 100644 --- a/include/llvm/CodeGen/MachineJumpTableInfo.h +++ b/include/llvm/CodeGen/MachineJumpTableInfo.h @@ -1,9 +1,8 @@ //===-- CodeGen/MachineJumpTableInfo.h - Abstract Jump Tables --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/MachineLoopInfo.h b/include/llvm/CodeGen/MachineLoopInfo.h index 917fb90380f5..da6df59c739c 100644 --- a/include/llvm/CodeGen/MachineLoopInfo.h +++ b/include/llvm/CodeGen/MachineLoopInfo.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/MachineLoopInfo.h - Natural Loop Calculator -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/MachineMemOperand.h b/include/llvm/CodeGen/MachineMemOperand.h index 078ef7ca510c..65f706302bc2 100644 --- a/include/llvm/CodeGen/MachineMemOperand.h +++ b/include/llvm/CodeGen/MachineMemOperand.h @@ -1,9 +1,8 @@ //==- llvm/CodeGen/MachineMemOperand.h - MachineMemOperand class -*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,8 +18,6 @@ #include "llvm/ADT/BitmaskEnum.h" #include "llvm/ADT/PointerUnion.h" #include "llvm/CodeGen/PseudoSourceValue.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/Metadata.h" #include "llvm/IR/Value.h" // PointerLikeTypeTraits #include "llvm/Support/AtomicOrdering.h" #include "llvm/Support/DataTypes.h" @@ -223,6 +220,9 @@ public: /// Return the size in bytes of the memory reference. uint64_t getSize() const { return Size; } + /// Return the size in bits of the memory reference. + uint64_t getSizeInBits() const { return Size * 8; } + /// Return the minimum known alignment in bytes of the actual memory /// reference. uint64_t getAlignment() const; @@ -267,13 +267,13 @@ public: bool isAtomic() const { return getOrdering() != AtomicOrdering::NotAtomic; } /// Returns true if this memory operation doesn't have any ordering - /// constraints other than normal aliasing. Volatile and atomic memory - /// operations can't be reordered. - /// - /// Currently, we don't model the difference between volatile and atomic - /// operations. They should retain their ordering relative to all memory - /// operations. - bool isUnordered() const { return !isVolatile(); } + /// constraints other than normal aliasing. Volatile and (ordered) atomic + /// memory operations can't be reordered. + bool isUnordered() const { + return (getOrdering() == AtomicOrdering::NotAtomic || + getOrdering() == AtomicOrdering::Unordered) && + !isVolatile(); + } /// Update this MachineMemOperand to reflect the alignment of MMO, if it has a /// greater alignment. This must only be used when the new alignment applies diff --git a/include/llvm/CodeGen/MachineModuleInfo.h b/include/llvm/CodeGen/MachineModuleInfo.h index 4371420bc7a2..4ff5c7fd013a 100644 --- a/include/llvm/CodeGen/MachineModuleInfo.h +++ b/include/llvm/CodeGen/MachineModuleInfo.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/MachineModuleInfo.h ------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -114,10 +113,9 @@ class MachineModuleInfo : public ImmutablePass { /// True if debugging information is available in this module. bool DbgInfoAvailable; - /// True if this module calls VarArg function with floating-point arguments. - /// This is used to emit an undefined reference to _fltused on Windows - /// targets. - bool UsesVAFloatArgument; + /// True if this module is being built for windows/msvc, and uses floating + /// point. This is used to emit an undefined reference to _fltused. + bool UsesMSVCFloatingPoint; /// True if the module calls the __morestack function indirectly, as is /// required under the large code model on x86. This is used to emit @@ -152,6 +150,8 @@ public: bool doInitialization(Module &) override; bool doFinalization(Module &) override; + const LLVMTargetMachine &getTarget() const { return TM; } + const MCContext &getContext() const { return Context; } MCContext &getContext() { return Context; } @@ -187,13 +187,9 @@ public: bool hasDebugInfo() const { return DbgInfoAvailable; } void setDebugInfoAvailability(bool avail) { DbgInfoAvailable = avail; } - bool usesVAFloatArgument() const { - return UsesVAFloatArgument; - } + bool usesMSVCFloatingPoint() const { return UsesMSVCFloatingPoint; } - void setUsesVAFloatArgument(bool b) { - UsesVAFloatArgument = b; - } + void setUsesMSVCFloatingPoint(bool b) { UsesMSVCFloatingPoint = b; } bool usesMorestackAddr() const { return UsesMorestackAddr; @@ -258,14 +254,6 @@ public: /// \} }; // End class MachineModuleInfo -//===- MMI building helpers -----------------------------------------------===// - -/// Determine if any floating-point values are being passed to this variadic -/// function, and set the MachineModuleInfo's usesVAFloatArgument flag if so. -/// This flag is used to emit an undefined reference to _fltused on Windows, -/// which will link in MSVCRT's floating-point support. -void computeUsesVAFloatArgument(const CallInst &I, MachineModuleInfo &MMI); - } // end namespace llvm #endif // LLVM_CODEGEN_MACHINEMODULEINFO_H diff --git a/include/llvm/CodeGen/MachineModuleInfoImpls.h b/include/llvm/CodeGen/MachineModuleInfoImpls.h index 17df1fa792b7..746e92239613 100644 --- a/include/llvm/CodeGen/MachineModuleInfoImpls.h +++ b/include/llvm/CodeGen/MachineModuleInfoImpls.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/MachineModuleInfoImpls.h --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/MachineOperand.h b/include/llvm/CodeGen/MachineOperand.h index 53e8889d118a..2152c7582e5a 100644 --- a/include/llvm/CodeGen/MachineOperand.h +++ b/include/llvm/CodeGen/MachineOperand.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/MachineOperand.h - MachineOperand class ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,6 +14,7 @@ #define LLVM_CODEGEN_MACHINEOPERAND_H #include "llvm/ADT/DenseMap.h" +#include "llvm/CodeGen/Register.h" #include "llvm/IR/Intrinsics.h" #include "llvm/Support/DataTypes.h" #include "llvm/Support/LowLevelTypeImpl.h" @@ -346,9 +346,9 @@ public: //===--------------------------------------------------------------------===// /// getReg - Returns the register number. - unsigned getReg() const { + Register getReg() const { assert(isReg() && "This is not a register operand!"); - return SmallContents.RegNo; + return Register(SmallContents.RegNo); } unsigned getSubReg() const { @@ -684,6 +684,11 @@ public: Contents.RegMask = RegMaskPtr; } + void setPredicate(unsigned Predicate) { + assert(isPredicate() && "Wrong MachineOperand mutator"); + Contents.Pred = Predicate; + } + //===--------------------------------------------------------------------===// // Other methods. //===--------------------------------------------------------------------===// @@ -714,6 +719,10 @@ public: /// ChangeToES - Replace this operand with a new external symbol operand. void ChangeToES(const char *SymName, unsigned char TargetFlags = 0); + /// ChangeToGA - Replace this operand with a new global address operand. + void ChangeToGA(const GlobalValue *GV, int64_t Offset, + unsigned char TargetFlags = 0); + /// ChangeToMCSymbol - Replace this operand with a new MC symbol operand. void ChangeToMCSymbol(MCSymbol *Sym); diff --git a/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h b/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h index a7ce870400c2..a461a299917c 100644 --- a/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h +++ b/include/llvm/CodeGen/MachineOptimizationRemarkEmitter.h @@ -1,9 +1,8 @@ ///===- MachineOptimizationRemarkEmitter.h - Opt Diagnostics -*- C++ -*----===// /// -/// The LLVM Compiler Infrastructure -/// -/// This file is distributed under the University of Illinois Open Source -/// License. See LICENSE.TXT for details. +/// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +/// See https://llvm.org/LICENSE.txt for license information. +/// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception /// ///===---------------------------------------------------------------------===// /// \file @@ -159,9 +158,10 @@ public: /// (1) to filter trivial false positives or (2) to provide more context so /// that non-trivial false positives can be quickly detected by the user. bool allowExtraAnalysis(StringRef PassName) const { - return (MF.getFunction().getContext().getDiagnosticsOutputFile() || - MF.getFunction().getContext() - .getDiagHandlerPtr()->isAnyRemarkEnabled(PassName)); + return ( + MF.getFunction().getContext().getRemarkStreamer() || + MF.getFunction().getContext().getDiagHandlerPtr()->isAnyRemarkEnabled( + PassName)); } /// Take a lambda that returns a remark which will be emitted. Second @@ -172,8 +172,11 @@ public: // remarks enabled. We can't currently check whether remarks are requested // for the calling pass since that requires actually building the remark. - if (MF.getFunction().getContext().getDiagnosticsOutputFile() || - MF.getFunction().getContext().getDiagHandlerPtr()->isAnyRemarkEnabled()) { + if (MF.getFunction().getContext().getRemarkStreamer() || + MF.getFunction() + .getContext() + .getDiagHandlerPtr() + ->isAnyRemarkEnabled()) { auto R = RemarkBuilder(); emit((DiagnosticInfoOptimizationBase &)R); } diff --git a/include/llvm/CodeGen/MachineOutliner.h b/include/llvm/CodeGen/MachineOutliner.h index bfd1e994053a..3868fa415579 100644 --- a/include/llvm/CodeGen/MachineOutliner.h +++ b/include/llvm/CodeGen/MachineOutliner.h @@ -1,9 +1,8 @@ //===---- MachineOutliner.h - Outliner data structures ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -172,13 +171,13 @@ public: /// Represents the size of a sequence in bytes. (Some instructions vary /// widely in size, so just counting the instructions isn't very useful.) - unsigned SequenceSize; + unsigned SequenceSize = 0; /// Target-defined overhead of constructing a frame for this function. - unsigned FrameOverhead; + unsigned FrameOverhead = 0; /// Target-defined identifier for constructing a frame for this function. - unsigned FrameConstructionID; + unsigned FrameConstructionID = 0; /// Return the number of candidates for this \p OutlinedFunction. unsigned getOccurrenceCount() const { return Candidates.size(); } diff --git a/include/llvm/CodeGen/MachinePassRegistry.h b/include/llvm/CodeGen/MachinePassRegistry.h index a031c92d914f..f5b3723db0aa 100644 --- a/include/llvm/CodeGen/MachinePassRegistry.h +++ b/include/llvm/CodeGen/MachinePassRegistry.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/MachinePassRegistry.h -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/MachinePipeliner.h b/include/llvm/CodeGen/MachinePipeliner.h index 38cb33e90e63..03ca53072685 100644 --- a/include/llvm/CodeGen/MachinePipeliner.h +++ b/include/llvm/CodeGen/MachinePipeliner.h @@ -1,9 +1,8 @@ //===- MachinePipeliner.h - Machine Software Pipeliner Pass -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -63,6 +62,8 @@ public: const InstrItineraryData *InstrItins; const TargetInstrInfo *TII = nullptr; RegisterClassInfo RegClassInfo; + bool disabledByPragma = false; + unsigned II_setByPragma = 0; #ifndef NDEBUG static int NumTries; @@ -100,6 +101,7 @@ private: bool canPipelineLoop(MachineLoop &L); bool scheduleLoop(MachineLoop &L); bool swingModuloScheduler(MachineLoop &L); + void setPragmaPipelineOptions(MachineLoop &L); }; /// This class builds the dependence graph for the instructions in a loop, @@ -108,11 +110,14 @@ class SwingSchedulerDAG : public ScheduleDAGInstrs { MachinePipeliner &Pass; /// The minimum initiation interval between iterations for this schedule. unsigned MII = 0; + /// The maximum initiation interval between iterations for this schedule. + unsigned MAX_II = 0; /// Set to true if a valid pipelined schedule is found for the loop. bool Scheduled = false; MachineLoop &Loop; LiveIntervals &LIS; const RegisterClassInfo &RegClassInfo; + unsigned II_setByPragma = 0; /// A toplogical ordering of the SUnits, which is needed for changing /// dependences and iterating over the SUnits. @@ -190,9 +195,9 @@ class SwingSchedulerDAG : public ScheduleDAGInstrs { public: SwingSchedulerDAG(MachinePipeliner &P, MachineLoop &L, LiveIntervals &lis, - const RegisterClassInfo &rci) + const RegisterClassInfo &rci, unsigned II) : ScheduleDAGInstrs(*P.MF, P.MLI, false), Pass(P), Loop(L), LIS(lis), - RegClassInfo(rci), Topo(SUnits, &ExitSU) { + RegClassInfo(rci), II_setByPragma(II), Topo(SUnits, &ExitSU) { P.MF->getSubtarget().getSMSMutations(Mutations); if (SwpEnableCopyToPhi) Mutations.push_back(llvm::make_unique()); @@ -253,9 +258,6 @@ public: return 0; } - /// Set the Minimum Initiation Interval for this schedule attempt. - void setMII(unsigned mii) { MII = mii; } - void applyInstrChange(MachineInstr *MI, SMSchedule &Schedule); void fixupRegisterOverlaps(std::deque &Instrs); @@ -316,9 +318,9 @@ private: MBBVectorTy &EpilogBBs); void splitLifetimes(MachineBasicBlock *KernelBB, MBBVectorTy &EpilogBBs, SMSchedule &Schedule); - void addBranches(MBBVectorTy &PrologBBs, MachineBasicBlock *KernelBB, - MBBVectorTy &EpilogBBs, SMSchedule &Schedule, - ValueMapTy *VRMap); + void addBranches(MachineBasicBlock &PreheaderBB, MBBVectorTy &PrologBBs, + MachineBasicBlock *KernelBB, MBBVectorTy &EpilogBBs, + SMSchedule &Schedule, ValueMapTy *VRMap); bool computeDelta(MachineInstr &MI, unsigned &Delta); void updateMemOperands(MachineInstr &NewMI, MachineInstr &OldMI, unsigned Num); @@ -346,6 +348,10 @@ private: unsigned &OffsetPos, unsigned &NewBase, int64_t &NewOffset); void postprocessDAG(); + /// Set the Minimum Initiation Interval for this schedule attempt. + void setMII(unsigned ResMII, unsigned RecMII); + /// Set the Maximum Initiation Interval for this schedule attempt. + void setMAX_II(); }; /// A NodeSet contains a set of SUnit DAG nodes with additional information @@ -457,6 +463,56 @@ public: #endif }; +// 16 was selected based on the number of ProcResource kinds for all +// existing Subtargets, so that SmallVector don't need to resize too often. +static const int DefaultProcResSize = 16; + +class ResourceManager { +private: + const MCSubtargetInfo *STI; + const MCSchedModel &SM; + const bool UseDFA; + std::unique_ptr DFAResources; + /// Each processor resource is associated with a so-called processor resource + /// mask. This vector allows to correlate processor resource IDs with + /// processor resource masks. There is exactly one element per each processor + /// resource declared by the scheduling model. + llvm::SmallVector ProcResourceMasks; + + llvm::SmallVector ProcResourceCount; + +public: + ResourceManager(const TargetSubtargetInfo *ST) + : STI(ST), SM(ST->getSchedModel()), UseDFA(ST->useDFAforSMS()), + ProcResourceMasks(SM.getNumProcResourceKinds(), 0), + ProcResourceCount(SM.getNumProcResourceKinds(), 0) { + if (UseDFA) + DFAResources.reset(ST->getInstrInfo()->CreateTargetScheduleState(*ST)); + initProcResourceVectors(SM, ProcResourceMasks); + } + + void initProcResourceVectors(const MCSchedModel &SM, + SmallVectorImpl &Masks); + /// Check if the resources occupied by a MCInstrDesc are available in + /// the current state. + bool canReserveResources(const MCInstrDesc *MID) const; + + /// Reserve the resources occupied by a MCInstrDesc and change the current + /// state to reflect that change. + void reserveResources(const MCInstrDesc *MID); + + /// Check if the resources occupied by a machine instruction are available + /// in the current state. + bool canReserveResources(const MachineInstr &MI) const; + + /// Reserve the resources occupied by a machine instruction and change the + /// current state to reflect that change. + void reserveResources(const MachineInstr &MI); + + /// Reset the state + void clearResources(); +}; + /// This class represents the scheduled code. The main data structure is a /// map from scheduled cycle to instructions. During scheduling, the /// data structure explicitly represents all stages/iterations. When @@ -495,12 +551,11 @@ private: /// Virtual register information. MachineRegisterInfo &MRI; - std::unique_ptr Resources; + ResourceManager ProcItinResources; public: SMSchedule(MachineFunction *mf) - : ST(mf->getSubtarget()), MRI(mf->getRegInfo()), - Resources(ST.getInstrInfo()->CreateTargetScheduleState(ST)) {} + : ST(mf->getSubtarget()), MRI(mf->getRegInfo()), ProcItinResources(&ST) {} void reset() { ScheduledInstrs.clear(); diff --git a/include/llvm/CodeGen/MachinePostDominators.h b/include/llvm/CodeGen/MachinePostDominators.h index c6a41598ce32..b67e6b52ac8f 100644 --- a/include/llvm/CodeGen/MachinePostDominators.h +++ b/include/llvm/CodeGen/MachinePostDominators.h @@ -1,9 +1,8 @@ -//=- llvm/CodeGen/MachineDominators.h ----------------------------*- C++ -*-==// +//===- llvm/CodeGen/MachinePostDominators.h ----------------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/MachineRegionInfo.h b/include/llvm/CodeGen/MachineRegionInfo.h index 8394b58d0a16..6d9fb9b9100a 100644 --- a/include/llvm/CodeGen/MachineRegionInfo.h +++ b/include/llvm/CodeGen/MachineRegionInfo.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/MachineRegionInfo.h -------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/CodeGen/MachineRegisterInfo.h b/include/llvm/CodeGen/MachineRegisterInfo.h index fef010a23ef9..b5deed1f5010 100644 --- a/include/llvm/CodeGen/MachineRegisterInfo.h +++ b/include/llvm/CodeGen/MachineRegisterInfo.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/MachineRegisterInfo.h -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -562,9 +561,14 @@ public: } /// hasOneNonDBGUse - Return true if there is exactly one non-Debug - /// instruction using the specified register. + /// use of the specified register. bool hasOneNonDBGUse(unsigned RegNo) const; + /// hasOneNonDBGUse - Return true if there is exactly one non-Debug + /// instruction using the specified register. Said instruction may have + /// multiple uses. + bool hasOneNonDBGUser(unsigned RegNo) const; + /// replaceRegWith - Replace all instances of FromReg with ToReg in the /// machine function. This is like llvm-level X->replaceAllUsesWith(Y), /// except that it also changes any definitions of the register as well. @@ -713,12 +717,12 @@ public: /// createVirtualRegister - Create and return a new virtual register in the /// function with the specified register class. - unsigned createVirtualRegister(const TargetRegisterClass *RegClass, + Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name = ""); /// Create and return a new virtual register in the function with the same /// attributes as the given register. - unsigned cloneVirtualRegister(unsigned VReg, StringRef Name = ""); + Register cloneVirtualRegister(Register VReg, StringRef Name = ""); /// Get the low-level type of \p Reg or LLT{} if Reg is not a generic /// (target independent) virtual register. @@ -733,7 +737,7 @@ public: /// Create and return a new generic virtual register with low-level /// type \p Ty. - unsigned createGenericVirtualRegister(LLT Ty, StringRef Name = ""); + Register createGenericVirtualRegister(LLT Ty, StringRef Name = ""); /// Remove all types associated to virtual registers (after instruction /// selection and constraining of all generic virtual registers). diff --git a/include/llvm/CodeGen/MachineSSAUpdater.h b/include/llvm/CodeGen/MachineSSAUpdater.h index 5e91246b402c..0319ec774671 100644 --- a/include/llvm/CodeGen/MachineSSAUpdater.h +++ b/include/llvm/CodeGen/MachineSSAUpdater.h @@ -1,9 +1,8 @@ //===- MachineSSAUpdater.h - Unstructured SSA Update Tool -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/MachineScheduler.h b/include/llvm/CodeGen/MachineScheduler.h index 4bc31ae7c61a..75a334f61ad0 100644 --- a/include/llvm/CodeGen/MachineScheduler.h +++ b/include/llvm/CodeGen/MachineScheduler.h @@ -1,9 +1,8 @@ //===- MachineScheduler.h - MachineInstr Scheduling Pass --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -265,10 +264,6 @@ protected: LiveIntervals *LIS; std::unique_ptr SchedImpl; - /// Topo - A topological ordering for SUnits which permits fast IsReachable - /// and similar queries. - ScheduleDAGTopologicalSort Topo; - /// Ordered list of DAG postprocessing steps. std::vector> Mutations; @@ -292,7 +287,7 @@ public: ScheduleDAGMI(MachineSchedContext *C, std::unique_ptr S, bool RemoveKillFlags) : ScheduleDAGInstrs(*C->MF, C->MLI, RemoveKillFlags), AA(C->AA), - LIS(C->LIS), SchedImpl(std::move(S)), Topo(SUnits, &ExitSU) {} + LIS(C->LIS), SchedImpl(std::move(S)) {} // Provide a vtable anchor ~ScheduleDAGMI() override; @@ -320,17 +315,6 @@ public: Mutations.push_back(std::move(Mutation)); } - /// True if an edge can be added from PredSU to SuccSU without creating - /// a cycle. - bool canAddEdge(SUnit *SuccSU, SUnit *PredSU); - - /// Add a DAG edge to the given SU with the given predecessor - /// dependence data. - /// - /// \returns true if the edge may be added without creating a cycle OR if an - /// equivalent edge already existed (false indicates failure). - bool addEdge(SUnit *SuccSU, const SDep &PredDep); - MachineBasicBlock::iterator top() const { return CurrentTop; } MachineBasicBlock::iterator bottom() const { return CurrentBottom; } @@ -682,6 +666,10 @@ private: // scheduled instruction. SmallVector ReservedCycles; + // For each PIdx, stores first index into ReservedCycles that corresponds to + // it. + SmallVector ReservedCyclesIndex; + #ifndef NDEBUG // Remember the greatest possible stall as an upper bound on the number of // times we should retry the pending queue because of a hazard. @@ -756,7 +744,11 @@ public: /// cycle. unsigned getLatencyStallCycles(SUnit *SU); - unsigned getNextResourceCycle(unsigned PIdx, unsigned Cycles); + unsigned getNextResourceCycleByInstance(unsigned InstanceIndex, + unsigned Cycles); + + std::pair getNextResourceCycle(unsigned PIdx, + unsigned Cycles); bool checkHazard(SUnit *SU); @@ -1015,6 +1007,7 @@ protected: /// Callbacks from ScheduleDAGMI: /// initPolicy -> initialize(DAG) -> registerRoots -> pickNode ... class PostGenericScheduler : public GenericSchedulerBase { +protected: ScheduleDAGMI *DAG; SchedBoundary Top; SmallVector BotRoots; diff --git a/include/llvm/CodeGen/MachineTraceMetrics.h b/include/llvm/CodeGen/MachineTraceMetrics.h index 9d8db393ca92..025989504177 100644 --- a/include/llvm/CodeGen/MachineTraceMetrics.h +++ b/include/llvm/CodeGen/MachineTraceMetrics.h @@ -1,9 +1,8 @@ //===- lib/CodeGen/MachineTraceMetrics.h - Super-scalar metrics -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/MacroFusion.h b/include/llvm/CodeGen/MacroFusion.h index a77226ddaf33..3a140fe63fde 100644 --- a/include/llvm/CodeGen/MacroFusion.h +++ b/include/llvm/CodeGen/MacroFusion.h @@ -1,9 +1,8 @@ //===- MacroFusion.h - Macro Fusion -----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/PBQP/CostAllocator.h b/include/llvm/CodeGen/PBQP/CostAllocator.h index bde451ae1fcc..0d6d8a31317b 100644 --- a/include/llvm/CodeGen/PBQP/CostAllocator.h +++ b/include/llvm/CodeGen/PBQP/CostAllocator.h @@ -1,9 +1,8 @@ //===- CostAllocator.h - PBQP Cost Allocator --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/PBQP/Graph.h b/include/llvm/CodeGen/PBQP/Graph.h index a6d88b057dcb..c2cd6dadae5f 100644 --- a/include/llvm/CodeGen/PBQP/Graph.h +++ b/include/llvm/CodeGen/PBQP/Graph.h @@ -1,9 +1,8 @@ //===- Graph.h - PBQP Graph -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/PBQP/Math.h b/include/llvm/CodeGen/PBQP/Math.h index d1432a3053c4..8b014ccbb07b 100644 --- a/include/llvm/CodeGen/PBQP/Math.h +++ b/include/llvm/CodeGen/PBQP/Math.h @@ -1,9 +1,8 @@ //===- Math.h - PBQP Vector and Matrix classes ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/CodeGen/PBQP/ReductionRules.h b/include/llvm/CodeGen/PBQP/ReductionRules.h index 21b99027970d..51822d082bad 100644 --- a/include/llvm/CodeGen/PBQP/ReductionRules.h +++ b/include/llvm/CodeGen/PBQP/ReductionRules.h @@ -1,9 +1,8 @@ //===- ReductionRules.h - Reduction Rules -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/PBQP/Solution.h b/include/llvm/CodeGen/PBQP/Solution.h index 4d4379fbc2c2..d5b1474f0f4c 100644 --- a/include/llvm/CodeGen/PBQP/Solution.h +++ b/include/llvm/CodeGen/PBQP/Solution.h @@ -1,9 +1,8 @@ //===- Solution.h - PBQP Solution -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/PBQPRAConstraint.h b/include/llvm/CodeGen/PBQPRAConstraint.h index 995467dc56d8..876ab97a669f 100644 --- a/include/llvm/CodeGen/PBQPRAConstraint.h +++ b/include/llvm/CodeGen/PBQPRAConstraint.h @@ -1,9 +1,8 @@ -//===- RegAllocPBQP.h -------------------------------------------*- C++ -*-===// +//===- llvm/CodeGen/PBQPRAConstraint.h --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/ParallelCG.h b/include/llvm/CodeGen/ParallelCG.h index dbf09ea31e20..a44715d4fc4f 100644 --- a/include/llvm/CodeGen/ParallelCG.h +++ b/include/llvm/CodeGen/ParallelCG.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/ParallelCG.h - Parallel code generation ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/Passes.h b/include/llvm/CodeGen/Passes.h index acf1ebb5bc83..d92ee93268e7 100644 --- a/include/llvm/CodeGen/Passes.h +++ b/include/llvm/CodeGen/Passes.h @@ -1,9 +1,8 @@ //===-- Passes.h - Target independent code generation passes ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -346,8 +345,9 @@ namespace llvm { /// pointer or stack pointer index addressing. extern char &LocalStackSlotAllocationID; - /// ExpandISelPseudos - This pass expands pseudo-instructions. - extern char &ExpandISelPseudosID; + /// This pass expands pseudo-instructions, reserves registers and adjusts + /// machine frame information. + extern char &FinalizeISelID; /// UnpackMachineBundles - This pass unpack machine instruction bundles. extern char &UnpackMachineBundlesID; @@ -447,6 +447,9 @@ namespace llvm { /// Creates CFI Instruction Inserter pass. \see CFIInstrInserter.cpp FunctionPass *createCFIInstrInserter(); + /// Create Hardware Loop pass. \see HardwareLoops.cpp + FunctionPass *createHardwareLoopsPass(); + } // End llvm namespace #endif diff --git a/include/llvm/CodeGen/PreISelIntrinsicLowering.h b/include/llvm/CodeGen/PreISelIntrinsicLowering.h index b7f83e515b7e..73d7d779e55b 100644 --- a/include/llvm/CodeGen/PreISelIntrinsicLowering.h +++ b/include/llvm/CodeGen/PreISelIntrinsicLowering.h @@ -1,9 +1,8 @@ //===- PreISelIntrinsicLowering.h - Pre-ISel intrinsic lowering pass ------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/PseudoSourceValue.h b/include/llvm/CodeGen/PseudoSourceValue.h index f66191bc9fb4..4b3cc9145a13 100644 --- a/include/llvm/CodeGen/PseudoSourceValue.h +++ b/include/llvm/CodeGen/PseudoSourceValue.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/PseudoSourceValue.h ------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -16,7 +15,6 @@ #include "llvm/ADT/StringMap.h" #include "llvm/IR/GlobalValue.h" -#include "llvm/IR/Value.h" #include "llvm/IR/ValueMap.h" #include @@ -124,7 +122,7 @@ public: bool mayAlias(const MachineFrameInfo *) const override; }; -/// A specialized pseudo soruce value for holding GlobalValue values. +/// A specialized pseudo source value for holding GlobalValue values. class GlobalValuePseudoSourceValue : public CallEntryPseudoSourceValue { const GlobalValue *GV; diff --git a/include/llvm/CodeGen/ReachingDefAnalysis.h b/include/llvm/CodeGen/ReachingDefAnalysis.h index b21b745c8fd1..a599fb62f5e2 100644 --- a/include/llvm/CodeGen/ReachingDefAnalysis.h +++ b/include/llvm/CodeGen/ReachingDefAnalysis.h @@ -1,15 +1,14 @@ //==--- llvm/CodeGen/ReachingDefAnalysis.h - Reaching Def Analysis -*- C++ -*---==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // /// \file Reaching Defs Analysis pass. /// -/// This pass tracks for each instruction what is the “closest” reaching def of +/// This pass tracks for each instruction what is the "closest" reaching def of /// a given register. It is used by BreakFalseDeps (for clearance calculation) /// and ExecutionDomainFix (for arbitrating conflicting domains). /// diff --git a/include/llvm/CodeGen/RegAllocPBQP.h b/include/llvm/CodeGen/RegAllocPBQP.h index ba9763077d09..f7f92248f4ce 100644 --- a/include/llvm/CodeGen/RegAllocPBQP.h +++ b/include/llvm/CodeGen/RegAllocPBQP.h @@ -1,9 +1,8 @@ //===- RegAllocPBQP.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/RegAllocRegistry.h b/include/llvm/CodeGen/RegAllocRegistry.h index b518fbb9c9da..9a63674689b3 100644 --- a/include/llvm/CodeGen/RegAllocRegistry.h +++ b/include/llvm/CodeGen/RegAllocRegistry.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/RegAllocRegistry.h --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -23,29 +22,30 @@ class FunctionPass; //===----------------------------------------------------------------------===// /// -/// RegisterRegAlloc class - Track the registration of register allocators. +/// RegisterRegAllocBase class - Track the registration of register allocators. /// //===----------------------------------------------------------------------===// -class RegisterRegAlloc : public MachinePassRegistryNode { +template +class RegisterRegAllocBase : public MachinePassRegistryNode { public: using FunctionPassCtor = FunctionPass *(*)(); static MachinePassRegistry Registry; - RegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C) + RegisterRegAllocBase(const char *N, const char *D, FunctionPassCtor C) : MachinePassRegistryNode(N, D, C) { Registry.Add(this); } - ~RegisterRegAlloc() { Registry.Remove(this); } + ~RegisterRegAllocBase() { Registry.Remove(this); } // Accessors. - RegisterRegAlloc *getNext() const { - return (RegisterRegAlloc *)MachinePassRegistryNode::getNext(); + SubClass *getNext() const { + return static_cast(MachinePassRegistryNode::getNext()); } - static RegisterRegAlloc *getList() { - return (RegisterRegAlloc *)Registry.getList(); + static SubClass *getList() { + return static_cast(Registry.getList()); } static FunctionPassCtor getDefault() { return Registry.getDefault(); } @@ -57,6 +57,17 @@ public: } }; +class RegisterRegAlloc : public RegisterRegAllocBase { +public: + RegisterRegAlloc(const char *N, const char *D, FunctionPassCtor C) + : RegisterRegAllocBase(N, D, C) {} +}; + +/// RegisterRegAlloc's global Registry tracks allocator registration. +template +MachinePassRegistry +RegisterRegAllocBase::Registry; + } // end namespace llvm #endif // LLVM_CODEGEN_REGALLOCREGISTRY_H diff --git a/include/llvm/CodeGen/Register.h b/include/llvm/CodeGen/Register.h new file mode 100644 index 000000000000..907c1a99e56f --- /dev/null +++ b/include/llvm/CodeGen/Register.h @@ -0,0 +1,60 @@ +//===-- llvm/CodeGen/Register.h ---------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_REGISTER_H +#define LLVM_CODEGEN_REGISTER_H + +#include + +namespace llvm { + +/// Wrapper class representing virtual and physical registers. Should be passed +/// by value. +class Register { + unsigned Reg; + +public: + Register(unsigned Val = 0): Reg(Val) {} + + /// Return true if the specified register number is in the virtual register + /// namespace. + bool isVirtual() const { + return int(Reg) < 0; + } + + /// Return true if the specified register number is in the physical register + /// namespace. + bool isPhysical() const { + return int(Reg) > 0; + } + + /// Convert a virtual register number to a 0-based index. The first virtual + /// register in a function will get the index 0. + unsigned virtRegIndex() const { + assert(isVirtual() && "Not a virtual register"); + return Reg & ~(1u << 31); + } + + /// Convert a 0-based index to a virtual register number. + /// This is the inverse operation of VirtReg2IndexFunctor below. + static Register index2VirtReg(unsigned Index) { + return Register(Index | (1u << 31)); + } + + operator unsigned() const { + return Reg; + } + + bool isValid() const { + return Reg != 0; + } +}; + +} + +#endif diff --git a/include/llvm/CodeGen/RegisterClassInfo.h b/include/llvm/CodeGen/RegisterClassInfo.h index 97113c575815..14af5c4d090d 100644 --- a/include/llvm/CodeGen/RegisterClassInfo.h +++ b/include/llvm/CodeGen/RegisterClassInfo.h @@ -1,9 +1,8 @@ //===- RegisterClassInfo.h - Dynamic Register Class Info --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/RegisterPressure.h b/include/llvm/CodeGen/RegisterPressure.h index 79054b9e33b7..5bbaa03fd751 100644 --- a/include/llvm/CodeGen/RegisterPressure.h +++ b/include/llvm/CodeGen/RegisterPressure.h @@ -1,9 +1,8 @@ //===- RegisterPressure.h - Dynamic Register Pressure -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -132,10 +131,6 @@ public: } }; -template <> struct isPodLike { - static const bool value = true; -}; - /// List of PressureChanges in order of increasing, unique PSetID. /// /// Use a small fixed number, because we can fit more PressureChanges in an diff --git a/include/llvm/CodeGen/RegisterScavenging.h b/include/llvm/CodeGen/RegisterScavenging.h index b6bd028a8cac..9c48df82f07d 100644 --- a/include/llvm/CodeGen/RegisterScavenging.h +++ b/include/llvm/CodeGen/RegisterScavenging.h @@ -1,9 +1,8 @@ //===- RegisterScavenging.h - Machine register scavenging -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -158,10 +157,15 @@ public: /// Returns the scavenged register. /// This is deprecated as it depends on the quality of the kill flags being /// present; Use scavengeRegisterBackwards() instead! + /// + /// If \p AllowSpill is false, fail if a spill is required to make the + /// register available, and return NoRegister. unsigned scavengeRegister(const TargetRegisterClass *RC, - MachineBasicBlock::iterator I, int SPAdj); - unsigned scavengeRegister(const TargetRegisterClass *RegClass, int SPAdj) { - return scavengeRegister(RegClass, MBBI, SPAdj); + MachineBasicBlock::iterator I, int SPAdj, + bool AllowSpill = true); + unsigned scavengeRegister(const TargetRegisterClass *RegClass, int SPAdj, + bool AllowSpill = true) { + return scavengeRegister(RegClass, MBBI, SPAdj, AllowSpill); } /// Make a register of the specific register class available from the current @@ -170,9 +174,13 @@ public: /// SPAdj is the stack adjustment due to call frame, it's passed along to /// eliminateFrameIndex(). /// Returns the scavenged register. + /// + /// If \p AllowSpill is false, fail if a spill is required to make the + /// register available, and return NoRegister. unsigned scavengeRegisterBackwards(const TargetRegisterClass &RC, MachineBasicBlock::iterator To, - bool RestoreAfter, int SPAdj); + bool RestoreAfter, int SPAdj, + bool AllowSpill = true); /// Tell the scavenger a register is used. void setRegUsed(unsigned Reg, LaneBitmask LaneMask = LaneBitmask::getAll()); diff --git a/include/llvm/CodeGen/RegisterUsageInfo.h b/include/llvm/CodeGen/RegisterUsageInfo.h index efecc61d9c30..33554550b9dc 100644 --- a/include/llvm/CodeGen/RegisterUsageInfo.h +++ b/include/llvm/CodeGen/RegisterUsageInfo.h @@ -1,9 +1,8 @@ //==- RegisterUsageInfo.h - Register Usage Informartion Storage --*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/CodeGen/ResourcePriorityQueue.h b/include/llvm/CodeGen/ResourcePriorityQueue.h index 8d582ee298b6..81587a3170ce 100644 --- a/include/llvm/CodeGen/ResourcePriorityQueue.h +++ b/include/llvm/CodeGen/ResourcePriorityQueue.h @@ -1,9 +1,8 @@ //===----- ResourcePriorityQueue.h - A DFA-oriented priority queue -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/RuntimeLibcalls.h b/include/llvm/CodeGen/RuntimeLibcalls.h index 28567a1ce437..f71f39e5bf03 100644 --- a/include/llvm/CodeGen/RuntimeLibcalls.h +++ b/include/llvm/CodeGen/RuntimeLibcalls.h @@ -1,9 +1,8 @@ //===-- CodeGen/RuntimeLibcalls.h - Runtime Library Calls -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/SDNodeProperties.td b/include/llvm/CodeGen/SDNodeProperties.td index 83bbab2fdc8d..d25e0bda26a9 100644 --- a/include/llvm/CodeGen/SDNodeProperties.td +++ b/include/llvm/CodeGen/SDNodeProperties.td @@ -1,9 +1,8 @@ //===- SDNodeProperties.td - Common code for DAG isels ---*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/CodeGen/ScheduleDAG.h b/include/llvm/CodeGen/ScheduleDAG.h index 0870d67db390..e004f3bf2cc1 100644 --- a/include/llvm/CodeGen/ScheduleDAG.h +++ b/include/llvm/CodeGen/ScheduleDAG.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/ScheduleDAG.h - Common Base Class -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -239,9 +238,6 @@ class TargetRegisterInfo; void dump(const TargetRegisterInfo *TRI = nullptr) const; }; - template <> - struct isPodLike { static const bool value = true; }; - /// Scheduling unit. This is a node in the scheduling DAG. class SUnit { private: @@ -418,7 +414,7 @@ class TargetRegisterInfo; /// dirty. void setDepthToAtLeast(unsigned NewDepth); - /// If NewDepth is greater than this node's depth value, set it to be + /// If NewHeight is greater than this node's height value, set it to be /// the new height value. This also recursively marks predecessor nodes /// dirty. void setHeightToAtLeast(unsigned NewHeight); @@ -695,6 +691,12 @@ class TargetRegisterInfo; std::vector &SUnits; SUnit *ExitSU; + // Have any new nodes been added? + bool Dirty = false; + + // Outstanding added edges, that have not been applied to the ordering. + SmallVector, 16> Updates; + /// Maps topological index to the node number. std::vector Index2Node; /// Maps the node number to its topological index. @@ -714,6 +716,11 @@ class TargetRegisterInfo; /// Assigns the topological index to the node n. void Allocate(int n, int index); + /// Fix the ordering, by either recomputing from scratch or by applying + /// any outstanding updates. Uses a heuristic to estimate what will be + /// cheaper. + void FixOrder(); + public: ScheduleDAGTopologicalSort(std::vector &SUnits, SUnit *ExitSU); @@ -738,11 +745,19 @@ class TargetRegisterInfo; /// added from SUnit \p X to SUnit \p Y. void AddPred(SUnit *Y, SUnit *X); + /// Queues an update to the topological ordering to accommodate an edge to + /// be added from SUnit \p X to SUnit \p Y. + void AddPredQueued(SUnit *Y, SUnit *X); + /// Updates the topological ordering to accommodate an an edge to be /// removed from the specified node \p N from the predecessors of the /// current node \p M. void RemovePred(SUnit *M, SUnit *N); + /// Mark the ordering as temporarily broken, after a new node has been + /// added. + void MarkDirty() { Dirty = true; } + typedef std::vector::iterator iterator; typedef std::vector::const_iterator const_iterator; iterator begin() { return Index2Node.begin(); } diff --git a/include/llvm/CodeGen/ScheduleDAGInstrs.h b/include/llvm/CodeGen/ScheduleDAGInstrs.h index daad18125db9..3e3b604acbac 100644 --- a/include/llvm/CodeGen/ScheduleDAGInstrs.h +++ b/include/llvm/CodeGen/ScheduleDAGInstrs.h @@ -1,9 +1,8 @@ //===- ScheduleDAGInstrs.h - MachineInstr Scheduling ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -235,6 +234,11 @@ namespace llvm { /// For an unanalyzable memory access, this Value is used in maps. UndefValue *UnknownValue; + + /// Topo - A topological ordering for SUnits which permits fast IsReachable + /// and similar queries. + ScheduleDAGTopologicalSort Topo; + using DbgValueVector = std::vector>; /// Remember instruction that precedes DBG_VALUE. @@ -339,6 +343,17 @@ namespace llvm { /// Fixes register kill flags that scheduling has made invalid. void fixupKills(MachineBasicBlock &MBB); + /// True if an edge can be added from PredSU to SuccSU without creating + /// a cycle. + bool canAddEdge(SUnit *SuccSU, SUnit *PredSU); + + /// Add a DAG edge to the given SU with the given predecessor + /// dependence data. + /// + /// \returns true if the edge may be added without creating a cycle OR if an + /// equivalent edge already existed (false indicates failure). + bool addEdge(SUnit *SuccSU, const SDep &PredDep); + protected: void initSUnits(); void addPhysRegDataDeps(SUnit *SU, unsigned OperIdx); diff --git a/include/llvm/CodeGen/ScheduleDAGMutation.h b/include/llvm/CodeGen/ScheduleDAGMutation.h index 5c236427e0b8..d1dd72859a38 100644 --- a/include/llvm/CodeGen/ScheduleDAGMutation.h +++ b/include/llvm/CodeGen/ScheduleDAGMutation.h @@ -1,9 +1,8 @@ //===- ScheduleDAGMutation.h - MachineInstr Scheduling ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/ScheduleDFS.h b/include/llvm/CodeGen/ScheduleDFS.h index 3ecc033ac35a..d60deab95f5d 100644 --- a/include/llvm/CodeGen/ScheduleDFS.h +++ b/include/llvm/CodeGen/ScheduleDFS.h @@ -1,9 +1,8 @@ -//===- ScheduleDAGILP.h - ILP metric for ScheduleDAGInstrs ------*- C++ -*-===// +//===- ScheduleDFS.h - ILP metric for ScheduleDAGInstrs ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/ScheduleHazardRecognizer.h b/include/llvm/CodeGen/ScheduleHazardRecognizer.h index ace4a2d836ca..37590f496ca2 100644 --- a/include/llvm/CodeGen/ScheduleHazardRecognizer.h +++ b/include/llvm/CodeGen/ScheduleHazardRecognizer.h @@ -1,9 +1,8 @@ //=- llvm/CodeGen/ScheduleHazardRecognizer.h - Scheduling Support -*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/SchedulerRegistry.h b/include/llvm/CodeGen/SchedulerRegistry.h index fbe559f25556..0ccfaafd9e50 100644 --- a/include/llvm/CodeGen/SchedulerRegistry.h +++ b/include/llvm/CodeGen/SchedulerRegistry.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/SchedulerRegistry.h -------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/ScoreboardHazardRecognizer.h b/include/llvm/CodeGen/ScoreboardHazardRecognizer.h index 3f75d108f282..ac67f3008fa7 100644 --- a/include/llvm/CodeGen/ScoreboardHazardRecognizer.h +++ b/include/llvm/CodeGen/ScoreboardHazardRecognizer.h @@ -1,9 +1,8 @@ //=- llvm/CodeGen/ScoreboardHazardRecognizer.h - Schedule Support -*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/SelectionDAG.h b/include/llvm/CodeGen/SelectionDAG.h index 67fe87fc96af..12a970847021 100644 --- a/include/llvm/CodeGen/SelectionDAG.h +++ b/include/llvm/CodeGen/SelectionDAG.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/SelectionDAG.h - InstSelection DAG ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -268,6 +267,10 @@ class SelectionDAG { /// Tracks dbg_value and dbg_label information through SDISel. SDDbgInfo *DbgInfo; + using CallSiteInfo = MachineFunction::CallSiteInfo; + using CallSiteInfoImpl = MachineFunction::CallSiteInfoImpl; + DenseMap SDCallSiteInfo; + uint16_t NextPersistentId = 0; public: @@ -298,6 +301,9 @@ public: /// The node N that was updated. virtual void NodeUpdated(SDNode *N); + + /// The node N that was inserted. + virtual void NodeInserted(SDNode *N); }; struct DAGNodeDeletedListener : public DAGUpdateListener { @@ -404,6 +410,7 @@ public: const TargetLowering &getTargetLoweringInfo() const { return *TLI; } const TargetLibraryInfo &getLibInfo() const { return *LibInfo; } const SelectionDAGTargetInfo &getSelectionDAGInfo() const { return *TSI; } + const LegacyDivergenceAnalysis *getDivergenceAnalysis() const { return DA; } LLVMContext *getContext() const {return Context; } OptimizationRemarkEmitter &getORE() const { return *ORE; } @@ -573,6 +580,9 @@ public: bool isTarget = false, bool isOpaque = false); SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget = false); + SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL, + bool LegalTypes = true); + SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque = false) { return getConstant(Val, DL, VT, true, isOpaque); @@ -789,6 +799,16 @@ public: /// value assuming it was the smaller SrcTy value. SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT); + /// Convert Op, which must be of integer type, to the integer type VT, by + /// either truncating it or performing either zero or sign extension as + /// appropriate extension for the pointer's semantics. + SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT); + + /// Return the expression required to extend the Op as a pointer value + /// assuming it was the smaller SrcTy value. This may be either a zero extend + /// or a sign extend. + SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT); + /// Convert Op, which must be of integer type, to the integer type VT, /// by using an extension appropriate for the target's /// BooleanContent for type OpVT or truncating it. @@ -971,6 +991,10 @@ public: /// Try to simplify a shift into 1 of its operands or a constant. SDValue simplifyShift(SDValue X, SDValue Y); + /// Try to simplify a floating-point binary operation into 1 of its operands + /// or a constant. + SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y); + /// VAArg produces a result and token chain, and takes a pointer /// and a source value as input. SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, @@ -980,22 +1004,12 @@ public: /// valid Opcodes. ISD::ATOMIC_CMO_SWAP produces the value loaded and a /// chain result. ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS produces the value loaded, /// a success flag (initially i1), and a chain. - SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, - SDVTList VTs, SDValue Chain, SDValue Ptr, - SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, - unsigned Alignment, AtomicOrdering SuccessOrdering, - AtomicOrdering FailureOrdering, - SyncScope::ID SSID); SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, MachineMemOperand *MMO); /// Gets a node for an atomic op, produces result (if relevant) /// and chain and takes 2 operands. - SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, - SDValue Ptr, SDValue Val, const Value *PtrVal, - unsigned Alignment, AtomicOrdering Ordering, - SyncScope::ID SSID); SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO); @@ -1021,12 +1035,19 @@ public: unsigned Align = 0, MachineMemOperand::Flags Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore, - unsigned Size = 0); + unsigned Size = 0, + const AAMDNodes &AAInfo = AAMDNodes()); SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef Ops, EVT MemVT, MachineMemOperand *MMO); + /// Creates a LifetimeSDNode that starts (`IsStart==true`) or ends + /// (`IsStart==false`) the lifetime of the portion of `FrameIndex` between + /// offsets `Offset` and `Offset + Size`. + SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain, + int FrameIndex, int64_t Size, int64_t Offset = -1); + /// Create a MERGE_VALUES node from the given operands. SDValue getMergeValues(ArrayRef Ops, const SDLoc &dl); @@ -1154,6 +1175,11 @@ public: SDValue Op3, SDValue Op4, SDValue Op5); SDNode *UpdateNodeOperands(SDNode *N, ArrayRef Ops); + /// Creates a new TokenFactor containing \p Vals. If \p Vals contains 64k + /// values or more, move values into new TokenFactors in 64k-1 blocks, until + /// the final TokenFactor has less than 64k operands. + SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl &Vals); + /// *Mutate* the specified machine node's memory references to the provided /// list. void setNodeMemRefs(MachineSDNode *N, @@ -1358,21 +1384,20 @@ public: /// with this SelectionDAG. bool hasDebugValues() const { return !DbgInfo->empty(); } - SDDbgInfo::DbgIterator DbgBegin() { return DbgInfo->DbgBegin(); } - SDDbgInfo::DbgIterator DbgEnd() { return DbgInfo->DbgEnd(); } + SDDbgInfo::DbgIterator DbgBegin() const { return DbgInfo->DbgBegin(); } + SDDbgInfo::DbgIterator DbgEnd() const { return DbgInfo->DbgEnd(); } - SDDbgInfo::DbgIterator ByvalParmDbgBegin() { + SDDbgInfo::DbgIterator ByvalParmDbgBegin() const { return DbgInfo->ByvalParmDbgBegin(); } - - SDDbgInfo::DbgIterator ByvalParmDbgEnd() { + SDDbgInfo::DbgIterator ByvalParmDbgEnd() const { return DbgInfo->ByvalParmDbgEnd(); } - SDDbgInfo::DbgLabelIterator DbgLabelBegin() { + SDDbgInfo::DbgLabelIterator DbgLabelBegin() const { return DbgInfo->DbgLabelBegin(); } - SDDbgInfo::DbgLabelIterator DbgLabelEnd() { + SDDbgInfo::DbgLabelIterator DbgLabelEnd() const { return DbgInfo->DbgLabelEnd(); } @@ -1395,27 +1420,42 @@ public: const SDNode *N2); SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, - SDNode *Cst1, SDNode *Cst2); + SDNode *N1, SDNode *N2); SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, - const ConstantSDNode *Cst1, - const ConstantSDNode *Cst2); + const ConstantSDNode *C1, + const ConstantSDNode *C2); SDValue FoldConstantVectorArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef Ops, const SDNodeFlags Flags = SDNodeFlags()); + /// Fold floating-point operations with 2 operands when both operands are + /// constants and/or undefined. + SDValue foldConstantFPMath(unsigned Opcode, const SDLoc &DL, EVT VT, + SDValue N1, SDValue N2); + /// Constant fold a setcc to true or false. SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl); - /// See if the specified operand can be simplified with the knowledge that only - /// the bits specified by Mask are used. If so, return the simpler operand, - /// otherwise return a null SDValue. + /// See if the specified operand can be simplified with the knowledge that + /// only the bits specified by DemandedBits are used. If so, return the + /// simpler operand, otherwise return a null SDValue. + /// + /// (This exists alongside SimplifyDemandedBits because GetDemandedBits can + /// simplify nodes with multiple uses more aggressively.) + SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits); + + /// See if the specified operand can be simplified with the knowledge that + /// only the bits specified by DemandedBits are used in the elements specified + /// by DemandedElts. If so, return the simpler operand, otherwise return a + /// null SDValue. /// /// (This exists alongside SimplifyDemandedBits because GetDemandedBits can /// simplify nodes with multiple uses more aggressively.) - SDValue GetDemandedBits(SDValue V, const APInt &Mask); + SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits, + const APInt &DemandedElts); /// Return true if the sign bit of Op is known to be zero. /// We use this predicate to simplify operations downstream. @@ -1424,8 +1464,19 @@ public: /// Return true if 'Op & Mask' is known to be zero. We /// use this predicate to simplify operations downstream. Op and Mask are /// known to be the same type. - bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth = 0) - const; + bool MaskedValueIsZero(SDValue Op, const APInt &Mask, + unsigned Depth = 0) const; + + /// Return true if 'Op & Mask' is known to be zero in DemandedElts. We + /// use this predicate to simplify operations downstream. Op and Mask are + /// known to be the same type. + bool MaskedValueIsZero(SDValue Op, const APInt &Mask, + const APInt &DemandedElts, unsigned Depth = 0) const; + + /// Return true if '(Op & Mask) == Mask'. + /// Op and Mask are known to be the same type. + bool MaskedValueIsAllOnes(SDValue Op, const APInt &Mask, + unsigned Depth = 0) const; /// Determine which bits of Op are known to be either zero or one and return /// them in Known. For vectors, the known bits are those that are shared by @@ -1525,6 +1576,13 @@ public: /// Test whether \p V has a splatted value. bool isSplatValue(SDValue V, bool AllowUndefs = false); + /// If V is a splatted value, return the source vector and its splat index. + SDValue getSplatSourceVector(SDValue V, int &SplatIndex); + + /// If V is a splat vector, return its scalar source operand by extracting + /// that element from the source vector. + SDValue getSplatValue(SDValue V); + /// Match a binop + shuffle pyramid that represents a horizontal reduction /// over the elements of a vector starting from the EXTRACT_VECTOR_ELT node /p /// Extract. The reduction must use one of the opcodes listed in /p @@ -1542,6 +1600,11 @@ public: /// vector op and fill the end of the resulting vector with UNDEFS. SDValue UnrollVectorOp(SDNode *N, unsigned ResNE = 0); + /// Like UnrollVectorOp(), but for the [US](ADD|SUB|MUL)O family of opcodes. + /// This is a separate function because those opcodes have two results. + std::pair UnrollVectorOverflowOp(SDNode *N, + unsigned ResNE = 0); + /// Return true if loads are next to each other and can be /// merged. Check that both are nonvolatile and if LD is loading /// 'Bytes' bytes from a location that is 'Dist' units away from the @@ -1576,6 +1639,9 @@ public: return SplitVector(N->getOperand(OpNo), SDLoc(N)); } + /// Widen the vector up to the next power of two using INSERT_SUBVECTOR. + SDValue WidenVector(const SDValue &N, const SDLoc &DL); + /// Append the extracted elements from Start to Count out of the vector Op /// in Args. If Count is 0, all of the elements will be extracted. void ExtractVectorElements(SDValue Op, SmallVectorImpl &Args, @@ -1597,6 +1663,17 @@ public: isConstantFPBuildVectorOrConstantFP(N); } + void addCallSiteInfo(const SDNode *CallNode, CallSiteInfoImpl &&CallInfo) { + SDCallSiteInfo[CallNode] = std::move(CallInfo); + } + + CallSiteInfo getSDCallSiteInfo(const SDNode *CallNode) { + auto I = SDCallSiteInfo.find(CallNode); + if (I != SDCallSiteInfo.end()) + return std::move(I->second); + return CallSiteInfo(); + } + private: void InsertNode(SDNode *N); bool RemoveNodeFromCSEMaps(SDNode *N); diff --git a/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h b/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h index 2b2c48d57bc0..4ee58333495b 100644 --- a/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h +++ b/include/llvm/CodeGen/SelectionDAGAddressAnalysis.h @@ -1,9 +1,8 @@ //===- SelectionDAGAddressAnalysis.h - DAG Address Analysis -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -34,11 +33,13 @@ class BaseIndexOffset { private: SDValue Base; SDValue Index; - int64_t Offset = 0; + Optional Offset; bool IsIndexSignExt = false; public: BaseIndexOffset() = default; + BaseIndexOffset(SDValue Base, SDValue Index, bool IsIndexSignExt) + : Base(Base), Index(Index), Offset(), IsIndexSignExt(IsIndexSignExt) {} BaseIndexOffset(SDValue Base, SDValue Index, int64_t Offset, bool IsIndexSignExt) : Base(Base), Index(Index), Offset(Offset), @@ -48,6 +49,13 @@ public: SDValue getBase() const { return Base; } SDValue getIndex() { return Index; } SDValue getIndex() const { return Index; } + bool hasValidOffset() const { return Offset.hasValue(); } + + // Returns true if `Other` and `*this` are both some offset from the same base + // pointer. In that case, `Off` is set to the offset between `*this` and + // `Other` (negative if `Other` is before `*this`). + bool equalBaseIndex(const BaseIndexOffset &Other, const SelectionDAG &DAG, + int64_t &Off) const; bool equalBaseIndex(const BaseIndexOffset &Other, const SelectionDAG &DAG) const { @@ -55,11 +63,31 @@ public: return equalBaseIndex(Other, DAG, Off); } - bool equalBaseIndex(const BaseIndexOffset &Other, const SelectionDAG &DAG, - int64_t &Off) const; + // Returns true if `Other` (with size `OtherSize`) can be proven to be fully + // contained in `*this` (with size `Size`). + bool contains(const SelectionDAG &DAG, int64_t BitSize, + const BaseIndexOffset &Other, int64_t OtherBitSize, + int64_t &BitOffset) const; + + bool contains(const SelectionDAG &DAG, int64_t BitSize, + const BaseIndexOffset &Other, int64_t OtherBitSize) const { + int64_t BitOffset; + return contains(DAG, BitSize, Other, OtherBitSize, BitOffset); + } + + // Returns true `Op0` and `Op1` can be proven to alias/not alias, in + // which case `IsAlias` is set to true/false. + static bool computeAliasing(const SDNode *Op0, + const Optional NumBytes0, + const SDNode *Op1, + const Optional NumBytes1, + const SelectionDAG &DAG, bool &IsAlias); + + /// Parses tree in N for base, index, offset addresses. + static BaseIndexOffset match(const SDNode *N, const SelectionDAG &DAG); - /// Parses tree in Ptr for base, index, offset addresses. - static BaseIndexOffset match(const LSBaseSDNode *N, const SelectionDAG &DAG); + void print(raw_ostream& OS) const; + void dump() const; }; } // end namespace llvm diff --git a/include/llvm/CodeGen/SelectionDAGISel.h b/include/llvm/CodeGen/SelectionDAGISel.h index 6758c55c696a..147c325342fc 100644 --- a/include/llvm/CodeGen/SelectionDAGISel.h +++ b/include/llvm/CodeGen/SelectionDAGISel.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/SelectionDAGISel.h - Common Base Class------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -35,6 +34,7 @@ namespace llvm { class TargetLibraryInfo; class FunctionLoweringInfo; class ScheduleHazardRecognizer; + class SwiftErrorValueTracking; class GCFunctionInfo; class ScheduleDAGSDNodes; class LoadInst; @@ -46,6 +46,7 @@ public: TargetMachine &TM; const TargetLibraryInfo *LibInfo; FunctionLoweringInfo *FuncInfo; + SwiftErrorValueTracking *SwiftError; MachineFunction *MF; MachineRegisterInfo *RegInfo; SelectionDAG *CurDAG; @@ -144,10 +145,12 @@ public: OPC_CheckInteger, OPC_CheckChild0Integer, OPC_CheckChild1Integer, OPC_CheckChild2Integer, OPC_CheckChild3Integer, OPC_CheckChild4Integer, - OPC_CheckCondCode, + OPC_CheckCondCode, OPC_CheckChild2CondCode, OPC_CheckValueType, OPC_CheckComplexPat, OPC_CheckAndImm, OPC_CheckOrImm, + OPC_CheckImmAllOnesV, + OPC_CheckImmAllZerosV, OPC_CheckFoldableChainNode, OPC_EmitInteger, @@ -303,7 +306,7 @@ public: private: // Calls to these functions are generated by tblgen. - void Select_INLINEASM(SDNode *N); + void Select_INLINEASM(SDNode *N, bool Branch); void Select_READ_REGISTER(SDNode *Op); void Select_WRITE_REGISTER(SDNode *Op); void Select_UNDEF(SDNode *N); diff --git a/include/llvm/CodeGen/SelectionDAGNodes.h b/include/llvm/CodeGen/SelectionDAGNodes.h index 10f284179084..5aab9643e09d 100644 --- a/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/include/llvm/CodeGen/SelectionDAGNodes.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -184,6 +183,7 @@ public: inline unsigned getNumOperands() const; inline const SDValue &getOperand(unsigned i) const; inline uint64_t getConstantOperandVal(unsigned i) const; + inline const APInt &getConstantOperandAPInt(unsigned i) const; inline bool isTargetMemoryOpcode() const; inline bool isTargetOpcode() const; inline bool isMachineOpcode() const; @@ -232,7 +232,6 @@ template<> struct DenseMapInfo { return LHS == RHS; } }; -template <> struct isPodLike { static const bool value = true; }; /// Allow casting operators to work directly on /// SDValues as if they were SDNode*'s. @@ -369,6 +368,13 @@ private: bool ApproximateFuncs : 1; bool AllowReassociation : 1; + // We assume instructions do not raise floating-point exceptions by default, + // and only those marked explicitly may do so. We could choose to represent + // this via a positive "FPExcept" flags like on the MI level, but having a + // negative "NoFPExcept" flag here (that defaults to true) makes the flag + // intersection logic more straightforward. + bool NoFPExcept : 1; + public: /// Default constructor turns off all optimization flags. SDNodeFlags() @@ -376,7 +382,7 @@ public: Exact(false), NoNaNs(false), NoInfs(false), NoSignedZeros(false), AllowReciprocal(false), VectorReduction(false), AllowContract(false), ApproximateFuncs(false), - AllowReassociation(false) {} + AllowReassociation(false), NoFPExcept(true) {} /// Propagate the fast-math-flags from an IR FPMathOperator. void copyFMF(const FPMathOperator &FPMO) { @@ -439,6 +445,10 @@ public: setDefined(); AllowReassociation = b; } + void setFPExcept(bool b) { + setDefined(); + NoFPExcept = !b; + } // These are accessors for each flag. bool hasNoUnsignedWrap() const { return NoUnsignedWrap; } @@ -452,9 +462,10 @@ public: bool hasAllowContract() const { return AllowContract; } bool hasApproximateFuncs() const { return ApproximateFuncs; } bool hasAllowReassociation() const { return AllowReassociation; } + bool hasFPExcept() const { return !NoFPExcept; } bool isFast() const { - return NoSignedZeros && AllowReciprocal && NoNaNs && NoInfs && + return NoSignedZeros && AllowReciprocal && NoNaNs && NoInfs && NoFPExcept && AllowContract && ApproximateFuncs && AllowReassociation; } @@ -474,6 +485,7 @@ public: AllowContract &= Flags.AllowContract; ApproximateFuncs &= Flags.ApproximateFuncs; AllowReassociation &= Flags.AllowReassociation; + NoFPExcept &= Flags.NoFPExcept; } }; @@ -489,6 +501,17 @@ protected: // SubclassData. These are designed to fit within a uint16_t so they pack // with NodeType. +#if defined(_AIX) && (!defined(__GNUC__) || defined(__ibmxl__)) +// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words +// and give the `pack` pragma push semantics. +#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)") +#define END_TWO_BYTE_PACK() _Pragma("pack(pop)") +#else +#define BEGIN_TWO_BYTE_PACK() +#define END_TWO_BYTE_PACK() +#endif + +BEGIN_TWO_BYTE_PACK() class SDNodeBitfields { friend class SDNode; friend class MemIntrinsicSDNode; @@ -561,6 +584,9 @@ protected: LoadSDNodeBitfields LoadSDNodeBits; StoreSDNodeBitfields StoreSDNodeBits; }; +END_TWO_BYTE_PACK() +#undef BEGIN_TWO_BYTE_PACK +#undef END_TWO_BYTE_PACK // RawSDNodeBits must cover the entirety of the union. This means that all of // the union's members must have size <= RawSDNodeBits. We write the RHS as @@ -678,6 +704,8 @@ public: case ISD::STRICT_FFLOOR: case ISD::STRICT_FROUND: case ISD::STRICT_FTRUNC: + case ISD::STRICT_FP_ROUND: + case ISD::STRICT_FP_EXTEND: return true; } } @@ -898,9 +926,17 @@ public: /// Return the number of values used by this operation. unsigned getNumOperands() const { return NumOperands; } + /// Return the maximum number of operands that a SDNode can hold. + static constexpr size_t getMaxNumOperands() { + return std::numeric_limits::max(); + } + /// Helper method returns the integer value of a ConstantSDNode operand. inline uint64_t getConstantOperandVal(unsigned Num) const; + /// Helper method returns the APInt of a ConstantSDNode operand. + inline const APInt &getConstantOperandAPInt(unsigned Num) const; + const SDValue &getOperand(unsigned Num) const { assert(Num < NumOperands && "Invalid child # of SDNode!"); return OperandList[Num]; @@ -1128,6 +1164,10 @@ inline uint64_t SDValue::getConstantOperandVal(unsigned i) const { return Node->getConstantOperandVal(i); } +inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const { + return Node->getConstantOperandAPInt(i); +} + inline bool SDValue::isTargetOpcode() const { return Node->isTargetOpcode(); } @@ -1356,6 +1396,8 @@ public: N->getOpcode() == ISD::ATOMIC_LOAD_MAX || N->getOpcode() == ISD::ATOMIC_LOAD_UMIN || N->getOpcode() == ISD::ATOMIC_LOAD_UMAX || + N->getOpcode() == ISD::ATOMIC_LOAD_FADD || + N->getOpcode() == ISD::ATOMIC_LOAD_FSUB || N->getOpcode() == ISD::ATOMIC_LOAD || N->getOpcode() == ISD::ATOMIC_STORE || N->getOpcode() == ISD::MLOAD || @@ -1372,7 +1414,10 @@ class AtomicSDNode : public MemSDNode { public: AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL, EVT MemVT, MachineMemOperand *MMO) - : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {} + : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) { + assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || + MMO->isAtomic()) && "then why are we using an AtomicSDNode?"); + } const SDValue &getBasePtr() const { return getOperand(1); } const SDValue &getVal() const { return getOperand(2); } @@ -1408,6 +1453,8 @@ public: N->getOpcode() == ISD::ATOMIC_LOAD_MAX || N->getOpcode() == ISD::ATOMIC_LOAD_UMIN || N->getOpcode() == ISD::ATOMIC_LOAD_UMAX || + N->getOpcode() == ISD::ATOMIC_LOAD_FADD || + N->getOpcode() == ISD::ATOMIC_LOAD_FSUB || N->getOpcode() == ISD::ATOMIC_LOAD || N->getOpcode() == ISD::ATOMIC_STORE; } @@ -1467,14 +1514,16 @@ public: bool isSplat() const { return isSplatMask(Mask, getValueType(0)); } - int getSplatIndex() const { + int getSplatIndex() const { assert(isSplat() && "Cannot get splat index for non-splat!"); EVT VT = getValueType(0); - for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) { + for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) if (Mask[i] >= 0) return Mask[i]; - } - llvm_unreachable("Splat with all undef indices?"); + + // We can choose any index value here and be correct because all elements + // are undefined. Return 0 for better potential for callers to simplify. + return 0; } static bool isSplatMask(const int *Mask, EVT VT); @@ -1536,6 +1585,10 @@ uint64_t SDNode::getConstantOperandVal(unsigned Num) const { return cast(getOperand(Num))->getZExtValue(); } +const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const { + return cast(getOperand(Num))->getAPIntValue(); +} + class ConstantFPSDNode : public SDNode { friend class SelectionDAG; @@ -1603,20 +1656,36 @@ SDValue peekThroughBitcasts(SDValue V); /// If \p V is not a bitcasted one-use value, it is returned as-is. SDValue peekThroughOneUseBitcasts(SDValue V); +/// Return the non-extracted vector source operand of \p V if it exists. +/// If \p V is not an extracted subvector, it is returned as-is. +SDValue peekThroughExtractSubvectors(SDValue V); + /// Returns true if \p V is a bitwise not operation. Assumes that an all ones /// constant is canonicalized to be operand 1. -bool isBitwiseNot(SDValue V); +bool isBitwiseNot(SDValue V, bool AllowUndefs = false); /// Returns the SDNode if it is a constant splat BuildVector or constant int. -ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false); +ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false, + bool AllowTruncation = false); + +/// Returns the SDNode if it is a demanded constant splat BuildVector or +/// constant int. +ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts, + bool AllowUndefs = false, + bool AllowTruncation = false); /// Returns the SDNode if it is a constant splat BuildVector or constant float. ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false); +/// Returns the SDNode if it is a demanded constant splat BuildVector or +/// constant float. +ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts, + bool AllowUndefs = false); + /// Return true if the value is a constant 0 integer or a splatted vector of -/// a constant 0 integer (with no undefs). +/// a constant 0 integer (with no undefs by default). /// Build vector implicit truncation is not an issue for null values. -bool isNullOrNullSplat(SDValue V); +bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false); /// Return true if the value is a constant 1 integer or a splatted vector of a /// constant 1 integer (with no undefs). @@ -1673,6 +1742,38 @@ public: } }; +/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate +/// the offet and size that are started/ended in the underlying FrameIndex. +class LifetimeSDNode : public SDNode { + friend class SelectionDAG; + int64_t Size; + int64_t Offset; // -1 if offset is unknown. + + LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, + SDVTList VTs, int64_t Size, int64_t Offset) + : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {} +public: + int64_t getFrameIndex() const { + return cast(getOperand(1))->getIndex(); + } + + bool hasOffset() const { return Offset >= 0; } + int64_t getOffset() const { + assert(hasOffset() && "offset is unknown"); + return Offset; + } + int64_t getSize() const { + assert(hasOffset() && "offset is unknown"); + return Size; + } + + // Methods to support isa and dyn_cast + static bool classof(const SDNode *N) { + return N->getOpcode() == ISD::LIFETIME_START || + N->getOpcode() == ISD::LIFETIME_END; + } +}; + class JumpTableSDNode : public SDNode { friend class SelectionDAG; @@ -1818,12 +1919,31 @@ public: unsigned MinSplatBits = 0, bool isBigEndian = false) const; + /// Returns the demanded splatted value or a null value if this is not a + /// splat. + /// + /// The DemandedElts mask indicates the elements that must be in the splat. + /// If passed a non-null UndefElements bitvector, it will resize it to match + /// the vector width and set the bits where elements are undef. + SDValue getSplatValue(const APInt &DemandedElts, + BitVector *UndefElements = nullptr) const; + /// Returns the splatted value or a null value if this is not a splat. /// /// If passed a non-null UndefElements bitvector, it will resize it to match /// the vector width and set the bits where elements are undef. SDValue getSplatValue(BitVector *UndefElements = nullptr) const; + /// Returns the demanded splatted constant or null if this is not a constant + /// splat. + /// + /// The DemandedElts mask indicates the elements that must be in the splat. + /// If passed a non-null UndefElements bitvector, it will resize it to match + /// the vector width and set the bits where elements are undef. + ConstantSDNode * + getConstantSplatNode(const APInt &DemandedElts, + BitVector *UndefElements = nullptr) const; + /// Returns the splatted constant or null if this is not a constant /// splat. /// @@ -1832,6 +1952,16 @@ public: ConstantSDNode * getConstantSplatNode(BitVector *UndefElements = nullptr) const; + /// Returns the demanded splatted constant FP or null if this is not a + /// constant FP splat. + /// + /// The DemandedElts mask indicates the elements that must be in the splat. + /// If passed a non-null UndefElements bitvector, it will resize it to match + /// the vector width and set the bits where elements are undef. + ConstantFPSDNode * + getConstantFPSplatNode(const APInt &DemandedElts, + BitVector *UndefElements = nullptr) const; + /// Returns the splatted constant FP or null if this is not a constant /// FP splat. /// @@ -1956,8 +2086,10 @@ class LabelSDNode : public SDNode { MCSymbol *Label; - LabelSDNode(unsigned Order, const DebugLoc &dl, MCSymbol *L) - : SDNode(ISD::EH_LABEL, Order, dl, getSDVTList(MVT::Other)), Label(L) {} + LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L) + : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) { + assert(LabelSDNode::classof(this) && "not a label opcode"); + } public: MCSymbol *getLabel() const { return Label; } @@ -2049,6 +2181,8 @@ public: : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { LSBaseSDNodeBits.AddressingMode = AM; assert(getAddressingMode() == AM && "Value truncated"); + assert((!MMO->isAtomic() || MMO->isVolatile()) && + "use an AtomicSDNode instead for non-volatile atomics"); } const SDValue &getOffset() const { @@ -2473,18 +2607,6 @@ namespace ISD { cast(N)->getAddressingMode() == ISD::UNINDEXED; } - /// Return true if the node is a math/logic binary operator. This corresponds - /// to the IR function of the same name. - inline bool isBinaryOp(const SDNode *N) { - auto Op = N->getOpcode(); - return (Op == ISD::ADD || Op == ISD::SUB || Op == ISD::MUL || - Op == ISD::AND || Op == ISD::OR || Op == ISD::XOR || - Op == ISD::SHL || Op == ISD::SRL || Op == ISD::SRA || - Op == ISD::SDIV || Op == ISD::UDIV || Op == ISD::SREM || - Op == ISD::UREM || Op == ISD::FADD || Op == ISD::FSUB || - Op == ISD::FMUL || Op == ISD::FDIV || Op == ISD::FREM); - } - /// Attempt to match a unary predicate against a scalar/splat constant or /// every element of a constant BUILD_VECTOR. /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match. @@ -2495,10 +2617,11 @@ namespace ISD { /// Attempt to match a binary predicate against a pair of scalar/splat /// constants or every element of a pair of constant BUILD_VECTORs. /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match. + /// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match. bool matchBinaryPredicate( SDValue LHS, SDValue RHS, std::function Match, - bool AllowUndefs = false); + bool AllowUndefs = false, bool AllowTypeMismatch = false); } // end namespace ISD } // end namespace llvm diff --git a/include/llvm/CodeGen/SelectionDAGTargetInfo.h b/include/llvm/CodeGen/SelectionDAGTargetInfo.h index 45c1df48a5e6..6f6a9a5ae269 100644 --- a/include/llvm/CodeGen/SelectionDAGTargetInfo.h +++ b/include/llvm/CodeGen/SelectionDAGTargetInfo.h @@ -1,9 +1,8 @@ //==- llvm/CodeGen/SelectionDAGTargetInfo.h - SelectionDAG Info --*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -148,6 +147,14 @@ public: return std::make_pair(SDValue(), SDValue()); } + virtual SDValue EmitTargetCodeForSetTag(SelectionDAG &DAG, const SDLoc &dl, + SDValue Chain, SDValue Addr, + SDValue Size, + MachinePointerInfo DstPtrInfo, + bool ZeroData) const { + return SDValue(); + } + // Return true when the decision to generate FMA's (or FMS, FMLA etc) rather // than FMUL and ADD is delegated to the machine combiner. virtual bool generateFMAsInMachineCombiner(CodeGenOpt::Level OptLevel) const { diff --git a/include/llvm/CodeGen/SlotIndexes.h b/include/llvm/CodeGen/SlotIndexes.h index 8c8a7be459fd..2b32a4d30dff 100644 --- a/include/llvm/CodeGen/SlotIndexes.h +++ b/include/llvm/CodeGen/SlotIndexes.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/SlotIndexes.h - Slot indexes representation -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -302,8 +301,6 @@ class raw_ostream; } }; - template <> struct isPodLike { static const bool value = true; }; - inline raw_ostream& operator<<(raw_ostream &os, SlotIndex li) { li.print(os); return os; @@ -311,20 +308,6 @@ class raw_ostream; using IdxMBBPair = std::pair; - inline bool operator<(SlotIndex V, const IdxMBBPair &IM) { - return V < IM.first; - } - - inline bool operator<(const IdxMBBPair &IM, SlotIndex V) { - return IM.first < V; - } - - struct Idx2MBBCompare { - bool operator()(const IdxMBBPair &LHS, const IdxMBBPair &RHS) const { - return LHS.first < RHS.first; - } - }; - /// SlotIndexes pass. /// /// This pass assigns indexes to each instruction. @@ -336,10 +319,6 @@ class raw_ostream; using IndexList = ilist; IndexList indexList; -#ifdef EXPENSIVE_CHECKS - IndexList graveyardList; -#endif // EXPENSIVE_CHECKS - MachineFunction *mf; using Mi2IndexMap = DenseMap; @@ -368,7 +347,7 @@ class raw_ostream; public: static char ID; - SlotIndexes() : MachineFunctionPass(ID) { + SlotIndexes() : MachineFunctionPass(ID), mf(nullptr) { initializeSlotIndexesPass(*PassRegistry::getPassRegistry()); } @@ -385,9 +364,6 @@ class raw_ostream; /// Dump the indexes. void dump() const; - /// Renumber the index list, providing space for new instructions. - void renumberIndexes(); - /// Repair indexes after adding and removing instructions. void repairIndexesInRange(MachineBasicBlock *MBB, MachineBasicBlock::iterator Begin, @@ -516,7 +492,9 @@ class raw_ostream; /// Move iterator to the next IdxMBBPair where the SlotIndex is greater or /// equal to \p To. MBBIndexIterator advanceMBBIndex(MBBIndexIterator I, SlotIndex To) const { - return std::lower_bound(I, idx2MBBMap.end(), To); + return std::partition_point( + I, idx2MBBMap.end(), + [=](const IdxMBBPair &IM) { return IM.first < To; }); } /// Get an iterator pointing to the IdxMBBPair with the biggest SlotIndex @@ -552,29 +530,6 @@ class raw_ostream; return J->second; } - /// Returns the MBB covering the given range, or null if the range covers - /// more than one basic block. - MachineBasicBlock* getMBBCoveringRange(SlotIndex start, SlotIndex end) const { - - assert(start < end && "Backwards ranges not allowed."); - MBBIndexIterator itr = findMBBIndex(start); - if (itr == MBBIndexEnd()) { - itr = std::prev(itr); - return itr->second; - } - - // Check that we don't cross the boundary into this block. - if (itr->first < end) - return nullptr; - - itr = std::prev(itr); - - if (itr->first <= start) - return itr->second; - - return nullptr; - } - /// Insert the given machine instruction into the mapping. Returns the /// assigned index. /// If Late is set and there are null indexes between mi's neighboring @@ -680,33 +635,7 @@ class raw_ostream; idx2MBBMap.push_back(IdxMBBPair(startIdx, mbb)); renumberIndexes(newItr); - llvm::sort(idx2MBBMap, Idx2MBBCompare()); - } - - /// Free the resources that were required to maintain a SlotIndex. - /// - /// Once an index is no longer needed (for instance because the instruction - /// at that index has been moved), the resources required to maintain the - /// index can be relinquished to reduce memory use and improve renumbering - /// performance. Any remaining SlotIndex objects that point to the same - /// index are left 'dangling' (much the same as a dangling pointer to a - /// freed object) and should not be accessed, except to destruct them. - /// - /// Like dangling pointers, access to dangling SlotIndexes can cause - /// painful-to-track-down bugs, especially if the memory for the index - /// previously pointed to has been re-used. To detect dangling SlotIndex - /// bugs, build with EXPENSIVE_CHECKS=1. This will cause "erased" indexes to - /// be retained in a graveyard instead of being freed. Operations on indexes - /// in the graveyard will trigger an assertion. - void eraseIndex(SlotIndex index) { - IndexListEntry *entry = index.listEntry(); -#ifdef EXPENSIVE_CHECKS - indexList.remove(entry); - graveyardList.push_back(entry); - entry->setPoison(); -#else - indexList.erase(entry); -#endif + llvm::sort(idx2MBBMap, less_first()); } }; diff --git a/include/llvm/CodeGen/StackMaps.h b/include/llvm/CodeGen/StackMaps.h index 8be9ae378557..d7d88de6f682 100644 --- a/include/llvm/CodeGen/StackMaps.h +++ b/include/llvm/CodeGen/StackMaps.h @@ -1,9 +1,8 @@ //===- StackMaps.h - StackMaps ----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/CodeGen/StackProtector.h b/include/llvm/CodeGen/StackProtector.h index a506ac636a17..2bdf4425e24a 100644 --- a/include/llvm/CodeGen/StackProtector.h +++ b/include/llvm/CodeGen/StackProtector.h @@ -1,9 +1,8 @@ //===- StackProtector.h - Stack Protector Insertion -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -62,12 +61,6 @@ private: /// protection when -fstack-protection is used. unsigned SSPBufferSize = 0; - /// VisitedPHIs - The set of PHI nodes visited when determining - /// if a variable's reference has been taken. This set - /// is maintained to ensure we don't visit the same PHI node multiple - /// times. - SmallPtrSet VisitedPHIs; - // A prologue is generated. bool HasPrologue = false; diff --git a/include/llvm/CodeGen/SwiftErrorValueTracking.h b/include/llvm/CodeGen/SwiftErrorValueTracking.h new file mode 100644 index 000000000000..fb7a12853c09 --- /dev/null +++ b/include/llvm/CodeGen/SwiftErrorValueTracking.h @@ -0,0 +1,110 @@ +//===- SwiftErrorValueTracking.h - Track swifterror VReg vals --*- C++ -*--===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This implements a limited mem2reg-like analysis to promote uses of function +// arguments and allocas marked with swiftalloc from memory into virtual +// registers tracked by this class. +// +//===----------------------------------------------------------------------===// + +#ifndef SWIFTERRORVALUETRACKING_H +#define SWIFTERRORVALUETRACKING_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/Register.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/DebugLoc.h" +#include +#include +#include + + +namespace llvm { + class Function; + class MachineBasicBlock; + class MachineFunction; + class MachineInstr; + class TargetInstrInfo; + class TargetLowering; + +class SwiftErrorValueTracking { + // Some useful objects to reduce the number of function arguments needed. + MachineFunction *MF; + const Function *Fn; + const TargetLowering *TLI; + const TargetInstrInfo *TII; + + /// A map from swifterror value in a basic block to the virtual register it is + /// currently represented by. + DenseMap, Register> + VRegDefMap; + + /// A list of upward exposed vreg uses that need to be satisfied by either a + /// copy def or a phi node at the beginning of the basic block representing + /// the predecessor(s) swifterror value. + DenseMap, Register> + VRegUpwardsUse; + + /// A map from instructions that define/use a swifterror value to the virtual + /// register that represents that def/use. + llvm::DenseMap, Register> + VRegDefUses; + + /// The swifterror argument of the current function. + const Value *SwiftErrorArg; + + using SwiftErrorValues = SmallVector; + /// A function can only have a single swifterror argument. And if it does + /// have a swifterror argument, it must be the first entry in + /// SwiftErrorVals. + SwiftErrorValues SwiftErrorVals; + +public: + /// Initialize data structures for specified new function. + void setFunction(MachineFunction &MF); + + /// Get the (unique) function argument that was marked swifterror, or nullptr + /// if this function has no swifterror args. + const Value *getFunctionArg() const { + return SwiftErrorArg; + } + + /// Get or create the swifterror value virtual register in + /// VRegDefMap for this basic block. + Register getOrCreateVReg(const MachineBasicBlock *, const Value *); + + /// Set the swifterror virtual register in the VRegDefMap for this + /// basic block. + void setCurrentVReg(const MachineBasicBlock *MBB, const Value *, Register); + + /// Get or create the swifterror value virtual register for a def of a + /// swifterror by an instruction. + Register getOrCreateVRegDefAt(const Instruction *, const MachineBasicBlock *, + const Value *); + + /// Get or create the swifterror value virtual register for a use of a + /// swifterror by an instruction. + Register getOrCreateVRegUseAt(const Instruction *, const MachineBasicBlock *, + const Value *); + + /// Create initial definitions of swifterror values in the entry block of the + /// current function. + bool createEntriesInEntryBlock(DebugLoc DbgLoc); + + /// Propagate assigned swifterror vregs through a function, synthesizing PHI + /// nodes when needed to maintain consistency. + void propagateVRegs(); + + void preassignVRegs(MachineBasicBlock *MBB, BasicBlock::const_iterator Begin, + BasicBlock::const_iterator End); +}; + +} + +#endif diff --git a/include/llvm/CodeGen/SwitchLoweringUtils.h b/include/llvm/CodeGen/SwitchLoweringUtils.h new file mode 100644 index 000000000000..62134dc792f7 --- /dev/null +++ b/include/llvm/CodeGen/SwitchLoweringUtils.h @@ -0,0 +1,297 @@ +//===- SwitchLoweringUtils.h - Switch Lowering ------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_SWITCHLOWERINGUTILS_H +#define LLVM_CODEGEN_SWITCHLOWERINGUTILS_H + +#include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/SelectionDAGNodes.h" +#include "llvm/CodeGen/TargetLowering.h" +#include "llvm/IR/Constants.h" +#include "llvm/Support/BranchProbability.h" + +namespace llvm { + +class FunctionLoweringInfo; +class MachineBasicBlock; + +namespace SwitchCG { + +enum CaseClusterKind { + /// A cluster of adjacent case labels with the same destination, or just one + /// case. + CC_Range, + /// A cluster of cases suitable for jump table lowering. + CC_JumpTable, + /// A cluster of cases suitable for bit test lowering. + CC_BitTests +}; + +/// A cluster of case labels. +struct CaseCluster { + CaseClusterKind Kind; + const ConstantInt *Low, *High; + union { + MachineBasicBlock *MBB; + unsigned JTCasesIndex; + unsigned BTCasesIndex; + }; + BranchProbability Prob; + + static CaseCluster range(const ConstantInt *Low, const ConstantInt *High, + MachineBasicBlock *MBB, BranchProbability Prob) { + CaseCluster C; + C.Kind = CC_Range; + C.Low = Low; + C.High = High; + C.MBB = MBB; + C.Prob = Prob; + return C; + } + + static CaseCluster jumpTable(const ConstantInt *Low, const ConstantInt *High, + unsigned JTCasesIndex, BranchProbability Prob) { + CaseCluster C; + C.Kind = CC_JumpTable; + C.Low = Low; + C.High = High; + C.JTCasesIndex = JTCasesIndex; + C.Prob = Prob; + return C; + } + + static CaseCluster bitTests(const ConstantInt *Low, const ConstantInt *High, + unsigned BTCasesIndex, BranchProbability Prob) { + CaseCluster C; + C.Kind = CC_BitTests; + C.Low = Low; + C.High = High; + C.BTCasesIndex = BTCasesIndex; + C.Prob = Prob; + return C; + } +}; + +using CaseClusterVector = std::vector; +using CaseClusterIt = CaseClusterVector::iterator; + +/// Sort Clusters and merge adjacent cases. +void sortAndRangeify(CaseClusterVector &Clusters); + +struct CaseBits { + uint64_t Mask = 0; + MachineBasicBlock *BB = nullptr; + unsigned Bits = 0; + BranchProbability ExtraProb; + + CaseBits() = default; + CaseBits(uint64_t mask, MachineBasicBlock *bb, unsigned bits, + BranchProbability Prob) + : Mask(mask), BB(bb), Bits(bits), ExtraProb(Prob) {} +}; + +using CaseBitsVector = std::vector; + +/// This structure is used to communicate between SelectionDAGBuilder and +/// SDISel for the code generation of additional basic blocks needed by +/// multi-case switch statements. +struct CaseBlock { + // For the GISel interface. + struct PredInfoPair { + CmpInst::Predicate Pred; + // Set when no comparison should be emitted. + bool NoCmp; + }; + union { + // The condition code to use for the case block's setcc node. + // Besides the integer condition codes, this can also be SETTRUE, in which + // case no comparison gets emitted. + ISD::CondCode CC; + struct PredInfoPair PredInfo; + }; + + // The LHS/MHS/RHS of the comparison to emit. + // Emit by default LHS op RHS. MHS is used for range comparisons: + // If MHS is not null: (LHS <= MHS) and (MHS <= RHS). + const Value *CmpLHS, *CmpMHS, *CmpRHS; + + // The block to branch to if the setcc is true/false. + MachineBasicBlock *TrueBB, *FalseBB; + + // The block into which to emit the code for the setcc and branches. + MachineBasicBlock *ThisBB; + + /// The debug location of the instruction this CaseBlock was + /// produced from. + SDLoc DL; + DebugLoc DbgLoc; + + // Branch weights. + BranchProbability TrueProb, FalseProb; + + // Constructor for SelectionDAG. + CaseBlock(ISD::CondCode cc, const Value *cmplhs, const Value *cmprhs, + const Value *cmpmiddle, MachineBasicBlock *truebb, + MachineBasicBlock *falsebb, MachineBasicBlock *me, SDLoc dl, + BranchProbability trueprob = BranchProbability::getUnknown(), + BranchProbability falseprob = BranchProbability::getUnknown()) + : CC(cc), CmpLHS(cmplhs), CmpMHS(cmpmiddle), CmpRHS(cmprhs), + TrueBB(truebb), FalseBB(falsebb), ThisBB(me), DL(dl), + TrueProb(trueprob), FalseProb(falseprob) {} + + // Constructor for GISel. + CaseBlock(CmpInst::Predicate pred, bool nocmp, const Value *cmplhs, + const Value *cmprhs, const Value *cmpmiddle, + MachineBasicBlock *truebb, MachineBasicBlock *falsebb, + MachineBasicBlock *me, DebugLoc dl, + BranchProbability trueprob = BranchProbability::getUnknown(), + BranchProbability falseprob = BranchProbability::getUnknown()) + : PredInfo({pred, nocmp}), CmpLHS(cmplhs), CmpMHS(cmpmiddle), + CmpRHS(cmprhs), TrueBB(truebb), FalseBB(falsebb), ThisBB(me), + DbgLoc(dl), TrueProb(trueprob), FalseProb(falseprob) {} +}; + +struct JumpTable { + /// The virtual register containing the index of the jump table entry + /// to jump to. + unsigned Reg; + /// The JumpTableIndex for this jump table in the function. + unsigned JTI; + /// The MBB into which to emit the code for the indirect jump. + MachineBasicBlock *MBB; + /// The MBB of the default bb, which is a successor of the range + /// check MBB. This is when updating PHI nodes in successors. + MachineBasicBlock *Default; + + JumpTable(unsigned R, unsigned J, MachineBasicBlock *M, MachineBasicBlock *D) + : Reg(R), JTI(J), MBB(M), Default(D) {} +}; +struct JumpTableHeader { + APInt First; + APInt Last; + const Value *SValue; + MachineBasicBlock *HeaderBB; + bool Emitted; + bool OmitRangeCheck; + + JumpTableHeader(APInt F, APInt L, const Value *SV, MachineBasicBlock *H, + bool E = false) + : First(std::move(F)), Last(std::move(L)), SValue(SV), HeaderBB(H), + Emitted(E), OmitRangeCheck(false) {} +}; +using JumpTableBlock = std::pair; + +struct BitTestCase { + uint64_t Mask; + MachineBasicBlock *ThisBB; + MachineBasicBlock *TargetBB; + BranchProbability ExtraProb; + + BitTestCase(uint64_t M, MachineBasicBlock *T, MachineBasicBlock *Tr, + BranchProbability Prob) + : Mask(M), ThisBB(T), TargetBB(Tr), ExtraProb(Prob) {} +}; + +using BitTestInfo = SmallVector; + +struct BitTestBlock { + APInt First; + APInt Range; + const Value *SValue; + unsigned Reg; + MVT RegVT; + bool Emitted; + bool ContiguousRange; + MachineBasicBlock *Parent; + MachineBasicBlock *Default; + BitTestInfo Cases; + BranchProbability Prob; + BranchProbability DefaultProb; + + BitTestBlock(APInt F, APInt R, const Value *SV, unsigned Rg, MVT RgVT, bool E, + bool CR, MachineBasicBlock *P, MachineBasicBlock *D, + BitTestInfo C, BranchProbability Pr) + : First(std::move(F)), Range(std::move(R)), SValue(SV), Reg(Rg), + RegVT(RgVT), Emitted(E), ContiguousRange(CR), Parent(P), Default(D), + Cases(std::move(C)), Prob(Pr) {} +}; + +/// Return the range of value within a range. +uint64_t getJumpTableRange(const CaseClusterVector &Clusters, unsigned First, + unsigned Last); + +/// Return the number of cases within a range. +uint64_t getJumpTableNumCases(const SmallVectorImpl &TotalCases, + unsigned First, unsigned Last); + +struct SwitchWorkListItem { + MachineBasicBlock *MBB; + CaseClusterIt FirstCluster; + CaseClusterIt LastCluster; + const ConstantInt *GE; + const ConstantInt *LT; + BranchProbability DefaultProb; +}; +using SwitchWorkList = SmallVector; + +class SwitchLowering { +public: + SwitchLowering(FunctionLoweringInfo &funcinfo) : FuncInfo(funcinfo) {} + + void init(const TargetLowering &tli, const TargetMachine &tm, + const DataLayout &dl) { + TLI = &tli; + TM = &tm; + DL = &dl; + } + + /// Vector of CaseBlock structures used to communicate SwitchInst code + /// generation information. + std::vector SwitchCases; + + /// Vector of JumpTable structures used to communicate SwitchInst code + /// generation information. + std::vector JTCases; + + /// Vector of BitTestBlock structures used to communicate SwitchInst code + /// generation information. + std::vector BitTestCases; + + void findJumpTables(CaseClusterVector &Clusters, const SwitchInst *SI, + MachineBasicBlock *DefaultMBB); + + bool buildJumpTable(const CaseClusterVector &Clusters, unsigned First, + unsigned Last, const SwitchInst *SI, + MachineBasicBlock *DefaultMBB, CaseCluster &JTCluster); + + + void findBitTestClusters(CaseClusterVector &Clusters, const SwitchInst *SI); + + /// Build a bit test cluster from Clusters[First..Last]. Returns false if it + /// decides it's not a good idea. + bool buildBitTests(CaseClusterVector &Clusters, unsigned First, unsigned Last, + const SwitchInst *SI, CaseCluster &BTCluster); + + virtual void addSuccessorWithProb( + MachineBasicBlock *Src, MachineBasicBlock *Dst, + BranchProbability Prob = BranchProbability::getUnknown()) = 0; + + virtual ~SwitchLowering() = default; + +private: + const TargetLowering *TLI; + const TargetMachine *TM; + const DataLayout *DL; + FunctionLoweringInfo &FuncInfo; +}; + +} // namespace SwitchCG +} // namespace llvm + +#endif // LLVM_CODEGEN_SWITCHLOWERINGUTILS_H + diff --git a/include/llvm/CodeGen/TailDuplicator.h b/include/llvm/CodeGen/TailDuplicator.h index be6562c85f2e..358798d5ed60 100644 --- a/include/llvm/CodeGen/TailDuplicator.h +++ b/include/llvm/CodeGen/TailDuplicator.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/TailDuplicator.h ----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/TargetCallingConv.h b/include/llvm/CodeGen/TargetCallingConv.h index 7d138f585171..aebeeecbe506 100644 --- a/include/llvm/CodeGen/TargetCallingConv.h +++ b/include/llvm/CodeGen/TargetCallingConv.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/TargetCallingConv.h - Calling Convention ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -46,9 +45,12 @@ namespace ISD { unsigned IsInConsecutiveRegsLast : 1; unsigned IsInConsecutiveRegs : 1; unsigned IsCopyElisionCandidate : 1; ///< Argument copy elision candidate + unsigned IsPointer : 1; unsigned ByValSize; ///< Byval struct size + unsigned PointerAddrSpace; ///< Address space of pointer argument + public: ArgFlagsTy() : IsZExt(0), IsSExt(0), IsInReg(0), IsSRet(0), IsByVal(0), IsNest(0), @@ -56,8 +58,9 @@ namespace ISD { IsSwiftSelf(0), IsSwiftError(0), IsHva(0), IsHvaStart(0), IsSecArgPass(0), ByValAlign(0), OrigAlign(0), IsInConsecutiveRegsLast(0), IsInConsecutiveRegs(0), - IsCopyElisionCandidate(0), ByValSize(0) { - static_assert(sizeof(*this) == 2 * sizeof(unsigned), "flags are too big"); + IsCopyElisionCandidate(0), IsPointer(0), ByValSize(0), + PointerAddrSpace(0) { + static_assert(sizeof(*this) == 3 * sizeof(unsigned), "flags are too big"); } bool isZExt() const { return IsZExt; } @@ -114,6 +117,9 @@ namespace ISD { bool isCopyElisionCandidate() const { return IsCopyElisionCandidate; } void setCopyElisionCandidate() { IsCopyElisionCandidate = 1; } + bool isPointer() const { return IsPointer; } + void setPointer() { IsPointer = 1; } + unsigned getByValAlign() const { return (1U << ByValAlign) / 2; } void setByValAlign(unsigned A) { ByValAlign = Log2_32(A) + 1; @@ -128,7 +134,10 @@ namespace ISD { unsigned getByValSize() const { return ByValSize; } void setByValSize(unsigned S) { ByValSize = S; } - }; + + unsigned getPointerAddrSpace() const { return PointerAddrSpace; } + void setPointerAddrSpace(unsigned AS) { PointerAddrSpace = AS; } +}; /// InputArg - This struct carries flags and type information about a /// single incoming (formal) argument or incoming (from the perspective diff --git a/include/llvm/CodeGen/TargetFrameLowering.h b/include/llvm/CodeGen/TargetFrameLowering.h index b4d1da941433..878c9ffd2b51 100644 --- a/include/llvm/CodeGen/TargetFrameLowering.h +++ b/include/llvm/CodeGen/TargetFrameLowering.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/TargetFrameLowering.h ----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,6 +14,7 @@ #define LLVM_CODEGEN_TARGETFRAMELOWERING_H #include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/ADT/StringSwitch.h" #include #include @@ -24,6 +24,14 @@ namespace llvm { class MachineFunction; class RegScavenger; +namespace TargetStackID { + enum Value { + Default = 0, + SGPRSpill = 1, + NoAlloc = 255 + }; +} + /// Information about stack frame layout on the target. It holds the direction /// of stack growth, the known stack alignment on entry to each function, and /// the offset to the locals area. @@ -262,6 +270,17 @@ public: return getFrameIndexReference(MF, FI, FrameReg); } + /// getNonLocalFrameIndexReference - This method returns the offset used to + /// reference a frame index location. The offset can be from either FP/BP/SP + /// based on which base register is returned by llvm.localaddress. + virtual int getNonLocalFrameIndexReference(const MachineFunction &MF, + int FI) const { + // By default, dispatch to getFrameIndexReference. Interested targets can + // override this. + unsigned FrameReg; + return getFrameIndexReference(MF, FI, FrameReg); + } + /// This method determines which of the registers reported by /// TargetRegisterInfo::getCalleeSavedRegs() should actually get saved. /// The default implementation checks populates the \p SavedRegs bitset with @@ -335,6 +354,16 @@ public: return true; } + virtual bool isSupportedStackID(TargetStackID::Value ID) const { + switch (ID) { + default: + return false; + case TargetStackID::Default: + case TargetStackID::NoAlloc: + return true; + } + } + /// Check if given function is safe for not having callee saved registers. /// This is used when interprocedural register allocation is enabled. static bool isSafeForNoCSROpt(const Function &F) { diff --git a/include/llvm/CodeGen/TargetInstrInfo.h b/include/llvm/CodeGen/TargetInstrInfo.h index 961b90e9bc12..25b04f8c019a 100644 --- a/include/llvm/CodeGen/TargetInstrInfo.h +++ b/include/llvm/CodeGen/TargetInstrInfo.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/TargetInstrInfo.h - Instruction Info --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -27,6 +26,7 @@ #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineOutliner.h" #include "llvm/CodeGen/PseudoSourceValue.h" +#include "llvm/CodeGen/VirtRegMap.h" #include "llvm/MC/MCInstrInfo.h" #include "llvm/Support/BranchProbability.h" #include "llvm/Support/ErrorHandling.h" @@ -81,6 +81,7 @@ public: /// Given a machine instruction descriptor, returns the register /// class constraint for OpNum, or NULL. + virtual const TargetRegisterClass *getRegClass(const MCInstrDesc &MCID, unsigned OpNum, const TargetRegisterInfo *TRI, const MachineFunction &MF) const; @@ -429,6 +430,13 @@ public: RegSubRegPair(unsigned Reg = 0, unsigned SubReg = 0) : Reg(Reg), SubReg(SubReg) {} + + bool operator==(const RegSubRegPair& P) const { + return Reg == P.Reg && SubReg == P.SubReg; + } + bool operator!=(const RegSubRegPair& P) const { + return !(*this == P); + } }; /// A pair composed of a pair of a register and a sub-register index, @@ -663,8 +671,9 @@ public: /// is finished. Return the value/register of the new loop count. We need /// this function when peeling off one or more iterations of a loop. This /// function assumes the nth iteration is peeled first. - virtual unsigned reduceLoopCount(MachineBasicBlock &MBB, MachineInstr *IndVar, - MachineInstr &Cmp, + virtual unsigned reduceLoopCount(MachineBasicBlock &MBB, + MachineBasicBlock &PreHeader, + MachineInstr *IndVar, MachineInstr &Cmp, SmallVectorImpl &Cond, SmallVectorImpl &PrevInsts, unsigned Iter, unsigned MaxIter) const { @@ -926,9 +935,12 @@ public: /// operand folded, otherwise NULL is returned. /// The new instruction is inserted before MI, and the client is responsible /// for removing the old instruction. + /// If VRM is passed, the assigned physregs can be inspected by target to + /// decide on using an opcode (note that those assignments can still change). MachineInstr *foldMemoryOperand(MachineInstr &MI, ArrayRef Ops, int FI, - LiveIntervals *LIS = nullptr) const; + LiveIntervals *LIS = nullptr, + VirtRegMap *VRM = nullptr) const; /// Same as the previous version except it allows folding of any load and /// store from / to any address, not just from a specific stack slot. @@ -1018,7 +1030,8 @@ protected: foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI, ArrayRef Ops, MachineBasicBlock::iterator InsertPt, int FrameIndex, - LiveIntervals *LIS = nullptr) const { + LiveIntervals *LIS = nullptr, + VirtRegMap *VRM = nullptr) const { return nullptr; } @@ -1138,8 +1151,9 @@ public: /// Get the base operand and byte offset of an instruction that reads/writes /// memory. - virtual bool getMemOperandWithOffset(MachineInstr &MI, - MachineOperand *&BaseOp, int64_t &Offset, + virtual bool getMemOperandWithOffset(const MachineInstr &MI, + const MachineOperand *&BaseOp, + int64_t &Offset, const TargetRegisterInfo *TRI) const { return false; } @@ -1164,8 +1178,8 @@ public: /// or /// DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI)); /// to TargetPassConfig::createMachineScheduler() to have an effect. - virtual bool shouldClusterMemOps(MachineOperand &BaseOp1, - MachineOperand &BaseOp2, + virtual bool shouldClusterMemOps(const MachineOperand &BaseOp1, + const MachineOperand &BaseOp2, unsigned NumLoads) const { llvm_unreachable("target did not implement shouldClusterMemOps()"); } @@ -1253,8 +1267,9 @@ public: /// Measure the specified inline asm to determine an approximation of its /// length. - virtual unsigned getInlineAsmLength(const char *Str, - const MCAsmInfo &MAI) const; + virtual unsigned getInlineAsmLength( + const char *Str, const MCAsmInfo &MAI, + const TargetSubtargetInfo *STI = nullptr) const; /// Allocate and return a hazard recognizer to use for this target when /// scheduling the machine instructions before register allocation. @@ -1542,7 +1557,8 @@ public: /// See also MachineInstr::mayAlias, which is implemented on top of this /// function. virtual bool - areMemAccessesTriviallyDisjoint(MachineInstr &MIa, MachineInstr &MIb, + areMemAccessesTriviallyDisjoint(const MachineInstr &MIa, + const MachineInstr &MIb, AliasAnalysis *AA = nullptr) const { assert((MIa.mayLoad() || MIa.mayStore()) && "MIa must load from or modify a memory location"); diff --git a/include/llvm/CodeGen/TargetLowering.h b/include/llvm/CodeGen/TargetLowering.h index 23dbaac03ebe..d5cca60bb1b2 100644 --- a/include/llvm/CodeGen/TargetLowering.h +++ b/include/llvm/CodeGen/TargetLowering.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/TargetLowering.h - Target Lowering Info -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -189,13 +188,18 @@ public: bool IsSwiftSelf : 1; bool IsSwiftError : 1; uint16_t Alignment = 0; + Type *ByValType = nullptr; ArgListEntry() : IsSExt(false), IsZExt(false), IsInReg(false), IsSRet(false), IsNest(false), IsByVal(false), IsInAlloca(false), IsReturned(false), IsSwiftSelf(false), IsSwiftError(false) {} - void setAttributes(ImmutableCallSite *CS, unsigned ArgIdx); + void setAttributes(const CallBase *Call, unsigned ArgIdx); + + void setAttributes(ImmutableCallSite *CS, unsigned ArgIdx) { + return setAttributes(cast(CS->getInstruction()), ArgIdx); + } }; using ArgListTy = std::vector; @@ -235,7 +239,14 @@ public: /// Return the pointer type for the given address space, defaults to /// the pointer type from the data layout. /// FIXME: The default needs to be removed once all the code is updated. - MVT getPointerTy(const DataLayout &DL, uint32_t AS = 0) const { + virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS = 0) const { + return MVT::getIntegerVT(DL.getPointerSizeInBits(AS)); + } + + /// Return the in-memory pointer type for the given address space, defaults to + /// the pointer type from the data layout. FIXME: The default needs to be + /// removed once all the code is updated. + MVT getPointerMemTy(const DataLayout &DL, uint32_t AS = 0) const { return MVT::getIntegerVT(DL.getPointerSizeInBits(AS)); } @@ -291,6 +302,9 @@ public: // The default action for one element vectors is to scalarize if (VT.getVectorNumElements() == 1) return TypeScalarizeVector; + // The default action for an odd-width vector is to widen. + if (!VT.isPow2VectorType()) + return TypeWidenVector; // The default action for other vectors is to promote return TypePromoteInteger; } @@ -387,8 +401,9 @@ public: /// efficiently, casting the load to a smaller vector of larger types and /// loading is more efficient, however, this can be undone by optimizations in /// dag combiner. - virtual bool isLoadBitCastBeneficial(EVT LoadVT, - EVT BitcastVT) const { + virtual bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT, + const SelectionDAG &DAG, + const MachineMemOperand &MMO) const { // Don't do if we could do an indexed load on the original type, but not on // the new one. if (!LoadVT.isSimple() || !BitcastVT.isSimple()) @@ -402,14 +417,18 @@ public: getTypeToPromoteTo(ISD::LOAD, LoadMVT) == BitcastVT.getSimpleVT()) return false; - return true; + bool Fast = false; + return allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), BitcastVT, + MMO, &Fast) && Fast; } /// Return true if the following transform is beneficial: /// (store (y (conv x)), y*)) -> (store x, (x*)) - virtual bool isStoreBitCastBeneficial(EVT StoreVT, EVT BitcastVT) const { + virtual bool isStoreBitCastBeneficial(EVT StoreVT, EVT BitcastVT, + const SelectionDAG &DAG, + const MachineMemOperand &MMO) const { // Default to the same logic as loads. - return isLoadBitCastBeneficial(StoreVT, BitcastVT); + return isLoadBitCastBeneficial(StoreVT, BitcastVT, DAG, MMO); } /// Return true if it is expected to be cheaper to do a store of a non-zero @@ -421,10 +440,12 @@ public: return false; } - /// Allow store merging after legalization in addition to before legalization. - /// This may catch stores that do not exist earlier (eg, stores created from - /// intrinsics). - virtual bool mergeStoresAfterLegalization() const { return true; } + /// Allow store merging for the specified type after legalization in addition + /// to before legalization. This may transform stores that do not exist + /// earlier (for example, stores created from intrinsics). + virtual bool mergeStoresAfterLegalization(EVT MemVT) const { + return true; + } /// Returns if it's reasonable to merge stores to MemVT size. virtual bool canMergeStoresTo(unsigned AS, EVT MemVT, @@ -521,13 +542,22 @@ public: /// There are two ways to clear extreme bits (either low or high): /// Mask: x & (-1 << y) (the instcombine canonical form) /// Shifts: x >> y << y - /// Return true if the variant with 2 shifts is preferred. + /// Return true if the variant with 2 variable shifts is preferred. /// Return false if there is no preference. - virtual bool preferShiftsToClearExtremeBits(SDValue X) const { + virtual bool shouldFoldMaskToVariableShiftPair(SDValue X) const { // By default, let's assume that no one prefers shifts. return false; } + /// Return true if it is profitable to fold a pair of shifts into a mask. + /// This is usually true on most targets. But some targets, like Thumb1, + /// have immediate shift instructions, but no immediate "and" instruction; + /// this makes the fold unprofitable. + virtual bool shouldFoldConstantShiftPairToMask(const SDNode *N, + CombineLevel Level) const { + return true; + } + /// Should we tranform the IR-optimal check for whether given truncation /// down into KeptBits would be truncating or not: /// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits) @@ -541,6 +571,16 @@ public: return false; } + /// These two forms are equivalent: + /// sub %y, (xor %x, -1) + /// add (add %x, 1), %y + /// The variant with two add's is IR-canonical. + /// Some targets may prefer one to the other. + virtual bool preferIncOfAddToSubOfNot(EVT VT) const { + // By default, let's assume that everyone prefers the form with two add's. + return true; + } + /// Return true if the target wants to use the optimization that /// turns ext(promotableInst1(...(promotableInstN(load)))) into /// promotedInst1(...(promotedInstN(ext(load)))). @@ -560,11 +600,6 @@ public: return false; } - /// Return true if target supports floating point exceptions. - bool hasFloatingPointExceptions() const { - return HasFloatingPointExceptions; - } - /// Return true if target always beneficiates from combining into FMA for a /// given value type. This must typically return false on targets where FMA /// takes more cycles to execute than FADD. @@ -619,12 +654,21 @@ public: /// Return the register class that should be used for the specified value /// type. - virtual const TargetRegisterClass *getRegClassFor(MVT VT) const { + virtual const TargetRegisterClass *getRegClassFor(MVT VT, bool isDivergent = false) const { + (void)isDivergent; const TargetRegisterClass *RC = RegClassForVT[VT.SimpleTy]; assert(RC && "This value type is not natively supported!"); return RC; } + /// Allows target to decide about the register class of the + /// specific value that is live outside the defining block. + /// Returns true if the value needs uniform register class. + virtual bool requiresUniformRegister(MachineFunction &MF, + const Value *) const { + return false; + } + /// Return the 'representative' register class for the specified value /// type. /// @@ -643,6 +687,13 @@ public: return RepRegClassCostForVT[VT.SimpleTy]; } + /// Return true if SHIFT instructions should be expanded to SHIFT_PARTS + /// instructions, and false if a library call is preferred (e.g for code-size + /// reasons). + virtual bool shouldExpandShift(SelectionDAG &DAG, SDNode *N) const { + return true; + } + /// Return true if the target has native support for the specified value type. /// This means that it has a register that directly holds it without /// promotions or expansions. @@ -768,7 +819,8 @@ public: /// Returns true if the target can instruction select the specified FP /// immediate natively. If false, the legalizer will materialize the FP /// immediate as a load from a constant pool. - virtual bool isFPImmLegal(const APFloat &/*Imm*/, EVT /*VT*/) const { + virtual bool isFPImmLegal(const APFloat & /*Imm*/, EVT /*VT*/, + bool ForCodeSize = false) const { return false; } @@ -830,6 +882,8 @@ public: default: llvm_unreachable("Unexpected fixed point operation."); case ISD::SMULFIX: + case ISD::SMULFIXSAT: + case ISD::UMULFIX: Supported = isSupportedFixedPointOperation(Op, VT, Scale); break; } @@ -865,6 +919,8 @@ public: case ISD::STRICT_FFLOOR: EqOpc = ISD::FFLOOR; break; case ISD::STRICT_FROUND: EqOpc = ISD::FROUND; break; case ISD::STRICT_FTRUNC: EqOpc = ISD::FTRUNC; break; + case ISD::STRICT_FP_ROUND: EqOpc = ISD::FP_ROUND; break; + case ISD::STRICT_FP_EXTEND: EqOpc = ISD::FP_EXTEND; break; } auto Action = getOperationAction(EqOpc, VT); @@ -931,21 +987,20 @@ public: /// Return true if lowering to a jump table is suitable for a set of case /// clusters which may contain \p NumCases cases, \p Range range of values. - /// FIXME: This function check the maximum table size and density, but the - /// minimum size is not checked. It would be nice if the minimum size is - /// also combined within this function. Currently, the minimum size check is - /// performed in findJumpTable() in SelectionDAGBuiler and - /// getEstimatedNumberOfCaseClusters() in BasicTTIImpl. virtual bool isSuitableForJumpTable(const SwitchInst *SI, uint64_t NumCases, uint64_t Range) const { - const bool OptForSize = SI->getParent()->getParent()->optForSize(); + // FIXME: This function check the maximum table size and density, but the + // minimum size is not checked. It would be nice if the minimum size is + // also combined within this function. Currently, the minimum size check is + // performed in findJumpTable() in SelectionDAGBuiler and + // getEstimatedNumberOfCaseClusters() in BasicTTIImpl. + const bool OptForSize = SI->getParent()->getParent()->hasOptSize(); const unsigned MinDensity = getMinimumJumpTableDensity(OptForSize); - const unsigned MaxJumpTableSize = - OptForSize || getMaximumJumpTableSize() == 0 - ? UINT_MAX - : getMaximumJumpTableSize(); - // Check whether a range of clusters is dense enough for a jump table. - if (Range <= MaxJumpTableSize && + const unsigned MaxJumpTableSize = getMaximumJumpTableSize(); + + // Check whether the number of cases is small enough and + // the range is dense enough for a jump table. + if ((OptForSize || Range <= MaxJumpTableSize) && (NumCases * 100 >= Range * MinDensity)) { return true; } @@ -1140,24 +1195,42 @@ public: EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown = false) const { // Lower scalar pointers to native pointer types. - if (PointerType *PTy = dyn_cast(Ty)) + if (auto *PTy = dyn_cast(Ty)) return getPointerTy(DL, PTy->getAddressSpace()); - if (Ty->isVectorTy()) { - VectorType *VTy = cast(Ty); - Type *Elm = VTy->getElementType(); + if (auto *VTy = dyn_cast(Ty)) { + Type *EltTy = VTy->getElementType(); // Lower vectors of pointers to native pointer types. + if (auto *PTy = dyn_cast(EltTy)) { + EVT PointerTy(getPointerTy(DL, PTy->getAddressSpace())); + EltTy = PointerTy.getTypeForEVT(Ty->getContext()); + } + return EVT::getVectorVT(Ty->getContext(), EVT::getEVT(EltTy, false), + VTy->getNumElements()); + } + + return EVT::getEVT(Ty, AllowUnknown); + } + + EVT getMemValueType(const DataLayout &DL, Type *Ty, + bool AllowUnknown = false) const { + // Lower scalar pointers to native pointer types. + if (PointerType *PTy = dyn_cast(Ty)) + return getPointerMemTy(DL, PTy->getAddressSpace()); + else if (VectorType *VTy = dyn_cast(Ty)) { + Type *Elm = VTy->getElementType(); if (PointerType *PT = dyn_cast(Elm)) { - EVT PointerTy(getPointerTy(DL, PT->getAddressSpace())); + EVT PointerTy(getPointerMemTy(DL, PT->getAddressSpace())); Elm = PointerTy.getTypeForEVT(Ty->getContext()); } - return EVT::getVectorVT(Ty->getContext(), EVT::getEVT(Elm, false), VTy->getNumElements()); } - return EVT::getEVT(Ty, AllowUnknown); + + return getValueType(DL, Ty, AllowUnknown); } + /// Return the MVT corresponding to this LLVM type. See getValueType. MVT getSimpleValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown = false) const { @@ -1327,18 +1400,6 @@ public: return OptSize ? MaxLoadsPerMemcmpOptSize : MaxLoadsPerMemcmp; } - /// For memcmp expansion when the memcmp result is only compared equal or - /// not-equal to 0, allow up to this number of load pairs per block. As an - /// example, this may allow 'memcmp(a, b, 3) == 0' in a single block: - /// a0 = load2bytes &a[0] - /// b0 = load2bytes &b[0] - /// a2 = load1byte &a[2] - /// b2 = load1byte &b[2] - /// r = cmp eq (a0 ^ b0 | a2 ^ b2), 0 - virtual unsigned getMemcmpEqZeroLoadsPerBlock() const { - return 1; - } - /// Get maximum # of store operations permitted for llvm.memmove /// /// This function returns the maximum number of store operations permitted @@ -1358,10 +1419,10 @@ public: /// copy/move/set is converted to a sequence of store operations. Its use /// helps to ensure that such replacements don't generate code that causes an /// alignment error (trap) on the target machine. - virtual bool allowsMisalignedMemoryAccesses(EVT, - unsigned AddrSpace = 0, - unsigned Align = 1, - bool * /*Fast*/ = nullptr) const { + virtual bool allowsMisalignedMemoryAccesses( + EVT, unsigned AddrSpace = 0, unsigned Align = 1, + MachineMemOperand::Flags Flags = MachineMemOperand::MONone, + bool * /*Fast*/ = nullptr) const { return false; } @@ -1369,8 +1430,18 @@ public: /// given address space and alignment. If the access is allowed, the optional /// final parameter returns if the access is also fast (as defined by the /// target). + bool + allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, + unsigned AddrSpace = 0, unsigned Alignment = 1, + MachineMemOperand::Flags Flags = MachineMemOperand::MONone, + bool *Fast = nullptr) const; + + /// Return true if the target supports a memory access of this type for the + /// given MachineMemOperand. If the access is allowed, the optional + /// final parameter returns if the access is also fast (as defined by the + /// target). bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, - unsigned AddrSpace = 0, unsigned Alignment = 1, + const MachineMemOperand &MMO, bool *Fast = nullptr) const; /// Returns the target specific optimal type for load and store operations as @@ -1384,12 +1455,11 @@ public: /// zero. 'MemcpyStrSrc' indicates whether the memcpy source is constant so it /// does not need to be loaded. It returns EVT::Other if the type should be /// determined using generic target-independent logic. - virtual EVT getOptimalMemOpType(uint64_t /*Size*/, - unsigned /*DstAlign*/, unsigned /*SrcAlign*/, - bool /*IsMemset*/, - bool /*ZeroMemset*/, - bool /*MemcpyStrSrc*/, - MachineFunction &/*MF*/) const { + virtual EVT + getOptimalMemOpType(uint64_t /*Size*/, unsigned /*DstAlign*/, + unsigned /*SrcAlign*/, bool /*IsMemset*/, + bool /*ZeroMemset*/, bool /*MemcpyStrSrc*/, + const AttributeList & /*FuncAttributes*/) const { return MVT::Other; } @@ -1515,7 +1585,7 @@ public: /// performs validation and error handling, returns the function. Otherwise, /// returns nullptr. Must be previously inserted by insertSSPDeclarations. /// Should be used only when getIRStackGuard returns nullptr. - virtual Value *getSSPStackGuardCheck(const Module &M) const; + virtual Function *getSSPStackGuardCheck(const Module &M) const; protected: Value *getDefaultSafeStackPointerLocation(IRBuilder<> &IRB, @@ -1537,8 +1607,9 @@ public: } /// Returns true if a cast from SrcAS to DestAS is "cheap", such that e.g. we - /// are happy to sink it into basic blocks. - virtual bool isCheapAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const { + /// are happy to sink it into basic blocks. A cast may be free, but not + /// necessarily a no-op. e.g. a free truncate from a 64-bit to 32-bit pointer. + virtual bool isFreeAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const { return isNoopAddrSpaceCast(SrcAS, DestAS); } @@ -1716,8 +1787,9 @@ public: /// Returns how the IR-level AtomicExpand pass should expand the given /// AtomicRMW, if at all. Default is to never expand. - virtual AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *) const { - return AtomicExpansionKind::None; + virtual AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const { + return RMW->isFloatingPointOperation() ? + AtomicExpansionKind::CmpXChg : AtomicExpansionKind::None; } /// On some platforms, an AtomicRMW that never actually modifies the value @@ -1762,6 +1834,8 @@ public: Action != TypeSplitVector; } + virtual bool isProfitableToCombineMinNumMaxNum(EVT VT) const { return true; } + /// Return true if a select of constants (select Cond, C1, C2) should be /// transformed into simple math ops with the condition value. For example: /// select Cond, C1, C1-1 --> add (zext Cond), C1-1 @@ -1865,12 +1939,6 @@ protected: /// control. void setJumpIsExpensive(bool isExpensive = true); - /// Tells the code generator that this target supports floating point - /// exceptions and cares about preserving floating point exception behavior. - void setHasFloatingPointExceptions(bool FPExceptions = true) { - HasFloatingPointExceptions = FPExceptions; - } - /// Tells the code generator which bitwidths to bypass. void addBypassSlowDiv(unsigned int SlowBitWidth, unsigned int FastBitWidth) { BypassSlowDivWidths[SlowBitWidth] = FastBitWidth; @@ -2159,6 +2227,8 @@ public: case ISD::UADDSAT: case ISD::FMINNUM: case ISD::FMAXNUM: + case ISD::FMINNUM_IEEE: + case ISD::FMAXNUM_IEEE: case ISD::FMINIMUM: case ISD::FMAXIMUM: return true; @@ -2166,6 +2236,30 @@ public: } } + /// Return true if the node is a math/logic binary operator. + virtual bool isBinOp(unsigned Opcode) const { + // A commutative binop must be a binop. + if (isCommutativeBinOp(Opcode)) + return true; + // These are non-commutative binops. + switch (Opcode) { + case ISD::SUB: + case ISD::SHL: + case ISD::SRL: + case ISD::SRA: + case ISD::SDIV: + case ISD::UDIV: + case ISD::SREM: + case ISD::UREM: + case ISD::FSUB: + case ISD::FDIV: + case ISD::FREM: + return true; + default: + return false; + } + } + /// Return true if it's free to truncate a value of type FromTy to type /// ToTy. e.g. On x86 it's free to truncate a i32 value in register EAX to i16 /// by referencing its sub-register AX. @@ -2270,6 +2364,16 @@ public: return false; } + /// Return true if sinking I's operands to the same basic block as I is + /// profitable, e.g. because the operands can be folded into a target + /// instruction during instruction selection. After calling the function + /// \p Ops contains the Uses to sink ordered by dominance (dominating users + /// come first). + virtual bool shouldSinkOperands(Instruction *I, + SmallVectorImpl &Ops) const { + return false; + } + /// Return true if the target supplies and combines to a paired load /// two loaded values of type LoadedType next to each other in memory. /// RequiredAlignment gives the minimal alignment constraints that must be met @@ -2415,6 +2519,31 @@ public: return false; } + /// Return true if extraction of a scalar element from the given vector type + /// at the given index is cheap. For example, if scalar operations occur on + /// the same register file as vector operations, then an extract element may + /// be a sub-register rename rather than an actual instruction. + virtual bool isExtractVecEltCheap(EVT VT, unsigned Index) const { + return false; + } + + /// Try to convert math with an overflow comparison into the corresponding DAG + /// node operation. Targets may want to override this independently of whether + /// the operation is legal/custom for the given type because it may obscure + /// matching of other patterns. + virtual bool shouldFormOverflowOp(unsigned Opcode, EVT VT) const { + // TODO: The default logic is inherited from code in CodeGenPrepare. + // The opcode should not make a difference by default? + if (Opcode != ISD::UADDO) + return false; + + // Allow the transform as long as we have an integer type that is not + // obviously illegal and unsupported. + if (VT.isVector()) + return false; + return VT.isSimple() || !isOperationExpand(Opcode, VT); + } + // Return true if it is profitable to use a scalar input to a BUILD_VECTOR // even if the vector itself has multiple uses. virtual bool aggressivelyPreferBuildVectorSources(EVT VecVT) const { @@ -2495,10 +2624,6 @@ private: /// predication. bool JumpIsExpensive; - /// Whether the target supports or cares about preserving floating point - /// exception behavior. - bool HasFloatingPointExceptions; - /// This target prefers to use _setjmp to implement llvm.setjmp. /// /// Defaults to false. @@ -2834,11 +2959,10 @@ public: /// Returns a pair of (return value, chain). /// It is an error to pass RTLIB::UNKNOWN_LIBCALL as \p LC. - std::pair makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, - EVT RetVT, ArrayRef Ops, - bool isSigned, const SDLoc &dl, - bool doesNotReturn = false, - bool isReturnValueUsed = true) const; + std::pair makeLibCall( + SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef Ops, + bool isSigned, const SDLoc &dl, bool doesNotReturn = false, + bool isReturnValueUsed = true, bool isPostTypeLegalization = false) const; /// Check whether parameters to a call that are passed in callee saved /// registers are the same as from the calling function. This needs to be @@ -2876,6 +3000,20 @@ public: } }; + /// Determines the optimal series of memory ops to replace the memset / memcpy. + /// Return true if the number of memory ops is below the threshold (Limit). + /// It returns the types of the sequence of memory ops to perform + /// memset / memcpy by reference. + bool findOptimalMemOpLowering(std::vector &MemOps, + unsigned Limit, uint64_t Size, + unsigned DstAlign, unsigned SrcAlign, + bool IsMemset, + bool ZeroMemset, + bool MemcpyStrSrc, + bool AllowOverlap, + unsigned DstAS, unsigned SrcAS, + const AttributeList &FuncAttributes) const; + /// Check to see if the specified operand of the specified instruction is a /// constant integer. If so, check to see if there are any bits set in the /// constant that are not demanded. If so, shrink the constant and return @@ -3001,6 +3139,10 @@ public: TargetLoweringOpt &TLO, unsigned Depth = 0) const; + /// This method returns the constant pool value that will be loaded by LD. + /// NOTE: You must check for implicit extensions of the constant by LD. + virtual const Constant *getTargetConstantFromLoad(LoadSDNode *LD) const; + /// If \p SNaN is false, \returns true if \p Op is known to never be any /// NaN. If \p sNaN is true, returns if \p Op is known to never be a signaling /// NaN. @@ -3088,15 +3230,6 @@ public: return true; } - /// Return true if it is profitable to fold a pair of shifts into a mask. - /// This is usually true on most targets. But some targets, like Thumb1, - /// have immediate shift instructions, but no immediate "and" instruction; - /// this makes the fold unprofitable. - virtual bool shouldFoldShiftPairToMask(const SDNode *N, - CombineLevel Level) const { - return true; - } - // Return true if it is profitable to combine a BUILD_VECTOR with a stride-pattern // to a shuffle and a truncate. // Example of such a combine: @@ -3430,6 +3563,15 @@ public: return false; } + /// For most targets, an LLVM type must be broken down into multiple + /// smaller types. Usually the halves are ordered according to the endianness + /// but for some platform that would break. So this method will default to + /// matching the endianness but can be overridden. + virtual bool + shouldSplitFunctionArgumentsAsLittleEndian(const DataLayout &DL) const { + return DL.isLittleEndian(); + } + /// Returns a 0 terminated array of registers that can be safely used as /// scratch registers. virtual const MCPhysReg *getScratchRegisters(CallingConv::ID CC) const { @@ -3638,6 +3780,12 @@ public: std::vector &Ops, SelectionDAG &DAG) const; + // Lower custom output constraints. If invalid, return SDValue(). + virtual SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag, + SDLoc DL, + const AsmOperandInfo &OpInfo, + SelectionDAG &DAG) const; + //===--------------------------------------------------------------------===// // Div utility functions // @@ -3840,8 +3988,26 @@ public: /// Method for building the DAG expansion of ISD::SMULFIX. This method accepts /// integers as its arguments. - SDValue getExpandedFixedPointMultiplication(SDNode *Node, - SelectionDAG &DAG) const; + SDValue expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const; + + /// Method for building the DAG expansion of ISD::U(ADD|SUB)O. Expansion + /// always suceeds and populates the Result and Overflow arguments. + void expandUADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow, + SelectionDAG &DAG) const; + + /// Method for building the DAG expansion of ISD::S(ADD|SUB)O. Expansion + /// always suceeds and populates the Result and Overflow arguments. + void expandSADDSUBO(SDNode *Node, SDValue &Result, SDValue &Overflow, + SelectionDAG &DAG) const; + + /// Method for building the DAG expansion of ISD::[US]MULO. Returns whether + /// expansion was successful and populates the Result and Overflow arguments. + bool expandMULO(SDNode *Node, SDValue &Result, SDValue &Overflow, + SelectionDAG &DAG) const; + + /// Expand a VECREDUCE_* into an explicit calculation. If Count is specified, + /// only the first Count elements of the vector are used. + SDValue expandVecReduce(SDNode *Node, SelectionDAG &DAG) const; //===--------------------------------------------------------------------===// // Instruction Emitting Hooks @@ -3894,14 +4060,23 @@ public: SDValue lowerCmpEqZeroToCtlzSrl(SDValue Op, SelectionDAG &DAG) const; private: - SDValue simplifySetCCWithAnd(EVT VT, SDValue N0, SDValue N1, - ISD::CondCode Cond, DAGCombinerInfo &DCI, - const SDLoc &DL) const; + SDValue foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, + const SDLoc &DL, DAGCombinerInfo &DCI) const; + SDValue foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1, ISD::CondCode Cond, + const SDLoc &DL, DAGCombinerInfo &DCI) const; SDValue optimizeSetCCOfSignedTruncationCheck(EVT SCCVT, SDValue N0, SDValue N1, ISD::CondCode Cond, DAGCombinerInfo &DCI, const SDLoc &DL) const; + + SDValue prepareUREMEqFold(EVT SETCCVT, SDValue REMNode, + SDValue CompTargetNode, ISD::CondCode Cond, + DAGCombinerInfo &DCI, const SDLoc &DL, + SmallVectorImpl &Created) const; + SDValue buildUREMEqFold(EVT SETCCVT, SDValue REMNode, SDValue CompTargetNode, + ISD::CondCode Cond, DAGCombinerInfo &DCI, + const SDLoc &DL) const; }; /// Given an LLVM IR type and return type attributes, compute the return value diff --git a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h index 052d1f8bc686..a1fb81cb009d 100644 --- a/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h +++ b/include/llvm/CodeGen/TargetLoweringObjectFileImpl.h @@ -1,9 +1,8 @@ //==- llvm/CodeGen/TargetLoweringObjectFileImpl.h - Object Info --*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/TargetOpcodes.h b/include/llvm/CodeGen/TargetOpcodes.h index d0d959c4ae11..080a244f6f69 100644 --- a/include/llvm/CodeGen/TargetOpcodes.h +++ b/include/llvm/CodeGen/TargetOpcodes.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/TargetOpcodes.h - Target Indep Opcodes -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/TargetPassConfig.h b/include/llvm/CodeGen/TargetPassConfig.h index 3288711a335d..0bd82aafac37 100644 --- a/include/llvm/CodeGen/TargetPassConfig.h +++ b/include/llvm/CodeGen/TargetPassConfig.h @@ -1,9 +1,8 @@ //===- TargetPassConfig.h - Code Generation pass options --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,6 +24,7 @@ class LLVMTargetMachine; struct MachineSchedContext; class PassConfigImpl; class ScheduleDAGInstrs; +class CSEConfigBase; // The old pass manager infrastructure is hidden in a legacy namespace now. namespace legacy { @@ -75,9 +75,6 @@ public: } }; -template <> struct isPodLike { - static const bool value = true; -}; /// Target-Independent Code Generator Pass Configuration Options. /// @@ -319,6 +316,13 @@ public: /// when GlobalISel failed and isGlobalISelAbortEnabled is false. virtual bool reportDiagnosticWhenGlobalISelFallback() const; + /// Check whether continuous CSE should be enabled in GISel passes. + /// By default, it's enabled for non O0 levels. + virtual bool isGISelCSEEnabled() const; + + /// Returns the CSEConfig object to use for the current optimization level. + virtual std::unique_ptr getCSEConfig() const; + protected: // Helper to verify the analysis is really immutable. void setOpt(bool &Opt, bool Val); @@ -360,11 +364,11 @@ protected: /// addFastRegAlloc - Add the minimum set of target-independent passes that /// are required for fast register allocation. - virtual void addFastRegAlloc(FunctionPass *RegAllocPass); + virtual void addFastRegAlloc(); /// addOptimizedRegAlloc - Add passes related to register allocation. /// LLVMTargetMachine provides standard regalloc passes for most targets. - virtual void addOptimizedRegAlloc(FunctionPass *RegAllocPass); + virtual void addOptimizedRegAlloc(); /// addPreRewrite - Add passes to the optimized register allocation pipeline /// after register allocation is complete, but before virtual registers are @@ -374,10 +378,18 @@ protected: /// after RABasic or RAGreedy, they should take advantage of LiveRegMatrix. /// When these passes run, VirtRegMap contains legal physreg assignments for /// all virtual registers. + /// + /// Note if the target overloads addRegAssignAndRewriteOptimized, this may not + /// be honored. This is also not generally used for the the fast variant, + /// where the allocation and rewriting are done in one pass. virtual bool addPreRewrite() { return false; } + /// Add passes to be run immediately after virtual registers are rewritten + /// to physical registers. + virtual void addPostRewrite() { } + /// This method may be implemented by targets that want to run passes after /// register allocation pass pipeline but before prolog-epilog insertion. virtual void addPostRegAlloc() { } @@ -431,7 +443,12 @@ protected: /// addMachinePasses helper to create the target-selected or overriden /// regalloc pass. - FunctionPass *createRegAllocPass(bool Optimized); + virtual FunctionPass *createRegAllocPass(bool Optimized); + + /// Add core register alloator passes which do the actual register assignment + /// and rewriting. \returns true if any passes were added. + virtual bool addRegAssignmentFast(); + virtual bool addRegAssignmentOptimized(); }; } // end namespace llvm diff --git a/include/llvm/CodeGen/TargetRegisterInfo.h b/include/llvm/CodeGen/TargetRegisterInfo.h index 0fbff3137653..ddbd677b3eaa 100644 --- a/include/llvm/CodeGen/TargetRegisterInfo.h +++ b/include/llvm/CodeGen/TargetRegisterInfo.h @@ -1,9 +1,8 @@ //==- CodeGen/TargetRegisterInfo.h - Target Register Information -*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -521,6 +520,11 @@ public: /// function. Used by MachineRegisterInfo::isConstantPhysReg(). virtual bool isConstantPhysReg(unsigned PhysReg) const { return false; } + /// Returns true if the register class is considered divergent. + virtual bool isDivergentRegClass(const TargetRegisterClass *RC) const { + return false; + } + /// Physical registers that may be modified within a function but are /// guaranteed to be restored before any uses. This is useful for targets that /// have call sequences where a GOT register may be updated by the caller @@ -986,7 +990,7 @@ public: /// getFrameRegister - This method should return the register used as a base /// for values allocated in the current stack frame. - virtual unsigned getFrameRegister(const MachineFunction &MF) const = 0; + virtual Register getFrameRegister(const MachineFunction &MF) const = 0; /// Mark a register and all its aliases as reserved in the given set. void markSuperRegs(BitVector &RegisterSet, unsigned Reg) const; diff --git a/include/llvm/CodeGen/TargetSchedule.h b/include/llvm/CodeGen/TargetSchedule.h index 6173925e23a1..cce85c8d7b0d 100644 --- a/include/llvm/CodeGen/TargetSchedule.h +++ b/include/llvm/CodeGen/TargetSchedule.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/TargetSchedule.h - Sched Machine Model ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/TargetSubtargetInfo.h b/include/llvm/CodeGen/TargetSubtargetInfo.h index 968e4c4b8102..037fc3ed3243 100644 --- a/include/llvm/CodeGen/TargetSubtargetInfo.h +++ b/include/llvm/CodeGen/TargetSubtargetInfo.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/TargetSubtargetInfo.h - Target Information --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -43,6 +42,7 @@ class RegisterBankInfo; class SDep; class SelectionDAGTargetInfo; struct SubtargetFeatureKV; +struct SubtargetSubTypeKV; struct SubtargetInfoKV; class SUnit; class TargetFrameLowering; @@ -63,8 +63,7 @@ class TargetSubtargetInfo : public MCSubtargetInfo { protected: // Can only create subclasses... TargetSubtargetInfo(const Triple &TT, StringRef CPU, StringRef FS, ArrayRef PF, - ArrayRef PD, - const SubtargetInfoKV *ProcSched, + ArrayRef PD, const MCWriteProcResEntry *WPR, const MCWriteLatencyEntry *WL, const MCReadAdvanceEntry *RA, const InstrStage *IS, @@ -190,13 +189,13 @@ public: /// TargetLowering preference). It does not yet disable the postRA scheduler. virtual bool enableMachineScheduler() const; - /// Support printing of [latency:throughput] comment in output .S file. - virtual bool supportPrintSchedInfo() const { return false; } - /// True if the machine scheduler should disable the TLI preference /// for preRA scheduling with the source level scheduler. virtual bool enableMachineSchedDefaultSched() const { return true; } + /// True if the subtarget should run MachinePipeliner + virtual bool enableMachinePipeliner() const { return true; }; + /// True if the subtarget should enable joining global copies. /// /// By default this is enabled if the machine scheduler is enabled, but @@ -250,6 +249,10 @@ public: std::vector> &Mutations) const { } + /// Default to DFA for resource management, return false when target will use + /// ProcResource in InstrSchedModel instead. + virtual bool useDFAforSMS() const { return true; } + // For use with PostRAScheduling: get the minimum optimization level needed // to enable post-RA scheduling. virtual CodeGenOpt::Level getOptLevelToEnablePostRAScheduler() const { @@ -286,12 +289,16 @@ public: /// possible. virtual bool enableSubRegLiveness() const { return false; } - /// Returns string representation of scheduler comment - std::string getSchedInfoStr(const MachineInstr &MI) const; - std::string getSchedInfoStr(MCInst const &MCI) const override; - /// This is called after a .mir file was loaded. virtual void mirFileLoaded(MachineFunction &MF) const; + + /// True if the register allocator should use the allocation orders exactly as + /// written in the tablegen descriptions, false if it should allocate + /// the specified physical register later if is it callee-saved. + virtual bool ignoreCSRForAllocationOrder(const MachineFunction &MF, + unsigned PhysReg) const { + return false; + } }; } // end namespace llvm diff --git a/include/llvm/CodeGen/UnreachableBlockElim.h b/include/llvm/CodeGen/UnreachableBlockElim.h index 3e7afd4cd433..d52d7c3c5b49 100644 --- a/include/llvm/CodeGen/UnreachableBlockElim.h +++ b/include/llvm/CodeGen/UnreachableBlockElim.h @@ -1,9 +1,8 @@ //===-- UnreachableBlockElim.h - Remove unreachable blocks for codegen --===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/ValueTypes.h b/include/llvm/CodeGen/ValueTypes.h index d2ef4a94f8e2..c540c94f79d9 100644 --- a/include/llvm/CodeGen/ValueTypes.h +++ b/include/llvm/CodeGen/ValueTypes.h @@ -1,9 +1,8 @@ //===- CodeGen/ValueTypes.h - Low-Level Target independ. types --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/CodeGen/ValueTypes.td b/include/llvm/CodeGen/ValueTypes.td index 0abb4ece1d14..5818ac183fcc 100644 --- a/include/llvm/CodeGen/ValueTypes.td +++ b/include/llvm/CodeGen/ValueTypes.td @@ -1,9 +1,8 @@ //===- ValueTypes.td - ValueType definitions ---------------*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -63,89 +62,105 @@ def v32i16 : ValueType<512, 38>; // 32 x i16 vector value def v64i16 : ValueType<1024,39>; // 64 x i16 vector value def v128i16: ValueType<2048,40>; //128 x i16 vector value -def v1i32 : ValueType<32 , 41>; // 1 x i32 vector value -def v2i32 : ValueType<64 , 42>; // 2 x i32 vector value -def v4i32 : ValueType<128, 43>; // 4 x i32 vector value -def v8i32 : ValueType<256, 44>; // 8 x i32 vector value -def v16i32 : ValueType<512, 45>; // 16 x i32 vector value -def v32i32 : ValueType<1024,46>; // 32 x i32 vector value -def v64i32 : ValueType<2048,47>; // 64 x i32 vector value - -def v1i64 : ValueType<64 , 48>; // 1 x i64 vector value -def v2i64 : ValueType<128, 49>; // 2 x i64 vector value -def v4i64 : ValueType<256, 50>; // 4 x i64 vector value -def v8i64 : ValueType<512, 51>; // 8 x i64 vector value -def v16i64 : ValueType<1024,52>; // 16 x i64 vector value -def v32i64 : ValueType<2048,53>; // 32 x i64 vector value - -def v1i128 : ValueType<128, 54>; // 1 x i128 vector value - -def nxv1i1 : ValueType<1, 55>; // n x 1 x i1 vector value -def nxv2i1 : ValueType<2, 56>; // n x 2 x i1 vector value -def nxv4i1 : ValueType<4, 57>; // n x 4 x i1 vector value -def nxv8i1 : ValueType<8, 58>; // n x 8 x i1 vector value -def nxv16i1 : ValueType<16, 59>; // n x 16 x i1 vector value -def nxv32i1 : ValueType<32, 60>; // n x 32 x i1 vector value - -def nxv1i8 : ValueType<8, 61>; // n x 1 x i8 vector value -def nxv2i8 : ValueType<16, 62>; // n x 2 x i8 vector value -def nxv4i8 : ValueType<32, 63>; // n x 4 x i8 vector value -def nxv8i8 : ValueType<64, 64>; // n x 8 x i8 vector value -def nxv16i8 : ValueType<128, 65>; // n x 16 x i8 vector value -def nxv32i8 : ValueType<256, 66>; // n x 32 x i8 vector value - -def nxv1i16 : ValueType<16, 67>; // n x 1 x i16 vector value -def nxv2i16 : ValueType<32, 68>; // n x 2 x i16 vector value -def nxv4i16 : ValueType<64, 69>; // n x 4 x i16 vector value -def nxv8i16 : ValueType<128, 70>; // n x 8 x i16 vector value -def nxv16i16: ValueType<256, 71>; // n x 16 x i16 vector value -def nxv32i16: ValueType<512, 72>; // n x 32 x i16 vector value - -def nxv1i32 : ValueType<32, 73>; // n x 1 x i32 vector value -def nxv2i32 : ValueType<64, 74>; // n x 2 x i32 vector value -def nxv4i32 : ValueType<128, 75>; // n x 4 x i32 vector value -def nxv8i32 : ValueType<256, 76>; // n x 8 x i32 vector value -def nxv16i32: ValueType<512, 77>; // n x 16 x i32 vector value -def nxv32i32: ValueType<1024,78>; // n x 32 x i32 vector value - -def nxv1i64 : ValueType<64, 79>; // n x 1 x i64 vector value -def nxv2i64 : ValueType<128, 80>; // n x 2 x i64 vector value -def nxv4i64 : ValueType<256, 81>; // n x 4 x i64 vector value -def nxv8i64 : ValueType<512, 82>; // n x 8 x i64 vector value -def nxv16i64: ValueType<1024,83>; // n x 16 x i64 vector value -def nxv32i64: ValueType<2048,84>; // n x 32 x i64 vector value - -def v2f16 : ValueType<32 , 85>; // 2 x f16 vector value -def v4f16 : ValueType<64 , 86>; // 4 x f16 vector value -def v8f16 : ValueType<128, 87>; // 8 x f16 vector value -def v1f32 : ValueType<32 , 88>; // 1 x f32 vector value -def v2f32 : ValueType<64 , 89>; // 2 x f32 vector value -def v4f32 : ValueType<128, 90>; // 4 x f32 vector value -def v8f32 : ValueType<256, 91>; // 8 x f32 vector value -def v16f32 : ValueType<512, 92>; // 16 x f32 vector value -def v1f64 : ValueType<64, 93>; // 1 x f64 vector value -def v2f64 : ValueType<128, 94>; // 2 x f64 vector value -def v4f64 : ValueType<256, 95>; // 4 x f64 vector value -def v8f64 : ValueType<512, 96>; // 8 x f64 vector value - -def nxv2f16 : ValueType<32 , 97>; // n x 2 x f16 vector value -def nxv4f16 : ValueType<64 , 98>; // n x 4 x f16 vector value -def nxv8f16 : ValueType<128, 99>; // n x 8 x f16 vector value -def nxv1f32 : ValueType<32 , 100>; // n x 1 x f32 vector value -def nxv2f32 : ValueType<64 , 101>; // n x 2 x f32 vector value -def nxv4f32 : ValueType<128, 102>; // n x 4 x f32 vector value -def nxv8f32 : ValueType<256, 103>; // n x 8 x f32 vector value -def nxv16f32 : ValueType<512, 104>; // n x 16 x f32 vector value -def nxv1f64 : ValueType<64, 105>; // n x 1 x f64 vector value -def nxv2f64 : ValueType<128, 106>; // n x 2 x f64 vector value -def nxv4f64 : ValueType<256, 107>; // n x 4 x f64 vector value -def nxv8f64 : ValueType<512, 108>; // n x 8 x f64 vector value - -def x86mmx : ValueType<64 , 109>; // X86 MMX value -def FlagVT : ValueType<0 , 110>; // Pre-RA sched glue -def isVoid : ValueType<0 , 111>; // Produces no value -def untyped: ValueType<8 , 112>; // Produces an untyped value -def ExceptRef: ValueType<0, 113>; // WebAssembly's except_ref type +def v1i32 : ValueType<32 , 41>; // 1 x i32 vector value +def v2i32 : ValueType<64 , 42>; // 2 x i32 vector value +def v3i32 : ValueType<96 , 43>; // 3 x i32 vector value +def v4i32 : ValueType<128, 44>; // 4 x i32 vector value +def v5i32 : ValueType<160, 45>; // 5 x i32 vector value +def v8i32 : ValueType<256, 46>; // 8 x i32 vector value +def v16i32 : ValueType<512, 47>; // 16 x i32 vector value +def v32i32 : ValueType<1024,48>; // 32 x i32 vector value +def v64i32 : ValueType<2048,49>; // 64 x i32 vector value +def v128i32 : ValueType<4096,50>; // 128 x i32 vector value +def v256i32 : ValueType<8182,51>; // 256 x i32 vector value +def v512i32 : ValueType<16384,52>; // 512 x i32 vector value +def v1024i32 : ValueType<32768,53>; // 1024 x i32 vector value +def v2048i32 : ValueType<65536,54>; // 2048 x i32 vector value + +def v1i64 : ValueType<64 , 55>; // 1 x i64 vector value +def v2i64 : ValueType<128, 56>; // 2 x i64 vector value +def v4i64 : ValueType<256, 57>; // 4 x i64 vector value +def v8i64 : ValueType<512, 58>; // 8 x i64 vector value +def v16i64 : ValueType<1024,59>; // 16 x i64 vector value +def v32i64 : ValueType<2048,60>; // 32 x i64 vector value + +def v1i128 : ValueType<128, 61>; // 1 x i128 vector value + +def nxv1i1 : ValueType<1, 62>; // n x 1 x i1 vector value +def nxv2i1 : ValueType<2, 63>; // n x 2 x i1 vector value +def nxv4i1 : ValueType<4, 64>; // n x 4 x i1 vector value +def nxv8i1 : ValueType<8, 65>; // n x 8 x i1 vector value +def nxv16i1 : ValueType<16, 66>; // n x 16 x i1 vector value +def nxv32i1 : ValueType<32, 67>; // n x 32 x i1 vector value + +def nxv1i8 : ValueType<8, 68>; // n x 1 x i8 vector value +def nxv2i8 : ValueType<16, 69>; // n x 2 x i8 vector value +def nxv4i8 : ValueType<32, 70>; // n x 4 x i8 vector value +def nxv8i8 : ValueType<64, 71>; // n x 8 x i8 vector value +def nxv16i8 : ValueType<128, 72>; // n x 16 x i8 vector value +def nxv32i8 : ValueType<256, 73>; // n x 32 x i8 vector value + +def nxv1i16 : ValueType<16, 74>; // n x 1 x i16 vector value +def nxv2i16 : ValueType<32, 75>; // n x 2 x i16 vector value +def nxv4i16 : ValueType<64, 76>; // n x 4 x i16 vector value +def nxv8i16 : ValueType<128, 77>; // n x 8 x i16 vector value +def nxv16i16: ValueType<256, 78>; // n x 16 x i16 vector value +def nxv32i16: ValueType<512, 79>; // n x 32 x i16 vector value + +def nxv1i32 : ValueType<32, 80>; // n x 1 x i32 vector value +def nxv2i32 : ValueType<64, 81>; // n x 2 x i32 vector value +def nxv4i32 : ValueType<128, 82>; // n x 4 x i32 vector value +def nxv8i32 : ValueType<256, 83>; // n x 8 x i32 vector value +def nxv16i32: ValueType<512, 84>; // n x 16 x i32 vector value +def nxv32i32: ValueType<1024,85>; // n x 32 x i32 vector value + +def nxv1i64 : ValueType<64, 86>; // n x 1 x i64 vector value +def nxv2i64 : ValueType<128, 87>; // n x 2 x i64 vector value +def nxv4i64 : ValueType<256, 88>; // n x 4 x i64 vector value +def nxv8i64 : ValueType<512, 89>; // n x 8 x i64 vector value +def nxv16i64: ValueType<1024,90>; // n x 16 x i64 vector value +def nxv32i64: ValueType<2048,91>; // n x 32 x i64 vector value + +def v2f16 : ValueType<32 , 92>; // 2 x f16 vector value +def v4f16 : ValueType<64 , 93>; // 4 x f16 vector value +def v8f16 : ValueType<128, 94>; // 8 x f16 vector value +def v1f32 : ValueType<32 , 95>; // 1 x f32 vector value +def v2f32 : ValueType<64 , 96>; // 2 x f32 vector value +def v3f32 : ValueType<96 , 97>; // 3 x f32 vector value +def v4f32 : ValueType<128, 98>; // 4 x f32 vector value +def v5f32 : ValueType<160, 99>; // 5 x f32 vector value +def v8f32 : ValueType<256, 100>; // 8 x f32 vector value +def v16f32 : ValueType<512, 101>; // 16 x f32 vector value +def v32f32 : ValueType<1024, 102>; // 32 x f32 vector value +def v64f32 : ValueType<2048, 103>; // 64 x f32 vector value +def v128f32 : ValueType<4096, 104>; // 128 x f32 vector value +def v256f32 : ValueType<8182, 105>; // 256 x f32 vector value +def v512f32 : ValueType<16384, 106>; // 512 x f32 vector value +def v1024f32 : ValueType<32768, 107>; // 1024 x f32 vector value +def v2048f32 : ValueType<65536, 108>; // 2048 x f32 vector value +def v1f64 : ValueType<64, 109>; // 1 x f64 vector value +def v2f64 : ValueType<128, 110>; // 2 x f64 vector value +def v4f64 : ValueType<256, 111>; // 4 x f64 vector value +def v8f64 : ValueType<512, 112>; // 8 x f64 vector value + +def nxv2f16 : ValueType<32 , 113>; // n x 2 x f16 vector value +def nxv4f16 : ValueType<64 , 114>; // n x 4 x f16 vector value +def nxv8f16 : ValueType<128, 115>; // n x 8 x f16 vector value +def nxv1f32 : ValueType<32 , 116>; // n x 1 x f32 vector value +def nxv2f32 : ValueType<64 , 117>; // n x 2 x f32 vector value +def nxv4f32 : ValueType<128, 118>; // n x 4 x f32 vector value +def nxv8f32 : ValueType<256, 119>; // n x 8 x f32 vector value +def nxv16f32 : ValueType<512, 120>; // n x 16 x f32 vector value +def nxv1f64 : ValueType<64, 121>; // n x 1 x f64 vector value +def nxv2f64 : ValueType<128, 122>; // n x 2 x f64 vector value +def nxv4f64 : ValueType<256, 123>; // n x 4 x f64 vector value +def nxv8f64 : ValueType<512, 124>; // n x 8 x f64 vector value + +def x86mmx : ValueType<64 , 125>; // X86 MMX value +def FlagVT : ValueType<0 , 126>; // Pre-RA sched glue +def isVoid : ValueType<0 , 127>; // Produces no value +def untyped: ValueType<8 , 128>; // Produces an untyped value +def exnref: ValueType<0, 129>; // WebAssembly's exnref type def token : ValueType<0 , 248>; // TokenTy def MetadataVT: ValueType<0, 249>; // Metadata @@ -167,3 +182,14 @@ def iPTR : ValueType<0 , 254>; // Pseudo valuetype to represent "any type of any size". def Any : ValueType<0 , 255>; + +/// This class is for targets that want to use pointer types in patterns +/// with the GlobalISelEmitter. Targets must define their own pointer +/// derived from this class. The scalar argument should be an +/// integer type with the same bit size as the ponter. +/// e.g. def p0 : PtrValueType ; + +class PtrValueType : + ValueType { + int AddrSpace = addrspace; +} diff --git a/include/llvm/CodeGen/VirtRegMap.h b/include/llvm/CodeGen/VirtRegMap.h index 6a8e50a7e5f5..70eb048f05eb 100644 --- a/include/llvm/CodeGen/VirtRegMap.h +++ b/include/llvm/CodeGen/VirtRegMap.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/VirtRegMap.h - Virtual Register Map ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -68,8 +67,10 @@ class TargetInstrInfo; public: static char ID; - VirtRegMap() : MachineFunctionPass(ID), Virt2PhysMap(NO_PHYS_REG), - Virt2StackSlotMap(NO_STACK_SLOT), Virt2SplitMap(0) {} + VirtRegMap() + : MachineFunctionPass(ID), MRI(nullptr), TII(nullptr), TRI(nullptr), + MF(nullptr), Virt2PhysMap(NO_PHYS_REG), + Virt2StackSlotMap(NO_STACK_SLOT), Virt2SplitMap(0) {} VirtRegMap(const VirtRegMap &) = delete; VirtRegMap &operator=(const VirtRegMap &) = delete; @@ -98,8 +99,8 @@ class TargetInstrInfo; /// returns the physical register mapped to the specified /// virtual register - unsigned getPhys(unsigned virtReg) const { - assert(TargetRegisterInfo::isVirtualRegister(virtReg)); + Register getPhys(Register virtReg) const { + assert(virtReg.isVirtual()); return Virt2PhysMap[virtReg]; } diff --git a/include/llvm/CodeGen/WasmEHFuncInfo.h b/include/llvm/CodeGen/WasmEHFuncInfo.h index 219fff988f6e..887a1467b3e4 100644 --- a/include/llvm/CodeGen/WasmEHFuncInfo.h +++ b/include/llvm/CodeGen/WasmEHFuncInfo.h @@ -1,9 +1,8 @@ //===--- llvm/CodeGen/WasmEHFuncInfo.h --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -29,10 +28,6 @@ struct WasmEHFuncInfo { // When there is an entry , if an exception is not caught by A, it // should next unwind to the EH pad B. DenseMap EHPadUnwindMap; - // For entry , A is a BB with an instruction that may throw - // (invoke/cleanupret in LLVM IR, call/rethrow in the backend) and B is an EH - // pad that A unwinds to. - DenseMap ThrowUnwindMap; // Helper functions const BasicBlock *getEHPadUnwindDest(const BasicBlock *BB) const { @@ -41,18 +36,9 @@ struct WasmEHFuncInfo { void setEHPadUnwindDest(const BasicBlock *BB, const BasicBlock *Dest) { EHPadUnwindMap[BB] = Dest; } - const BasicBlock *getThrowUnwindDest(BasicBlock *BB) const { - return ThrowUnwindMap.lookup(BB).get(); - } - void setThrowUnwindDest(const BasicBlock *BB, const BasicBlock *Dest) { - ThrowUnwindMap[BB] = Dest; - } bool hasEHPadUnwindDest(const BasicBlock *BB) const { return EHPadUnwindMap.count(BB); } - bool hasThrowUnwindDest(const BasicBlock *BB) const { - return ThrowUnwindMap.count(BB); - } MachineBasicBlock *getEHPadUnwindDest(MachineBasicBlock *MBB) const { return EHPadUnwindMap.lookup(MBB).get(); @@ -60,18 +46,9 @@ struct WasmEHFuncInfo { void setEHPadUnwindDest(MachineBasicBlock *MBB, MachineBasicBlock *Dest) { EHPadUnwindMap[MBB] = Dest; } - MachineBasicBlock *getThrowUnwindDest(MachineBasicBlock *MBB) const { - return ThrowUnwindMap.lookup(MBB).get(); - } - void setThrowUnwindDest(MachineBasicBlock *MBB, MachineBasicBlock *Dest) { - ThrowUnwindMap[MBB] = Dest; - } bool hasEHPadUnwindDest(MachineBasicBlock *MBB) const { return EHPadUnwindMap.count(MBB); } - bool hasThrowUnwindDest(MachineBasicBlock *MBB) const { - return ThrowUnwindMap.count(MBB); - } }; // Analyze the IR in the given function to build WasmEHFuncInfo. diff --git a/include/llvm/CodeGen/WinEHFuncInfo.h b/include/llvm/CodeGen/WinEHFuncInfo.h index 8043024626a0..f098316de793 100644 --- a/include/llvm/CodeGen/WinEHFuncInfo.h +++ b/include/llvm/CodeGen/WinEHFuncInfo.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/WinEHFuncInfo.h -----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h b/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h index bd1743511ed4..0ac8b651939d 100644 --- a/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h +++ b/include/llvm/DebugInfo/CodeView/AppendingTypeTableBuilder.h @@ -1,9 +1,8 @@ //===- AppendingTypeTableBuilder.h -------------------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/CVRecord.h b/include/llvm/DebugInfo/CodeView/CVRecord.h index 11ca9ff108de..784c47e3bf5d 100644 --- a/include/llvm/DebugInfo/CodeView/CVRecord.h +++ b/include/llvm/DebugInfo/CodeView/CVRecord.h @@ -1,9 +1,8 @@ -//===- RecordIterator.h -----------------------------------------*- C++ -*-===// +//===- CVRecord.h -----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -25,17 +24,31 @@ namespace llvm { namespace codeview { +/// CVRecord is a fat pointer (base + size pair) to a symbol or type record. +/// Carrying the size separately instead of trusting the size stored in the +/// record prefix provides some extra safety and flexibility. template class CVRecord { public: - CVRecord() : Type(static_cast(0)) {} + CVRecord() = default; + + CVRecord(ArrayRef Data) : RecordData(Data) {} - CVRecord(Kind K, ArrayRef Data) : Type(K), RecordData(Data) {} + CVRecord(const RecordPrefix *P, size_t Size) + : RecordData(reinterpret_cast(P), Size) {} - bool valid() const { return Type != static_cast(0); } + bool valid() const { return kind() != Kind(0); } uint32_t length() const { return RecordData.size(); } - Kind kind() const { return Type; } + + Kind kind() const { + if (RecordData.size() < sizeof(RecordPrefix)) + return Kind(0); + return static_cast(static_cast( + reinterpret_cast(RecordData.data())->RecordKind)); + } + ArrayRef data() const { return RecordData; } + StringRef str_data() const { return StringRef(reinterpret_cast(RecordData.data()), RecordData.size()); @@ -45,7 +58,6 @@ public: return RecordData.drop_front(sizeof(RecordPrefix)); } - Kind Type; ArrayRef RecordData; }; @@ -72,8 +84,7 @@ Error forEachCodeViewRecord(ArrayRef StreamBuffer, Func F) { ArrayRef Data = StreamBuffer.take_front(RealLen); StreamBuffer = StreamBuffer.drop_front(RealLen); - Record R(static_cast((uint16_t)Prefix->RecordKind), - Data); + Record R(Data); if (auto EC = F(R)) return EC; } @@ -92,13 +103,12 @@ inline Expected> readCVRecordFromStream(BinaryStreamRef Stream, return std::move(EC); if (Prefix->RecordLen < 2) return make_error(cv_error_code::corrupt_record); - Kind K = static_cast(uint16_t(Prefix->RecordKind)); Reader.setOffset(Offset); ArrayRef RawData; if (auto EC = Reader.readBytes(RawData, Prefix->RecordLen + sizeof(uint16_t))) return std::move(EC); - return codeview::CVRecord(K, RawData); + return codeview::CVRecord(RawData); } } // end namespace codeview diff --git a/include/llvm/DebugInfo/CodeView/CVSymbolVisitor.h b/include/llvm/DebugInfo/CodeView/CVSymbolVisitor.h index 7c8cd121751a..1615ff41df12 100644 --- a/include/llvm/DebugInfo/CodeView/CVSymbolVisitor.h +++ b/include/llvm/DebugInfo/CodeView/CVSymbolVisitor.h @@ -1,9 +1,8 @@ //===- CVSymbolVisitor.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/CVTypeVisitor.h b/include/llvm/DebugInfo/CodeView/CVTypeVisitor.h index b765ba1abb4d..7d20bb0a7bde 100644 --- a/include/llvm/DebugInfo/CodeView/CVTypeVisitor.h +++ b/include/llvm/DebugInfo/CodeView/CVTypeVisitor.h @@ -1,9 +1,8 @@ //===- CVTypeVisitor.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -12,6 +11,7 @@ #include "llvm/DebugInfo/CodeView/CVRecord.h" #include "llvm/DebugInfo/CodeView/TypeRecord.h" +#include "llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h" #include "llvm/Support/Error.h" namespace llvm { @@ -31,6 +31,9 @@ enum VisitorDataSource { Error visitTypeRecord(CVType &Record, TypeIndex Index, TypeVisitorCallbacks &Callbacks, VisitorDataSource Source = VDS_BytesPresent); +Error visitTypeRecord(CVType &Record, TypeIndex Index, + TypeVisitorCallbackPipeline &Callbacks, + VisitorDataSource Source = VDS_BytesPresent); Error visitTypeRecord(CVType &Record, TypeVisitorCallbacks &Callbacks, VisitorDataSource Source = VDS_BytesPresent); diff --git a/include/llvm/DebugInfo/CodeView/CodeView.h b/include/llvm/DebugInfo/CodeView/CodeView.h index 8e0d9f608e93..c3acb05ea8b1 100644 --- a/include/llvm/DebugInfo/CodeView/CodeView.h +++ b/include/llvm/DebugInfo/CodeView/CodeView.h @@ -1,9 +1,8 @@ //===- CodeView.h -----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -160,9 +159,10 @@ enum SourceLanguage : uint8_t { MSIL = 0x0f, HLSL = 0x10, - /// The DMD compiler emits 'D' for the CV source language. Microsoft doesn't - /// have an enumerator for it yet. + /// The DMD & Swift compilers emit 'D' and 'S', respectively, for the CV + /// source language. Microsoft does not have enumerators for them yet. D = 'D', + Swift = 'S', }; /// These values correspond to the CV_call_e enumeration, and are documented @@ -304,6 +304,9 @@ enum class ModifierOptions : uint16_t { }; CV_DEFINE_ENUM_CLASS_FLAGS_OPERATORS(ModifierOptions) +// If the subsection kind has this bit set, then the linker should ignore it. +enum : uint32_t { SubsectionIgnoreFlag = 0x80000000 }; + enum class DebugSubsectionKind : uint32_t { None = 0, Symbols = 0xf1, @@ -509,9 +512,23 @@ enum class FrameCookieKind : uint8_t { // Corresponds to CV_HREG_e enum. enum class RegisterId : uint16_t { +#define CV_REGISTERS_ALL #define CV_REGISTER(name, value) name = value, #include "CodeViewRegisters.def" #undef CV_REGISTER +#undef CV_REGISTERS_ALL +}; + +// Register Ids are shared between architectures in CodeView. CPUType is needed +// to map register Id to name. +struct CPURegister { + CPURegister() = delete; + CPURegister(CPUType Cpu, codeview::RegisterId Reg) { + this->Cpu = Cpu; + this->Reg = Reg; + } + CPUType Cpu; + RegisterId Reg; }; /// Two-bit value indicating which register is the designated frame pointer diff --git a/include/llvm/DebugInfo/CodeView/CodeViewError.h b/include/llvm/DebugInfo/CodeView/CodeViewError.h index d4615d02220d..9990c8d05d1c 100644 --- a/include/llvm/DebugInfo/CodeView/CodeViewError.h +++ b/include/llvm/DebugInfo/CodeView/CodeViewError.h @@ -1,9 +1,8 @@ //===- CodeViewError.h - Error extensions for CodeView ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h b/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h index 94f104ff772c..00fb0cf4cc90 100644 --- a/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h +++ b/include/llvm/DebugInfo/CodeView/CodeViewRecordIO.h @@ -1,9 +1,8 @@ //===- CodeViewRecordIO.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -25,28 +24,65 @@ #include namespace llvm { + namespace codeview { +class CodeViewRecordStreamer { +public: + virtual void EmitBytes(StringRef Data) = 0; + virtual void EmitIntValue(uint64_t Value, unsigned Size) = 0; + virtual void EmitBinaryData(StringRef Data) = 0; + virtual void AddComment(const Twine &T) = 0; + virtual ~CodeViewRecordStreamer() = default; +}; + class CodeViewRecordIO { uint32_t getCurrentOffset() const { - return (isWriting()) ? Writer->getOffset() : Reader->getOffset(); + if (isWriting()) + return Writer->getOffset(); + else if (isReading()) + return Reader->getOffset(); + else + return 0; } public: + // deserializes records to structures explicit CodeViewRecordIO(BinaryStreamReader &Reader) : Reader(&Reader) {} + + // serializes records to buffer explicit CodeViewRecordIO(BinaryStreamWriter &Writer) : Writer(&Writer) {} + // writes records to assembly file using MC library interface + explicit CodeViewRecordIO(CodeViewRecordStreamer &Streamer) + : Streamer(&Streamer) {} + Error beginRecord(Optional MaxLength); Error endRecord(); - Error mapInteger(TypeIndex &TypeInd); + Error mapInteger(TypeIndex &TypeInd, const Twine &Comment = ""); - bool isReading() const { return Reader != nullptr; } - bool isWriting() const { return !isReading(); } + bool isStreaming() const { + return (Streamer != nullptr) && (Reader == nullptr) && (Writer == nullptr); + } + bool isReading() const { + return (Reader != nullptr) && (Streamer == nullptr) && (Writer == nullptr); + } + bool isWriting() const { + return (Writer != nullptr) && (Streamer == nullptr) && (Reader == nullptr); + } uint32_t maxFieldLength() const; template Error mapObject(T &Value) { + if (isStreaming()) { + StringRef BytesSR = + StringRef((reinterpret_cast(&Value)), sizeof(Value)); + Streamer->EmitBytes(BytesSR); + incrStreamedLen(sizeof(T)); + return Error::success(); + } + if (isWriting()) return Writer->writeObject(Value); @@ -57,41 +93,63 @@ public: return Error::success(); } - template Error mapInteger(T &Value) { + template Error mapInteger(T &Value, const Twine &Comment = "") { + if (isStreaming()) { + emitComment(Comment); + Streamer->EmitIntValue((int)Value, sizeof(T)); + incrStreamedLen(sizeof(T)); + return Error::success(); + } + if (isWriting()) return Writer->writeInteger(Value); return Reader->readInteger(Value); } - template Error mapEnum(T &Value) { - if (sizeof(Value) > maxFieldLength()) + template Error mapEnum(T &Value, const Twine &Comment = "") { + if (!isStreaming() && sizeof(Value) > maxFieldLength()) return make_error(cv_error_code::insufficient_buffer); using U = typename std::underlying_type::type; U X; - if (isWriting()) + + if (isWriting() || isStreaming()) X = static_cast(Value); - if (auto EC = mapInteger(X)) + if (auto EC = mapInteger(X, Comment)) return EC; + if (isReading()) Value = static_cast(X); + return Error::success(); } - Error mapEncodedInteger(int64_t &Value); - Error mapEncodedInteger(uint64_t &Value); - Error mapEncodedInteger(APSInt &Value); - Error mapStringZ(StringRef &Value); - Error mapGuid(GUID &Guid); + Error mapEncodedInteger(int64_t &Value, const Twine &Comment = ""); + Error mapEncodedInteger(uint64_t &Value, const Twine &Comment = ""); + Error mapEncodedInteger(APSInt &Value, const Twine &Comment = ""); + Error mapStringZ(StringRef &Value, const Twine &Comment = ""); + Error mapGuid(GUID &Guid, const Twine &Comment = ""); - Error mapStringZVectorZ(std::vector &Value); + Error mapStringZVectorZ(std::vector &Value, + const Twine &Comment = ""); template - Error mapVectorN(T &Items, const ElementMapper &Mapper) { + Error mapVectorN(T &Items, const ElementMapper &Mapper, + const Twine &Comment = "") { SizeType Size; - if (isWriting()) { + if (isStreaming()) { + Size = static_cast(Items.size()); + emitComment(Comment); + Streamer->EmitIntValue(Size, sizeof(Size)); + incrStreamedLen(sizeof(Size)); // add 1 for the delimiter + + for (auto &X : Items) { + if (auto EC = Mapper(*this, X)) + return EC; + } + } else if (isWriting()) { Size = static_cast(Items.size()); if (auto EC = Writer->writeInteger(Size)) return EC; @@ -115,8 +173,10 @@ public: } template - Error mapVectorTail(T &Items, const ElementMapper &Mapper) { - if (isWriting()) { + Error mapVectorTail(T &Items, const ElementMapper &Mapper, + const Twine &Comment = "") { + emitComment(Comment); + if (isStreaming() || isWriting()) { for (auto &Item : Items) { if (auto EC = Mapper(*this, Item)) return EC; @@ -133,16 +193,44 @@ public: return Error::success(); } - Error mapByteVectorTail(ArrayRef &Bytes); - Error mapByteVectorTail(std::vector &Bytes); + Error mapByteVectorTail(ArrayRef &Bytes, const Twine &Comment = ""); + Error mapByteVectorTail(std::vector &Bytes, + const Twine &Comment = ""); Error padToAlignment(uint32_t Align); Error skipPadding(); + uint64_t getStreamedLen() { + if (isStreaming()) + return StreamedLen; + return 0; + } + private: + void emitEncodedSignedInteger(const int64_t &Value, + const Twine &Comment = ""); + void emitEncodedUnsignedInteger(const uint64_t &Value, + const Twine &Comment = ""); Error writeEncodedSignedInteger(const int64_t &Value); Error writeEncodedUnsignedInteger(const uint64_t &Value); + void incrStreamedLen(const uint64_t &Len) { + if (isStreaming()) + StreamedLen += Len; + } + + void resetStreamedLen() { + if (isStreaming()) + StreamedLen = 4; // The record prefix is 4 bytes long + } + + void emitComment(const Twine &Comment) { + if (isStreaming()) { + Twine TComment(Comment); + Streamer->AddComment(TComment); + } + } + struct RecordLimit { uint32_t BeginOffset; Optional MaxLength; @@ -163,6 +251,8 @@ private: BinaryStreamReader *Reader = nullptr; BinaryStreamWriter *Writer = nullptr; + CodeViewRecordStreamer *Streamer = nullptr; + uint64_t StreamedLen = 0; }; } // end namespace codeview diff --git a/include/llvm/DebugInfo/CodeView/CodeViewRegisters.def b/include/llvm/DebugInfo/CodeView/CodeViewRegisters.def index fdfcf4d53a23..9767e49c44f5 100644 --- a/include/llvm/DebugInfo/CodeView/CodeViewRegisters.def +++ b/include/llvm/DebugInfo/CodeView/CodeViewRegisters.def @@ -1,9 +1,8 @@ //===-- CodeViewRegisters.def - CodeView registers --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,8 +14,15 @@ #define CV_REGISTER(name, value) #endif +#if !defined(CV_REGISTERS_ALL) && !defined(CV_REGISTERS_X86) && \ + !defined(CV_REGISTERS_ARM64) +#error Need include at least one register set. +#endif + // This currently only contains the "register subset shared by all processor -// types" (ERR etc.) and the x86 registers. +// types" (ERR etc.) and the x86/arm64 registers. + +#if defined(CV_REGISTERS_ALL) || defined(CV_REGISTERS_X86) // Some system headers define macros that conflict with our enums. Every // compiler supported by LLVM has the push_macro and pop_macro pragmas, so use @@ -357,3 +363,197 @@ CV_REGISTER(AMD64_K7, 765) #pragma pop_macro("CR2") #pragma pop_macro("CR3") #pragma pop_macro("CR4") + +#endif // defined(CV_REGISTERS_ALL) || defined(CV_REGISTERS_X86) + +#if defined(CV_REGISTERS_ALL) || defined(CV_REGISTERS_ARM64) + +// ARM64 registers + +CV_REGISTER(ARM64_NOREG, 0) + +// General purpose 32-bit integer registers + +CV_REGISTER(ARM64_W0, 10) +CV_REGISTER(ARM64_W1, 11) +CV_REGISTER(ARM64_W2, 12) +CV_REGISTER(ARM64_W3, 13) +CV_REGISTER(ARM64_W4, 14) +CV_REGISTER(ARM64_W5, 15) +CV_REGISTER(ARM64_W6, 16) +CV_REGISTER(ARM64_W7, 17) +CV_REGISTER(ARM64_W8, 18) +CV_REGISTER(ARM64_W9, 19) +CV_REGISTER(ARM64_W10, 20) +CV_REGISTER(ARM64_W11, 21) +CV_REGISTER(ARM64_W12, 22) +CV_REGISTER(ARM64_W13, 23) +CV_REGISTER(ARM64_W14, 24) +CV_REGISTER(ARM64_W15, 25) +CV_REGISTER(ARM64_W16, 26) +CV_REGISTER(ARM64_W17, 27) +CV_REGISTER(ARM64_W18, 28) +CV_REGISTER(ARM64_W19, 29) +CV_REGISTER(ARM64_W20, 30) +CV_REGISTER(ARM64_W21, 31) +CV_REGISTER(ARM64_W22, 32) +CV_REGISTER(ARM64_W23, 33) +CV_REGISTER(ARM64_W24, 34) +CV_REGISTER(ARM64_W25, 35) +CV_REGISTER(ARM64_W26, 36) +CV_REGISTER(ARM64_W27, 37) +CV_REGISTER(ARM64_W28, 38) +CV_REGISTER(ARM64_W29, 39) +CV_REGISTER(ARM64_W30, 40) +CV_REGISTER(ARM64_WZR, 41) + +// General purpose 64-bit integer registers + +CV_REGISTER(ARM64_X0, 50) +CV_REGISTER(ARM64_X1, 51) +CV_REGISTER(ARM64_X2, 52) +CV_REGISTER(ARM64_X3, 53) +CV_REGISTER(ARM64_X4, 54) +CV_REGISTER(ARM64_X5, 55) +CV_REGISTER(ARM64_X6, 56) +CV_REGISTER(ARM64_X7, 57) +CV_REGISTER(ARM64_X8, 58) +CV_REGISTER(ARM64_X9, 59) +CV_REGISTER(ARM64_X10, 60) +CV_REGISTER(ARM64_X11, 61) +CV_REGISTER(ARM64_X12, 62) +CV_REGISTER(ARM64_X13, 63) +CV_REGISTER(ARM64_X14, 64) +CV_REGISTER(ARM64_X15, 65) +CV_REGISTER(ARM64_X16, 66) +CV_REGISTER(ARM64_X17, 67) +CV_REGISTER(ARM64_X18, 68) +CV_REGISTER(ARM64_X19, 69) +CV_REGISTER(ARM64_X20, 70) +CV_REGISTER(ARM64_X21, 71) +CV_REGISTER(ARM64_X22, 72) +CV_REGISTER(ARM64_X23, 73) +CV_REGISTER(ARM64_X24, 74) +CV_REGISTER(ARM64_X25, 75) +CV_REGISTER(ARM64_X26, 76) +CV_REGISTER(ARM64_X27, 77) +CV_REGISTER(ARM64_X28, 78) +CV_REGISTER(ARM64_FP, 79) +CV_REGISTER(ARM64_LR, 80) +CV_REGISTER(ARM64_SP, 81) +CV_REGISTER(ARM64_ZR, 82) + +// status register + +CV_REGISTER(ARM64_NZCV, 90) + +// 32-bit floating point registers + +CV_REGISTER(ARM64_S0, 100) +CV_REGISTER(ARM64_S1, 101) +CV_REGISTER(ARM64_S2, 102) +CV_REGISTER(ARM64_S3, 103) +CV_REGISTER(ARM64_S4, 104) +CV_REGISTER(ARM64_S5, 105) +CV_REGISTER(ARM64_S6, 106) +CV_REGISTER(ARM64_S7, 107) +CV_REGISTER(ARM64_S8, 108) +CV_REGISTER(ARM64_S9, 109) +CV_REGISTER(ARM64_S10, 110) +CV_REGISTER(ARM64_S11, 111) +CV_REGISTER(ARM64_S12, 112) +CV_REGISTER(ARM64_S13, 113) +CV_REGISTER(ARM64_S14, 114) +CV_REGISTER(ARM64_S15, 115) +CV_REGISTER(ARM64_S16, 116) +CV_REGISTER(ARM64_S17, 117) +CV_REGISTER(ARM64_S18, 118) +CV_REGISTER(ARM64_S19, 119) +CV_REGISTER(ARM64_S20, 120) +CV_REGISTER(ARM64_S21, 121) +CV_REGISTER(ARM64_S22, 122) +CV_REGISTER(ARM64_S23, 123) +CV_REGISTER(ARM64_S24, 124) +CV_REGISTER(ARM64_S25, 125) +CV_REGISTER(ARM64_S26, 126) +CV_REGISTER(ARM64_S27, 127) +CV_REGISTER(ARM64_S28, 128) +CV_REGISTER(ARM64_S29, 129) +CV_REGISTER(ARM64_S30, 130) +CV_REGISTER(ARM64_S31, 131) + +// 64-bit floating point registers + +CV_REGISTER(ARM64_D0, 140) +CV_REGISTER(ARM64_D1, 141) +CV_REGISTER(ARM64_D2, 142) +CV_REGISTER(ARM64_D3, 143) +CV_REGISTER(ARM64_D4, 144) +CV_REGISTER(ARM64_D5, 145) +CV_REGISTER(ARM64_D6, 146) +CV_REGISTER(ARM64_D7, 147) +CV_REGISTER(ARM64_D8, 148) +CV_REGISTER(ARM64_D9, 149) +CV_REGISTER(ARM64_D10, 150) +CV_REGISTER(ARM64_D11, 151) +CV_REGISTER(ARM64_D12, 152) +CV_REGISTER(ARM64_D13, 153) +CV_REGISTER(ARM64_D14, 154) +CV_REGISTER(ARM64_D15, 155) +CV_REGISTER(ARM64_D16, 156) +CV_REGISTER(ARM64_D17, 157) +CV_REGISTER(ARM64_D18, 158) +CV_REGISTER(ARM64_D19, 159) +CV_REGISTER(ARM64_D20, 160) +CV_REGISTER(ARM64_D21, 161) +CV_REGISTER(ARM64_D22, 162) +CV_REGISTER(ARM64_D23, 163) +CV_REGISTER(ARM64_D24, 164) +CV_REGISTER(ARM64_D25, 165) +CV_REGISTER(ARM64_D26, 166) +CV_REGISTER(ARM64_D27, 167) +CV_REGISTER(ARM64_D28, 168) +CV_REGISTER(ARM64_D29, 169) +CV_REGISTER(ARM64_D30, 170) +CV_REGISTER(ARM64_D31, 171) + +// 128-bit SIMD registers + +CV_REGISTER(ARM64_Q0, 180) +CV_REGISTER(ARM64_Q1, 181) +CV_REGISTER(ARM64_Q2, 182) +CV_REGISTER(ARM64_Q3, 183) +CV_REGISTER(ARM64_Q4, 184) +CV_REGISTER(ARM64_Q5, 185) +CV_REGISTER(ARM64_Q6, 186) +CV_REGISTER(ARM64_Q7, 187) +CV_REGISTER(ARM64_Q8, 188) +CV_REGISTER(ARM64_Q9, 189) +CV_REGISTER(ARM64_Q10, 190) +CV_REGISTER(ARM64_Q11, 191) +CV_REGISTER(ARM64_Q12, 192) +CV_REGISTER(ARM64_Q13, 193) +CV_REGISTER(ARM64_Q14, 194) +CV_REGISTER(ARM64_Q15, 195) +CV_REGISTER(ARM64_Q16, 196) +CV_REGISTER(ARM64_Q17, 197) +CV_REGISTER(ARM64_Q18, 198) +CV_REGISTER(ARM64_Q19, 199) +CV_REGISTER(ARM64_Q20, 200) +CV_REGISTER(ARM64_Q21, 201) +CV_REGISTER(ARM64_Q22, 202) +CV_REGISTER(ARM64_Q23, 203) +CV_REGISTER(ARM64_Q24, 204) +CV_REGISTER(ARM64_Q25, 205) +CV_REGISTER(ARM64_Q26, 206) +CV_REGISTER(ARM64_Q27, 207) +CV_REGISTER(ARM64_Q28, 208) +CV_REGISTER(ARM64_Q29, 209) +CV_REGISTER(ARM64_Q30, 210) +CV_REGISTER(ARM64_Q31, 211) + +// Floating point status register + +CV_REGISTER(ARM64_FPSR, 220) + +#endif // defined(CV_REGISTERS_ALL) || defined(CV_REGISTERS_ARM64) diff --git a/include/llvm/DebugInfo/CodeView/CodeViewSymbols.def b/include/llvm/DebugInfo/CodeView/CodeViewSymbols.def index b5f1cc0198dc..4f8ccfdd16af 100644 --- a/include/llvm/DebugInfo/CodeView/CodeViewSymbols.def +++ b/include/llvm/DebugInfo/CodeView/CodeViewSymbols.def @@ -1,9 +1,8 @@ //===-- CodeViewSymbols.def - All CodeView leaf types -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -103,7 +102,6 @@ CV_SYMBOL(S_LPROCIA64_ST , 0x1015) CV_SYMBOL(S_GPROCIA64_ST , 0x1016) CV_SYMBOL(S_LOCALSLOT_ST , 0x1017) CV_SYMBOL(S_PARAMSLOT_ST , 0x1018) -CV_SYMBOL(S_ANNOTATION , 0x1019) CV_SYMBOL(S_GMANPROC_ST , 0x101a) CV_SYMBOL(S_LMANPROC_ST , 0x101b) CV_SYMBOL(S_RESERVED1 , 0x101c) @@ -255,6 +253,7 @@ SYMBOL_RECORD(S_LTHREAD32 , 0x1112, ThreadLocalDataSym) SYMBOL_RECORD_ALIAS(S_GTHREAD32 , 0x1113, GlobalTLS, ThreadLocalDataSym) SYMBOL_RECORD(S_UNAMESPACE , 0x1124, UsingNamespaceSym) +SYMBOL_RECORD(S_ANNOTATION , 0x1019, AnnotationSym) #undef CV_SYMBOL #undef SYMBOL_RECORD diff --git a/include/llvm/DebugInfo/CodeView/CodeViewTypes.def b/include/llvm/DebugInfo/CodeView/CodeViewTypes.def index e9a479dba496..a31111eb80a4 100644 --- a/include/llvm/DebugInfo/CodeView/CodeViewTypes.def +++ b/include/llvm/DebugInfo/CodeView/CodeViewTypes.def @@ -1,9 +1,8 @@ //===-- CodeViewTypes.def - All CodeView leaf types -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/DebugInfo/CodeView/ContinuationRecordBuilder.h b/include/llvm/DebugInfo/CodeView/ContinuationRecordBuilder.h index 7f851a2595dc..53ab2dd04aa7 100644 --- a/include/llvm/DebugInfo/CodeView/ContinuationRecordBuilder.h +++ b/include/llvm/DebugInfo/CodeView/ContinuationRecordBuilder.h @@ -1,9 +1,8 @@ //===- ContinuationRecordBuilder.h ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -62,4 +61,4 @@ public: } // namespace codeview } // namespace llvm -#endif \ No newline at end of file +#endif diff --git a/include/llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h b/include/llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h index 78b284563afd..01f83676afdf 100644 --- a/include/llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h +++ b/include/llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h @@ -1,9 +1,8 @@ //===- DebugChecksumsSubsection.h -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/DebugCrossExSubsection.h b/include/llvm/DebugInfo/CodeView/DebugCrossExSubsection.h index 2f9e9814d998..64a78a7cef21 100644 --- a/include/llvm/DebugInfo/CodeView/DebugCrossExSubsection.h +++ b/include/llvm/DebugInfo/CodeView/DebugCrossExSubsection.h @@ -1,9 +1,8 @@ //===- DebugCrossExSubsection.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/DebugCrossImpSubsection.h b/include/llvm/DebugInfo/CodeView/DebugCrossImpSubsection.h index 8be7ef265c82..e7683cb2a9c4 100644 --- a/include/llvm/DebugInfo/CodeView/DebugCrossImpSubsection.h +++ b/include/llvm/DebugInfo/CodeView/DebugCrossImpSubsection.h @@ -1,9 +1,8 @@ -//===- DebugCrossExSubsection.h ---------------------------------*- C++ -*-===// +//===- DebugCrossImpSubsection.h --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h b/include/llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h index 847d93f0e985..d5cd640231f9 100644 --- a/include/llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h +++ b/include/llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h @@ -1,9 +1,8 @@ //===- DebugFrameDataSubsection.h ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h b/include/llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h index b88c0eae1de2..9fd88a64873a 100644 --- a/include/llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h +++ b/include/llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h @@ -1,9 +1,8 @@ //===- DebugInlineeLinesSubsection.h ----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -71,6 +70,11 @@ public: } Error initialize(BinaryStreamReader Reader); + Error initialize(BinaryStreamRef Section) { + return initialize(BinaryStreamReader(Section)); + } + + bool valid() const { return Lines.valid(); } bool hasExtraFiles() const; Iterator begin() const { return Lines.begin(); } @@ -78,7 +82,7 @@ public: private: InlineeLinesSignature Signature; - VarStreamArray Lines; + LinesArray Lines; }; class DebugInlineeLinesSubsection final : public DebugSubsection { diff --git a/include/llvm/DebugInfo/CodeView/DebugLinesSubsection.h b/include/llvm/DebugInfo/CodeView/DebugLinesSubsection.h index 53044b6c3dc8..1f8e56c5311f 100644 --- a/include/llvm/DebugInfo/CodeView/DebugLinesSubsection.h +++ b/include/llvm/DebugInfo/CodeView/DebugLinesSubsection.h @@ -1,9 +1,8 @@ //===- DebugLinesSubsection.h -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/DebugStringTableSubsection.h b/include/llvm/DebugInfo/CodeView/DebugStringTableSubsection.h index bebc960223cc..6e5b8adddd4a 100644 --- a/include/llvm/DebugInfo/CodeView/DebugStringTableSubsection.h +++ b/include/llvm/DebugInfo/CodeView/DebugStringTableSubsection.h @@ -1,9 +1,8 @@ //===- DebugStringTableSubsection.h - CodeView String Table -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/DebugSubsection.h b/include/llvm/DebugInfo/CodeView/DebugSubsection.h index e427e0006a55..66272870efda 100644 --- a/include/llvm/DebugInfo/CodeView/DebugSubsection.h +++ b/include/llvm/DebugInfo/CodeView/DebugSubsection.h @@ -1,9 +1,8 @@ //===- DebugSubsection.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/DebugSubsectionRecord.h b/include/llvm/DebugInfo/CodeView/DebugSubsectionRecord.h index fc0cf0d1d90e..bcb379f00d68 100644 --- a/include/llvm/DebugInfo/CodeView/DebugSubsectionRecord.h +++ b/include/llvm/DebugInfo/CodeView/DebugSubsectionRecord.h @@ -1,9 +1,8 @@ //===- DebugSubsectionRecord.h ----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/DebugSubsectionVisitor.h b/include/llvm/DebugInfo/CodeView/DebugSubsectionVisitor.h index 75f749dfa933..720b1b49581f 100644 --- a/include/llvm/DebugInfo/CodeView/DebugSubsectionVisitor.h +++ b/include/llvm/DebugInfo/CodeView/DebugSubsectionVisitor.h @@ -1,9 +1,8 @@ //===- DebugSubsectionVisitor.h -----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/DebugSymbolRVASubsection.h b/include/llvm/DebugInfo/CodeView/DebugSymbolRVASubsection.h index a4c04b55eb4c..91b740ce6b9a 100644 --- a/include/llvm/DebugInfo/CodeView/DebugSymbolRVASubsection.h +++ b/include/llvm/DebugInfo/CodeView/DebugSymbolRVASubsection.h @@ -1,9 +1,8 @@ //===- DebugSymbolRVASubsection.h -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/DebugSymbolsSubsection.h b/include/llvm/DebugInfo/CodeView/DebugSymbolsSubsection.h index dfda7deb6cb4..784fc59484b9 100644 --- a/include/llvm/DebugInfo/CodeView/DebugSymbolsSubsection.h +++ b/include/llvm/DebugInfo/CodeView/DebugSymbolsSubsection.h @@ -1,9 +1,8 @@ //===- DebugSymbolsSubsection.h --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/DebugUnknownSubsection.h b/include/llvm/DebugInfo/CodeView/DebugUnknownSubsection.h index ea9a96ca8d68..fa7df325499f 100644 --- a/include/llvm/DebugInfo/CodeView/DebugUnknownSubsection.h +++ b/include/llvm/DebugInfo/CodeView/DebugUnknownSubsection.h @@ -1,9 +1,8 @@ //===- DebugUnknownSubsection.h -----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/EnumTables.h b/include/llvm/DebugInfo/CodeView/EnumTables.h index ee0f0f7c6023..ed126ed9e2ff 100644 --- a/include/llvm/DebugInfo/CodeView/EnumTables.h +++ b/include/llvm/DebugInfo/CodeView/EnumTables.h @@ -1,9 +1,8 @@ //===- EnumTables.h - Enum to string conversion tables ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -21,7 +20,7 @@ namespace codeview { ArrayRef> getSymbolTypeNames(); ArrayRef> getTypeLeafNames(); -ArrayRef> getRegisterNames(); +ArrayRef> getRegisterNames(CPUType Cpu); ArrayRef> getPublicSymFlagNames(); ArrayRef> getProcSymFlagNames(); ArrayRef> getLocalFlagNames(); diff --git a/include/llvm/DebugInfo/CodeView/Formatters.h b/include/llvm/DebugInfo/CodeView/Formatters.h index 278ad02a39cd..7d04a6a89bef 100644 --- a/include/llvm/DebugInfo/CodeView/Formatters.h +++ b/include/llvm/DebugInfo/CodeView/Formatters.h @@ -1,9 +1,8 @@ //===- Formatters.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/FunctionId.h b/include/llvm/DebugInfo/CodeView/FunctionId.h index 1af3da810b5a..bc102278819c 100644 --- a/include/llvm/DebugInfo/CodeView/FunctionId.h +++ b/include/llvm/DebugInfo/CodeView/FunctionId.h @@ -1,9 +1,8 @@ //===- FunctionId.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/GUID.h b/include/llvm/DebugInfo/CodeView/GUID.h index a055ce9e2e45..5f807e6f7eeb 100644 --- a/include/llvm/DebugInfo/CodeView/GUID.h +++ b/include/llvm/DebugInfo/CodeView/GUID.h @@ -1,9 +1,8 @@ //===- GUID.h ---------------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h b/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h index c4704168ed34..a43ce20edde6 100644 --- a/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h +++ b/include/llvm/DebugInfo/CodeView/GlobalTypeTableBuilder.h @@ -1,9 +1,8 @@ //===- GlobalTypeTableBuilder.h ----------------------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -74,14 +73,30 @@ public: CreateFunc Create) { auto Result = HashedRecords.try_emplace(Hash, nextTypeIndex()); - if (LLVM_UNLIKELY(Result.second)) { + if (LLVM_UNLIKELY(Result.second /*inserted*/ || + Result.first->second.isSimple())) { uint8_t *Stable = RecordStorage.Allocate(RecordSize); MutableArrayRef Data(Stable, RecordSize); - SeenRecords.push_back(Create(Data)); + ArrayRef StableRecord = Create(Data); + if (StableRecord.empty()) { + // Records with forward references into the Type stream will be deferred + // for insertion at a later time, on the second pass. + Result.first->getSecond() = TypeIndex(SimpleTypeKind::NotTranslated); + return TypeIndex(SimpleTypeKind::NotTranslated); + } + if (Result.first->second.isSimple()) { + assert(Result.first->second.getIndex() == + (uint32_t)SimpleTypeKind::NotTranslated); + // On the second pass, update with index to remapped record. The + // (initially misbehaved) record will now come *after* other records + // resolved in the first pass, with proper *back* references in the + // stream. + Result.first->second = nextTypeIndex(); + } + SeenRecords.push_back(StableRecord); SeenHashes.push_back(Hash); } - // Update the caller's copy of Record to point a stable copy. return Result.first->second; } diff --git a/include/llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h b/include/llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h index 383f7dd9fb6a..4e03627e9580 100644 --- a/include/llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h +++ b/include/llvm/DebugInfo/CodeView/LazyRandomTypeCollection.h @@ -1,9 +1,8 @@ //===- LazyRandomTypeCollection.h -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/Line.h b/include/llvm/DebugInfo/CodeView/Line.h index ac229c337513..eb2aa154df1b 100644 --- a/include/llvm/DebugInfo/CodeView/Line.h +++ b/include/llvm/DebugInfo/CodeView/Line.h @@ -1,9 +1,8 @@ //===- Line.h ---------------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h b/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h index 9030918ebbb3..1b2f6d29a9b6 100644 --- a/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h +++ b/include/llvm/DebugInfo/CodeView/MergingTypeTableBuilder.h @@ -1,9 +1,8 @@ //===- MergingTypeTableBuilder.h ---------------------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/RecordName.h b/include/llvm/DebugInfo/CodeView/RecordName.h index b022108df3d6..cc09db8933bd 100644 --- a/include/llvm/DebugInfo/CodeView/RecordName.h +++ b/include/llvm/DebugInfo/CodeView/RecordName.h @@ -1,9 +1,8 @@ //===- RecordName.h ------------------------------------------- *- C++ --*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/RecordSerialization.h b/include/llvm/DebugInfo/CodeView/RecordSerialization.h index 36237e1a4d9e..36c0f2fbd8fa 100644 --- a/include/llvm/DebugInfo/CodeView/RecordSerialization.h +++ b/include/llvm/DebugInfo/CodeView/RecordSerialization.h @@ -1,9 +1,8 @@ //===- RecordSerialization.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -32,6 +31,9 @@ using llvm::support::ulittle32_t; enum : unsigned { MaxRecordLength = 0xFF00 }; struct RecordPrefix { + RecordPrefix() = default; + explicit RecordPrefix(uint16_t Kind) : RecordLen(2), RecordKind(Kind) {} + ulittle16_t RecordLen; // Record length, starting from &RecordKind. ulittle16_t RecordKind; // Record kind enum (SymRecordKind or TypeRecordKind) }; diff --git a/include/llvm/DebugInfo/CodeView/SimpleTypeSerializer.h b/include/llvm/DebugInfo/CodeView/SimpleTypeSerializer.h index a85d9270186b..3ca09b445a30 100644 --- a/include/llvm/DebugInfo/CodeView/SimpleTypeSerializer.h +++ b/include/llvm/DebugInfo/CodeView/SimpleTypeSerializer.h @@ -1,9 +1,8 @@ //===- SimpleTypeSerializer.h -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/StringsAndChecksums.h b/include/llvm/DebugInfo/CodeView/StringsAndChecksums.h index 22a333e631a0..22a283e785e1 100644 --- a/include/llvm/DebugInfo/CodeView/StringsAndChecksums.h +++ b/include/llvm/DebugInfo/CodeView/StringsAndChecksums.h @@ -1,9 +1,8 @@ //===- StringsAndChecksums.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h b/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h index 6b5dd2d20d17..62761cb87c81 100644 --- a/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h +++ b/include/llvm/DebugInfo/CodeView/SymbolDeserializer.h @@ -1,9 +1,8 @@ //===- SymbolDeserializer.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/SymbolDumpDelegate.h b/include/llvm/DebugInfo/CodeView/SymbolDumpDelegate.h index 823636c398de..12f45dcb21ff 100644 --- a/include/llvm/DebugInfo/CodeView/SymbolDumpDelegate.h +++ b/include/llvm/DebugInfo/CodeView/SymbolDumpDelegate.h @@ -1,9 +1,8 @@ //===-- SymbolDumpDelegate.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/SymbolDumper.h b/include/llvm/DebugInfo/CodeView/SymbolDumper.h index 215da2e2b522..d832a48b1265 100644 --- a/include/llvm/DebugInfo/CodeView/SymbolDumper.h +++ b/include/llvm/DebugInfo/CodeView/SymbolDumper.h @@ -1,9 +1,8 @@ //===-- SymbolDumper.h - CodeView symbol info dumper ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/SymbolRecord.h b/include/llvm/DebugInfo/CodeView/SymbolRecord.h index b58825c4a788..5e9a7432b9b6 100644 --- a/include/llvm/DebugInfo/CodeView/SymbolRecord.h +++ b/include/llvm/DebugInfo/CodeView/SymbolRecord.h @@ -1,9 +1,8 @@ //===- SymbolRecord.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -14,6 +13,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/iterator.h" #include "llvm/ADT/iterator_range.h" #include "llvm/DebugInfo/CodeView/CVRecord.h" #include "llvm/DebugInfo/CodeView/CodeView.h" @@ -156,15 +156,19 @@ public: uint32_t RecordOffset; }; -struct BinaryAnnotationIterator { - struct AnnotationData { - BinaryAnnotationsOpCode OpCode; - StringRef Name; - uint32_t U1; - uint32_t U2; - int32_t S1; - }; +struct DecodedAnnotation { + StringRef Name; + ArrayRef Bytes; + BinaryAnnotationsOpCode OpCode; + uint32_t U1 = 0; + uint32_t U2 = 0; + int32_t S1 = 0; +}; +struct BinaryAnnotationIterator + : public iterator_facade_base { BinaryAnnotationIterator() = default; BinaryAnnotationIterator(ArrayRef Annotations) : Data(Annotations) {} BinaryAnnotationIterator(const BinaryAnnotationIterator &Other) @@ -174,10 +178,6 @@ struct BinaryAnnotationIterator { return Data == Other.Data; } - bool operator!=(const BinaryAnnotationIterator &Other) const { - return !(*this == Other); - } - BinaryAnnotationIterator &operator=(const BinaryAnnotationIterator Other) { Data = Other.Data; return *this; @@ -194,13 +194,7 @@ struct BinaryAnnotationIterator { return *this; } - BinaryAnnotationIterator operator++(int) { - BinaryAnnotationIterator Orig(*this); - ++(*this); - return Orig; - } - - const AnnotationData &operator*() { + const DecodedAnnotation &operator*() { ParseCurrentAnnotation(); return Current.getValue(); } @@ -242,17 +236,17 @@ private: (ThirdByte << 8) | FourthByte; return -1; - }; + } static int32_t DecodeSignedOperand(uint32_t Operand) { if (Operand & 1) return -(Operand >> 1); return Operand >> 1; - }; + } static int32_t DecodeSignedOperand(ArrayRef &Annotations) { return DecodeSignedOperand(GetCompressedAnnotation(Annotations)); - }; + } bool ParseCurrentAnnotation() { if (Current.hasValue()) @@ -260,7 +254,7 @@ private: Next = Data; uint32_t Op = GetCompressedAnnotation(Next); - AnnotationData Result; + DecodedAnnotation Result; Result.OpCode = static_cast(Op); switch (Result.OpCode) { case BinaryAnnotationsOpCode::Invalid: @@ -325,11 +319,12 @@ private: break; } } + Result.Bytes = Data.take_front(Data.size() - Next.size()); Current = Result; return true; } - Optional Current; + Optional Current; ArrayRef Data; ArrayRef Next; }; @@ -974,7 +969,7 @@ class UsingNamespaceSym : public SymbolRecord { public: explicit UsingNamespaceSym(SymbolRecordKind Kind) : SymbolRecord(Kind) {} explicit UsingNamespaceSym(uint32_t RecordOffset) - : SymbolRecord(SymbolRecordKind::RegRelativeSym), + : SymbolRecord(SymbolRecordKind::UsingNamespaceSym), RecordOffset(RecordOffset) {} StringRef Name; @@ -983,6 +978,19 @@ public: }; // S_ANNOTATION +class AnnotationSym : public SymbolRecord { +public: + explicit AnnotationSym(SymbolRecordKind Kind) : SymbolRecord(Kind) {} + explicit AnnotationSym(uint32_t RecordOffset) + : SymbolRecord(SymbolRecordKind::AnnotationSym), + RecordOffset(RecordOffset) {} + + uint32_t CodeOffset = 0; + uint16_t Segment = 0; + std::vector Strings; + + uint32_t RecordOffset; +}; using CVSymbol = CVRecord; using CVSymbolArray = VarStreamArray; diff --git a/include/llvm/DebugInfo/CodeView/SymbolRecordHelpers.h b/include/llvm/DebugInfo/CodeView/SymbolRecordHelpers.h index 3713fe118eaa..57dbc56c0769 100644 --- a/include/llvm/DebugInfo/CodeView/SymbolRecordHelpers.h +++ b/include/llvm/DebugInfo/CodeView/SymbolRecordHelpers.h @@ -1,9 +1,8 @@ //===- SymbolRecordHelpers.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/SymbolRecordMapping.h b/include/llvm/DebugInfo/CodeView/SymbolRecordMapping.h index 391e8f127665..34368b6185d6 100644 --- a/include/llvm/DebugInfo/CodeView/SymbolRecordMapping.h +++ b/include/llvm/DebugInfo/CodeView/SymbolRecordMapping.h @@ -1,9 +1,8 @@ //===- SymbolRecordMapping.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/SymbolSerializer.h b/include/llvm/DebugInfo/CodeView/SymbolSerializer.h index f4d8ab0c3c2e..b805b6595e80 100644 --- a/include/llvm/DebugInfo/CodeView/SymbolSerializer.h +++ b/include/llvm/DebugInfo/CodeView/SymbolSerializer.h @@ -1,9 +1,8 @@ //===- SymbolSerializer.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -52,8 +51,8 @@ public: template static CVSymbol writeOneSymbol(SymType &Sym, BumpPtrAllocator &Storage, CodeViewContainer Container) { - CVSymbol Result; - Result.Type = static_cast(Sym.Kind); + RecordPrefix Prefix{uint16_t(Sym.Kind)}; + CVSymbol Result(&Prefix, sizeof(Prefix)); SymbolSerializer Serializer(Storage, Container); consumeError(Serializer.visitSymbolBegin(Result)); consumeError(Serializer.visitKnownRecord(Result, Sym)); diff --git a/include/llvm/DebugInfo/CodeView/SymbolVisitorCallbackPipeline.h b/include/llvm/DebugInfo/CodeView/SymbolVisitorCallbackPipeline.h index e29511a67b7f..145d63a6fe61 100644 --- a/include/llvm/DebugInfo/CodeView/SymbolVisitorCallbackPipeline.h +++ b/include/llvm/DebugInfo/CodeView/SymbolVisitorCallbackPipeline.h @@ -1,9 +1,8 @@ //===- SymbolVisitorCallbackPipeline.h --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/SymbolVisitorCallbacks.h b/include/llvm/DebugInfo/CodeView/SymbolVisitorCallbacks.h index 0816f7c62656..1a4d5b9d31df 100644 --- a/include/llvm/DebugInfo/CodeView/SymbolVisitorCallbacks.h +++ b/include/llvm/DebugInfo/CodeView/SymbolVisitorCallbacks.h @@ -1,9 +1,8 @@ //===- SymbolVisitorCallbacks.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/SymbolVisitorDelegate.h b/include/llvm/DebugInfo/CodeView/SymbolVisitorDelegate.h index a2a3c6f18fba..368d8b288315 100644 --- a/include/llvm/DebugInfo/CodeView/SymbolVisitorDelegate.h +++ b/include/llvm/DebugInfo/CodeView/SymbolVisitorDelegate.h @@ -1,9 +1,8 @@ //===-- SymbolVisitorDelegate.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/TypeCollection.h b/include/llvm/DebugInfo/CodeView/TypeCollection.h index e9fc9b0de8ef..58b1dd058c1a 100644 --- a/include/llvm/DebugInfo/CodeView/TypeCollection.h +++ b/include/llvm/DebugInfo/CodeView/TypeCollection.h @@ -1,9 +1,8 @@ //===- TypeCollection.h - A collection of CodeView type records -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/TypeDeserializer.h b/include/llvm/DebugInfo/CodeView/TypeDeserializer.h index 9887d901773a..081de32dd02c 100644 --- a/include/llvm/DebugInfo/CodeView/TypeDeserializer.h +++ b/include/llvm/DebugInfo/CodeView/TypeDeserializer.h @@ -1,9 +1,8 @@ //===- TypeDeserializer.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -59,7 +58,7 @@ public: TypeRecordKind K = static_cast(uint16_t(Prefix->RecordKind)); T Record(K); - CVType CVT(static_cast(K), Data); + CVType CVT(Data); if (auto EC = deserializeAs(CVT, Record)) return std::move(EC); return Record; @@ -112,14 +111,14 @@ class FieldListDeserializer : public TypeVisitorCallbacks { public: explicit FieldListDeserializer(BinaryStreamReader &Reader) : Mapping(Reader) { - CVType FieldList; - FieldList.Type = TypeLeafKind::LF_FIELDLIST; + RecordPrefix Pre(static_cast(TypeLeafKind::LF_FIELDLIST)); + CVType FieldList(&Pre, sizeof(Pre)); consumeError(Mapping.Mapping.visitTypeBegin(FieldList)); } ~FieldListDeserializer() override { - CVType FieldList; - FieldList.Type = TypeLeafKind::LF_FIELDLIST; + RecordPrefix Pre(static_cast(TypeLeafKind::LF_FIELDLIST)); + CVType FieldList(&Pre, sizeof(Pre)); consumeError(Mapping.Mapping.visitTypeEnd(FieldList)); } diff --git a/include/llvm/DebugInfo/CodeView/TypeDumpVisitor.h b/include/llvm/DebugInfo/CodeView/TypeDumpVisitor.h index afb8b3636361..41a219ae5a7b 100644 --- a/include/llvm/DebugInfo/CodeView/TypeDumpVisitor.h +++ b/include/llvm/DebugInfo/CodeView/TypeDumpVisitor.h @@ -1,9 +1,8 @@ //===-- TypeDumpVisitor.h - CodeView type info dumper -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/TypeHashing.h b/include/llvm/DebugInfo/CodeView/TypeHashing.h index 1f732d29a538..b0a16cccbff3 100644 --- a/include/llvm/DebugInfo/CodeView/TypeHashing.h +++ b/include/llvm/DebugInfo/CodeView/TypeHashing.h @@ -1,9 +1,8 @@ //===- TypeHashing.h ---------------------------------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -85,6 +84,8 @@ struct GloballyHashedType { } std::array Hash; + bool empty() const { return *(const uint64_t*)Hash.data() == 0; } + /// Given a sequence of bytes representing a record, compute a global hash for /// this record. Due to the nature of global hashes incorporating the hashes /// of referenced records, this function requires a list of types and ids @@ -108,8 +109,33 @@ struct GloballyHashedType { template static std::vector hashTypes(Range &&Records) { std::vector Hashes; - for (const auto &R : Records) - Hashes.push_back(hashType(R, Hashes, Hashes)); + bool UnresolvedRecords = false; + for (const auto &R : Records) { + GloballyHashedType H = hashType(R, Hashes, Hashes); + if (H.empty()) + UnresolvedRecords = true; + Hashes.push_back(H); + } + + // In some rare cases, there might be records with forward references in the + // stream. Several passes might be needed to fully hash each record in the + // Type stream. However this occurs on very small OBJs generated by MASM, + // with a dozen records at most. Therefore this codepath isn't + // time-critical, as it isn't taken in 99% of cases. + while (UnresolvedRecords) { + UnresolvedRecords = false; + auto HashIt = Hashes.begin(); + for (const auto &R : Records) { + if (HashIt->empty()) { + GloballyHashedType H = hashType(R, Hashes, Hashes); + if (H.empty()) + UnresolvedRecords = true; + else + *HashIt = H; + } + ++HashIt; + } + } return Hashes; } diff --git a/include/llvm/DebugInfo/CodeView/TypeIndex.h b/include/llvm/DebugInfo/CodeView/TypeIndex.h index 58463a6b13df..b9e2562bfc2b 100644 --- a/include/llvm/DebugInfo/CodeView/TypeIndex.h +++ b/include/llvm/DebugInfo/CodeView/TypeIndex.h @@ -1,9 +1,8 @@ //===- TypeIndex.h ----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/TypeIndexDiscovery.h b/include/llvm/DebugInfo/CodeView/TypeIndexDiscovery.h index c424a09ece89..469768787274 100644 --- a/include/llvm/DebugInfo/CodeView/TypeIndexDiscovery.h +++ b/include/llvm/DebugInfo/CodeView/TypeIndexDiscovery.h @@ -1,9 +1,8 @@ //===- TypeIndexDiscovery.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/TypeRecord.h b/include/llvm/DebugInfo/CodeView/TypeRecord.h index 7b4a30ee622d..b147dd6c3d05 100644 --- a/include/llvm/DebugInfo/CodeView/TypeRecord.h +++ b/include/llvm/DebugInfo/CodeView/TypeRecord.h @@ -1,9 +1,8 @@ //===- TypeRecord.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/TypeRecordHelpers.h b/include/llvm/DebugInfo/CodeView/TypeRecordHelpers.h index 389472ed1aea..e84704d99ddc 100644 --- a/include/llvm/DebugInfo/CodeView/TypeRecordHelpers.h +++ b/include/llvm/DebugInfo/CodeView/TypeRecordHelpers.h @@ -1,9 +1,8 @@ //===- TypeRecordHelpers.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/TypeRecordMapping.h b/include/llvm/DebugInfo/CodeView/TypeRecordMapping.h index cbe8d6066bb9..4c309c10ff0c 100644 --- a/include/llvm/DebugInfo/CodeView/TypeRecordMapping.h +++ b/include/llvm/DebugInfo/CodeView/TypeRecordMapping.h @@ -1,9 +1,8 @@ //===- TypeRecordMapping.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -24,9 +23,11 @@ class TypeRecordMapping : public TypeVisitorCallbacks { public: explicit TypeRecordMapping(BinaryStreamReader &Reader) : IO(Reader) {} explicit TypeRecordMapping(BinaryStreamWriter &Writer) : IO(Writer) {} + explicit TypeRecordMapping(CodeViewRecordStreamer &Streamer) : IO(Streamer) {} using TypeVisitorCallbacks::visitTypeBegin; Error visitTypeBegin(CVType &Record) override; + Error visitTypeBegin(CVType &Record, TypeIndex Index) override; Error visitTypeEnd(CVType &Record) override; Error visitMemberBegin(CVMemberRecord &Record) override; diff --git a/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h b/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h index 0b9f54ec60bf..d0506cce8176 100644 --- a/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h +++ b/include/llvm/DebugInfo/CodeView/TypeStreamMerger.h @@ -1,9 +1,8 @@ //===- TypeStreamMerger.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/TypeSymbolEmitter.h b/include/llvm/DebugInfo/CodeView/TypeSymbolEmitter.h index dfba83d62fce..4f2e5deb10b4 100644 --- a/include/llvm/DebugInfo/CodeView/TypeSymbolEmitter.h +++ b/include/llvm/DebugInfo/CodeView/TypeSymbolEmitter.h @@ -1,9 +1,8 @@ //===- TypeSymbolEmitter.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/TypeTableCollection.h b/include/llvm/DebugInfo/CodeView/TypeTableCollection.h index 80326a0ffd39..5cbe3400e029 100644 --- a/include/llvm/DebugInfo/CodeView/TypeTableCollection.h +++ b/include/llvm/DebugInfo/CodeView/TypeTableCollection.h @@ -1,9 +1,8 @@ //===- TypeTableCollection.h ---------------------------------- *- C++ --*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h b/include/llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h index 126fb8abb0da..169715be2d52 100644 --- a/include/llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h +++ b/include/llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h @@ -1,9 +1,8 @@ //===- TypeVisitorCallbackPipeline.h ----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -83,6 +82,11 @@ public: Pipeline.push_back(&Callbacks); } + void addCallbackToPipelineFront(TypeVisitorCallbacks &Callbacks) { + auto CallBackItr = Pipeline.begin(); + Pipeline.insert(CallBackItr, &Callbacks); + } + #define TYPE_RECORD(EnumName, EnumVal, Name) \ Error visitKnownRecord(CVType &CVR, Name##Record &Record) override { \ return visitKnownRecordImpl(CVR, Record); \ diff --git a/include/llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h b/include/llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h index d7a473306bc2..33f8b1f24b1b 100644 --- a/include/llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h +++ b/include/llvm/DebugInfo/CodeView/TypeVisitorCallbacks.h @@ -1,9 +1,8 @@ //===- TypeVisitorCallbacks.h -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/DIContext.h b/include/llvm/DebugInfo/DIContext.h index 85e96402a246..d2a5318179eb 100644 --- a/include/llvm/DebugInfo/DIContext.h +++ b/include/llvm/DebugInfo/DIContext.h @@ -1,9 +1,8 @@ //===- DIContext.h ----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -98,11 +97,10 @@ public: void addFrame(const DILineInfo &Frame) { Frames.push_back(Frame); } - + void resize(unsigned i) { Frames.resize(i); } - }; /// Container for description of a global variable. @@ -114,6 +112,16 @@ struct DIGlobal { DIGlobal() : Name("") {} }; +struct DILocal { + std::string FunctionName; + std::string Name; + std::string DeclFile; + uint64_t DeclLine = 0; + Optional FrameOffset; + Optional Size; + Optional TagOffset; +}; + /// A DINameKind is passed to name search methods to specify a /// preference regarding the type of name resolution the caller wants. enum class DINameKind { None, ShortName, LinkageName }; @@ -158,7 +166,8 @@ enum DIDumpType : unsigned { /// dumped. struct DIDumpOptions { unsigned DumpType = DIDT_All; - unsigned RecurseDepth = -1U; + unsigned ChildRecurseDepth = -1U; + unsigned ParentRecurseDepth = -1U; uint16_t Version = 0; // DWARF version to assume when extracting. uint8_t AddrSize = 4; // Address byte size to assume when extracting. bool ShowAddresses = true; @@ -172,15 +181,18 @@ struct DIDumpOptions { /// Return default option set for printing a single DIE without children. static DIDumpOptions getForSingleDIE() { DIDumpOptions Opts; - Opts.RecurseDepth = 0; + Opts.ChildRecurseDepth = 0; + Opts.ParentRecurseDepth = 0; return Opts; } /// Return the options with RecurseDepth set to 0 unless explicitly required. DIDumpOptions noImplicitRecursion() const { DIDumpOptions Opts = *this; - if (RecurseDepth == -1U && !ShowChildren) - Opts.RecurseDepth = 0; + if (ChildRecurseDepth == -1U && !ShowChildren) + Opts.ChildRecurseDepth = 0; + if (ParentRecurseDepth == -1U && !ShowParents) + Opts.ParentRecurseDepth = 0; return Opts; } }; @@ -204,12 +216,18 @@ public: return true; } - virtual DILineInfo getLineInfoForAddress(uint64_t Address, + virtual DILineInfo getLineInfoForAddress( + object::SectionedAddress Address, DILineInfoSpecifier Specifier = DILineInfoSpecifier()) = 0; - virtual DILineInfoTable getLineInfoForAddressRange(uint64_t Address, - uint64_t Size, DILineInfoSpecifier Specifier = DILineInfoSpecifier()) = 0; - virtual DIInliningInfo getInliningInfoForAddress(uint64_t Address, + virtual DILineInfoTable getLineInfoForAddressRange( + object::SectionedAddress Address, uint64_t Size, DILineInfoSpecifier Specifier = DILineInfoSpecifier()) = 0; + virtual DIInliningInfo getInliningInfoForAddress( + object::SectionedAddress Address, + DILineInfoSpecifier Specifier = DILineInfoSpecifier()) = 0; + + virtual std::vector + getLocalsForAddress(object::SectionedAddress Address) = 0; private: const DIContextKind Kind; diff --git a/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h b/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h index 84b23398b8cc..ccf2891c2e21 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h +++ b/include/llvm/DebugInfo/DWARF/DWARFAbbreviationDeclaration.h @@ -1,9 +1,8 @@ //===- DWARFAbbreviationDeclaration.h ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h b/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h index 1d448728338f..303375703d2e 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h +++ b/include/llvm/DebugInfo/DWARF/DWARFAcceleratorTable.h @@ -1,9 +1,8 @@ //===- DWARFAcceleratorTable.h ----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -72,7 +71,7 @@ public: : AccelSection(AccelSection), StringSection(StringSection) {} virtual ~DWARFAcceleratorTable(); - virtual llvm::Error extract() = 0; + virtual Error extract() = 0; virtual void dump(raw_ostream &OS) const = 0; DWARFAcceleratorTable(const DWARFAcceleratorTable &) = delete; @@ -175,7 +174,7 @@ public: DataExtractor StringSection) : DWARFAcceleratorTable(AccelSection, StringSection) {} - llvm::Error extract() override; + Error extract() override; uint32_t getNumBuckets(); uint32_t getNumHashes(); uint32_t getSizeHdr(); @@ -223,7 +222,7 @@ public: /// referenced by the name table and interpreted with the help of the /// abbreviation table. class DWARFDebugNames : public DWARFAcceleratorTable { - /// The fixed-size part of a Dwarf 5 Name Index header + /// The fixed-size part of a DWARF v5 Name Index header struct HeaderPOD { uint32_t UnitLength; uint16_t Version; @@ -242,7 +241,7 @@ public: class NameIterator; class ValueIterator; - /// Dwarf 5 Name Index header. + /// DWARF v5 Name Index header. struct Header : public HeaderPOD { SmallString<8> AugmentationString; @@ -349,7 +348,7 @@ private: }; public: - /// A single entry in the Name Table (Dwarf 5 sect. 6.1.1.4.6) of the Name + /// A single entry in the Name Table (DWARF v5 sect. 6.1.1.4.6) of the Name /// Index. class NameTableEntry { DataExtractor StrData; @@ -381,7 +380,7 @@ public: uint32_t getEntryOffset() const { return EntryOffset; } }; - /// Represents a single accelerator table within the Dwarf 5 .debug_names + /// Represents a single accelerator table within the DWARF v5 .debug_names /// section. class NameIndex { DenseSet Abbrevs; @@ -460,7 +459,7 @@ public: NameIterator begin() const { return NameIterator(this, 1); } NameIterator end() const { return NameIterator(this, getNameCount() + 1); } - llvm::Error extract(); + Error extract(); uint32_t getUnitOffset() const { return Base; } uint32_t getNextUnitOffset() const { return Base + 4 + Hdr.UnitLength; } void dump(ScopedPrinter &W) const; @@ -580,7 +579,7 @@ public: DataExtractor StringSection) : DWARFAcceleratorTable(AccelSection, StringSection) {} - llvm::Error extract() override; + Error extract() override; void dump(raw_ostream &OS) const override; /// Look up all entries in the accelerator table matching \c Key. diff --git a/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h b/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h index 5a7df5c353e8..2d5f9f3c7658 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h +++ b/include/llvm/DebugInfo/DWARF/DWARFAddressRange.h @@ -1,9 +1,8 @@ //===- DWARFAddressRange.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -43,12 +42,6 @@ struct DWARFAddressRange { return LowPC < RHS.HighPC && RHS.LowPC < HighPC; } - /// Returns true if [LowPC, HighPC) fully contains [RHS.LowPC, RHS.HighPC). - bool contains(const DWARFAddressRange &RHS) const { - assert(valid() && RHS.valid()); - return LowPC <= RHS.LowPC && RHS.HighPC <= HighPC; - } - void dump(raw_ostream &OS, uint32_t AddressSize, DIDumpOptions DumpOpts = {}) const; }; diff --git a/include/llvm/DebugInfo/DWARF/DWARFAttribute.h b/include/llvm/DebugInfo/DWARF/DWARFAttribute.h index f0672bb0ca75..c8ad19ad6bf6 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFAttribute.h +++ b/include/llvm/DebugInfo/DWARF/DWARFAttribute.h @@ -1,9 +1,8 @@ //===- DWARFAttribute.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -28,13 +27,10 @@ struct DWARFAttribute { /// The debug info/types section byte size of the data for this attribute. uint32_t ByteSize = 0; /// The attribute enumeration of this attribute. - dwarf::Attribute Attr; + dwarf::Attribute Attr = dwarf::Attribute(0); /// The form and value for this attribute. DWARFFormValue Value; - DWARFAttribute(uint32_t O, dwarf::Attribute A = dwarf::Attribute(0), - dwarf::Form F = dwarf::Form(0)) : Attr(A), Value(F) {} - bool isValid() const { return Offset != 0 && Attr != dwarf::Attribute(0); } @@ -43,12 +39,9 @@ struct DWARFAttribute { return isValid(); } - void clear() { - Offset = 0; - ByteSize = 0; - Attr = dwarf::Attribute(0); - Value = DWARFFormValue(); - } + /// Identifies DWARF attributes that may contain a reference to a + /// DWARF expression. + static bool mayHaveLocationDescription(dwarf::Attribute Attr); }; } // end namespace llvm diff --git a/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h b/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h index 33797419a7b8..16b9bfb5de56 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h +++ b/include/llvm/DebugInfo/DWARF/DWARFCompileUnit.h @@ -1,9 +1,8 @@ //===- DWARFCompileUnit.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/DWARF/DWARFContext.h b/include/llvm/DebugInfo/DWARF/DWARFContext.h index dbb6be04544b..23cf21c3523f 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFContext.h +++ b/include/llvm/DebugInfo/DWARF/DWARFContext.h @@ -1,9 +1,8 @@ //===- DWARFContext.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===/ @@ -318,15 +317,23 @@ public: /// Get the compilation unit, the function DIE and lexical block DIE for the /// given address where applicable. + /// TODO: change input parameter from "uint64_t Address" + /// into "SectionedAddress Address" DIEsForAddress getDIEsForAddress(uint64_t Address); - DILineInfo getLineInfoForAddress(uint64_t Address, + DILineInfo getLineInfoForAddress( + object::SectionedAddress Address, DILineInfoSpecifier Specifier = DILineInfoSpecifier()) override; - DILineInfoTable getLineInfoForAddressRange(uint64_t Address, uint64_t Size, + DILineInfoTable getLineInfoForAddressRange( + object::SectionedAddress Address, uint64_t Size, DILineInfoSpecifier Specifier = DILineInfoSpecifier()) override; - DIInliningInfo getInliningInfoForAddress(uint64_t Address, + DIInliningInfo getInliningInfoForAddress( + object::SectionedAddress Address, DILineInfoSpecifier Specifier = DILineInfoSpecifier()) override; + std::vector + getLocalsForAddress(object::SectionedAddress Address) override; + bool isLittleEndian() const { return DObj->isLittleEndian(); } static bool isSupportedVersion(unsigned version) { return version == 2 || version == 3 || version == 4 || version == 5; @@ -367,7 +374,11 @@ public: private: /// Return the compile unit which contains instruction with provided /// address. + /// TODO: change input parameter from "uint64_t Address" + /// into "SectionedAddress Address" DWARFCompileUnit *getCompileUnitForAddress(uint64_t Address); + void addLocalsForDie(DWARFCompileUnit *CU, DWARFDie Subprogram, DWARFDie Die, + std::vector &Result); }; } // end namespace llvm diff --git a/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h b/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h index 1ed087520b30..7c2a159b71fa 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDataExtractor.h @@ -1,9 +1,8 @@ //===- DWARFDataExtractor.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h b/include/llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h index d277ec382ba5..28fd8484b4a9 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugAbbrev.h @@ -1,9 +1,8 @@ //===- DWARFDebugAbbrev.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugAddr.h b/include/llvm/DebugInfo/DWARF/DWARFDebugAddr.h index ffbd1b06d1e2..a98bf282fe7c 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugAddr.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugAddr.h @@ -1,9 +1,8 @@ //===- DWARFDebugAddr.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h b/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h index ab46fac39f7c..5b6c578bc3bf 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugArangeSet.h @@ -1,9 +1,8 @@ //===- DWARFDebugArangeSet.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugAranges.h b/include/llvm/DebugInfo/DWARF/DWARFDebugAranges.h index ea71a50f3270..03223fbc80a9 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugAranges.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugAranges.h @@ -1,9 +1,8 @@ //===- DWARFDebugAranges.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -50,10 +49,6 @@ private: return -1ULL; } - bool containsAddress(uint64_t Address) const { - return LowPC <= Address && Address < HighPC(); - } - bool operator<(const Range &other) const { return LowPC < other.LowPC; } diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h b/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h index 7dc07d774aba..d960f4bc9b1c 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugFrame.h @@ -1,9 +1,8 @@ //===- DWARFDebugFrame.h - Parsing of .debug_frame --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h b/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h index 88c8f57bc33c..f50063b24370 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugInfoEntry.h @@ -1,9 +1,8 @@ //===- DWARFDebugInfoEntry.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h b/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h index d50af5a057f1..e7425c192373 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugLine.h @@ -1,9 +1,8 @@ //===- DWARFDebugLine.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -122,6 +121,17 @@ public: return LineBase + (int8_t)LineRange - 1; } + /// Get DWARF-version aware access to the file name entry at the provided + /// index. + const llvm::DWARFDebugLine::FileNameEntry & + getFileNameEntry(uint64_t Index) const; + + bool hasFileAtIndex(uint64_t FileIndex) const; + + bool getFileNameByIndex(uint64_t FileIndex, StringRef CompDir, + DILineInfoSpecifier::FileLineInfoKind Kind, + std::string &Result) const; + void clear(); void dump(raw_ostream &OS, DIDumpOptions DumpOptions) const; Error parse(const DWARFDataExtractor &DebugLineData, uint32_t *OffsetPtr, @@ -140,12 +150,16 @@ public: static void dumpTableHeader(raw_ostream &OS); static bool orderByAddress(const Row &LHS, const Row &RHS) { - return LHS.Address < RHS.Address; + return std::tie(LHS.Address.SectionIndex, LHS.Address.Address) < + std::tie(RHS.Address.SectionIndex, RHS.Address.Address); } /// The program-counter value corresponding to a machine instruction - /// generated by the compiler. - uint64_t Address; + /// generated by the compiler and section index pointing to the section + /// containg this PC. If relocation information is present then section + /// index is the index of the section which contains above address. + /// Otherwise this is object::SectionedAddress::Undef value. + object::SectionedAddress Address; /// An unsigned integer indicating a source line number. Lines are numbered /// beginning at 1. The compiler may emit the value 0 in cases where an /// instruction cannot be attributed to any source line. @@ -193,21 +207,29 @@ public: /// and is described by line table rows [FirstRowIndex, LastRowIndex). uint64_t LowPC; uint64_t HighPC; + /// If relocation information is present then this is the index of the + /// section which contains above addresses. Otherwise this is + /// object::SectionedAddress::Undef value. + uint64_t SectionIndex; unsigned FirstRowIndex; unsigned LastRowIndex; bool Empty; void reset(); - static bool orderByLowPC(const Sequence &LHS, const Sequence &RHS) { - return LHS.LowPC < RHS.LowPC; + static bool orderByHighPC(const Sequence &LHS, const Sequence &RHS) { + return std::tie(LHS.SectionIndex, LHS.HighPC) < + std::tie(RHS.SectionIndex, RHS.HighPC); } bool isValid() const { return !Empty && (LowPC < HighPC) && (FirstRowIndex < LastRowIndex); } - bool containsPC(uint64_t PC) const { return (LowPC <= PC && PC < HighPC); } + bool containsPC(object::SectionedAddress PC) const { + return SectionIndex == PC.SectionIndex && + (LowPC <= PC.Address && PC.Address < HighPC); + } }; struct LineTable { @@ -224,22 +246,30 @@ public: /// Returns the index of the row with file/line info for a given address, /// or UnknownRowIndex if there is no such row. - uint32_t lookupAddress(uint64_t Address) const; + uint32_t lookupAddress(object::SectionedAddress Address) const; - bool lookupAddressRange(uint64_t Address, uint64_t Size, + bool lookupAddressRange(object::SectionedAddress Address, uint64_t Size, std::vector &Result) const; - bool hasFileAtIndex(uint64_t FileIndex) const; + bool hasFileAtIndex(uint64_t FileIndex) const { + return Prologue.hasFileAtIndex(FileIndex); + } /// Extracts filename by its index in filename table in prologue. + /// In Dwarf 4, the files are 1-indexed and the current compilation file + /// name is not represented in the list. In DWARF v5, the files are + /// 0-indexed and the primary source file has the index 0. /// Returns true on success. - bool getFileNameByIndex(uint64_t FileIndex, const char *CompDir, + bool getFileNameByIndex(uint64_t FileIndex, StringRef CompDir, DILineInfoSpecifier::FileLineInfoKind Kind, - std::string &Result) const; + std::string &Result) const { + return Prologue.getFileNameByIndex(FileIndex, CompDir, Kind, Result); + } /// Fills the Result argument with the file and line information /// corresponding to Address. Returns true on success. - bool getFileLineInfoForAddress(uint64_t Address, const char *CompDir, + bool getFileLineInfoForAddress(object::SectionedAddress Address, + const char *CompDir, DILineInfoSpecifier::FileLineInfoKind Kind, DILineInfo &Result) const; @@ -264,10 +294,15 @@ public: private: uint32_t findRowInSeq(const DWARFDebugLine::Sequence &Seq, - uint64_t Address) const; + object::SectionedAddress Address) const; Optional getSourceByIndex(uint64_t FileIndex, DILineInfoSpecifier::FileLineInfoKind Kind) const; + + uint32_t lookupAddressImpl(object::SectionedAddress Address) const; + + bool lookupAddressRangeImpl(object::SectionedAddress Address, uint64_t Size, + std::vector &Result) const; }; const LineTable *getLineTable(uint32_t Offset) const; @@ -334,13 +369,10 @@ private: ParsingState(struct LineTable *LT); void resetRowAndSequence(); - void appendRowToMatrix(uint32_t Offset); + void appendRowToMatrix(); /// Line table we're currently parsing. struct LineTable *LineTable; - /// The row number that starts at zero for the prologue, and increases for - /// each row added to the matrix. - unsigned RowNumber = 0; struct Row Row; struct Sequence Sequence; }; diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h b/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h index da2098e15402..cced6048e811 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugLoc.h @@ -1,9 +1,8 @@ //===- DWARFDebugLoc.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -42,7 +41,7 @@ public: SmallVector Entries; /// Dump this list on OS. void dump(raw_ostream &OS, bool IsLittleEndian, unsigned AddressSize, - const MCRegisterInfo *MRI, uint64_t BaseAddress, + const MCRegisterInfo *MRI, DWARFUnit *U, uint64_t BaseAddress, unsigned Indent) const; }; @@ -87,7 +86,7 @@ public: SmallVector Entries; void dump(raw_ostream &OS, uint64_t BaseAddr, bool IsLittleEndian, unsigned AddressSize, const MCRegisterInfo *RegInfo, - unsigned Indent) const; + DWARFUnit *U, unsigned Indent) const; }; private: diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h b/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h index bfe2fc3ac02d..a6c125990ca7 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugMacro.h @@ -1,9 +1,8 @@ //===- DWARFDebugMacro.h ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h b/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h index 9e1656eb1615..99e91ca90319 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugPubTable.h @@ -1,9 +1,8 @@ //===- DWARFDebugPubTable.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h b/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h index bc26edf00647..a66f60292343 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugRangeList.h @@ -1,9 +1,8 @@ //===- DWARFDebugRangeList.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -77,7 +76,7 @@ public: /// list. Has to be passed base address of the compile unit referencing this /// range list. DWARFAddressRangesVector - getAbsoluteRanges(llvm::Optional BaseAddr) const; + getAbsoluteRanges(llvm::Optional BaseAddr) const; }; } // end namespace llvm diff --git a/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h b/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h index 5cc8d789e598..167ddde3ec3d 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDebugRnglists.h @@ -1,9 +1,8 @@ //===- DWARFDebugRnglists.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -38,7 +37,7 @@ struct RangeListEntry : public DWARFListEntryBase { Error extract(DWARFDataExtractor Data, uint32_t End, uint32_t *OffsetPtr); void dump(raw_ostream &OS, uint8_t AddrSize, uint8_t MaxEncodingStringLength, uint64_t &CurrentBase, DIDumpOptions DumpOpts, - llvm::function_ref(uint32_t)> + llvm::function_ref(uint32_t)> LookupPooledAddress) const; bool isSentinel() const { return EntryKind == dwarf::DW_RLE_end_of_list; } }; @@ -48,7 +47,7 @@ class DWARFDebugRnglist : public DWARFListType { public: /// Build a DWARFAddressRangesVector from a rangelist. DWARFAddressRangesVector - getAbsoluteRanges(llvm::Optional BaseAddr, + getAbsoluteRanges(llvm::Optional BaseAddr, DWARFUnit &U) const; }; diff --git a/include/llvm/DebugInfo/DWARF/DWARFDie.h b/include/llvm/DebugInfo/DWARF/DWARFDie.h index 56d46cd739a2..21e68f983bb3 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFDie.h +++ b/include/llvm/DebugInfo/DWARF/DWARFDie.h @@ -1,9 +1,8 @@ //===- DWARFDie.h -----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/DWARF/DWARFExpression.h b/include/llvm/DebugInfo/DWARF/DWARFExpression.h index 3fad68a9b48b..f066dd58d606 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFExpression.h +++ b/include/llvm/DebugInfo/DWARF/DWARFExpression.h @@ -1,9 +1,8 @@ //===--- DWARFExpression.h - DWARF Expression handling ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -42,7 +41,8 @@ public: SizeAddr = 5, SizeRefAddr = 6, SizeBlock = 7, ///< Preceding operand contains block size - SignBit = 0x8, + BaseTypeRef = 8, + SignBit = 0x80, SignedSize1 = SignBit | Size1, SignedSize2 = SignBit | Size2, SignedSize4 = SignBit | Size4, @@ -55,7 +55,8 @@ public: DwarfNA, ///< Serves as a marker for unused entries Dwarf2 = 2, Dwarf3, - Dwarf4 + Dwarf4, + Dwarf5 }; /// Description of the encoding of one expression Op. @@ -78,17 +79,20 @@ public: bool Error; uint32_t EndOffset; uint64_t Operands[2]; + uint32_t OperandEndOffsets[2]; public: Description &getDescription() { return Desc; } uint8_t getCode() { return Opcode; } uint64_t getRawOperand(unsigned Idx) { return Operands[Idx]; } + uint32_t getOperandEndOffset(unsigned Idx) { return OperandEndOffsets[Idx]; } uint32_t getEndOffset() { return EndOffset; } bool extract(DataExtractor Data, uint16_t Version, uint8_t AddressSize, uint32_t Offset); bool isError() { return Error; } - bool print(raw_ostream &OS, const DWARFExpression *U, - const MCRegisterInfo *RegInfo, bool isEH); + bool print(raw_ostream &OS, const DWARFExpression *Expr, + const MCRegisterInfo *RegInfo, DWARFUnit *U, bool isEH); + bool verify(DWARFUnit *U); }; /// An iterator to go through the expression operations. @@ -125,15 +129,17 @@ public: DWARFExpression(DataExtractor Data, uint16_t Version, uint8_t AddressSize) : Data(Data), Version(Version), AddressSize(AddressSize) { - assert(AddressSize == 8 || AddressSize == 4); + assert(AddressSize == 8 || AddressSize == 4 || AddressSize == 2); } iterator begin() const { return iterator(this, 0); } iterator end() const { return iterator(this, Data.getData().size()); } - void print(raw_ostream &OS, const MCRegisterInfo *RegInfo, + void print(raw_ostream &OS, const MCRegisterInfo *RegInfo, DWARFUnit *U, bool IsEH = false) const; + bool verify(DWARFUnit *U); + private: DataExtractor Data; uint16_t Version; diff --git a/include/llvm/DebugInfo/DWARF/DWARFFormValue.h b/include/llvm/DebugInfo/DWARF/DWARFFormValue.h index 727e853c09fb..731e71ed9eae 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFFormValue.h +++ b/include/llvm/DebugInfo/DWARF/DWARFFormValue.h @@ -1,9 +1,8 @@ //===- DWARFFormValue.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -42,6 +41,9 @@ public: private: struct ValueType { ValueType() { uval = 0; } + ValueType(int64_t V) : sval(V) {} + ValueType(uint64_t V) : uval(V) {} + ValueType(const char *V) : cstr(V) {} union { uint64_t uval; @@ -56,26 +58,28 @@ private: ValueType Value; /// Contains all data for the form. const DWARFUnit *U = nullptr; /// Remember the DWARFUnit at extract time. const DWARFContext *C = nullptr; /// Context for extract time. + + DWARFFormValue(dwarf::Form F, ValueType V) : Form(F), Value(V) {} + public: DWARFFormValue(dwarf::Form F = dwarf::Form(0)) : Form(F) {} + static DWARFFormValue createFromSValue(dwarf::Form F, int64_t V); + static DWARFFormValue createFromUValue(dwarf::Form F, uint64_t V); + static DWARFFormValue createFromPValue(dwarf::Form F, const char *V); + static DWARFFormValue createFromBlockValue(dwarf::Form F, + ArrayRef D); + static DWARFFormValue createFromUnit(dwarf::Form F, const DWARFUnit *Unit, + uint32_t *OffsetPtr); + dwarf::Form getForm() const { return Form; } uint64_t getRawUValue() const { return Value.uval; } - void setForm(dwarf::Form F) { Form = F; } - void setUValue(uint64_t V) { Value.uval = V; } - void setSValue(int64_t V) { Value.sval = V; } - void setPValue(const char *V) { Value.cstr = V; } - - void setBlockValue(const ArrayRef &Data) { - Value.data = Data.data(); - setUValue(Data.size()); - } bool isFormClass(FormClass FC) const; const DWARFUnit *getUnit() const { return U; } void dump(raw_ostream &OS, DIDumpOptions DumpOpts = DIDumpOptions()) const; void dumpSectionedAddress(raw_ostream &OS, DIDumpOptions DumpOpts, - SectionedAddress SA) const; + object::SectionedAddress SA) const; static void dumpAddressSection(const DWARFObject &Obj, raw_ostream &OS, DIDumpOptions DumpOpts, uint64_t SectionIndex); @@ -100,11 +104,16 @@ public: /// getAsFoo functions below return the extracted value as Foo if only /// DWARFFormValue has form class is suitable for representing Foo. Optional getAsReference() const; + struct UnitOffset { + DWARFUnit *Unit; + uint64_t Offset; + }; + Optional getAsRelativeReference() const; Optional getAsUnsignedConstant() const; Optional getAsSignedConstant() const; Optional getAsCString() const; Optional getAsAddress() const; - Optional getAsSectionedAddress() const; + Optional getAsSectionedAddress() const; Optional getAsSectionOffset() const; Optional> getAsBlock() const; Optional getAsCStringOffset() const; @@ -155,6 +164,19 @@ inline Optional toString(const Optional &V) { return None; } +/// Take an optional DWARFFormValue and try to extract a string value from it. +/// +/// \param V and optional DWARFFormValue to attempt to extract the value from. +/// \returns an optional value that contains a value if the form value +/// was valid and was a string. +inline StringRef toStringRef(const Optional &V, + StringRef Default = {}) { + if (V) + if (auto S = V->getAsCString()) + return *S; + return Default; +} + /// Take an optional DWARFFormValue and extract a string value from it. /// /// \param V and optional DWARFFormValue to attempt to extract the value from. @@ -242,7 +264,7 @@ inline Optional toAddress(const Optional &V) { return None; } -inline Optional +inline Optional toSectionedAddress(const Optional &V) { if (V) return V->getAsSectionedAddress(); diff --git a/include/llvm/DebugInfo/DWARF/DWARFGdbIndex.h b/include/llvm/DebugInfo/DWARF/DWARFGdbIndex.h index 073e02903c39..38cd42ddb883 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFGdbIndex.h +++ b/include/llvm/DebugInfo/DWARF/DWARFGdbIndex.h @@ -1,9 +1,8 @@ //===- DWARFGdbIndex.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/DWARF/DWARFListTable.h b/include/llvm/DebugInfo/DWARF/DWARFListTable.h index 9b987314f209..a1ea69b040f0 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFListTable.h +++ b/include/llvm/DebugInfo/DWARF/DWARFListTable.h @@ -1,9 +1,8 @@ //===- DWARFListTable.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -158,7 +157,7 @@ public: uint8_t getAddrSize() const { return Header.getAddrSize(); } void dump(raw_ostream &OS, - llvm::function_ref(uint32_t)> + llvm::function_ref(uint32_t)> LookupPooledAddress, DIDumpOptions DumpOpts = {}) const; @@ -235,7 +234,7 @@ Error DWARFListType::extract(DWARFDataExtractor Data, template void DWARFListTableBase::dump( raw_ostream &OS, - llvm::function_ref(uint32_t)> + llvm::function_ref(uint32_t)> LookupPooledAddress, DIDumpOptions DumpOpts) const { Header.dump(OS, DumpOpts); diff --git a/include/llvm/DebugInfo/DWARF/DWARFObject.h b/include/llvm/DebugInfo/DWARF/DWARFObject.h index d611b5d075c8..1bba74a25d0e 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFObject.h +++ b/include/llvm/DebugInfo/DWARF/DWARFObject.h @@ -1,9 +1,8 @@ //===- DWARFObject.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===-----------------------------------------------------------------------===/ diff --git a/include/llvm/DebugInfo/DWARF/DWARFRelocMap.h b/include/llvm/DebugInfo/DWARF/DWARFRelocMap.h index f51838424614..3add711943d0 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFRelocMap.h +++ b/include/llvm/DebugInfo/DWARF/DWARFRelocMap.h @@ -1,9 +1,8 @@ //===- DWARFRelocMap.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -11,6 +10,7 @@ #define LLVM_DEBUGINFO_DWARF_DWARFRELOCMAP_H #include "llvm/ADT/DenseMap.h" +#include "llvm/Object/RelocationResolver.h" #include namespace llvm { @@ -19,7 +19,11 @@ namespace llvm { /// Section index is -1LL if relocation points to absolute symbol. struct RelocAddrEntry { uint64_t SectionIndex; - uint64_t Value; + object::RelocationRef Reloc; + uint64_t SymbolValue; + Optional Reloc2; + uint64_t SymbolValue2; + object::RelocationResolver Resolver; }; /// In place of applying the relocations to the data we've read from disk we use diff --git a/include/llvm/DebugInfo/DWARF/DWARFSection.h b/include/llvm/DebugInfo/DWARF/DWARFSection.h index 7f8235965297..054524d368ed 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFSection.h +++ b/include/llvm/DebugInfo/DWARF/DWARFSection.h @@ -1,9 +1,8 @@ //===- DWARFSection.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -23,11 +22,6 @@ struct SectionName { bool IsNameUnique; }; -struct SectionedAddress { - uint64_t Address; - uint64_t SectionIndex; -}; - } // end namespace llvm #endif // LLVM_DEBUGINFO_DWARF_DWARFSECTION_H diff --git a/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h b/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h index 8ca5ba13fc23..90d89375fd35 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h +++ b/include/llvm/DebugInfo/DWARF/DWARFTypeUnit.h @@ -1,9 +1,8 @@ //===- DWARFTypeUnit.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/DWARF/DWARFUnit.h b/include/llvm/DebugInfo/DWARF/DWARFUnit.h index 79c3ce1106d5..f9f90db31890 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFUnit.h +++ b/include/llvm/DebugInfo/DWARF/DWARFUnit.h @@ -1,9 +1,8 @@ //===- DWARFUnit.h ----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -49,7 +48,7 @@ class DWARFUnitHeader { uint32_t Offset = 0; // Version, address size, and DWARF format. dwarf::FormParams FormParams; - uint32_t Length = 0; + uint64_t Length = 0; uint64_t AbbrOffset = 0; // For DWO units only. @@ -83,7 +82,7 @@ public: uint8_t getDwarfOffsetByteSize() const { return FormParams.getDwarfOffsetByteSize(); } - uint32_t getLength() const { return Length; } + uint64_t getLength() const { return Length; } uint64_t getAbbrOffset() const { return AbbrOffset; } Optional getDWOId() const { return DWOId; } void setDWOId(uint64_t Id) { @@ -98,8 +97,11 @@ public: return UnitType == dwarf::DW_UT_type || UnitType == dwarf::DW_UT_split_type; } uint8_t getSize() const { return Size; } - // FIXME: Support DWARF64. - uint32_t getNextUnitOffset() const { return Offset + Length + 4; } + uint32_t getNextUnitOffset() const { + return Offset + Length + + (FormParams.Format == llvm::dwarf::DwarfFormat::DWARF64 ? 4 : 0) + + FormParams.getDwarfOffsetByteSize(); + } }; const DWARFUnitIndex &getDWARFUnitIndex(DWARFContext &Context, @@ -173,6 +175,7 @@ struct StrOffsetsContributionDescriptor { StrOffsetsContributionDescriptor(uint64_t Base, uint64_t Size, uint8_t Version, dwarf::DwarfFormat Format) : Base(Base), Size(Size), FormParams({Version, 0, Format}) {} + StrOffsetsContributionDescriptor() = default; uint8_t getVersion() const { return FormParams.Version; } dwarf::DwarfFormat getFormat() const { return FormParams.Format; } @@ -182,7 +185,7 @@ struct StrOffsetsContributionDescriptor { /// Determine whether a contribution to the string offsets table is /// consistent with the relevant section size and that its length is /// a multiple of the size of one of its entries. - Optional + Expected validateContributionSize(DWARFDataExtractor &DA); }; @@ -218,7 +221,7 @@ class DWARFUnit { Optional RngListTable; mutable const DWARFAbbreviationDeclarationSet *Abbrevs; - llvm::Optional BaseAddr; + llvm::Optional BaseAddr; /// The compile unit debug information entry items. std::vector DieArray; @@ -247,14 +250,14 @@ protected: /// Find the unit's contribution to the string offsets table and determine its /// length and form. The given offset is expected to be derived from the unit /// DIE's DW_AT_str_offsets_base attribute. - Optional + Expected> determineStringOffsetsTableContribution(DWARFDataExtractor &DA); /// Find the unit's contribution to the string offsets table and determine its /// length and form. The given offset is expected to be 0 in a dwo file or, /// in a dwp file, the start of the unit's contribution to the string offsets /// table section (as determined by the index table). - Optional + Expected> determineStringOffsetsTableContributionDWO(DWARFDataExtractor &DA); public: @@ -305,7 +308,8 @@ public: RangeSectionBase = Base; } - Optional getAddrOffsetSectionItem(uint32_t Index) const; + Optional + getAddrOffsetSectionItem(uint32_t Index) const; Optional getStringOffsetSectionItem(uint32_t Index) const; DWARFDataExtractor getDebugInfoExtractor() const; @@ -376,7 +380,7 @@ public: llvm_unreachable("Invalid UnitType."); } - llvm::Optional getBaseAddress(); + llvm::Optional getBaseAddress(); DWARFDie getUnitDIE(bool ExtractUnitDIEOnly = true) { extractDIEsIfNeeded(ExtractUnitDIEOnly); @@ -385,6 +389,13 @@ public: return DWARFDie(this, &DieArray[0]); } + DWARFDie getNonSkeletonUnitDIE(bool ExtractUnitDIEOnly = true) { + parseDWO(); + if (DWO) + return DWO->getUnitDIE(ExtractUnitDIEOnly); + return getUnitDIE(ExtractUnitDIEOnly); + } + const char *getCompilationDir(); Optional getDWOId() { extractDIEsIfNeeded(/*CUDieOnly*/ true); @@ -462,13 +473,12 @@ public: DWARFDie getDIEForOffset(uint32_t Offset) { extractDIEsIfNeeded(false); assert(!DieArray.empty()); - auto it = std::lower_bound( - DieArray.begin(), DieArray.end(), Offset, - [](const DWARFDebugInfoEntry &LHS, uint32_t Offset) { - return LHS.getOffset() < Offset; + auto It = + llvm::partition_point(DieArray, [=](const DWARFDebugInfoEntry &DIE) { + return DIE.getOffset() < Offset; }); - if (it != DieArray.end() && it->getOffset() == Offset) - return DWARFDie(this, &*it); + if (It != DieArray.end() && It->getOffset() == Offset) + return DWARFDie(this, &*It); return DWARFDie(); } diff --git a/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h b/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h index 16be5f9401c0..fc8c707c512e 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h +++ b/include/llvm/DebugInfo/DWARF/DWARFUnitIndex.h @@ -1,9 +1,8 @@ //===- DWARFUnitIndex.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/DWARF/DWARFVerifier.h b/include/llvm/DebugInfo/DWARF/DWARFVerifier.h index e47fbea5646e..f1268f220272 100644 --- a/include/llvm/DebugInfo/DWARF/DWARFVerifier.h +++ b/include/llvm/DebugInfo/DWARF/DWARFVerifier.h @@ -1,9 +1,8 @@ //===- DWARFVerifier.h ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/GSYM/FileEntry.h b/include/llvm/DebugInfo/GSYM/FileEntry.h new file mode 100644 index 000000000000..228b4efa0656 --- /dev/null +++ b/include/llvm/DebugInfo/GSYM/FileEntry.h @@ -0,0 +1,68 @@ +//===- FileEntry.h ----------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_GSYM_FILEENTRY_H +#define LLVM_DEBUGINFO_GSYM_FILEENTRY_H + +#include "llvm/ADT/DenseMapInfo.h" +#include "llvm/ADT/Hashing.h" +#include +#include +#include + +namespace llvm { +namespace gsym { + +/// Files in GSYM are contained in FileEntry structs where we split the +/// directory and basename into two different strings in the string +/// table. This allows paths to shared commont directory and filename +/// strings and saves space. +struct FileEntry { + + /// Offsets in the string table. + /// @{ + uint32_t Dir = 0; + uint32_t Base = 0; + /// @} + + FileEntry() = default; + FileEntry(uint32_t D, uint32_t B) : Dir(D), Base(B) {} + + // Implement operator== so that FileEntry can be used as key in + // unordered containers. + bool operator==(const FileEntry &RHS) const { + return Base == RHS.Base && Dir == RHS.Dir; + }; + bool operator!=(const FileEntry &RHS) const { + return Base != RHS.Base || Dir != RHS.Dir; + }; +}; + +} // namespace gsym + +template <> struct DenseMapInfo { + static inline gsym::FileEntry getEmptyKey() { + uint32_t key = DenseMapInfo::getEmptyKey(); + return gsym::FileEntry(key, key); + } + static inline gsym::FileEntry getTombstoneKey() { + uint32_t key = DenseMapInfo::getTombstoneKey(); + return gsym::FileEntry(key, key); + } + static unsigned getHashValue(const gsym::FileEntry &Val) { + return llvm::hash_combine(DenseMapInfo::getHashValue(Val.Dir), + DenseMapInfo::getHashValue(Val.Base)); + } + static bool isEqual(const gsym::FileEntry &LHS, const gsym::FileEntry &RHS) { + return LHS == RHS; + } +}; + +} // namespace llvm +#endif // #ifndef LLVM_DEBUGINFO_GSYM_FILEENTRY_H diff --git a/include/llvm/DebugInfo/GSYM/FunctionInfo.h b/include/llvm/DebugInfo/GSYM/FunctionInfo.h new file mode 100644 index 000000000000..eedb1e638fd1 --- /dev/null +++ b/include/llvm/DebugInfo/GSYM/FunctionInfo.h @@ -0,0 +1,107 @@ +//===- FunctionInfo.h -------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_GSYM_FUNCTIONINFO_H +#define LLVM_DEBUGINFO_GSYM_FUNCTIONINFO_H + +#include "llvm/DebugInfo/GSYM/InlineInfo.h" +#include "llvm/DebugInfo/GSYM/LineEntry.h" +#include "llvm/DebugInfo/GSYM/Range.h" +#include "llvm/DebugInfo/GSYM/StringTable.h" +#include +#include + +namespace llvm { +class raw_ostream; +namespace gsym { + +/// Function information in GSYM files encodes information for one +/// contiguous address range. The name of the function is encoded as +/// a string table offset and allows multiple functions with the same +/// name to share the name string in the string table. Line tables are +/// stored in a sorted vector of gsym::LineEntry objects and are split +/// into line tables for each function. If a function has a discontiguous +/// range, it will be split into two gsym::FunctionInfo objects. If the +/// function has inline functions, the information will be encoded in +/// the "Inline" member, see gsym::InlineInfo for more information. +struct FunctionInfo { + AddressRange Range; + uint32_t Name; ///< String table offset in the string table. + std::vector Lines; + InlineInfo Inline; + + FunctionInfo(uint64_t Addr = 0, uint64_t Size = 0, uint32_t N = 0) + : Range(Addr, Addr + Size), Name(N) {} + + bool hasRichInfo() const { + /// Returns whether we have something else than range and name. When + /// converting information from a symbol table and from debug info, we + /// might end up with multiple FunctionInfo objects for the same range + /// and we need to be able to tell which one is the better object to use. + return !Lines.empty() || Inline.isValid(); + } + + bool isValid() const { + /// Address and size can be zero and there can be no line entries for a + /// symbol so the only indication this entry is valid is if the name is + /// not zero. This can happen when extracting information from symbol + /// tables that do not encode symbol sizes. In that case only the + /// address and name will be filled in. + return Name != 0; + } + + uint64_t startAddress() const { return Range.Start; } + uint64_t endAddress() const { return Range.End; } + uint64_t size() const { return Range.size(); } + void setStartAddress(uint64_t Addr) { Range.Start = Addr; } + void setEndAddress(uint64_t Addr) { Range.End = Addr; } + void setSize(uint64_t Size) { Range.End = Range.Start + Size; } + + void clear() { + Range = {0, 0}; + Name = 0; + Lines.clear(); + Inline.clear(); + } +}; + +inline bool operator==(const FunctionInfo &LHS, const FunctionInfo &RHS) { + return LHS.Range == RHS.Range && LHS.Name == RHS.Name && + LHS.Lines == RHS.Lines && LHS.Inline == RHS.Inline; +} +inline bool operator!=(const FunctionInfo &LHS, const FunctionInfo &RHS) { + return !(LHS == RHS); +} +/// This sorting will order things consistently by address range first, but then +/// followed by inlining being valid and line tables. We might end up with a +/// FunctionInfo from debug info that will have the same range as one from the +/// symbol table, but we want to quickly be able to sort and use the best version +/// when creating the final GSYM file. +inline bool operator<(const FunctionInfo &LHS, const FunctionInfo &RHS) { + // First sort by address range + if (LHS.Range != RHS.Range) + return LHS.Range < RHS.Range; + + // Then sort by inline + if (LHS.Inline.isValid() != RHS.Inline.isValid()) + return RHS.Inline.isValid(); + + // If the number of lines is the same, then compare line table entries + if (LHS.Lines.size() == RHS.Lines.size()) + return LHS.Lines < RHS.Lines; + // Then sort by number of line table entries (more is better) + return LHS.Lines.size() < RHS.Lines.size(); +} + +raw_ostream &operator<<(raw_ostream &OS, const FunctionInfo &R); + +} // namespace gsym +} // namespace llvm + +#endif // #ifndef LLVM_DEBUGINFO_GSYM_FUNCTIONINFO_H diff --git a/include/llvm/DebugInfo/GSYM/InlineInfo.h b/include/llvm/DebugInfo/GSYM/InlineInfo.h new file mode 100644 index 000000000000..222430622932 --- /dev/null +++ b/include/llvm/DebugInfo/GSYM/InlineInfo.h @@ -0,0 +1,78 @@ +//===- InlineInfo.h ---------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_GSYM_INLINEINFO_H +#define LLVM_DEBUGINFO_GSYM_INLINEINFO_H + +#include "llvm/ADT/Optional.h" +#include "llvm/DebugInfo/GSYM/Range.h" +#include +#include + + +namespace llvm { +class raw_ostream; + +namespace gsym { + +/// Inline information stores the name of the inline function along with +/// an array of address ranges. It also stores the call file and call line +/// that called this inline function. This allows us to unwind inline call +/// stacks back to the inline or concrete function that called this +/// function. Inlined functions contained in this function are stored in the +/// "Children" variable. All address ranges must be sorted and all address +/// ranges of all children must be contained in the ranges of this function. +/// Any clients that encode information will need to ensure the ranges are +/// all contined correctly or lookups could fail. Add ranges in these objects +/// must be contained in the top level FunctionInfo address ranges as well. +struct InlineInfo { + + uint32_t Name; ///< String table offset in the string table. + uint32_t CallFile; ///< 1 based file index in the file table. + uint32_t CallLine; ///< Source line number. + AddressRanges Ranges; + std::vector Children; + InlineInfo() : Name(0), CallFile(0), CallLine(0) {} + void clear() { + Name = 0; + CallFile = 0; + CallLine = 0; + Ranges.clear(); + Children.clear(); + } + bool isValid() const { return !Ranges.empty(); } + + using InlineArray = std::vector; + + /// Lookup an address in the InlineInfo object + /// + /// This function is used to symbolicate an inline call stack and can + /// turn one address in the program into one or more inline call stacks + /// and have the stack trace show the original call site from + /// non-inlined code. + /// + /// \param Addr the address to lookup + /// + /// \returns optional vector of InlineInfo objects that describe the + /// inline call stack for a given address, false otherwise. + llvm::Optional getInlineStack(uint64_t Addr) const; +}; + +inline bool operator==(const InlineInfo &LHS, const InlineInfo &RHS) { + return LHS.Name == RHS.Name && LHS.CallFile == RHS.CallFile && + LHS.CallLine == RHS.CallLine && LHS.Ranges == RHS.Ranges && + LHS.Children == RHS.Children; +} + +raw_ostream &operator<<(raw_ostream &OS, const InlineInfo &FI); + +} // namespace gsym +} // namespace llvm + +#endif // #ifndef LLVM_DEBUGINFO_GSYM_INLINEINFO_H diff --git a/include/llvm/DebugInfo/GSYM/LineEntry.h b/include/llvm/DebugInfo/GSYM/LineEntry.h new file mode 100644 index 000000000000..6b9380940bd3 --- /dev/null +++ b/include/llvm/DebugInfo/GSYM/LineEntry.h @@ -0,0 +1,48 @@ +//===- LineEntry.h ----------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_GSYM_LINEENTRY_H +#define LLVM_DEBUGINFO_GSYM_LINEENTRY_H + +#include "llvm/DebugInfo/GSYM/Range.h" + +namespace llvm { +namespace gsym { + +/// Line entries are used to encode the line tables in FunctionInfo objects. +/// They are stored as a sorted vector of these objects and store the +/// address, file and line of the line table row for a given address. The +/// size of a line table entry is calculated by looking at the next entry +/// in the FunctionInfo's vector of entries. +struct LineEntry { + uint64_t Addr; ///< Start address of this line entry. + uint32_t File; ///< 1 based index of file in FileTable + uint32_t Line; ///< Source line number. + LineEntry(uint64_t A = 0, uint32_t F = 0, uint32_t L = 0) + : Addr(A), File(F), Line(L) {} + bool isValid() { return File != 0; } +}; + +inline raw_ostream &operator<<(raw_ostream &OS, const LineEntry &LE) { + return OS << "addr=" << HEX64(LE.Addr) << ", file=" << format("%3u", LE.File) + << ", line=" << format("%3u", LE.Line); +} + +inline bool operator==(const LineEntry &LHS, const LineEntry &RHS) { + return LHS.Addr == RHS.Addr && LHS.File == RHS.File && LHS.Line == RHS.Line; +} +inline bool operator!=(const LineEntry &LHS, const LineEntry &RHS) { + return !(LHS == RHS); +} +inline bool operator<(const LineEntry &LHS, const LineEntry &RHS) { + return LHS.Addr < RHS.Addr; +} +} // namespace gsym +} // namespace llvm +#endif // #ifndef LLVM_DEBUGINFO_GSYM_LINEENTRY_H diff --git a/include/llvm/DebugInfo/GSYM/Range.h b/include/llvm/DebugInfo/GSYM/Range.h new file mode 100644 index 000000000000..772ff244c5b7 --- /dev/null +++ b/include/llvm/DebugInfo/GSYM/Range.h @@ -0,0 +1,87 @@ +//===- AddressRange.h -------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_GSYM_RANGE_H +#define LLVM_DEBUGINFO_GSYM_RANGE_H + +#include "llvm/Support/Format.h" +#include "llvm/Support/raw_ostream.h" +#include +#include + +#define HEX8(v) llvm::format_hex(v, 4) +#define HEX16(v) llvm::format_hex(v, 6) +#define HEX32(v) llvm::format_hex(v, 10) +#define HEX64(v) llvm::format_hex(v, 18) + +namespace llvm { +class raw_ostream; + +namespace gsym { + +/// A class that represents an address range. The range is specified using +/// a start and an end address. +struct AddressRange { + uint64_t Start; + uint64_t End; + AddressRange() : Start(0), End(0) {} + AddressRange(uint64_t S, uint64_t E) : Start(S), End(E) {} + uint64_t size() const { return End - Start; } + bool contains(uint64_t Addr) const { return Start <= Addr && Addr < End; } + bool intersects(const AddressRange &R) const { + return Start < R.End && R.Start < End; + } + + bool operator==(const AddressRange &R) const { + return Start == R.Start && End == R.End; + } + bool operator!=(const AddressRange &R) const { + return !(*this == R); + } + bool operator<(const AddressRange &R) const { + return std::make_pair(Start, End) < std::make_pair(R.Start, R.End); + } +}; + +raw_ostream &operator<<(raw_ostream &OS, const AddressRange &R); + +/// The AddressRanges class helps normalize address range collections. +/// This class keeps a sorted vector of AddressRange objects and can perform +/// insertions and searches efficiently. The address ranges are always sorted +/// and never contain any invalid or empty address ranges. This allows us to +/// emit address ranges into the GSYM file efficiently. Intersecting address +/// ranges are combined during insertion so that we can emit the most compact +/// representation for address ranges when writing to disk. +class AddressRanges { +protected: + using Collection = std::vector; + Collection Ranges; +public: + void clear() { Ranges.clear(); } + bool empty() const { return Ranges.empty(); } + bool contains(uint64_t Addr) const; + void insert(AddressRange Range); + size_t size() const { return Ranges.size(); } + bool operator==(const AddressRanges &RHS) const { + return Ranges == RHS.Ranges; + } + const AddressRange &operator[](size_t i) const { + assert(i < Ranges.size()); + return Ranges[i]; + } + Collection::const_iterator begin() const { return Ranges.begin(); } + Collection::const_iterator end() const { return Ranges.end(); } +}; + +raw_ostream &operator<<(raw_ostream &OS, const AddressRanges &AR); + +} // namespace gsym +} // namespace llvm + +#endif // #ifndef LLVM_DEBUGINFO_GSYM_RANGE_H diff --git a/include/llvm/DebugInfo/GSYM/StringTable.h b/include/llvm/DebugInfo/GSYM/StringTable.h new file mode 100644 index 000000000000..0001b8b82743 --- /dev/null +++ b/include/llvm/DebugInfo/GSYM/StringTable.h @@ -0,0 +1,54 @@ +//===- StringTable.h --------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_GSYM_STRINGTABLE_H +#define LLVM_DEBUGINFO_GSYM_STRINGTABLE_H + +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/DebugInfo/GSYM/Range.h" +#include +#include + + +namespace llvm { +namespace gsym { + +/// String tables in GSYM files are required to start with an empty +/// string at offset zero. Strings must be UTF8 NULL terminated strings. +struct StringTable { + StringRef Data; + StringTable() : Data() {} + StringTable(StringRef D) : Data(D) {} + StringRef operator[](size_t Offset) const { return getString(Offset); } + StringRef getString(uint32_t Offset) const { + if (Offset < Data.size()) { + auto End = Data.find('\0', Offset); + return Data.substr(Offset, End - Offset); + } + return StringRef(); + } + void clear() { Data = StringRef(); } +}; + +inline raw_ostream &operator<<(raw_ostream &OS, const StringTable &S) { + OS << "String table:\n"; + uint32_t Offset = 0; + const size_t Size = S.Data.size(); + while (Offset < Size) { + StringRef Str = S.getString(Offset); + OS << HEX32(Offset) << ": \"" << Str << "\"\n"; + Offset += Str.size() + 1; + } + return OS; +} + +} // namespace gsym +} // namespace llvm +#endif // #ifndef LLVM_DEBUGINFO_GSYM_STRINGTABLE_H diff --git a/include/llvm/DebugInfo/MSF/IMSFFile.h b/include/llvm/DebugInfo/MSF/IMSFFile.h index f98e715e6b15..7e80f96b89ae 100644 --- a/include/llvm/DebugInfo/MSF/IMSFFile.h +++ b/include/llvm/DebugInfo/MSF/IMSFFile.h @@ -1,9 +1,8 @@ //===- IMSFFile.h - Abstract base class for an MSF file ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/MSF/MSFBuilder.h b/include/llvm/DebugInfo/MSF/MSFBuilder.h index 3de98c4ecba8..282870f5b3f1 100644 --- a/include/llvm/DebugInfo/MSF/MSFBuilder.h +++ b/include/llvm/DebugInfo/MSF/MSFBuilder.h @@ -1,9 +1,8 @@ //===- MSFBuilder.h - MSF Directory & Metadata Builder ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/MSF/MSFCommon.h b/include/llvm/DebugInfo/MSF/MSFCommon.h index 2db2b71df4a7..83331b14b8af 100644 --- a/include/llvm/DebugInfo/MSF/MSFCommon.h +++ b/include/llvm/DebugInfo/MSF/MSFCommon.h @@ -1,9 +1,8 @@ //===- MSFCommon.h - Common types and functions for MSF files ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/MSF/MSFError.h b/include/llvm/DebugInfo/MSF/MSFError.h index 5c043a7837b3..fbc4e6928536 100644 --- a/include/llvm/DebugInfo/MSF/MSFError.h +++ b/include/llvm/DebugInfo/MSF/MSFError.h @@ -1,9 +1,8 @@ //===- MSFError.h - Error extensions for MSF Files --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/MSF/MappedBlockStream.h b/include/llvm/DebugInfo/MSF/MappedBlockStream.h index f65e52922da7..593d781b990e 100644 --- a/include/llvm/DebugInfo/MSF/MappedBlockStream.h +++ b/include/llvm/DebugInfo/MSF/MappedBlockStream.h @@ -1,9 +1,8 @@ //==- MappedBlockStream.h - Discontiguous stream data in an MSF --*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/ConcreteSymbolEnumerator.h b/include/llvm/DebugInfo/PDB/ConcreteSymbolEnumerator.h index ac7f19637ab1..49ba20af7263 100644 --- a/include/llvm/DebugInfo/PDB/ConcreteSymbolEnumerator.h +++ b/include/llvm/DebugInfo/PDB/ConcreteSymbolEnumerator.h @@ -1,9 +1,8 @@ //===- ConcreteSymbolEnumerator.h -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/DIA/DIADataStream.h b/include/llvm/DebugInfo/PDB/DIA/DIADataStream.h index 881d7329ab66..f05b58c55507 100644 --- a/include/llvm/DebugInfo/PDB/DIA/DIADataStream.h +++ b/include/llvm/DebugInfo/PDB/DIA/DIADataStream.h @@ -1,9 +1,8 @@ //===- DIADataStream.h - DIA implementation of IPDBDataStream ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAEnumDebugStreams.h b/include/llvm/DebugInfo/PDB/DIA/DIAEnumDebugStreams.h index 1f129052d034..8a00ad45291a 100644 --- a/include/llvm/DebugInfo/PDB/DIA/DIAEnumDebugStreams.h +++ b/include/llvm/DebugInfo/PDB/DIA/DIAEnumDebugStreams.h @@ -1,9 +1,8 @@ //==- DIAEnumDebugStreams.h - DIA Debug Stream Enumerator impl ---*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAEnumFrameData.h b/include/llvm/DebugInfo/PDB/DIA/DIAEnumFrameData.h index f3b02f07e648..bd417c0746b1 100644 --- a/include/llvm/DebugInfo/PDB/DIA/DIAEnumFrameData.h +++ b/include/llvm/DebugInfo/PDB/DIA/DIAEnumFrameData.h @@ -1,9 +1,8 @@ //==- DIAEnumFrameData.h --------------------------------------- -*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAEnumInjectedSources.h b/include/llvm/DebugInfo/PDB/DIA/DIAEnumInjectedSources.h index 4669a8d31196..1f75ca27c4f8 100644 --- a/include/llvm/DebugInfo/PDB/DIA/DIAEnumInjectedSources.h +++ b/include/llvm/DebugInfo/PDB/DIA/DIAEnumInjectedSources.h @@ -1,9 +1,8 @@ //==- DIAEnumInjectedSources.h - DIA Injected Sources Enumerator -*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAEnumLineNumbers.h b/include/llvm/DebugInfo/PDB/DIA/DIAEnumLineNumbers.h index f1cb6268a26d..8800baac105d 100644 --- a/include/llvm/DebugInfo/PDB/DIA/DIAEnumLineNumbers.h +++ b/include/llvm/DebugInfo/PDB/DIA/DIAEnumLineNumbers.h @@ -1,9 +1,8 @@ //==- DIAEnumLineNumbers.h - DIA Line Number Enumerator impl -----*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAEnumSectionContribs.h b/include/llvm/DebugInfo/PDB/DIA/DIAEnumSectionContribs.h index ac2ae317d263..be8613bfba9d 100644 --- a/include/llvm/DebugInfo/PDB/DIA/DIAEnumSectionContribs.h +++ b/include/llvm/DebugInfo/PDB/DIA/DIAEnumSectionContribs.h @@ -1,9 +1,8 @@ //==- DIAEnumSectionContribs.h --------------------------------- -*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAEnumSourceFiles.h b/include/llvm/DebugInfo/PDB/DIA/DIAEnumSourceFiles.h index dac3df06a178..61278994ed36 100644 --- a/include/llvm/DebugInfo/PDB/DIA/DIAEnumSourceFiles.h +++ b/include/llvm/DebugInfo/PDB/DIA/DIAEnumSourceFiles.h @@ -1,9 +1,8 @@ //==- DIAEnumSourceFiles.h - DIA Source File Enumerator impl -----*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAEnumSymbols.h b/include/llvm/DebugInfo/PDB/DIA/DIAEnumSymbols.h index 9689859ae0f8..f55342cea2e5 100644 --- a/include/llvm/DebugInfo/PDB/DIA/DIAEnumSymbols.h +++ b/include/llvm/DebugInfo/PDB/DIA/DIAEnumSymbols.h @@ -1,9 +1,8 @@ //==- DIAEnumSymbols.h - DIA Symbol Enumerator impl --------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAEnumTables.h b/include/llvm/DebugInfo/PDB/DIA/DIAEnumTables.h index f4f856ebb6fd..057cb06fc8ca 100644 --- a/include/llvm/DebugInfo/PDB/DIA/DIAEnumTables.h +++ b/include/llvm/DebugInfo/PDB/DIA/DIAEnumTables.h @@ -1,9 +1,8 @@ //===- DIAEnumTables.h - DIA Tables Enumerator Impl -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAError.h b/include/llvm/DebugInfo/PDB/DIA/DIAError.h index 2b33a65a0a14..96d960599f7e 100644 --- a/include/llvm/DebugInfo/PDB/DIA/DIAError.h +++ b/include/llvm/DebugInfo/PDB/DIA/DIAError.h @@ -1,9 +1,8 @@ //===- DIAError.h - Error extensions for PDB DIA implementation -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAFrameData.h b/include/llvm/DebugInfo/PDB/DIA/DIAFrameData.h index 0ce6cfc93030..c04f7cd00836 100644 --- a/include/llvm/DebugInfo/PDB/DIA/DIAFrameData.h +++ b/include/llvm/DebugInfo/PDB/DIA/DIAFrameData.h @@ -1,9 +1,8 @@ //===- DIAFrameData.h - DIA Impl. of IPDBFrameData ---------------- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAInjectedSource.h b/include/llvm/DebugInfo/PDB/DIA/DIAInjectedSource.h index 635508da84ea..67963a06d939 100644 --- a/include/llvm/DebugInfo/PDB/DIA/DIAInjectedSource.h +++ b/include/llvm/DebugInfo/PDB/DIA/DIAInjectedSource.h @@ -1,9 +1,8 @@ //===- DIAInjectedSource.h - DIA impl for IPDBInjectedSource ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -26,7 +25,7 @@ public: std::string getFileName() const override; std::string getObjectFileName() const override; std::string getVirtualFileName() const override; - PDB_SourceCompression getCompression() const override; + uint32_t getCompression() const override; std::string getCode() const override; private: diff --git a/include/llvm/DebugInfo/PDB/DIA/DIALineNumber.h b/include/llvm/DebugInfo/PDB/DIA/DIALineNumber.h index a59e3a19c8c2..d8bb27220763 100644 --- a/include/llvm/DebugInfo/PDB/DIA/DIALineNumber.h +++ b/include/llvm/DebugInfo/PDB/DIA/DIALineNumber.h @@ -1,9 +1,8 @@ //===- DIALineNumber.h - DIA implementation of IPDBLineNumber ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/DIA/DIARawSymbol.h b/include/llvm/DebugInfo/PDB/DIA/DIARawSymbol.h index 5d4f855c63ca..7f201d3a4e36 100644 --- a/include/llvm/DebugInfo/PDB/DIA/DIARawSymbol.h +++ b/include/llvm/DebugInfo/PDB/DIA/DIARawSymbol.h @@ -1,9 +1,8 @@ //===- DIARawSymbol.h - DIA implementation of IPDBRawSymbol ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/DIA/DIASectionContrib.h b/include/llvm/DebugInfo/PDB/DIA/DIASectionContrib.h index 4688f1f91a89..0972831e8b16 100644 --- a/include/llvm/DebugInfo/PDB/DIA/DIASectionContrib.h +++ b/include/llvm/DebugInfo/PDB/DIA/DIASectionContrib.h @@ -1,9 +1,8 @@ //===- DIASectionContrib.h - DIA Impl. of IPDBSectionContrib ------ C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/DIA/DIASession.h b/include/llvm/DebugInfo/PDB/DIA/DIASession.h index 592e061a8d83..6f62e6061f56 100644 --- a/include/llvm/DebugInfo/PDB/DIA/DIASession.h +++ b/include/llvm/DebugInfo/PDB/DIA/DIASession.h @@ -1,9 +1,8 @@ //===- DIASession.h - DIA implementation of IPDBSession ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/DIA/DIASourceFile.h b/include/llvm/DebugInfo/PDB/DIA/DIASourceFile.h index 1088ea54981c..96edfc9f9e29 100644 --- a/include/llvm/DebugInfo/PDB/DIA/DIASourceFile.h +++ b/include/llvm/DebugInfo/PDB/DIA/DIASourceFile.h @@ -1,9 +1,8 @@ //===- DIASourceFile.h - DIA implementation of IPDBSourceFile ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/DIA/DIASupport.h b/include/llvm/DebugInfo/PDB/DIA/DIASupport.h index 92ebc04ae5a4..1a7c2f3aeeab 100644 --- a/include/llvm/DebugInfo/PDB/DIA/DIASupport.h +++ b/include/llvm/DebugInfo/PDB/DIA/DIASupport.h @@ -1,9 +1,8 @@ //===- DIASupport.h - Common header includes for DIA ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Common defines and header includes for all LLVMDebugInfoPDBDIA. The diff --git a/include/llvm/DebugInfo/PDB/DIA/DIATable.h b/include/llvm/DebugInfo/PDB/DIA/DIATable.h index ce93fa0b86c3..65396a042f06 100644 --- a/include/llvm/DebugInfo/PDB/DIA/DIATable.h +++ b/include/llvm/DebugInfo/PDB/DIA/DIATable.h @@ -1,9 +1,8 @@ //===- DIATable.h - DIA implementation of IPDBTable -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/DIA/DIAUtils.h b/include/llvm/DebugInfo/PDB/DIA/DIAUtils.h index aa843e05de70..5e01d8f10a6e 100644 --- a/include/llvm/DebugInfo/PDB/DIA/DIAUtils.h +++ b/include/llvm/DebugInfo/PDB/DIA/DIAUtils.h @@ -1,9 +1,8 @@ //===- DIAUtils.h - Utility functions for working with DIA ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/GenericError.h b/include/llvm/DebugInfo/PDB/GenericError.h index 997f13f5f30e..ec85d92d2a92 100644 --- a/include/llvm/DebugInfo/PDB/GenericError.h +++ b/include/llvm/DebugInfo/PDB/GenericError.h @@ -1,9 +1,8 @@ -//===- Error.h - system_error extensions for PDB ----------------*- C++ -*-===// +//===- GenericError.h - system_error extensions for PDB ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/IPDBDataStream.h b/include/llvm/DebugInfo/PDB/IPDBDataStream.h index 0d7a286a11a6..4d0589a87915 100644 --- a/include/llvm/DebugInfo/PDB/IPDBDataStream.h +++ b/include/llvm/DebugInfo/PDB/IPDBDataStream.h @@ -1,9 +1,8 @@ //===- IPDBDataStream.h - base interface for child enumerator ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/IPDBEnumChildren.h b/include/llvm/DebugInfo/PDB/IPDBEnumChildren.h index 7017f2600e9b..bfa67d39bc76 100644 --- a/include/llvm/DebugInfo/PDB/IPDBEnumChildren.h +++ b/include/llvm/DebugInfo/PDB/IPDBEnumChildren.h @@ -1,9 +1,8 @@ //===- IPDBEnumChildren.h - base interface for child enumerator -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/IPDBFrameData.h b/include/llvm/DebugInfo/PDB/IPDBFrameData.h index 74679215b880..24138b380db4 100644 --- a/include/llvm/DebugInfo/PDB/IPDBFrameData.h +++ b/include/llvm/DebugInfo/PDB/IPDBFrameData.h @@ -1,9 +1,8 @@ //===- IPDBFrameData.h - base interface for frame data ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/IPDBInjectedSource.h b/include/llvm/DebugInfo/PDB/IPDBInjectedSource.h index e75d64af92bb..d5b36f9846b5 100644 --- a/include/llvm/DebugInfo/PDB/IPDBInjectedSource.h +++ b/include/llvm/DebugInfo/PDB/IPDBInjectedSource.h @@ -1,16 +1,14 @@ //===- IPDBInjectedSource.h - base class for PDB injected file --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef LLVM_DEBUGINFO_PDB_IPDBINJECTEDSOURCE_H #define LLVM_DEBUGINFO_PDB_IPDBINJECTEDSOURCE_H -#include "PDBTypes.h" #include "llvm/Support/raw_ostream.h" #include #include @@ -33,7 +31,10 @@ public: virtual std::string getFileName() const = 0; virtual std::string getObjectFileName() const = 0; virtual std::string getVirtualFileName() const = 0; - virtual PDB_SourceCompression getCompression() const = 0; + // The returned value depends on the PDB producer, + // but 0 is guaranteed to mean "no compression". + // The enum PDB_SourceCompression lists known return values. + virtual uint32_t getCompression() const = 0; virtual std::string getCode() const = 0; }; } // namespace pdb diff --git a/include/llvm/DebugInfo/PDB/IPDBLineNumber.h b/include/llvm/DebugInfo/PDB/IPDBLineNumber.h index e20080f2fbfc..77e88999497e 100644 --- a/include/llvm/DebugInfo/PDB/IPDBLineNumber.h +++ b/include/llvm/DebugInfo/PDB/IPDBLineNumber.h @@ -1,9 +1,8 @@ //===- IPDBLineNumber.h - base interface for PDB line no. info ---*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/IPDBRawSymbol.h b/include/llvm/DebugInfo/PDB/IPDBRawSymbol.h index 7c818d7cadeb..b24e712e3b78 100644 --- a/include/llvm/DebugInfo/PDB/IPDBRawSymbol.h +++ b/include/llvm/DebugInfo/PDB/IPDBRawSymbol.h @@ -1,9 +1,8 @@ //===- IPDBRawSymbol.h - base interface for PDB symbol types ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/IPDBSectionContrib.h b/include/llvm/DebugInfo/PDB/IPDBSectionContrib.h index 4fda62404672..c5cf4bbe5560 100644 --- a/include/llvm/DebugInfo/PDB/IPDBSectionContrib.h +++ b/include/llvm/DebugInfo/PDB/IPDBSectionContrib.h @@ -1,9 +1,8 @@ //==- IPDBSectionContrib.h - Interfaces for PDB SectionContribs --*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/IPDBSession.h b/include/llvm/DebugInfo/PDB/IPDBSession.h index 88fd02c0a345..aa8d9c76d63e 100644 --- a/include/llvm/DebugInfo/PDB/IPDBSession.h +++ b/include/llvm/DebugInfo/PDB/IPDBSession.h @@ -1,9 +1,8 @@ //===- IPDBSession.h - base interface for a PDB symbol context --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/IPDBSourceFile.h b/include/llvm/DebugInfo/PDB/IPDBSourceFile.h index 3676c4030b13..d7e49fb70580 100644 --- a/include/llvm/DebugInfo/PDB/IPDBSourceFile.h +++ b/include/llvm/DebugInfo/PDB/IPDBSourceFile.h @@ -1,9 +1,8 @@ //===- IPDBSourceFile.h - base interface for a PDB source file --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/IPDBTable.h b/include/llvm/DebugInfo/PDB/IPDBTable.h index 4561c4e847b2..55ca230d58c4 100644 --- a/include/llvm/DebugInfo/PDB/IPDBTable.h +++ b/include/llvm/DebugInfo/PDB/IPDBTable.h @@ -1,9 +1,8 @@ //===- IPDBTable.h - Base Interface for a PDB Symbol Context ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h b/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h index 9eef4041d0a1..568f0c98c559 100644 --- a/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h +++ b/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h @@ -1,9 +1,8 @@ //===- DbiModuleDescriptor.h - PDB module information -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h b/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h index ac7f741afefa..4f5d28bbd05a 100644 --- a/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h +++ b/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.h @@ -1,9 +1,8 @@ //===- DbiModuleDescriptorBuilder.h - PDB module information ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/DbiModuleList.h b/include/llvm/DebugInfo/PDB/Native/DbiModuleList.h index 5f6e7ab92a96..14223273c898 100644 --- a/include/llvm/DebugInfo/PDB/Native/DbiModuleList.h +++ b/include/llvm/DebugInfo/PDB/Native/DbiModuleList.h @@ -1,9 +1,8 @@ //===- DbiModuleList.h - PDB module information list ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/DbiStream.h b/include/llvm/DebugInfo/PDB/Native/DbiStream.h index a3ca607efbef..7d75c159b7ae 100644 --- a/include/llvm/DebugInfo/PDB/Native/DbiStream.h +++ b/include/llvm/DebugInfo/PDB/Native/DbiStream.h @@ -1,9 +1,8 @@ //===- DbiStream.h - PDB Dbi Stream (Stream 3) Access -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -11,6 +10,7 @@ #define LLVM_DEBUGINFO_PDB_RAW_PDBDBISTREAM_H #include "llvm/DebugInfo/CodeView/DebugSubsection.h" +#include "llvm/DebugInfo/CodeView/DebugFrameDataSubsection.h" #include "llvm/DebugInfo/MSF/MappedBlockStream.h" #include "llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h" #include "llvm/DebugInfo/PDB/Native/DbiModuleList.h" @@ -80,7 +80,10 @@ public: FixedStreamArray getSectionHeaders() const; - FixedStreamArray getFpoRecords(); + bool hasOldFpoRecords() const; + FixedStreamArray getOldFpoRecords() const; + bool hasNewFpoRecords() const; + const codeview::DebugFrameDataSubsectionRef &getNewFpoRecords() const; FixedStreamArray getSectionMap() const; void visitSectionContributions(ISectionContribVisitor &Visitor) const; @@ -91,7 +94,11 @@ private: Error initializeSectionContributionData(); Error initializeSectionHeadersData(PDBFile *Pdb); Error initializeSectionMapData(); - Error initializeFpoRecords(PDBFile *Pdb); + Error initializeOldFpoRecords(PDBFile *Pdb); + Error initializeNewFpoRecords(PDBFile *Pdb); + + Expected> + createIndexedStreamForHeaderType(PDBFile *Pdb, DbgHeaderType Type) const; std::unique_ptr Stream; @@ -117,8 +124,11 @@ private: std::unique_ptr SectionHeaderStream; FixedStreamArray SectionHeaders; - std::unique_ptr FpoStream; - FixedStreamArray FpoRecords; + std::unique_ptr OldFpoStream; + FixedStreamArray OldFpoRecords; + + std::unique_ptr NewFpoStream; + codeview::DebugFrameDataSubsectionRef NewFpoRecords; const DbiStreamHeader *Header; }; diff --git a/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h b/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h index b538de576677..d9be238af07b 100644 --- a/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h +++ b/include/llvm/DebugInfo/PDB/Native/DbiStreamBuilder.h @@ -1,9 +1,8 @@ //===- DbiStreamBuilder.h - PDB Dbi Stream Creation -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/EnumTables.h b/include/llvm/DebugInfo/PDB/Native/EnumTables.h index c018445630fe..70161fadf7d2 100644 --- a/include/llvm/DebugInfo/PDB/Native/EnumTables.h +++ b/include/llvm/DebugInfo/PDB/Native/EnumTables.h @@ -1,9 +1,8 @@ //===- EnumTables.h - Enum to string conversion tables ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/Formatters.h b/include/llvm/DebugInfo/PDB/Native/Formatters.h index 7d5eab2e2a09..29c957eeb5e0 100644 --- a/include/llvm/DebugInfo/PDB/Native/Formatters.h +++ b/include/llvm/DebugInfo/PDB/Native/Formatters.h @@ -1,9 +1,8 @@ //===- Formatters.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h b/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h index 4c39ca762b5b..a49795600028 100644 --- a/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h +++ b/include/llvm/DebugInfo/PDB/Native/GSIStreamBuilder.h @@ -1,9 +1,8 @@ //===- GSIStreamBuilder.h - PDB Publics/Globals Stream Creation -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h b/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h index 7f84564ee988..404baaa25077 100644 --- a/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h +++ b/include/llvm/DebugInfo/PDB/Native/GlobalsStream.h @@ -1,9 +1,8 @@ //===- GlobalsStream.h - PDB Index of Symbols by Name -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/Hash.h b/include/llvm/DebugInfo/PDB/Native/Hash.h index 1f11d43ecdd4..b048d878a12c 100644 --- a/include/llvm/DebugInfo/PDB/Native/Hash.h +++ b/include/llvm/DebugInfo/PDB/Native/Hash.h @@ -1,9 +1,8 @@ //===- Hash.h - PDB hash functions ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/HashTable.h b/include/llvm/DebugInfo/PDB/Native/HashTable.h index 34cc6179688b..aa38417bcf4c 100644 --- a/include/llvm/DebugInfo/PDB/Native/HashTable.h +++ b/include/llvm/DebugInfo/PDB/Native/HashTable.h @@ -1,9 +1,8 @@ //===- HashTable.h - PDB Hash Table -----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -32,21 +31,21 @@ namespace pdb { Error readSparseBitVector(BinaryStreamReader &Stream, SparseBitVector<> &V); Error writeSparseBitVector(BinaryStreamWriter &Writer, SparseBitVector<> &Vec); -template class HashTable; +template class HashTable; -template +template class HashTableIterator - : public iterator_facade_base, + : public iterator_facade_base, std::forward_iterator_tag, - std::pair> { - friend HashTable; + const std::pair> { + friend HashTable; - HashTableIterator(const HashTable &Map, uint32_t Index, + HashTableIterator(const HashTable &Map, uint32_t Index, bool IsEnd) : Map(&Map), Index(Index), IsEnd(IsEnd) {} public: - HashTableIterator(const HashTable &Map) : Map(&Map) { + HashTableIterator(const HashTable &Map) : Map(&Map) { int I = Map.Present.find_first(); if (I == -1) { Index = 0; @@ -73,6 +72,12 @@ public: assert(Map->Present.test(Index)); return Map->Buckets[Index]; } + + // Implement postfix op++ in terms of prefix op++ by using the superclass + // implementation. + using iterator_facade_base, + std::forward_iterator_tag, + const std::pair>::operator++; HashTableIterator &operator++() { while (Index < Map->Buckets.size()) { ++Index; @@ -88,24 +93,13 @@ private: bool isEnd() const { return IsEnd; } uint32_t index() const { return Index; } - const HashTable *Map; + const HashTable *Map; uint32_t Index; bool IsEnd; }; -template struct PdbHashTraits {}; - -template <> struct PdbHashTraits { - uint32_t hashLookupKey(uint32_t N) const { return N; } - uint32_t storageKeyToLookupKey(uint32_t N) const { return N; } - uint32_t lookupKeyToStorageKey(uint32_t N) { return N; } -}; - -template > +template class HashTable { - using iterator = HashTableIterator; - friend iterator; - struct Header { support::ulittle32_t Size; support::ulittle32_t Capacity; @@ -114,10 +108,11 @@ class HashTable { using BucketList = std::vector>; public: - HashTable() { Buckets.resize(8); } + using const_iterator = HashTableIterator; + friend const_iterator; - explicit HashTable(TraitsT Traits) : HashTable(8, std::move(Traits)) {} - HashTable(uint32_t Capacity, TraitsT Traits) : Traits(Traits) { + HashTable() { Buckets.resize(8); } + explicit HashTable(uint32_t Capacity) { Buckets.resize(Capacity); } @@ -144,7 +139,7 @@ public: return EC; if (Present.intersects(Deleted)) return make_error(raw_error_code::corrupt_file, - "Present bit vector interesects deleted!"); + "Present bit vector intersects deleted!"); for (uint32_t P : Present) { if (auto EC = Stream.readInteger(Buckets[P].first)) @@ -217,19 +212,20 @@ public: uint32_t capacity() const { return Buckets.size(); } uint32_t size() const { return Present.count(); } - iterator begin() const { return iterator(*this); } - iterator end() const { return iterator(*this, 0, true); } + const_iterator begin() const { return const_iterator(*this); } + const_iterator end() const { return const_iterator(*this, 0, true); } /// Find the entry whose key has the specified hash value, using the specified /// traits defining hash function and equality. - template iterator find_as(const Key &K) const { + template + const_iterator find_as(const Key &K, TraitsT &Traits) const { uint32_t H = Traits.hashLookupKey(K) % capacity(); uint32_t I = H; Optional FirstUnused; do { if (isPresent(I)) { if (Traits.storageKeyToLookupKey(Buckets[I].first) == K) - return iterator(*this, I, false); + return const_iterator(*this, I, false); } else { if (!FirstUnused) FirstUnused = I; @@ -248,17 +244,19 @@ public: // table were Present. But this would violate the load factor constraints // that we impose, so it should never happen. assert(FirstUnused); - return iterator(*this, *FirstUnused, true); + return const_iterator(*this, *FirstUnused, true); } /// Set the entry using a key type that the specified Traits can convert /// from a real key to an internal key. - template bool set_as(const Key &K, ValueT V) { - return set_as_internal(K, std::move(V), None); + template + bool set_as(const Key &K, ValueT V, TraitsT &Traits) { + return set_as_internal(K, std::move(V), Traits, None); } - template ValueT get(const Key &K) const { - auto Iter = find_as(K); + template + ValueT get(const Key &K, TraitsT &Traits) const { + auto Iter = find_as(K, Traits); assert(Iter != end()); return (*Iter).second; } @@ -267,7 +265,6 @@ protected: bool isPresent(uint32_t K) const { return Present.test(K); } bool isDeleted(uint32_t K) const { return Deleted.test(K); } - TraitsT Traits; BucketList Buckets; mutable SparseBitVector<> Present; mutable SparseBitVector<> Deleted; @@ -275,9 +272,10 @@ protected: private: /// Set the entry using a key type that the specified Traits can convert /// from a real key to an internal key. - template - bool set_as_internal(const Key &K, ValueT V, Optional InternalKey) { - auto Entry = find_as(K); + template + bool set_as_internal(const Key &K, ValueT V, TraitsT &Traits, + Optional InternalKey) { + auto Entry = find_as(K, Traits); if (Entry != end()) { assert(isPresent(Entry.index())); assert(Traits.storageKeyToLookupKey(Buckets[Entry.index()].first) == K); @@ -294,15 +292,16 @@ private: Present.set(Entry.index()); Deleted.reset(Entry.index()); - grow(); + grow(Traits); - assert((find_as(K)) != end()); + assert((find_as(K, Traits)) != end()); return true; } static uint32_t maxLoad(uint32_t capacity) { return capacity * 2 / 3 + 1; } - void grow() { + template + void grow(TraitsT &Traits) { uint32_t S = size(); uint32_t MaxLoad = maxLoad(capacity()); if (S < maxLoad(capacity())) @@ -314,10 +313,11 @@ private: // Growing requires rebuilding the table and re-hashing every item. Make a // copy with a larger capacity, insert everything into the copy, then swap // it in. - HashTable NewMap(NewCapacity, Traits); + HashTable NewMap(NewCapacity); for (auto I : Present) { auto LookupKey = Traits.storageKeyToLookupKey(Buckets[I].first); - NewMap.set_as_internal(LookupKey, Buckets[I].second, Buckets[I].first); + NewMap.set_as_internal(LookupKey, Buckets[I].second, Traits, + Buckets[I].first); } Buckets.swap(NewMap.Buckets); diff --git a/include/llvm/DebugInfo/PDB/Native/ISectionContribVisitor.h b/include/llvm/DebugInfo/PDB/Native/ISectionContribVisitor.h index fb00d6ad4bc7..717dce2f2737 100644 --- a/include/llvm/DebugInfo/PDB/Native/ISectionContribVisitor.h +++ b/include/llvm/DebugInfo/PDB/Native/ISectionContribVisitor.h @@ -1,9 +1,8 @@ //===- ISectionContribVisitor.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/InfoStream.h b/include/llvm/DebugInfo/PDB/Native/InfoStream.h index 8c52b042f289..315b09356ae3 100644 --- a/include/llvm/DebugInfo/PDB/Native/InfoStream.h +++ b/include/llvm/DebugInfo/PDB/Native/InfoStream.h @@ -1,9 +1,8 @@ //===- InfoStream.h - PDB Info Stream (Stream 1) Access ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h b/include/llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h index 101127a355f5..208a37c45d49 100644 --- a/include/llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h +++ b/include/llvm/DebugInfo/PDB/Native/InfoStreamBuilder.h @@ -1,9 +1,8 @@ //===- InfoStreamBuilder.h - PDB Info Stream Creation -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/InjectedSourceStream.h b/include/llvm/DebugInfo/PDB/Native/InjectedSourceStream.h new file mode 100644 index 000000000000..d0cac3749bca --- /dev/null +++ b/include/llvm/DebugInfo/PDB/Native/InjectedSourceStream.h @@ -0,0 +1,44 @@ +//===- InjectedSourceStream.h - PDB Headerblock Stream Access ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_RAW_PDBINJECTEDSOURCESTREAM_H +#define LLVM_DEBUGINFO_PDB_RAW_PDBINJECTEDSOURCESTREAM_H + +#include "llvm/DebugInfo/PDB/Native/HashTable.h" +#include "llvm/DebugInfo/PDB/Native/RawTypes.h" +#include "llvm/Support/Error.h" + +namespace llvm { +namespace msf { +class MappedBlockStream; +} +namespace pdb { +class PDBFile; +class PDBStringTable; + +class InjectedSourceStream { +public: + InjectedSourceStream(std::unique_ptr Stream); + Error reload(const PDBStringTable &Strings); + + using const_iterator = HashTable::const_iterator; + const_iterator begin() const { return InjectedSourceTable.begin(); } + const_iterator end() const { return InjectedSourceTable.end(); } + + uint32_t size() const { return InjectedSourceTable.size(); } + +private: + std::unique_ptr Stream; + + const SrcHeaderBlockHeader* Header; + HashTable InjectedSourceTable; +}; +} +} + +#endif diff --git a/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h b/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h index 8d590df288f3..cb1ffc729512 100644 --- a/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h +++ b/include/llvm/DebugInfo/PDB/Native/ModuleDebugStream.h @@ -1,9 +1,8 @@ //===- ModuleDebugStream.h - PDB Module Info Stream Access ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -69,6 +68,8 @@ public: findChecksumsSubsection() const; private: + Error reloadSerialize(BinaryStreamReader &Reader); + DbiModuleDescriptor Mod; uint32_t Signature; diff --git a/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h b/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h index 01b8f1b5da56..1df059ffa9fd 100644 --- a/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h +++ b/include/llvm/DebugInfo/PDB/Native/NamedStreamMap.h @@ -1,9 +1,8 @@ //===- NamedStreamMap.h - PDB Named Stream Map ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -60,7 +59,7 @@ private: NamedStreamMapTraits HashTraits; /// Closed hash table from Offset -> StreamNumber, where Offset is the offset /// of the stream name in NamesBuffer. - HashTable OffsetIndexMap; + HashTable OffsetIndexMap; /// Buffer of string data. std::vector NamesBuffer; diff --git a/include/llvm/DebugInfo/PDB/Native/NativeCompilandSymbol.h b/include/llvm/DebugInfo/PDB/Native/NativeCompilandSymbol.h index 3cd465503044..50d437642d0f 100644 --- a/include/llvm/DebugInfo/PDB/Native/NativeCompilandSymbol.h +++ b/include/llvm/DebugInfo/PDB/Native/NativeCompilandSymbol.h @@ -1,9 +1,8 @@ //===- NativeCompilandSymbol.h - native impl for compiland syms -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/NativeEnumGlobals.h b/include/llvm/DebugInfo/PDB/Native/NativeEnumGlobals.h index 4442a1ec41fb..073878afd129 100644 --- a/include/llvm/DebugInfo/PDB/Native/NativeEnumGlobals.h +++ b/include/llvm/DebugInfo/PDB/Native/NativeEnumGlobals.h @@ -1,9 +1,8 @@ //==- NativeEnumGlobals.h - Native Global Enumerator impl --------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/NativeEnumInjectedSources.h b/include/llvm/DebugInfo/PDB/Native/NativeEnumInjectedSources.h new file mode 100644 index 000000000000..ca1e22bd82a2 --- /dev/null +++ b/include/llvm/DebugInfo/PDB/Native/NativeEnumInjectedSources.h @@ -0,0 +1,43 @@ +//==- NativeEnumInjectedSources.cpp - Native Injected Source Enumerator --*-==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEBUGINFO_PDB_NATIVE_NATIVEENUMINJECTEDSOURCES_H +#define LLVM_DEBUGINFO_PDB_NATIVE_NATIVEENUMINJECTEDSOURCES_H + +#include "llvm/DebugInfo/PDB/IPDBEnumChildren.h" +#include "llvm/DebugInfo/PDB/IPDBInjectedSource.h" +#include "llvm/DebugInfo/PDB/Native/InjectedSourceStream.h" + +namespace llvm { +namespace pdb { + +class InjectedSourceStream; +class PDBStringTable; + +class NativeEnumInjectedSources : public IPDBEnumChildren { +public: + NativeEnumInjectedSources(PDBFile &File, const InjectedSourceStream &IJS, + const PDBStringTable &Strings); + + uint32_t getChildCount() const override; + std::unique_ptr + getChildAtIndex(uint32_t Index) const override; + std::unique_ptr getNext() override; + void reset() override; + +private: + PDBFile &File; + const InjectedSourceStream &Stream; + const PDBStringTable &Strings; + InjectedSourceStream::const_iterator Cur; +}; + +} // namespace pdb +} // namespace llvm + +#endif diff --git a/include/llvm/DebugInfo/PDB/Native/NativeEnumModules.h b/include/llvm/DebugInfo/PDB/Native/NativeEnumModules.h index c268641a1008..94f1ee18ed9f 100644 --- a/include/llvm/DebugInfo/PDB/Native/NativeEnumModules.h +++ b/include/llvm/DebugInfo/PDB/Native/NativeEnumModules.h @@ -1,9 +1,8 @@ //==- NativeEnumModules.h - Native Module Enumerator impl --------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/NativeEnumTypes.h b/include/llvm/DebugInfo/PDB/Native/NativeEnumTypes.h index f8ac1655dc61..25c56567384f 100644 --- a/include/llvm/DebugInfo/PDB/Native/NativeEnumTypes.h +++ b/include/llvm/DebugInfo/PDB/Native/NativeEnumTypes.h @@ -1,9 +1,8 @@ //==- NativeEnumTypes.h - Native Type Enumerator impl ------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h b/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h index f4030da1d026..280358d02305 100644 --- a/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h +++ b/include/llvm/DebugInfo/PDB/Native/NativeExeSymbol.h @@ -1,9 +1,8 @@ //===- NativeExeSymbol.h - native impl for PDBSymbolExe ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h b/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h index 6505a7d39573..4133be220713 100644 --- a/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h +++ b/include/llvm/DebugInfo/PDB/Native/NativeRawSymbol.h @@ -1,9 +1,8 @@ //==- NativeRawSymbol.h - Native implementation of IPDBRawSymbol -*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/NativeSession.h b/include/llvm/DebugInfo/PDB/Native/NativeSession.h index 4878e47d3121..ee7d8cdec93b 100644 --- a/include/llvm/DebugInfo/PDB/Native/NativeSession.h +++ b/include/llvm/DebugInfo/PDB/Native/NativeSession.h @@ -1,9 +1,8 @@ //===- NativeSession.h - Native implementation of IPDBSession ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/NativeSymbolEnumerator.h b/include/llvm/DebugInfo/PDB/Native/NativeSymbolEnumerator.h index acc5eb8ff2c2..063585097899 100644 --- a/include/llvm/DebugInfo/PDB/Native/NativeSymbolEnumerator.h +++ b/include/llvm/DebugInfo/PDB/Native/NativeSymbolEnumerator.h @@ -1,9 +1,8 @@ //===- NativeSymbolEnumerator.h - info about enumerator values --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/NativeTypeArray.h b/include/llvm/DebugInfo/PDB/Native/NativeTypeArray.h index 10e68e6df450..262864fd709f 100644 --- a/include/llvm/DebugInfo/PDB/Native/NativeTypeArray.h +++ b/include/llvm/DebugInfo/PDB/Native/NativeTypeArray.h @@ -1,9 +1,8 @@ //===- NativeTypeArray.h ------------------------------------------ C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/NativeTypeBuiltin.h b/include/llvm/DebugInfo/PDB/Native/NativeTypeBuiltin.h index 725dfb89222f..8bb09f05d0bc 100644 --- a/include/llvm/DebugInfo/PDB/Native/NativeTypeBuiltin.h +++ b/include/llvm/DebugInfo/PDB/Native/NativeTypeBuiltin.h @@ -1,9 +1,8 @@ //===- NativeTypeBuiltin.h ---------------------------------------- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/NativeTypeEnum.h b/include/llvm/DebugInfo/PDB/Native/NativeTypeEnum.h index a5cbefc18111..2068c88fc74a 100644 --- a/include/llvm/DebugInfo/PDB/Native/NativeTypeEnum.h +++ b/include/llvm/DebugInfo/PDB/Native/NativeTypeEnum.h @@ -1,9 +1,8 @@ //===- NativeTypeEnum.h - info about enum type ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/NativeTypeFunctionSig.h b/include/llvm/DebugInfo/PDB/Native/NativeTypeFunctionSig.h index 1b1b87f6581f..a7ea287dffc8 100644 --- a/include/llvm/DebugInfo/PDB/Native/NativeTypeFunctionSig.h +++ b/include/llvm/DebugInfo/PDB/Native/NativeTypeFunctionSig.h @@ -1,9 +1,8 @@ //===- NativeTypeFunctionSig.h - info about function signature ---*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/NativeTypePointer.h b/include/llvm/DebugInfo/PDB/Native/NativeTypePointer.h index bcb7431fecf1..446f77db0f6c 100644 --- a/include/llvm/DebugInfo/PDB/Native/NativeTypePointer.h +++ b/include/llvm/DebugInfo/PDB/Native/NativeTypePointer.h @@ -1,9 +1,8 @@ //===- NativeTypePointer.h - info about pointer type -------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/NativeTypeTypedef.h b/include/llvm/DebugInfo/PDB/Native/NativeTypeTypedef.h index 06eb6fcf3764..fe8a6f7f2bda 100644 --- a/include/llvm/DebugInfo/PDB/Native/NativeTypeTypedef.h +++ b/include/llvm/DebugInfo/PDB/Native/NativeTypeTypedef.h @@ -1,9 +1,8 @@ //===- NativeTypeTypedef.h - info about typedef ------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/NativeTypeUDT.h b/include/llvm/DebugInfo/PDB/Native/NativeTypeUDT.h index 84821d8731be..8f4dee3e658c 100644 --- a/include/llvm/DebugInfo/PDB/Native/NativeTypeUDT.h +++ b/include/llvm/DebugInfo/PDB/Native/NativeTypeUDT.h @@ -1,9 +1,8 @@ //===- NativeTypeUDT.h - info about class/struct type ------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/NativeTypeVTShape.h b/include/llvm/DebugInfo/PDB/Native/NativeTypeVTShape.h index a996f34ef859..4ec0f9bf6b3d 100644 --- a/include/llvm/DebugInfo/PDB/Native/NativeTypeVTShape.h +++ b/include/llvm/DebugInfo/PDB/Native/NativeTypeVTShape.h @@ -1,9 +1,8 @@ //===- NativeTypeVTShape.h - info about virtual table shape ------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/PDBFile.h b/include/llvm/DebugInfo/PDB/Native/PDBFile.h index 5e39ac3e37b7..56de4030167d 100644 --- a/include/llvm/DebugInfo/PDB/Native/PDBFile.h +++ b/include/llvm/DebugInfo/PDB/Native/PDBFile.h @@ -1,9 +1,8 @@ //===- PDBFile.h - Low level interface to a PDB file ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -33,6 +32,7 @@ namespace pdb { class DbiStream; class GlobalsStream; class InfoStream; +class InjectedSourceStream; class PDBStringTable; class PDBFileBuilder; class PublicsStream; @@ -84,7 +84,12 @@ public: ArrayRef getDirectoryBlockArray() const; - std::unique_ptr createIndexedStream(uint16_t SN); + std::unique_ptr + createIndexedStream(uint16_t SN) const; + Expected> + safelyCreateIndexedStream(uint32_t StreamIndex) const; + Expected> + safelyCreateNamedStream(StringRef Name); msf::MSFStreamLayout getStreamLayout(uint32_t StreamIdx) const; msf::MSFStreamLayout getFpmStreamLayout() const; @@ -100,6 +105,7 @@ public: Expected getPDBPublicsStream(); Expected getPDBSymbolStream(); Expected getStringTable(); + Expected getInjectedSourceStream(); BumpPtrAllocator &getAllocator() { return Allocator; } @@ -111,15 +117,11 @@ public: bool hasPDBSymbolStream(); bool hasPDBTpiStream() const; bool hasPDBStringTable(); + bool hasPDBInjectedSourceStream(); uint32_t getPointerSize(); private: - Expected> - safelyCreateIndexedStream(const msf::MSFLayout &Layout, - BinaryStreamRef MsfData, - uint32_t StreamIndex) const; - std::string FilePath; BumpPtrAllocator &Allocator; @@ -136,6 +138,7 @@ private: std::unique_ptr Symbols; std::unique_ptr DirectoryStream; std::unique_ptr StringTableStream; + std::unique_ptr InjectedSources; std::unique_ptr Strings; }; } diff --git a/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h b/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h index 37458749a8d8..2abaa5f4cdc4 100644 --- a/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h +++ b/include/llvm/DebugInfo/PDB/Native/PDBFileBuilder.h @@ -1,9 +1,8 @@ //===- PDBFileBuilder.h - PDB File Creation ---------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -98,7 +97,7 @@ private: PDBStringTableBuilder Strings; StringTableHashTraits InjectedSourceHashTraits; - HashTable InjectedSourceTable; + HashTable InjectedSourceTable; SmallVector InjectedSources; diff --git a/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h b/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h index 29167c966d42..57f0b64a32a6 100644 --- a/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h +++ b/include/llvm/DebugInfo/PDB/Native/PDBStringTable.h @@ -1,9 +1,8 @@ //===- PDBStringTable.h - PDB String Table -----------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h b/include/llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h index 0f81c18eafe6..57267ef5c6c5 100644 --- a/include/llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h +++ b/include/llvm/DebugInfo/PDB/Native/PDBStringTableBuilder.h @@ -1,9 +1,8 @@ //===- PDBStringTableBuilder.h - PDB String Table Builder -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/DebugInfo/PDB/Native/PublicsStream.h b/include/llvm/DebugInfo/PDB/Native/PublicsStream.h index 2d0222a9071a..ee28d108df8b 100644 --- a/include/llvm/DebugInfo/PDB/Native/PublicsStream.h +++ b/include/llvm/DebugInfo/PDB/Native/PublicsStream.h @@ -1,9 +1,8 @@ //===- PublicsStream.h - PDB Public Symbol Stream -------- ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/RawConstants.h b/include/llvm/DebugInfo/PDB/Native/RawConstants.h index fbbd3318d958..0dde5ef66932 100644 --- a/include/llvm/DebugInfo/PDB/Native/RawConstants.h +++ b/include/llvm/DebugInfo/PDB/Native/RawConstants.h @@ -1,9 +1,8 @@ //===- RawConstants.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/RawError.h b/include/llvm/DebugInfo/PDB/Native/RawError.h index 97d11b4f20d1..aadb64c2e3f1 100644 --- a/include/llvm/DebugInfo/PDB/Native/RawError.h +++ b/include/llvm/DebugInfo/PDB/Native/RawError.h @@ -1,9 +1,8 @@ //===- RawError.h - Error extensions for raw PDB implementation -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/RawTypes.h b/include/llvm/DebugInfo/PDB/Native/RawTypes.h index 8f6d6611c032..6119e6e5db26 100644 --- a/include/llvm/DebugInfo/PDB/Native/RawTypes.h +++ b/include/llvm/DebugInfo/PDB/Native/RawTypes.h @@ -1,9 +1,8 @@ //===- RawTypes.h -----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -177,7 +176,7 @@ struct DbiStreamHeader { }; static_assert(sizeof(DbiStreamHeader) == 64, "Invalid DbiStreamHeader size!"); -/// The header preceeding the File Info Substream of the DBI stream. +/// The header preceding the File Info Substream of the DBI stream. struct FileInfoSubstreamHeader { /// Total # of modules, should match number of records in the ModuleInfo /// substream. @@ -208,7 +207,7 @@ struct ModInfoFlags { static const uint16_t TypeServerIndexShift = 8; }; -/// The header preceeding each entry in the Module Info substream of the DBI +/// The header preceding each entry in the Module Info substream of the DBI /// stream. Corresponds to the type MODI in the reference implementation. struct ModuleInfoHeader { /// Currently opened module. This field is a pointer in the reference @@ -273,7 +272,7 @@ struct PublicsStreamHeader { support::ulittle32_t NumSections; }; -// The header preceeding the global TPI stream. +// The header preceding the global TPI stream. // This corresponds to `HDR` in PDB/dbi/tpi.h. struct TpiStreamHeader { struct EmbeddedBuf { @@ -301,7 +300,7 @@ struct TpiStreamHeader { const uint32_t MinTpiHashBuckets = 0x1000; const uint32_t MaxTpiHashBuckets = 0x40000; -/// The header preceeding the global PDB Stream (Stream 1) +/// The header preceding the global PDB Stream (Stream 1) struct InfoStreamHeader { support::ulittle32_t Version; support::ulittle32_t Signature; @@ -309,7 +308,7 @@ struct InfoStreamHeader { codeview::GUID Guid; }; -/// The header preceeding the /names stream. +/// The header preceding the /names stream. struct PDBStringTableHeader { support::ulittle32_t Signature; // PDBStringTableSignature support::ulittle32_t HashVersion; // 1 or 2 @@ -342,7 +341,6 @@ struct SrcHeaderBlockEntry { short Padding; // Pad to 4 bytes. char Reserved[8]; }; - static_assert(sizeof(SrcHeaderBlockEntry) == 40, "Incorrect struct size!"); } // namespace pdb diff --git a/include/llvm/DebugInfo/PDB/Native/SymbolCache.h b/include/llvm/DebugInfo/PDB/Native/SymbolCache.h index 08e1d41e6ee9..0b15ab474f71 100644 --- a/include/llvm/DebugInfo/PDB/Native/SymbolCache.h +++ b/include/llvm/DebugInfo/PDB/Native/SymbolCache.h @@ -1,9 +1,8 @@ //==- SymbolCache.h - Cache of native symbols and ids ------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/SymbolStream.h b/include/llvm/DebugInfo/PDB/Native/SymbolStream.h index ae9f7d657b70..4fe1bd9734e4 100644 --- a/include/llvm/DebugInfo/PDB/Native/SymbolStream.h +++ b/include/llvm/DebugInfo/PDB/Native/SymbolStream.h @@ -1,9 +1,8 @@ //===- SymbolStream.cpp - PDB Symbol Stream Access --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/TpiHashing.h b/include/llvm/DebugInfo/PDB/Native/TpiHashing.h index c2996ccf1825..4ac60a80e701 100644 --- a/include/llvm/DebugInfo/PDB/Native/TpiHashing.h +++ b/include/llvm/DebugInfo/PDB/Native/TpiHashing.h @@ -1,9 +1,8 @@ //===- TpiHashing.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/TpiStream.h b/include/llvm/DebugInfo/PDB/Native/TpiStream.h index b76576a7a263..1b7fd2d54cb2 100644 --- a/include/llvm/DebugInfo/PDB/Native/TpiStream.h +++ b/include/llvm/DebugInfo/PDB/Native/TpiStream.h @@ -1,9 +1,8 @@ //===- TpiStream.cpp - PDB Type Info (TPI) Stream 2 Access ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h b/include/llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h index 411720d6f56b..72d98e9c2c4d 100644 --- a/include/llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h +++ b/include/llvm/DebugInfo/PDB/Native/TpiStreamBuilder.h @@ -1,9 +1,8 @@ //===- TpiStreamBuilder.h - PDB Tpi Stream Creation -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDB.h b/include/llvm/DebugInfo/PDB/PDB.h index 9f9da39ca6cc..6d734dc2f243 100644 --- a/include/llvm/DebugInfo/PDB/PDB.h +++ b/include/llvm/DebugInfo/PDB/PDB.h @@ -1,9 +1,8 @@ //===- PDB.h - base header file for creating a PDB reader -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBContext.h b/include/llvm/DebugInfo/PDB/PDBContext.h index 0ce49f5ef922..7b6793f0a639 100644 --- a/include/llvm/DebugInfo/PDB/PDBContext.h +++ b/include/llvm/DebugInfo/PDB/PDBContext.h @@ -1,9 +1,8 @@ //===-- PDBContext.h --------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===/ @@ -44,15 +43,18 @@ namespace pdb { void dump(raw_ostream &OS, DIDumpOptions DIDumpOpts) override; DILineInfo getLineInfoForAddress( - uint64_t Address, + object::SectionedAddress Address, DILineInfoSpecifier Specifier = DILineInfoSpecifier()) override; DILineInfoTable getLineInfoForAddressRange( - uint64_t Address, uint64_t Size, + object::SectionedAddress Address, uint64_t Size, DILineInfoSpecifier Specifier = DILineInfoSpecifier()) override; DIInliningInfo getInliningInfoForAddress( - uint64_t Address, + object::SectionedAddress Address, DILineInfoSpecifier Specifier = DILineInfoSpecifier()) override; + std::vector + getLocalsForAddress(object::SectionedAddress Address) override; + private: std::string getFunctionName(uint64_t Address, DINameKind NameKind) const; std::unique_ptr Session; diff --git a/include/llvm/DebugInfo/PDB/PDBExtras.h b/include/llvm/DebugInfo/PDB/PDBExtras.h index aaec71aa8c90..45aba013e7c8 100644 --- a/include/llvm/DebugInfo/PDB/PDBExtras.h +++ b/include/llvm/DebugInfo/PDB/PDBExtras.h @@ -1,9 +1,8 @@ //===- PDBExtras.h - helper functions and classes for PDBs ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -28,7 +27,8 @@ raw_ostream &operator<<(raw_ostream &OS, const PDB_VariantType &Value); raw_ostream &operator<<(raw_ostream &OS, const PDB_CallingConv &Conv); raw_ostream &operator<<(raw_ostream &OS, const PDB_BuiltinType &Type); raw_ostream &operator<<(raw_ostream &OS, const PDB_DataKind &Data); -raw_ostream &operator<<(raw_ostream &OS, const codeview::RegisterId &Reg); +raw_ostream &operator<<(raw_ostream &OS, + const llvm::codeview::CPURegister &CpuReg); raw_ostream &operator<<(raw_ostream &OS, const PDB_LocType &Loc); raw_ostream &operator<<(raw_ostream &OS, const codeview::ThunkOrdinal &Thunk); raw_ostream &operator<<(raw_ostream &OS, const PDB_Checksum &Checksum); @@ -37,13 +37,12 @@ raw_ostream &operator<<(raw_ostream &OS, const PDB_SymType &Tag); raw_ostream &operator<<(raw_ostream &OS, const PDB_MemberAccess &Access); raw_ostream &operator<<(raw_ostream &OS, const PDB_UdtType &Type); raw_ostream &operator<<(raw_ostream &OS, const PDB_Machine &Machine); -raw_ostream &operator<<(raw_ostream &OS, - const PDB_SourceCompression &Compression); raw_ostream &operator<<(raw_ostream &OS, const Variant &Value); raw_ostream &operator<<(raw_ostream &OS, const VersionInfo &Version); raw_ostream &operator<<(raw_ostream &OS, const TagStats &Stats); +raw_ostream& dumpPDBSourceCompression(raw_ostream& OS, uint32_t Compression); template void dumpSymbolField(raw_ostream &OS, StringRef Name, T Value, int Indent) { diff --git a/include/llvm/DebugInfo/PDB/PDBSymDumper.h b/include/llvm/DebugInfo/PDB/PDBSymDumper.h index c976935c48e0..f81b15f2353d 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymDumper.h +++ b/include/llvm/DebugInfo/PDB/PDBSymDumper.h @@ -1,9 +1,8 @@ //===- PDBSymDumper.h - base interface for PDB symbol dumper *- C++ -----*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbol.h b/include/llvm/DebugInfo/PDB/PDBSymbol.h index 3a74f7c3aace..d9004a8894d9 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbol.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbol.h @@ -1,9 +1,8 @@ //===- PDBSymbol.h - base class for user-facing symbol types -----*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolAnnotation.h b/include/llvm/DebugInfo/PDB/PDBSymbolAnnotation.h index ef00df15cb0a..c76466a97b66 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolAnnotation.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolAnnotation.h @@ -1,9 +1,8 @@ //===- PDBSymbolAnnotation.h - Accessors for querying PDB annotations ---*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef LLVM_DEBUGINFO_PDB_PDBSYMBOLANNOTATION_H diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolBlock.h b/include/llvm/DebugInfo/PDB/PDBSymbolBlock.h index 2cf9c72a8886..cf471450d989 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolBlock.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolBlock.h @@ -1,9 +1,8 @@ //===- PDBSymbolBlock.h - Accessors for querying PDB blocks -------------*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef LLVM_DEBUGINFO_PDB_PDBSYMBOLBLOCK_H diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolCompiland.h b/include/llvm/DebugInfo/PDB/PDBSymbolCompiland.h index 04dbd962ebd4..ca8b39d03f86 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolCompiland.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolCompiland.h @@ -1,9 +1,8 @@ //===- PDBSymbolCompiland.h - Accessors for querying PDB compilands -----*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef LLVM_DEBUGINFO_PDB_PDBSYMBOLCOMPILAND_H diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolCompilandDetails.h b/include/llvm/DebugInfo/PDB/PDBSymbolCompilandDetails.h index 3d651a464d94..b82bb6c0a352 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolCompilandDetails.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolCompilandDetails.h @@ -1,9 +1,8 @@ //===- PDBSymbolCompilandDetails.h - PDB compiland details ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolCompilandEnv.h b/include/llvm/DebugInfo/PDB/PDBSymbolCompilandEnv.h index ffc408314d9a..61607a03593d 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolCompilandEnv.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolCompilandEnv.h @@ -1,9 +1,8 @@ //===- PDBSymbolCompilandEnv.h - compiland environment variables *- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolCustom.h b/include/llvm/DebugInfo/PDB/PDBSymbolCustom.h index c29e4c31d3f3..75a86411643a 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolCustom.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolCustom.h @@ -1,9 +1,8 @@ //===- PDBSymbolCustom.h - compiler-specific types --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolData.h b/include/llvm/DebugInfo/PDB/PDBSymbolData.h index 217e1e976e6b..7e9b69d7cf4b 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolData.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolData.h @@ -1,9 +1,8 @@ //===- PDBSymbolData.h - PDB data (e.g. variable) accessors -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolExe.h b/include/llvm/DebugInfo/PDB/PDBSymbolExe.h index 366d0cf4777f..1a9fb240a248 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolExe.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolExe.h @@ -1,9 +1,8 @@ //===- PDBSymbolExe.h - Accessors for querying executables in a PDB ----*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h b/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h index 129e557c7f25..6be27c8d3bc7 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolFunc.h @@ -1,9 +1,8 @@ //===- PDBSymbolFunc.h - class representing a function instance -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h b/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h index 18db8a50fd1b..7152249cbd03 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugEnd.h @@ -1,9 +1,8 @@ //===- PDBSymbolFuncDebugEnd.h - function end bounds info -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h b/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h index 83d82f0cbcc5..3125c271d2e8 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolFuncDebugStart.h @@ -1,9 +1,8 @@ //===- PDBSymbolFuncDebugStart.h - function start bounds info ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h b/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h index 8b2617fcd757..3625e23f014f 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolLabel.h @@ -1,9 +1,8 @@ //===- PDBSymbolLabel.h - label info ----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h b/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h index 9def3edb469a..e2b2545d78ec 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolPublicSymbol.h @@ -1,9 +1,8 @@ //===- PDBSymbolPublicSymbol.h - public symbol info -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h b/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h index 7bb0555362db..274de8b0b16f 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolThunk.h @@ -1,9 +1,8 @@ //===- PDBSymbolThunk.h - Support for querying PDB thunks ---------------*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h index 488f668bdc10..c0215c9ee4b1 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeArray.h @@ -1,9 +1,8 @@ //===- PDBSymbolTypeArray.h - array type information ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h index 550deedd7504..bab292ee0d46 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeBaseClass.h @@ -1,9 +1,8 @@ //===- PDBSymbolTypeBaseClass.h - base class type information ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h index e07e88802b8f..7d94c3c97a2b 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeBuiltin.h @@ -1,9 +1,8 @@ //===- PDBSymbolTypeBuiltin.h - builtin type information --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeCustom.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeCustom.h index 0d8979c9c5c5..dc647aff48d3 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeCustom.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeCustom.h @@ -1,9 +1,8 @@ //===- PDBSymbolTypeCustom.h - custom compiler type information -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeDimension.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeDimension.h index 58292a63501f..7a9e43785d67 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeDimension.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeDimension.h @@ -1,9 +1,8 @@ //===- PDBSymbolTypeDimension.h - array dimension type info -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h index f463047bb5b5..3ac72801b202 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeEnum.h @@ -1,9 +1,8 @@ //===- PDBSymbolTypeEnum.h - enum type info ---------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeFriend.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeFriend.h index 5b940b0737af..c4d9dd6308a3 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeFriend.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeFriend.h @@ -1,9 +1,8 @@ //===- PDBSymbolTypeFriend.h - friend type info -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h index 074cb418fc82..22d3623496f2 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionArg.h @@ -1,9 +1,8 @@ //===- PDBSymbolTypeFunctionArg.h - function arg type info ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h index dfdf436197c3..a1491ca2e415 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeFunctionSig.h @@ -1,9 +1,8 @@ //===- PDBSymbolTypeFunctionSig.h - function signature type info *- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeManaged.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeManaged.h index d716abd640c6..6bc70bca82e7 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeManaged.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeManaged.h @@ -1,9 +1,8 @@ //===- PDBSymbolTypeManaged.h - managed type info ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h index 300d6722fc4d..b36f459e880c 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypePointer.h @@ -1,9 +1,8 @@ //===- PDBSymbolTypePointer.h - pointer type info ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h index d6e2a36486d5..2712d0617e0e 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeTypedef.h @@ -1,9 +1,8 @@ //===- PDBSymbolTypeTypedef.h - typedef type info ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h index 937dd6c87221..3e73ad7ac85a 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeUDT.h @@ -1,9 +1,8 @@ //===- PDBSymbolTypeUDT.h - UDT type info -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h index 6efce4bbd686..e8161d311ea7 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTable.h @@ -1,9 +1,8 @@ //===- PDBSymbolTypeVTable.h - VTable type info -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h b/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h index 8949052b0c0f..614060867042 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolTypeVTableShape.h @@ -1,9 +1,8 @@ //===- PDBSymbolTypeVTableShape.h - VTable shape info -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h b/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h index e935ac6ce0dc..cc29d38c2578 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolUnknown.h @@ -1,9 +1,8 @@ //===- PDBSymbolUnknown.h - unknown symbol type -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h b/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h index 4e8c99fc8d89..fd812cb2f793 100644 --- a/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h +++ b/include/llvm/DebugInfo/PDB/PDBSymbolUsingNamespace.h @@ -1,9 +1,8 @@ //===- PDBSymbolUsingNamespace.h - using namespace info ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/PDB/PDBTypes.h b/include/llvm/DebugInfo/PDB/PDBTypes.h index 917f3ed73910..c26d8d1ed10c 100644 --- a/include/llvm/DebugInfo/PDB/PDBTypes.h +++ b/include/llvm/DebugInfo/PDB/PDBTypes.h @@ -1,9 +1,8 @@ //===- PDBTypes.h - Defines enums for various fields contained in PDB ----====// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -127,6 +126,7 @@ enum class PDB_Machine { Am33 = 0x13, Amd64 = 0x8664, Arm = 0x1C0, + Arm64 = 0xaa64, ArmNT = 0x1C4, Ebc = 0xEBC, x86 = 0x14C, @@ -146,11 +146,69 @@ enum class PDB_Machine { WceMipsV2 = 0x169 }; -enum class PDB_SourceCompression { - None, - RunLengthEncoded, - Huffman, - LZ, +// A struct with an inner unnamed enum with explicit underlying type resuls +// in an enum class that can implicitly convert to the underlying type, which +// is convenient for this enum. +struct PDB_SourceCompression { + enum : uint32_t { + // No compression. Produced e.g. by `link.exe /natvis:foo.natvis`. + None, + // Not known what produces this. + RunLengthEncoded, + // Not known what produces this. + Huffman, + // Not known what produces this. + LZ, + // Produced e.g. by `csc /debug`. The encoded data is its own mini-stream + // with the following layout (in little endian): + // GUID LanguageTypeGuid; + // GUID LanguageVendorGuid; + // GUID DocumentTypeGuid; + // GUID HashFunctionGuid; + // uint32_t HashDataSize; + // uint32_t CompressedDataSize; + // Followed by HashDataSize bytes containing a hash checksum, + // followed by CompressedDataSize bytes containing source contents. + // + // CompressedDataSize can be 0, in this case only the hash data is present. + // (CompressedDataSize is != 0 e.g. if `/embed` is passed to csc.exe.) + // The compressed data format is: + // uint32_t UncompressedDataSize; + // If UncompressedDataSize is 0, the data is stored uncompressed and + // CompressedDataSize stores the uncompressed size. + // If UncompressedDataSize is != 0, then the data is in raw deflate + // encoding as described in rfc1951. + // + // A GUID is 16 bytes, stored in the usual + // uint32_t + // uint16_t + // uint16_t + // uint8_t[24] + // layout. + // + // Well-known GUIDs for LanguageTypeGuid are: + // 63a08714-fc37-11d2-904c-00c04fa302a1 C + // 3a12d0b7-c26c-11d0-b442-00a0244a1dd2 C++ + // 3f5162f8-07c6-11d3-9053-00c04fa302a1 C# + // af046cd1-d0e1-11d2-977c-00a0c9b4d50c Cobol + // ab4f38c9-b6e6-43ba-be3b-58080b2ccce3 F# + // 3a12d0b4-c26c-11d0-b442-00a0244a1dd2 Java + // 3a12d0b6-c26c-11d0-b442-00a0244a1dd2 JScript + // af046cd2-d0e1-11d2-977c-00a0c9b4d50c Pascal + // 3a12d0b8-c26c-11d0-b442-00a0244a1dd2 Visual Basic + // + // Well-known GUIDs for LanguageVendorGuid are: + // 994b45c4-e6e9-11d2-903f-00c04fa302a1 Microsoft + // + // Well-known GUIDs for DocumentTypeGuid are: + // 5a869d0b-6611-11d3-bd2a-0000f80849bd Text + // + // Well-known GUIDs for HashFunctionGuid are: + // 406ea660-64cf-4c82-b6f0-42d48172a799 MD5 (HashDataSize is 16) + // ff1816ec-aa5e-4d10-87f7-6f4963833460 SHA1 (HashDataSize is 20) + // 8829d00f-11b8-4213-878b-770e8597ac16 SHA256 (HashDataSize is 32) + DotNet = 101, + }; }; /// These values correspond to the CV_call_e enumeration, and are documented diff --git a/include/llvm/DebugInfo/PDB/UDTLayout.h b/include/llvm/DebugInfo/PDB/UDTLayout.h index c4234c191e21..c67b093b63c0 100644 --- a/include/llvm/DebugInfo/PDB/UDTLayout.h +++ b/include/llvm/DebugInfo/PDB/UDTLayout.h @@ -1,9 +1,8 @@ //===- UDTLayout.h - UDT layout info ----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/DebugInfo/Symbolize/DIPrinter.h b/include/llvm/DebugInfo/Symbolize/DIPrinter.h index ab82be3706d8..db7a61a8f160 100644 --- a/include/llvm/DebugInfo/Symbolize/DIPrinter.h +++ b/include/llvm/DebugInfo/Symbolize/DIPrinter.h @@ -1,9 +1,8 @@ //===- llvm/DebugInfo/Symbolize/DIPrinter.h ---------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -21,15 +20,22 @@ namespace llvm { struct DILineInfo; class DIInliningInfo; struct DIGlobal; +struct DILocal; namespace symbolize { class DIPrinter { +public: + enum class OutputStyle { LLVM, GNU }; + +private: raw_ostream &OS; bool PrintFunctionNames; bool PrintPretty; int PrintSourceContext; bool Verbose; + bool Basenames; + OutputStyle Style; void print(const DILineInfo &Info, bool Inlined); void printContext(const std::string &FileName, int64_t Line); @@ -37,14 +43,16 @@ class DIPrinter { public: DIPrinter(raw_ostream &OS, bool PrintFunctionNames = true, bool PrintPretty = false, int PrintSourceContext = 0, - bool Verbose = false) + bool Verbose = false, bool Basenames = false, + OutputStyle Style = OutputStyle::LLVM) : OS(OS), PrintFunctionNames(PrintFunctionNames), PrintPretty(PrintPretty), PrintSourceContext(PrintSourceContext), - Verbose(Verbose) {} + Verbose(Verbose), Basenames(Basenames), Style(Style) {} DIPrinter &operator<<(const DILineInfo &Info); DIPrinter &operator<<(const DIInliningInfo &Info); DIPrinter &operator<<(const DIGlobal &Global); + DIPrinter &operator<<(const DILocal &Local); }; } } diff --git a/include/llvm/DebugInfo/Symbolize/SymbolizableModule.h b/include/llvm/DebugInfo/Symbolize/SymbolizableModule.h index e576a91e887c..506ecc424b4c 100644 --- a/include/llvm/DebugInfo/Symbolize/SymbolizableModule.h +++ b/include/llvm/DebugInfo/Symbolize/SymbolizableModule.h @@ -1,9 +1,8 @@ //===- SymbolizableModule.h -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,13 +24,16 @@ class SymbolizableModule { public: virtual ~SymbolizableModule() = default; - virtual DILineInfo symbolizeCode(uint64_t ModuleOffset, + virtual DILineInfo symbolizeCode(object::SectionedAddress ModuleOffset, FunctionNameKind FNKind, bool UseSymbolTable) const = 0; - virtual DIInliningInfo symbolizeInlinedCode(uint64_t ModuleOffset, - FunctionNameKind FNKind, - bool UseSymbolTable) const = 0; - virtual DIGlobal symbolizeData(uint64_t ModuleOffset) const = 0; + virtual DIInliningInfo + symbolizeInlinedCode(object::SectionedAddress ModuleOffset, + FunctionNameKind FNKind, bool UseSymbolTable) const = 0; + virtual DIGlobal + symbolizeData(object::SectionedAddress ModuleOffset) const = 0; + virtual std::vector + symbolizeFrame(object::SectionedAddress ModuleOffset) const = 0; // Return true if this is a 32-bit x86 PE COFF module. virtual bool isWin32Module() const = 0; diff --git a/include/llvm/DebugInfo/Symbolize/Symbolize.h b/include/llvm/DebugInfo/Symbolize/Symbolize.h index 289148f569db..d3da28ca0b7b 100644 --- a/include/llvm/DebugInfo/Symbolize/Symbolize.h +++ b/include/llvm/DebugInfo/Symbolize/Symbolize.h @@ -1,9 +1,8 @@ //===- Symbolize.h ----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -36,35 +35,35 @@ using FunctionNameKind = DILineInfoSpecifier::FunctionNameKind; class LLVMSymbolizer { public: struct Options { - FunctionNameKind PrintFunctions; - bool UseSymbolTable : 1; - bool Demangle : 1; - bool RelativeAddresses : 1; + FunctionNameKind PrintFunctions = FunctionNameKind::LinkageName; + bool UseSymbolTable = true; + bool Demangle = true; + bool RelativeAddresses = false; std::string DefaultArch; std::vector DsymHints; - - Options(FunctionNameKind PrintFunctions = FunctionNameKind::LinkageName, - bool UseSymbolTable = true, bool Demangle = true, - bool RelativeAddresses = false, std::string DefaultArch = "") - : PrintFunctions(PrintFunctions), UseSymbolTable(UseSymbolTable), - Demangle(Demangle), RelativeAddresses(RelativeAddresses), - DefaultArch(std::move(DefaultArch)) {} + std::string FallbackDebugPath; + std::string DWPName; }; - LLVMSymbolizer(const Options &Opts = Options()) : Opts(Opts) {} + LLVMSymbolizer() = default; + LLVMSymbolizer(const Options &Opts) : Opts(Opts) {} ~LLVMSymbolizer() { flush(); } + Expected symbolizeCode(const ObjectFile &Obj, + object::SectionedAddress ModuleOffset); Expected symbolizeCode(const std::string &ModuleName, - uint64_t ModuleOffset, - StringRef DWPName = ""); - Expected symbolizeInlinedCode(const std::string &ModuleName, - uint64_t ModuleOffset, - StringRef DWPName = ""); + object::SectionedAddress ModuleOffset); + Expected + symbolizeInlinedCode(const std::string &ModuleName, + object::SectionedAddress ModuleOffset); Expected symbolizeData(const std::string &ModuleName, - uint64_t ModuleOffset); + object::SectionedAddress ModuleOffset); + Expected> + symbolizeFrame(const std::string &ModuleName, + object::SectionedAddress ModuleOffset); void flush(); static std::string @@ -74,14 +73,23 @@ public: private: // Bundles together object file with code/data and object file with // corresponding debug info. These objects can be the same. - using ObjectPair = std::pair; + using ObjectPair = std::pair; + + Expected + symbolizeCodeCommon(SymbolizableModule *Info, + object::SectionedAddress ModuleOffset); /// Returns a SymbolizableModule or an error if loading debug info failed. /// Only one attempt is made to load a module, and errors during loading are /// only reported once. Subsequent calls to get module info for a module that /// failed to load will return nullptr. Expected - getOrCreateModuleInfo(const std::string &ModuleName, StringRef DWPName = ""); + getOrCreateModuleInfo(const std::string &ModuleName); + + Expected + createModuleInfo(const ObjectFile *Obj, + std::unique_ptr Context, + StringRef ModuleName); ObjectFile *lookUpDsymFile(const std::string &Path, const MachOObjectFile *ExeObj, diff --git a/include/llvm/Demangle/Compiler.h b/include/llvm/Demangle/Compiler.h deleted file mode 100644 index 248d6e3a7faa..000000000000 --- a/include/llvm/Demangle/Compiler.h +++ /dev/null @@ -1,93 +0,0 @@ -//===--- Compiler.h ---------------------------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -// -// This file contains a variety of feature test macros copied from -// include/llvm/Support/Compiler.h so that LLVMDemangle does not need to take -// a dependency on LLVMSupport. -//===----------------------------------------------------------------------===// - -#ifndef LLVM_DEMANGLE_COMPILER_H -#define LLVM_DEMANGLE_COMPILER_H - -#ifdef _MSC_VER -// snprintf is implemented in VS 2015 -#if _MSC_VER < 1900 -#define snprintf _snprintf_s -#endif -#endif - -#ifndef __has_feature -#define __has_feature(x) 0 -#endif - -#ifndef __has_cpp_attribute -#define __has_cpp_attribute(x) 0 -#endif - -#ifndef __has_attribute -#define __has_attribute(x) 0 -#endif - -#ifndef __has_builtin -#define __has_builtin(x) 0 -#endif - -#ifndef LLVM_GNUC_PREREQ -#if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) -#define LLVM_GNUC_PREREQ(maj, min, patch) \ - ((__GNUC__ << 20) + (__GNUC_MINOR__ << 10) + __GNUC_PATCHLEVEL__ >= \ - ((maj) << 20) + ((min) << 10) + (patch)) -#elif defined(__GNUC__) && defined(__GNUC_MINOR__) -#define LLVM_GNUC_PREREQ(maj, min, patch) \ - ((__GNUC__ << 20) + (__GNUC_MINOR__ << 10) >= ((maj) << 20) + ((min) << 10)) -#else -#define LLVM_GNUC_PREREQ(maj, min, patch) 0 -#endif -#endif - -#if __has_attribute(used) || LLVM_GNUC_PREREQ(3, 1, 0) -#define LLVM_ATTRIBUTE_USED __attribute__((__used__)) -#else -#define LLVM_ATTRIBUTE_USED -#endif - -#if __has_builtin(__builtin_unreachable) || LLVM_GNUC_PREREQ(4, 5, 0) -#define LLVM_BUILTIN_UNREACHABLE __builtin_unreachable() -#elif defined(_MSC_VER) -#define LLVM_BUILTIN_UNREACHABLE __assume(false) -#endif - -#if __has_attribute(noinline) || LLVM_GNUC_PREREQ(3, 4, 0) -#define LLVM_ATTRIBUTE_NOINLINE __attribute__((noinline)) -#elif defined(_MSC_VER) -#define LLVM_ATTRIBUTE_NOINLINE __declspec(noinline) -#else -#define LLVM_ATTRIBUTE_NOINLINE -#endif - -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) -#define LLVM_DUMP_METHOD LLVM_ATTRIBUTE_NOINLINE LLVM_ATTRIBUTE_USED -#else -#define LLVM_DUMP_METHOD LLVM_ATTRIBUTE_NOINLINE -#endif - -#if __cplusplus > 201402L && __has_cpp_attribute(fallthrough) -#define LLVM_FALLTHROUGH [[fallthrough]] -#elif __has_cpp_attribute(gnu::fallthrough) -#define LLVM_FALLTHROUGH [[gnu::fallthrough]] -#elif !__cplusplus -// Workaround for llvm.org/PR23435, since clang 3.6 and below emit a spurious -// error when __has_cpp_attribute is given a scoped attribute in C mode. -#define LLVM_FALLTHROUGH -#elif __has_cpp_attribute(clang::fallthrough) -#define LLVM_FALLTHROUGH [[clang::fallthrough]] -#else -#define LLVM_FALLTHROUGH -#endif - -#endif diff --git a/include/llvm/Demangle/Demangle.h b/include/llvm/Demangle/Demangle.h index 4c9dc9569e18..6fea7ef13f11 100644 --- a/include/llvm/Demangle/Demangle.h +++ b/include/llvm/Demangle/Demangle.h @@ -1,9 +1,8 @@ //===--- Demangle.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -11,6 +10,7 @@ #define LLVM_DEMANGLE_DEMANGLE_H #include +#include namespace llvm { /// This is a llvm local version of __cxa_demangle. Other than the name and @@ -36,6 +36,13 @@ enum MSDemangleFlags { MSDF_None = 0, MSDF_DumpBackrefs = 1 << 0 }; char *microsoftDemangle(const char *mangled_name, char *buf, size_t *n, int *status, MSDemangleFlags Flags = MSDF_None); +/// Attempt to demangle a string using different demangling schemes. +/// The function uses heuristics to determine which demangling scheme to use. +/// \param MangledName - reference to string to demangle. +/// \returns - the demangled string, or a copy of the input string if no +/// demangling occurred. +std::string demangle(const std::string &MangledName); + /// "Partial" demangler. This supports demangling a string into an AST /// (typically an intermediate stage in itaniumDemangle) and querying certain /// properties or partially printing the demangled name. diff --git a/include/llvm/Demangle/DemangleConfig.h b/include/llvm/Demangle/DemangleConfig.h new file mode 100644 index 000000000000..73f89d357c85 --- /dev/null +++ b/include/llvm/Demangle/DemangleConfig.h @@ -0,0 +1,99 @@ +//===--- DemangleConfig.h ---------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains a variety of feature test macros copied from +// include/llvm/Support/Compiler.h so that LLVMDemangle does not need to take +// a dependency on LLVMSupport. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_DEMANGLE_COMPILER_H +#define LLVM_DEMANGLE_COMPILER_H + +#ifdef _MSC_VER +// snprintf is implemented in VS 2015 +#if _MSC_VER < 1900 +#define snprintf _snprintf_s +#endif +#endif + +#ifndef __has_feature +#define __has_feature(x) 0 +#endif + +#ifndef __has_cpp_attribute +#define __has_cpp_attribute(x) 0 +#endif + +#ifndef __has_attribute +#define __has_attribute(x) 0 +#endif + +#ifndef __has_builtin +#define __has_builtin(x) 0 +#endif + +#ifndef DEMANGLE_GNUC_PREREQ +#if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) +#define DEMANGLE_GNUC_PREREQ(maj, min, patch) \ + ((__GNUC__ << 20) + (__GNUC_MINOR__ << 10) + __GNUC_PATCHLEVEL__ >= \ + ((maj) << 20) + ((min) << 10) + (patch)) +#elif defined(__GNUC__) && defined(__GNUC_MINOR__) +#define DEMANGLE_GNUC_PREREQ(maj, min, patch) \ + ((__GNUC__ << 20) + (__GNUC_MINOR__ << 10) >= ((maj) << 20) + ((min) << 10)) +#else +#define DEMANGLE_GNUC_PREREQ(maj, min, patch) 0 +#endif +#endif + +#if __has_attribute(used) || DEMANGLE_GNUC_PREREQ(3, 1, 0) +#define DEMANGLE_ATTRIBUTE_USED __attribute__((__used__)) +#else +#define DEMANGLE_ATTRIBUTE_USED +#endif + +#if __has_builtin(__builtin_unreachable) || DEMANGLE_GNUC_PREREQ(4, 5, 0) +#define DEMANGLE_UNREACHABLE __builtin_unreachable() +#elif defined(_MSC_VER) +#define DEMANGLE_UNREACHABLE __assume(false) +#else +#define DEMANGLE_UNREACHABLE +#endif + +#if __has_attribute(noinline) || DEMANGLE_GNUC_PREREQ(3, 4, 0) +#define DEMANGLE_ATTRIBUTE_NOINLINE __attribute__((noinline)) +#elif defined(_MSC_VER) +#define DEMANGLE_ATTRIBUTE_NOINLINE __declspec(noinline) +#else +#define DEMANGLE_ATTRIBUTE_NOINLINE +#endif + +#if !defined(NDEBUG) +#define DEMANGLE_DUMP_METHOD DEMANGLE_ATTRIBUTE_NOINLINE DEMANGLE_ATTRIBUTE_USED +#else +#define DEMANGLE_DUMP_METHOD DEMANGLE_ATTRIBUTE_NOINLINE +#endif + +#if __cplusplus > 201402L && __has_cpp_attribute(fallthrough) +#define DEMANGLE_FALLTHROUGH [[fallthrough]] +#elif __has_cpp_attribute(gnu::fallthrough) +#define DEMANGLE_FALLTHROUGH [[gnu::fallthrough]] +#elif !__cplusplus +// Workaround for llvm.org/PR23435, since clang 3.6 and below emit a spurious +// error when __has_cpp_attribute is given a scoped attribute in C mode. +#define DEMANGLE_FALLTHROUGH +#elif __has_cpp_attribute(clang::fallthrough) +#define DEMANGLE_FALLTHROUGH [[clang::fallthrough]] +#else +#define DEMANGLE_FALLTHROUGH +#endif + +#define DEMANGLE_NAMESPACE_BEGIN namespace llvm { namespace itanium_demangle { +#define DEMANGLE_NAMESPACE_END } } + +#endif diff --git a/include/llvm/Demangle/ItaniumDemangle.h b/include/llvm/Demangle/ItaniumDemangle.h index 0b9187f30a5a..aaccb27e17a3 100644 --- a/include/llvm/Demangle/ItaniumDemangle.h +++ b/include/llvm/Demangle/ItaniumDemangle.h @@ -1,23 +1,26 @@ //===------------------------- ItaniumDemangle.h ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +//===----------------------------------------------------------------------===// +// +// Generic itanium demangler library. This file has two byte-per-byte identical +// copies in the source tree, one in libcxxabi, and the other in llvm. // //===----------------------------------------------------------------------===// -#ifndef LLVM_DEMANGLE_ITANIUMDEMANGLE_H -#define LLVM_DEMANGLE_ITANIUMDEMANGLE_H +#ifndef DEMANGLE_ITANIUMDEMANGLE_H +#define DEMANGLE_ITANIUMDEMANGLE_H // FIXME: (possibly) incomplete list of features that clang mangles that this // file does not yet support: // - C++ modules TS -#include "llvm/Demangle/Compiler.h" -#include "llvm/Demangle/StringView.h" -#include "llvm/Demangle/Utility.h" - +#include "DemangleConfig.h" +#include "StringView.h" +#include "Utility.h" #include #include #include @@ -86,6 +89,7 @@ X(InitListExpr) \ X(FoldExpr) \ X(ThrowExpr) \ + X(UUIDOfExpr) \ X(BoolExpr) \ X(IntegerCastExpr) \ X(IntegerLiteral) \ @@ -95,8 +99,8 @@ X(BracedExpr) \ X(BracedRangeExpr) -namespace llvm { -namespace itanium_demangle { +DEMANGLE_NAMESPACE_BEGIN + // Base class of all AST nodes. The AST is built by the parser, then is // traversed by the printLeft/Right functions to produce a demangled string. class Node { @@ -194,7 +198,7 @@ public: virtual ~Node() = default; #ifndef NDEBUG - LLVM_DUMP_METHOD void dump() const; + DEMANGLE_DUMP_METHOD void dump() const; #endif }; @@ -1278,7 +1282,7 @@ public: case SpecialSubKind::iostream: return StringView("basic_iostream"); } - LLVM_BUILTIN_UNREACHABLE; + DEMANGLE_UNREACHABLE; } void printLeft(OutputStream &S) const override { @@ -1330,7 +1334,7 @@ public: case SpecialSubKind::iostream: return StringView("iostream"); } - LLVM_BUILTIN_UNREACHABLE; + DEMANGLE_UNREACHABLE; } void printLeft(OutputStream &S) const override { @@ -1870,6 +1874,21 @@ public: } }; +// MSVC __uuidof extension, generated by clang in -fms-extensions mode. +class UUIDOfExpr : public Node { + Node *Operand; +public: + UUIDOfExpr(Node *Operand_) : Node(KUUIDOfExpr), Operand(Operand_) {} + + template void match(Fn F) const { F(Operand); } + + void printLeft(OutputStream &S) const override { + S << "__uuidof("; + Operand->print(S); + S << ")"; + } +}; + class BoolExpr : public Node { bool Value; @@ -2476,6 +2495,12 @@ AbstractManglingParser::parseUnnamedTypeName(NameState *) { return nullptr; return make(Params, Count); } + if (consumeIf("Ub")) { + (void)parseNumber(); + if (!consumeIf('_')) + return nullptr; + return make("'block-literal'"); + } return nullptr; } @@ -2785,11 +2810,13 @@ AbstractManglingParser::parseOperatorName(NameState *State) { // ::= C1 # complete object constructor // ::= C2 # base object constructor // ::= C3 # complete object allocating constructor -// extension ::= C5 # ? +// extension ::= C4 # gcc old-style "[unified]" constructor +// extension ::= C5 # the COMDAT used for ctors // ::= D0 # deleting destructor // ::= D1 # complete object destructor // ::= D2 # base object destructor -// extension ::= D5 # ? +// extension ::= D4 # gcc old-style "[unified]" destructor +// extension ::= D5 # the COMDAT used for dtors template Node * AbstractManglingParser::parseCtorDtorName(Node *&SoFar, @@ -2812,7 +2839,8 @@ AbstractManglingParser::parseCtorDtorName(Node *&SoFar, if (consumeIf('C')) { bool IsInherited = consumeIf('I'); - if (look() != '1' && look() != '2' && look() != '3' && look() != '5') + if (look() != '1' && look() != '2' && look() != '3' && look() != '4' && + look() != '5') return nullptr; int Variant = look() - '0'; ++First; @@ -2821,15 +2849,15 @@ AbstractManglingParser::parseCtorDtorName(Node *&SoFar, if (getDerived().parseName(State) == nullptr) return nullptr; } - return make(SoFar, false, Variant); + return make(SoFar, /*IsDtor=*/false, Variant); } - if (look() == 'D' && - (look(1) == '0' || look(1) == '1' || look(1) == '2' || look(1) == '5')) { + if (look() == 'D' && (look(1) == '0' || look(1) == '1' || look(1) == '2' || + look(1) == '4' || look(1) == '5')) { int Variant = look(1) - '0'; First += 2; if (State) State->CtorDtorConversion = true; - return make(SoFar, true, Variant); + return make(SoFar, /*IsDtor=*/true, Variant); } return nullptr; @@ -3467,7 +3495,7 @@ Node *AbstractManglingParser::parseType() { Result = getDerived().parseFunctionType(); break; } - LLVM_FALLTHROUGH; + DEMANGLE_FALLTHROUGH; } case 'U': { Result = getDerived().parseQualifiedType(); @@ -3564,7 +3592,11 @@ Node *AbstractManglingParser::parseType() { StringView Res = parseBareSourceName(); if (Res.empty()) return nullptr; - return make(Res); + // Typically, s are not considered substitution candidates, + // but the exception to that exception is vendor extended types (Itanium C++ + // ABI 5.9.1). + Result = make(Res); + break; } case 'D': switch (look(1)) { @@ -3592,6 +3624,10 @@ Node *AbstractManglingParser::parseType() { case 's': First += 2; return make("char16_t"); + // ::= Du # char8_t (C++2a, not yet in the Itanium spec) + case 'u': + First += 2; + return make("char8_t"); // ::= Da # auto (in dependent new-expressions) case 'a': First += 2; @@ -3754,7 +3790,7 @@ Node *AbstractManglingParser::parseType() { // substitution table. return Sub; } - LLVM_FALLTHROUGH; + DEMANGLE_FALLTHROUGH; } // ::= default: { @@ -4633,6 +4669,21 @@ Node *AbstractManglingParser::parseExpr() { case '9': return getDerived().parseUnresolvedName(); } + + if (consumeIf("u8__uuidoft")) { + Node *Ty = getDerived().parseType(); + if (!Ty) + return nullptr; + return make(Ty); + } + + if (consumeIf("u8__uuidofz")) { + Node *Ex = getDerived().parseExpr(); + if (!Ex) + return nullptr; + return make(Ex); + } + return nullptr; } @@ -5139,7 +5190,7 @@ AbstractManglingParser::parseTemplateArgs(bool TagTemplates) { // extension ::= ___Z _block_invoke_+ template Node *AbstractManglingParser::parse() { - if (consumeIf("_Z")) { + if (consumeIf("_Z") || consumeIf("__Z")) { Node *Encoding = getDerived().parseEncoding(); if (Encoding == nullptr) return nullptr; @@ -5152,7 +5203,7 @@ Node *AbstractManglingParser::parse() { return Encoding; } - if (consumeIf("___Z")) { + if (consumeIf("___Z") || consumeIf("____Z")) { Node *Encoding = getDerived().parseEncoding(); if (Encoding == nullptr || !consumeIf("_block_invoke")) return nullptr; @@ -5178,7 +5229,6 @@ struct ManglingParser : AbstractManglingParser, Alloc> { Alloc>::AbstractManglingParser; }; -} // namespace itanium_demangle -} // namespace llvm +DEMANGLE_NAMESPACE_END -#endif // LLVM_DEMANGLE_ITANIUMDEMANGLE_H +#endif // DEMANGLE_ITANIUMDEMANGLE_H diff --git a/include/llvm/Demangle/MicrosoftDemangle.h b/include/llvm/Demangle/MicrosoftDemangle.h index 97b918fc9459..382e79401c43 100644 --- a/include/llvm/Demangle/MicrosoftDemangle.h +++ b/include/llvm/Demangle/MicrosoftDemangle.h @@ -1,16 +1,15 @@ //===------------------------- MicrosoftDemangle.h --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef LLVM_DEMANGLE_MICROSOFT_DEMANGLE_H #define LLVM_DEMANGLE_MICROSOFT_DEMANGLE_H -#include "llvm/Demangle/Compiler.h" +#include "llvm/Demangle/DemangleConfig.h" #include "llvm/Demangle/MicrosoftDemangleNodes.h" #include "llvm/Demangle/StringView.h" #include "llvm/Demangle/Utility.h" @@ -56,24 +55,21 @@ public: } } - char *allocUnalignedBuffer(size_t Length) { - uint8_t *Buf = Head->Buf + Head->Used; - - Head->Used += Length; - if (Head->Used > Head->Capacity) { - // It's possible we need a buffer which is larger than our default unit - // size, so we need to be careful to add a node with capacity that is at - // least as large as what we need. - addNode(std::max(AllocUnit, Length)); - Head->Used = Length; - Buf = Head->Buf; - } + char *allocUnalignedBuffer(size_t Size) { + assert(Head && Head->Buf); + + uint8_t *P = Head->Buf + Head->Used; + + Head->Used += Size; + if (Head->Used <= Head->Capacity) + return reinterpret_cast(P); - return reinterpret_cast(Buf); + addNode(std::max(AllocUnit, Size)); + Head->Used = Size; + return reinterpret_cast(Head->Buf); } template T *allocArray(size_t Count) { - size_t Size = Count * sizeof(T); assert(Head && Head->Buf); @@ -84,17 +80,16 @@ public: size_t Adjustment = AlignedP - P; Head->Used += Size + Adjustment; - if (Head->Used < Head->Capacity) + if (Head->Used <= Head->Capacity) return new (PP) T[Count](); - addNode(AllocUnit); + addNode(std::max(AllocUnit, Size)); Head->Used = Size; return new (Head->Buf) T[Count](); } template T *alloc(Args &&... ConstructorArgs) { - - size_t Size = sizeof(T); + constexpr size_t Size = sizeof(T); assert(Head && Head->Buf); size_t P = (size_t)Head->Buf + Head->Used; @@ -104,9 +99,10 @@ public: size_t Adjustment = AlignedP - P; Head->Used += Size + Adjustment; - if (Head->Used < Head->Capacity) + if (Head->Used <= Head->Capacity) return new (PP) T(std::forward(ConstructorArgs)...); + static_assert(Size < AllocUnit, ""); addNode(AllocUnit); Head->Used = Size; return new (Head->Buf) T(std::forward(ConstructorArgs)...); @@ -160,6 +156,8 @@ public: private: SymbolNode *demangleEncodedSymbol(StringView &MangledName, QualifiedNameNode *QN); + SymbolNode *demangleDeclarator(StringView &MangledName); + SymbolNode *demangleMD5Name(StringView &MangledName); VariableSymbolNode *demangleVariableEncoding(StringView &MangledName, StorageClass SC); @@ -179,8 +177,9 @@ private: ArrayTypeNode *demangleArrayType(StringView &MangledName); + NodeArrayNode *demangleFunctionParameterList(StringView &MangledName, + bool &IsVariadic); NodeArrayNode *demangleTemplateParameterList(StringView &MangledName); - NodeArrayNode *demangleFunctionParameterList(StringView &MangledName); std::pair demangleNumber(StringView &MangledName); uint64_t demangleUnsigned(StringView &MangledName); @@ -207,6 +206,8 @@ private: NamedIdentifierNode *demangleBackRefName(StringView &MangledName); IdentifierNode *demangleTemplateInstantiationName(StringView &MangledName, NameBackrefBehavior NBB); + IntrinsicFunctionKind + translateIntrinsicFunctionCode(char CH, FunctionIdentifierCodeGroup Group); IdentifierNode *demangleFunctionIdentifierCode(StringView &MangledName); IdentifierNode * demangleFunctionIdentifierCode(StringView &MangledName, @@ -223,7 +224,7 @@ private: demangleSpecialTableSymbolNode(StringView &MangledName, SpecialIntrinsicKind SIK); LocalStaticGuardVariableNode * - demangleLocalStaticGuard(StringView &MangledName); + demangleLocalStaticGuard(StringView &MangledName, bool IsThread); VariableSymbolNode *demangleUntypedVariable(ArenaAllocator &Arena, StringView &MangledName, StringView VariableName); diff --git a/include/llvm/Demangle/MicrosoftDemangleNodes.h b/include/llvm/Demangle/MicrosoftDemangleNodes.h index 9e3478e9fd29..da9d9d5bfdc0 100644 --- a/include/llvm/Demangle/MicrosoftDemangleNodes.h +++ b/include/llvm/Demangle/MicrosoftDemangleNodes.h @@ -1,11 +1,30 @@ +//===- MicrosoftDemangleNodes.h ---------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines the AST nodes used in the MSVC demangler. +// +//===----------------------------------------------------------------------===// + #ifndef LLVM_SUPPORT_MICROSOFTDEMANGLENODES_H #define LLVM_SUPPORT_MICROSOFTDEMANGLENODES_H -#include "llvm/Demangle/Compiler.h" +#include "llvm/Demangle/DemangleConfig.h" #include "llvm/Demangle/StringView.h" #include +namespace llvm { +namespace itanium_demangle { class OutputStream; +} +} + +using llvm::itanium_demangle::OutputStream; +using llvm::itanium_demangle::StringView; namespace llvm { namespace ms_demangle { @@ -63,6 +82,7 @@ enum class PrimitiveKind { Char, Schar, Uchar, + Char8, Char16, Char32, Short, @@ -151,8 +171,8 @@ enum class IntrinsicFunctionKind : uint8_t { VectorCopyCtorIter, // ?__G vector copy constructor iterator VectorVbaseCopyCtorIter, // ?__H vector vbase copy constructor iterator ManVectorVbaseCopyCtorIter, // ?__I managed vector vbase copy constructor - CoAwait, // ?__L co_await - Spaceship, // operator<=> + CoAwait, // ?__L operator co_await + Spaceship, // ?__M operator<=> MaxIntrinsic }; @@ -324,7 +344,7 @@ struct FunctionSignatureNode : public TypeNode { // Function parameters NodeArrayNode *Params = nullptr; - // True if the function type is noexcept + // True if the function type is noexcept. bool IsNoexcept = false; }; @@ -389,6 +409,7 @@ struct LocalStaticGuardIdentifierNode : public IdentifierNode { void output(OutputStream &OS, OutputFlags Flags) const override; + bool IsThread = false; uint32_t ScopeIndex = 0; }; @@ -494,7 +515,7 @@ struct NodeArrayNode : public Node { void output(OutputStream &OS, OutputFlags Flags, StringView Separator) const; - Node **Nodes = 0; + Node **Nodes = nullptr; size_t Count = 0; }; @@ -602,4 +623,4 @@ struct FunctionSymbolNode : public SymbolNode { } // namespace ms_demangle } // namespace llvm -#endif \ No newline at end of file +#endif diff --git a/include/llvm/Demangle/README.txt b/include/llvm/Demangle/README.txt new file mode 100644 index 000000000000..514ff6dd16f2 --- /dev/null +++ b/include/llvm/Demangle/README.txt @@ -0,0 +1,52 @@ +Itanium Name Demangler Library +============================== + +Introduction +------------ + +This directory contains the generic itanium name demangler library. The main +purpose of the library is to demangle C++ symbols, i.e. convert the string +"_Z1fv" into "f()". You can also use the CRTP base ManglingParser to perform +some simple analysis on the mangled name, or (in LLVM) use the opaque +ItaniumPartialDemangler to query the demangled AST. + +Why are there multiple copies of the this library in the source tree? +--------------------------------------------------------------------- + +This directory is mirrored between libcxxabi/demangle and +llvm/include/llvm/Demangle. The simple reason for this is that both projects +need to demangle symbols, but neither can depend on each other. libcxxabi needs +the demangler to implement __cxa_demangle, which is part of the itanium ABI +spec. LLVM needs a copy for a bunch of places, but doesn't want to use the +system's __cxa_demangle because it a) might not be available (i.e., on Windows), +and b) probably isn't that up-to-date on the latest language features. + +The copy of the demangler in LLVM has some extra stuff that aren't needed in +libcxxabi (ie, the MSVC demangler, ItaniumPartialDemangler), which depend on the +shared generic components. Despite these differences, we want to keep the "core" +generic demangling library identical between both copies to simplify development +and testing. + +If you're working on the generic library, then do the work first in libcxxabi, +then run the cp-to-llvm.sh script in src/demangle. This script takes as an +argument the path to llvm, and re-copies the changes you made to libcxxabi over. +Note that this script just blindly overwrites all changes to the generic library +in llvm, so be careful. + +Because the core demangler needs to work in libcxxabi, everything needs to be +declared in an anonymous namespace (see DEMANGLE_NAMESPACE_BEGIN), and you can't +introduce any code that depends on the libcxx dylib. + +Hopefully, when LLVM becomes a monorepo, we can de-duplicate this code, and have +both LLVM and libcxxabi depend on a shared demangler library. + +Testing +------- + +The tests are split up between libcxxabi/test/{unit,}test_demangle.cpp, and +llvm/unittest/Demangle. The llvm directory should only get tests for stuff not +included in the core library. In the future though, we should probably move all +the tests to LLVM. + +It is also a really good idea to run libFuzzer after non-trivial changes, see +libcxxabi/fuzz/cxa_demangle_fuzzer.cpp and https://llvm.org/docs/LibFuzzer.html. diff --git a/include/llvm/Demangle/StringView.h b/include/llvm/Demangle/StringView.h index a89deda694c2..ceb6c7958066 100644 --- a/include/llvm/Demangle/StringView.h +++ b/include/llvm/Demangle/StringView.h @@ -1,22 +1,25 @@ //===--- StringView.h -------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +//===----------------------------------------------------------------------===// // +// FIXME: Use std::string_view instead when we support C++17. // -// This file contains a limited version of LLVM's StringView class. It is -// copied here so that LLVMDemangle need not take a dependency on LLVMSupport. //===----------------------------------------------------------------------===// -#ifndef LLVM_DEMANGLE_STRINGVIEW_H -#define LLVM_DEMANGLE_STRINGVIEW_H +#ifndef DEMANGLE_STRINGVIEW_H +#define DEMANGLE_STRINGVIEW_H +#include "DemangleConfig.h" #include #include #include +DEMANGLE_NAMESPACE_BEGIN + class StringView { const char *First; const char *Last; @@ -43,7 +46,7 @@ public: if (FindBegin < size()) { // Just forward to memchr, which is faster than a hand-rolled loop. if (const void *P = ::memchr(First + FindBegin, C, size() - FindBegin)) - return static_cast(P) - First; + return size_t(static_cast(P) - First); } return npos; } @@ -118,4 +121,6 @@ inline bool operator==(const StringView &LHS, const StringView &RHS) { std::equal(LHS.begin(), LHS.end(), RHS.begin()); } +DEMANGLE_NAMESPACE_END + #endif diff --git a/include/llvm/Demangle/Utility.h b/include/llvm/Demangle/Utility.h index 1d1601c81635..ec23859af46a 100644 --- a/include/llvm/Demangle/Utility.h +++ b/include/llvm/Demangle/Utility.h @@ -1,25 +1,27 @@ //===--- Utility.h ----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +//===----------------------------------------------------------------------===// // +// Provide some utility classes for use in the demangler(s). // -// This file contains several utility classes used by the demangle library. //===----------------------------------------------------------------------===// -#ifndef LLVM_DEMANGLE_UTILITY_H -#define LLVM_DEMANGLE_UTILITY_H +#ifndef DEMANGLE_UTILITY_H +#define DEMANGLE_UTILITY_H #include "StringView.h" - #include #include #include #include #include +DEMANGLE_NAMESPACE_BEGIN + // Stream that AST nodes write their string representation into after the AST // has been parsed. class OutputStream { @@ -184,4 +186,6 @@ inline bool initializeOutputStream(char *Buf, size_t *N, OutputStream &S, return true; } +DEMANGLE_NAMESPACE_END + #endif diff --git a/include/llvm/ExecutionEngine/ExecutionEngine.h b/include/llvm/ExecutionEngine/ExecutionEngine.h index b61cb24fa5fb..4fb6dad96387 100644 --- a/include/llvm/ExecutionEngine/ExecutionEngine.h +++ b/include/llvm/ExecutionEngine/ExecutionEngine.h @@ -1,9 +1,8 @@ //===- ExecutionEngine.h - Abstract Execution Engine Interface --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -22,6 +21,7 @@ #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/OrcV1Deprecation.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Module.h" #include "llvm/Object/Binary.h" @@ -635,7 +635,13 @@ public: } // Use OrcMCJITReplacement instead of MCJIT. Off by default. - void setUseOrcMCJITReplacement(bool UseOrcMCJITReplacement) { + LLVM_ATTRIBUTE_DEPRECATED( + inline void setUseOrcMCJITReplacement(bool UseOrcMCJITReplacement), + "ORCv1 utilities (including OrcMCJITReplacement) are deprecated. Please " + "use ORCv2/LLJIT instead (see docs/ORCv2.rst)"); + + void setUseOrcMCJITReplacement(ORCv1DeprecationAcknowledgement, + bool UseOrcMCJITReplacement) { this->UseOrcMCJITReplacement = UseOrcMCJITReplacement; } @@ -659,6 +665,10 @@ public: ExecutionEngine *create(TargetMachine *TM); }; +void EngineBuilder::setUseOrcMCJITReplacement(bool UseOrcMCJITReplacement) { + this->UseOrcMCJITReplacement = UseOrcMCJITReplacement; +} + // Create wrappers for C Binding types (see CBindingWrapping.h). DEFINE_SIMPLE_CONVERSION_FUNCTIONS(ExecutionEngine, LLVMExecutionEngineRef) diff --git a/include/llvm/ExecutionEngine/GenericValue.h b/include/llvm/ExecutionEngine/GenericValue.h index 504e30a018b6..1ca989da1b7e 100644 --- a/include/llvm/ExecutionEngine/GenericValue.h +++ b/include/llvm/ExecutionEngine/GenericValue.h @@ -1,9 +1,8 @@ //===- GenericValue.h - Represent any type of LLVM value --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ExecutionEngine/Interpreter.h b/include/llvm/ExecutionEngine/Interpreter.h index a14707840ad8..0749409766e3 100644 --- a/include/llvm/ExecutionEngine/Interpreter.h +++ b/include/llvm/ExecutionEngine/Interpreter.h @@ -1,9 +1,8 @@ //===-- Interpreter.h - Abstract Execution Engine Interface -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ExecutionEngine/JITEventListener.h b/include/llvm/ExecutionEngine/JITEventListener.h index 1b08379b8c3b..606b6f7cc128 100644 --- a/include/llvm/ExecutionEngine/JITEventListener.h +++ b/include/llvm/ExecutionEngine/JITEventListener.h @@ -1,9 +1,8 @@ //===- JITEventListener.h - Exposes events from JIT compilation -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h b/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h new file mode 100644 index 000000000000..8d2f641254b3 --- /dev/null +++ b/include/llvm/ExecutionEngine/JITLink/EHFrameSupport.h @@ -0,0 +1,80 @@ +//===--------- EHFrameSupport.h - JITLink eh-frame utils --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// EHFrame registration support for JITLink. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_EHFRAMESUPPORT_H +#define LLVM_EXECUTIONENGINE_JITLINK_EHFRAMESUPPORT_H + +#include "llvm/ADT/Triple.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/Support/Error.h" + +namespace llvm { +namespace jitlink { + +/// Registers all FDEs in the given eh-frame section with the current process. +Error registerEHFrameSection(const void *EHFrameSectionAddr); + +/// Deregisters all FDEs in the given eh-frame section with the current process. +Error deregisterEHFrameSection(const void *EHFrameSectionAddr); + +/// Supports registration/deregistration of EH-frames in a target process. +class EHFrameRegistrar { +public: + virtual ~EHFrameRegistrar(); + virtual Error registerEHFrames(JITTargetAddress EHFrameSectionAddr) = 0; + virtual Error deregisterEHFrames(JITTargetAddress EHFrameSectionAddr) = 0; +}; + +/// Registers / Deregisters EH-frames in the current process. +class InProcessEHFrameRegistrar final : public EHFrameRegistrar { +public: + /// Get a reference to the InProcessEHFrameRegistrar singleton. + static InProcessEHFrameRegistrar &getInstance(); + + InProcessEHFrameRegistrar(const InProcessEHFrameRegistrar &) = delete; + InProcessEHFrameRegistrar & + operator=(const InProcessEHFrameRegistrar &) = delete; + + InProcessEHFrameRegistrar(InProcessEHFrameRegistrar &&) = delete; + InProcessEHFrameRegistrar &operator=(InProcessEHFrameRegistrar &&) = delete; + + Error registerEHFrames(JITTargetAddress EHFrameSectionAddr) override { + return registerEHFrameSection( + jitTargetAddressToPointer(EHFrameSectionAddr)); + } + + Error deregisterEHFrames(JITTargetAddress EHFrameSectionAddr) override { + return deregisterEHFrameSection( + jitTargetAddressToPointer(EHFrameSectionAddr)); + } + +private: + InProcessEHFrameRegistrar(); +}; + +using StoreFrameAddressFunction = std::function; + +/// Creates a pass that records the address of the EH frame section. If no +/// eh-frame section is found, it will set EHFrameAddr to zero. +/// +/// Authors of JITLinkContexts can use this function to register a post-fixup +/// pass that records the address of the eh-frame section. This address can +/// be used after finalization to register and deregister the frame. +AtomGraphPassFunction +createEHFrameRecorderPass(const Triple &TT, + StoreFrameAddressFunction StoreFrameAddress); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_EHFRAMESUPPORT_H diff --git a/include/llvm/ExecutionEngine/JITLink/JITLink.h b/include/llvm/ExecutionEngine/JITLink/JITLink.h new file mode 100644 index 000000000000..be80d44ccf51 --- /dev/null +++ b/include/llvm/ExecutionEngine/JITLink/JITLink.h @@ -0,0 +1,930 @@ +//===------------ JITLink.h - JIT linker functionality ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contains generic JIT-linker types. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_JITLINK_H +#define LLVM_EXECUTIONENGINE_JITLINK_JITLINK_H + +#include "JITLinkMemoryManager.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/DenseSet.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/Triple.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/FormatVariadic.h" +#include "llvm/Support/MathExtras.h" +#include "llvm/Support/Memory.h" +#include "llvm/Support/MemoryBuffer.h" + +#include +#include +#include + +namespace llvm { +namespace jitlink { + +/// Base class for errors originating in JIT linker, e.g. missing relocation +/// support. +class JITLinkError : public ErrorInfo { +public: + static char ID; + + JITLinkError(Twine ErrMsg) : ErrMsg(ErrMsg.str()) {} + + void log(raw_ostream &OS) const override; + const std::string &getErrorMessage() const { return ErrMsg; } + std::error_code convertToErrorCode() const override; + +private: + std::string ErrMsg; +}; + +// Forward declare the Atom class. +class Atom; + +/// Edge class. Represents both object file relocations, as well as layout and +/// keep-alive constraints. +class Edge { +public: + using Kind = uint8_t; + + using GenericEdgeKind = enum : Kind { + Invalid, // Invalid edge value. + FirstKeepAlive, // Keeps target alive. Offset/addend zero. + KeepAlive = FirstKeepAlive, // Tag first edge kind that preserves liveness. + LayoutNext, // Layout constraint. Offset/Addend zero. + FirstRelocation // First architecture specific relocation. + }; + + using OffsetT = uint32_t; + using AddendT = int64_t; + + Edge(Kind K, OffsetT Offset, Atom &Target, AddendT Addend) + : Target(&Target), Offset(Offset), Addend(Addend), K(K) {} + + OffsetT getOffset() const { return Offset; } + Kind getKind() const { return K; } + void setKind(Kind K) { this->K = K; } + bool isRelocation() const { return K >= FirstRelocation; } + Kind getRelocation() const { + assert(isRelocation() && "Not a relocation edge"); + return K - FirstRelocation; + } + bool isKeepAlive() const { return K >= FirstKeepAlive; } + Atom &getTarget() const { return *Target; } + void setTarget(Atom &Target) { this->Target = &Target; } + AddendT getAddend() const { return Addend; } + void setAddend(AddendT Addend) { this->Addend = Addend; } + +private: + Atom *Target; + OffsetT Offset; + AddendT Addend; + Kind K = 0; +}; + +using EdgeVector = std::vector; + +const StringRef getGenericEdgeKindName(Edge::Kind K); + +/// Base Atom class. Used by absolute and undefined atoms. +class Atom { + friend class AtomGraph; + +protected: + /// Create a named (as yet unresolved) atom. + Atom(StringRef Name) + : Name(Name), IsDefined(false), IsLive(false), ShouldDiscard(false), + IsGlobal(false), IsAbsolute(false), IsCallable(false), + IsExported(false), IsWeak(false), HasLayoutNext(false), + IsCommon(false) {} + + /// Create an absolute symbol atom. + Atom(StringRef Name, JITTargetAddress Address) + : Name(Name), Address(Address), IsDefined(true), IsLive(false), + ShouldDiscard(false), IsGlobal(false), IsAbsolute(false), + IsCallable(false), IsExported(false), IsWeak(false), + HasLayoutNext(false), IsCommon(false) {} + +public: + /// Returns true if this atom has a name. + bool hasName() const { return Name != StringRef(); } + + /// Returns the name of this atom. + StringRef getName() const { return Name; } + + /// Returns the current target address of this atom. + /// The initial target address (for atoms that have one) will be taken from + /// the input object file's virtual address space. During the layout phase + /// of JIT linking the atom's address will be updated to point to its final + /// address in the JIT'd process. + JITTargetAddress getAddress() const { return Address; } + + /// Set the current target address of this atom. + void setAddress(JITTargetAddress Address) { this->Address = Address; } + + /// Returns true if this is a defined atom. + bool isDefined() const { return IsDefined; } + + /// Returns true if this atom is marked as live. + bool isLive() const { return IsLive; } + + /// Mark this atom as live. + /// + /// Note: Only defined and absolute atoms can be marked live. + void setLive(bool IsLive) { + assert((IsDefined || IsAbsolute || !IsLive) && + "Only defined and absolute atoms can be marked live"); + this->IsLive = IsLive; + } + + /// Returns true if this atom should be discarded during pruning. + bool shouldDiscard() const { return ShouldDiscard; } + + /// Mark this atom to be discarded. + /// + /// Note: Only defined and absolute atoms can be marked live. + void setShouldDiscard(bool ShouldDiscard) { + assert((IsDefined || IsAbsolute || !ShouldDiscard) && + "Only defined and absolute atoms can be marked live"); + this->ShouldDiscard = ShouldDiscard; + } + + /// Returns true if this definition is global (i.e. visible outside this + /// linkage unit). + /// + /// Note: This is distict from Exported, which means visibile outside the + /// JITDylib that this graph is being linked in to. + bool isGlobal() const { return IsGlobal; } + + /// Mark this atom as global. + void setGlobal(bool IsGlobal) { this->IsGlobal = IsGlobal; } + + /// Returns true if this atom represents an absolute symbol. + bool isAbsolute() const { return IsAbsolute; } + + /// Returns true if this atom is known to be callable. + /// + /// Primarily provided for easy interoperability with ORC, which uses the + /// JITSymbolFlags::Common flag to identify symbols that can be interposed + /// with stubs. + bool isCallable() const { return IsCallable; } + + /// Mark this atom as callable. + void setCallable(bool IsCallable) { + assert((IsDefined || IsAbsolute || !IsCallable) && + "Callable atoms must be defined or absolute"); + this->IsCallable = IsCallable; + } + + /// Returns true if this atom should appear in the symbol table of a final + /// linked image. + bool isExported() const { return IsExported; } + + /// Mark this atom as exported. + void setExported(bool IsExported) { + assert((!IsExported || ((IsDefined || IsAbsolute) && hasName())) && + "Exported atoms must have names"); + this->IsExported = IsExported; + } + + /// Returns true if this is a weak symbol. + bool isWeak() const { return IsWeak; } + + /// Mark this atom as weak. + void setWeak(bool IsWeak) { this->IsWeak = IsWeak; } + +private: + StringRef Name; + JITTargetAddress Address = 0; + + bool IsDefined : 1; + bool IsLive : 1; + bool ShouldDiscard : 1; + + bool IsGlobal : 1; + bool IsAbsolute : 1; + bool IsCallable : 1; + bool IsExported : 1; + bool IsWeak : 1; + +protected: + // These flags only make sense for DefinedAtom, but we can minimize the size + // of DefinedAtom by defining them here. + bool HasLayoutNext : 1; + bool IsCommon : 1; +}; + +// Forward declare DefinedAtom. +class DefinedAtom; + +raw_ostream &operator<<(raw_ostream &OS, const Atom &A); +void printEdge(raw_ostream &OS, const Atom &FixupAtom, const Edge &E, + StringRef EdgeKindName); + +/// Represents a section address range via a pair of DefinedAtom pointers to +/// the first and last atoms in the section. +class SectionRange { +public: + SectionRange() = default; + SectionRange(DefinedAtom *First, DefinedAtom *Last) + : First(First), Last(Last) {} + DefinedAtom *getFirstAtom() const { + assert((!Last || First) && "First can not be null if end is non-null"); + return First; + } + DefinedAtom *getLastAtom() const { + assert((First || !Last) && "Last can not be null if start is non-null"); + return Last; + } + bool isEmpty() const { + assert((First || !Last) && "Last can not be null if start is non-null"); + return !First; + } + JITTargetAddress getStart() const; + JITTargetAddress getEnd() const; + uint64_t getSize() const; + +private: + DefinedAtom *First = nullptr; + DefinedAtom *Last = nullptr; +}; + +/// Represents an object file section. +class Section { + friend class AtomGraph; + +private: + Section(StringRef Name, uint32_t Alignment, sys::Memory::ProtectionFlags Prot, + unsigned Ordinal, bool IsZeroFill) + : Name(Name), Alignment(Alignment), Prot(Prot), Ordinal(Ordinal), + IsZeroFill(IsZeroFill) { + assert(isPowerOf2_32(Alignment) && "Alignments must be a power of 2"); + } + + using DefinedAtomSet = DenseSet; + +public: + using atom_iterator = DefinedAtomSet::iterator; + using const_atom_iterator = DefinedAtomSet::const_iterator; + + ~Section(); + StringRef getName() const { return Name; } + uint32_t getAlignment() const { return Alignment; } + sys::Memory::ProtectionFlags getProtectionFlags() const { return Prot; } + unsigned getSectionOrdinal() const { return Ordinal; } + size_t getNextAtomOrdinal() { return ++NextAtomOrdinal; } + + bool isZeroFill() const { return IsZeroFill; } + + /// Returns an iterator over the atoms in the section (in no particular + /// order). + iterator_range atoms() { + return make_range(DefinedAtoms.begin(), DefinedAtoms.end()); + } + + /// Returns an iterator over the atoms in the section (in no particular + /// order). + iterator_range atoms() const { + return make_range(DefinedAtoms.begin(), DefinedAtoms.end()); + } + + /// Return the number of atoms in this section. + DefinedAtomSet::size_type atoms_size() { return DefinedAtoms.size(); } + + /// Return true if this section contains no atoms. + bool atoms_empty() const { return DefinedAtoms.empty(); } + + /// Returns the range of this section as the pair of atoms with the lowest + /// and highest target address. This operation is expensive, as it + /// must traverse all atoms in the section. + /// + /// Note: If the section is empty, both values will be null. The section + /// address will evaluate to null, and the size to zero. If the section + /// contains a single atom both values will point to it, the address will + /// evaluate to the address of that atom, and the size will be the size of + /// that atom. + SectionRange getRange() const; + +private: + void addAtom(DefinedAtom &DA) { + assert(!DefinedAtoms.count(&DA) && "Atom is already in this section"); + DefinedAtoms.insert(&DA); + } + + void removeAtom(DefinedAtom &DA) { + assert(DefinedAtoms.count(&DA) && "Atom is not in this section"); + DefinedAtoms.erase(&DA); + } + + StringRef Name; + uint32_t Alignment = 0; + sys::Memory::ProtectionFlags Prot; + unsigned Ordinal = 0; + unsigned NextAtomOrdinal = 0; + bool IsZeroFill = false; + DefinedAtomSet DefinedAtoms; +}; + +/// Defined atom class. Suitable for use by defined named and anonymous +/// atoms. +class DefinedAtom : public Atom { + friend class AtomGraph; + +private: + DefinedAtom(Section &Parent, JITTargetAddress Address, uint32_t Alignment) + : Atom("", Address), Parent(Parent), Ordinal(Parent.getNextAtomOrdinal()), + Alignment(Alignment) { + assert(isPowerOf2_32(Alignment) && "Alignments must be a power of two"); + } + + DefinedAtom(Section &Parent, StringRef Name, JITTargetAddress Address, + uint32_t Alignment) + : Atom(Name, Address), Parent(Parent), + Ordinal(Parent.getNextAtomOrdinal()), Alignment(Alignment) { + assert(isPowerOf2_32(Alignment) && "Alignments must be a power of two"); + } + +public: + using edge_iterator = EdgeVector::iterator; + + Section &getSection() const { return Parent; } + + uint64_t getSize() const { return Size; } + + StringRef getContent() const { + assert(!Parent.isZeroFill() && "Trying to get content for zero-fill atom"); + assert(Size <= std::numeric_limits::max() && + "Content size too large"); + return {ContentPtr, static_cast(Size)}; + } + void setContent(StringRef Content) { + assert(!Parent.isZeroFill() && "Calling setContent on zero-fill atom?"); + ContentPtr = Content.data(); + Size = Content.size(); + } + + bool isZeroFill() const { return Parent.isZeroFill(); } + + void setZeroFill(uint64_t Size) { + assert(Parent.isZeroFill() && !ContentPtr && + "Can't set zero-fill length of a non zero-fill atom"); + this->Size = Size; + } + + uint64_t getZeroFillSize() const { + assert(Parent.isZeroFill() && + "Can't get zero-fill length of a non zero-fill atom"); + return Size; + } + + uint32_t getAlignment() const { return Alignment; } + + bool hasLayoutNext() const { return HasLayoutNext; } + void setLayoutNext(DefinedAtom &Next) { + assert(!HasLayoutNext && "Atom already has layout-next constraint"); + HasLayoutNext = true; + Edges.push_back(Edge(Edge::LayoutNext, 0, Next, 0)); + } + DefinedAtom &getLayoutNext() { + assert(HasLayoutNext && "Atom does not have a layout-next constraint"); + DefinedAtom *Next = nullptr; + for (auto &E : edges()) + if (E.getKind() == Edge::LayoutNext) { + assert(E.getTarget().isDefined() && + "layout-next target atom must be a defined atom"); + Next = static_cast(&E.getTarget()); + break; + } + assert(Next && "Missing LayoutNext edge"); + return *Next; + } + + bool isCommon() const { return IsCommon; } + + void addEdge(Edge::Kind K, Edge::OffsetT Offset, Atom &Target, + Edge::AddendT Addend) { + assert(K != Edge::LayoutNext && + "Layout edges should be added via setLayoutNext"); + Edges.push_back(Edge(K, Offset, Target, Addend)); + } + + iterator_range edges() { + return make_range(Edges.begin(), Edges.end()); + } + size_t edges_size() const { return Edges.size(); } + bool edges_empty() const { return Edges.empty(); } + + unsigned getOrdinal() const { return Ordinal; } + +private: + void setCommon(uint64_t Size) { + assert(ContentPtr == 0 && "Atom already has content?"); + IsCommon = true; + setZeroFill(Size); + } + + EdgeVector Edges; + uint64_t Size = 0; + Section &Parent; + const char *ContentPtr = nullptr; + unsigned Ordinal = 0; + uint32_t Alignment = 0; +}; + +inline JITTargetAddress SectionRange::getStart() const { + return First ? First->getAddress() : 0; +} + +inline JITTargetAddress SectionRange::getEnd() const { + return Last ? Last->getAddress() + Last->getSize() : 0; +} + +inline uint64_t SectionRange::getSize() const { return getEnd() - getStart(); } + +inline SectionRange Section::getRange() const { + if (atoms_empty()) + return SectionRange(); + DefinedAtom *First = *DefinedAtoms.begin(), *Last = *DefinedAtoms.begin(); + for (auto *DA : atoms()) { + if (DA->getAddress() < First->getAddress()) + First = DA; + if (DA->getAddress() > Last->getAddress()) + Last = DA; + } + return SectionRange(First, Last); +} + +class AtomGraph { +private: + using SectionList = std::vector>; + using AddressToAtomMap = std::map; + using NamedAtomMap = DenseMap; + using ExternalAtomSet = DenseSet; + +public: + using external_atom_iterator = ExternalAtomSet::iterator; + + using section_iterator = pointee_iterator; + using const_section_iterator = pointee_iterator; + + template + class defined_atom_iterator_impl + : public iterator_facade_base< + defined_atom_iterator_impl, + std::forward_iterator_tag, T> { + public: + defined_atom_iterator_impl() = default; + + defined_atom_iterator_impl(SecItrT SI, SecItrT SE) + : SI(SI), SE(SE), + AI(SI != SE ? SI->atoms().begin() : Section::atom_iterator()) { + moveToNextAtomOrEnd(); + } + + bool operator==(const defined_atom_iterator_impl &RHS) const { + return (SI == RHS.SI) && (AI == RHS.AI); + } + + T operator*() const { + assert(AI != SI->atoms().end() && "Dereferencing end?"); + return *AI; + } + + defined_atom_iterator_impl operator++() { + ++AI; + moveToNextAtomOrEnd(); + return *this; + } + + private: + void moveToNextAtomOrEnd() { + while (SI != SE && AI == SI->atoms().end()) { + ++SI; + if (SI == SE) + AI = Section::atom_iterator(); + else + AI = SI->atoms().begin(); + } + } + + SecItrT SI, SE; + AtomItrT AI; + }; + + using defined_atom_iterator = + defined_atom_iterator_impl; + + using const_defined_atom_iterator = + defined_atom_iterator_impl; + + AtomGraph(std::string Name, unsigned PointerSize, + support::endianness Endianness) + : Name(std::move(Name)), PointerSize(PointerSize), + Endianness(Endianness) {} + + /// Returns the name of this graph (usually the name of the original + /// underlying MemoryBuffer). + const std::string &getName() { return Name; } + + /// Returns the pointer size for use in this graph. + unsigned getPointerSize() const { return PointerSize; } + + /// Returns the endianness of atom-content in this graph. + support::endianness getEndianness() const { return Endianness; } + + /// Create a section with the given name, protection flags, and alignment. + Section &createSection(StringRef Name, uint32_t Alignment, + sys::Memory::ProtectionFlags Prot, bool IsZeroFill) { + std::unique_ptr
Sec( + new Section(Name, Alignment, Prot, Sections.size(), IsZeroFill)); + Sections.push_back(std::move(Sec)); + return *Sections.back(); + } + + /// Add an external atom representing an undefined symbol in this graph. + Atom &addExternalAtom(StringRef Name) { + assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted"); + Atom *A = reinterpret_cast( + AtomAllocator.Allocate(sizeof(Atom), alignof(Atom))); + new (A) Atom(Name); + ExternalAtoms.insert(A); + NamedAtoms[Name] = A; + return *A; + } + + /// Add an external atom representing an absolute symbol. + Atom &addAbsoluteAtom(StringRef Name, JITTargetAddress Addr) { + assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted"); + Atom *A = reinterpret_cast( + AtomAllocator.Allocate(sizeof(Atom), alignof(Atom))); + new (A) Atom(Name, Addr); + AbsoluteAtoms.insert(A); + NamedAtoms[Name] = A; + return *A; + } + + /// Add an anonymous defined atom to the graph. + /// + /// Anonymous atoms have content but no name. They must have an address. + DefinedAtom &addAnonymousAtom(Section &Parent, JITTargetAddress Address, + uint32_t Alignment) { + DefinedAtom *A = reinterpret_cast( + AtomAllocator.Allocate(sizeof(DefinedAtom), alignof(DefinedAtom))); + new (A) DefinedAtom(Parent, Address, Alignment); + Parent.addAtom(*A); + getAddrToAtomMap()[A->getAddress()] = A; + return *A; + } + + /// Add a defined atom to the graph. + /// + /// Allocates and constructs a DefinedAtom instance with the given parent, + /// name, address, and alignment. + DefinedAtom &addDefinedAtom(Section &Parent, StringRef Name, + JITTargetAddress Address, uint32_t Alignment) { + assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted"); + DefinedAtom *A = reinterpret_cast( + AtomAllocator.Allocate(sizeof(DefinedAtom), alignof(DefinedAtom))); + new (A) DefinedAtom(Parent, Name, Address, Alignment); + Parent.addAtom(*A); + getAddrToAtomMap()[A->getAddress()] = A; + NamedAtoms[Name] = A; + return *A; + } + + /// Add a common symbol atom to the graph. + /// + /// Adds a common-symbol atom to the graph with the given parent, name, + /// address, alignment and size. + DefinedAtom &addCommonAtom(Section &Parent, StringRef Name, + JITTargetAddress Address, uint32_t Alignment, + uint64_t Size) { + assert(!NamedAtoms.count(Name) && "Duplicate named atom inserted"); + DefinedAtom *A = reinterpret_cast( + AtomAllocator.Allocate(sizeof(DefinedAtom), alignof(DefinedAtom))); + new (A) DefinedAtom(Parent, Name, Address, Alignment); + A->setCommon(Size); + Parent.addAtom(*A); + NamedAtoms[Name] = A; + return *A; + } + + iterator_range sections() { + return make_range(section_iterator(Sections.begin()), + section_iterator(Sections.end())); + } + + /// Returns the section with the given name if it exists, otherwise returns + /// null. + Section *findSectionByName(StringRef Name) { + for (auto &S : sections()) + if (S.getName() == Name) + return &S; + return nullptr; + } + + iterator_range external_atoms() { + return make_range(ExternalAtoms.begin(), ExternalAtoms.end()); + } + + iterator_range absolute_atoms() { + return make_range(AbsoluteAtoms.begin(), AbsoluteAtoms.end()); + } + + iterator_range defined_atoms() { + return make_range(defined_atom_iterator(Sections.begin(), Sections.end()), + defined_atom_iterator(Sections.end(), Sections.end())); + } + + iterator_range defined_atoms() const { + return make_range( + const_defined_atom_iterator(Sections.begin(), Sections.end()), + const_defined_atom_iterator(Sections.end(), Sections.end())); + } + + /// Returns the atom with the given name, which must exist in this graph. + Atom &getAtomByName(StringRef Name) { + auto I = NamedAtoms.find(Name); + assert(I != NamedAtoms.end() && "Name not in NamedAtoms map"); + return *I->second; + } + + /// Returns the atom with the given name, which must exist in this graph and + /// be a DefinedAtom. + DefinedAtom &getDefinedAtomByName(StringRef Name) { + auto &A = getAtomByName(Name); + assert(A.isDefined() && "Atom is not a defined atom"); + return static_cast(A); + } + + /// Search for the given atom by name. + /// Returns the atom (if found) or an error (if no atom with this name + /// exists). + Expected findAtomByName(StringRef Name) { + auto I = NamedAtoms.find(Name); + if (I == NamedAtoms.end()) + return make_error("No atom named " + Name); + return *I->second; + } + + /// Search for the given defined atom by name. + /// Returns the defined atom (if found) or an error (if no atom with this + /// name exists, or if one exists but is not a defined atom). + Expected findDefinedAtomByName(StringRef Name) { + auto I = NamedAtoms.find(Name); + if (I == NamedAtoms.end()) + return make_error("No atom named " + Name); + if (!I->second->isDefined()) + return make_error("Atom " + Name + + " exists but is not a " + "defined atom"); + return static_cast(*I->second); + } + + /// Returns the atom covering the given address, or an error if no such atom + /// exists. + /// + /// Returns null if no atom exists at the given address. + DefinedAtom *getAtomByAddress(JITTargetAddress Address) { + refreshAddrToAtomCache(); + + // If there are no defined atoms, bail out early. + if (AddrToAtomCache->empty()) + return nullptr; + + // Find the atom *after* the given address. + auto I = AddrToAtomCache->upper_bound(Address); + + // If this address falls before any known atom, bail out. + if (I == AddrToAtomCache->begin()) + return nullptr; + + // The atom we're looking for is the one before the atom we found. + --I; + + // Otherwise range check the atom that was found. + assert(!I->second->getContent().empty() && "Atom content not set"); + if (Address >= I->second->getAddress() + I->second->getContent().size()) + return nullptr; + + return I->second; + } + + /// Like getAtomByAddress, but returns an Error if the given address is not + /// covered by an atom, rather than a null pointer. + Expected findAtomByAddress(JITTargetAddress Address) { + if (auto *DA = getAtomByAddress(Address)) + return *DA; + return make_error("No atom at address " + + formatv("{0:x16}", Address)); + } + + // Remove the given external atom from the graph. + void removeExternalAtom(Atom &A) { + assert(!A.isDefined() && !A.isAbsolute() && "A is not an external atom"); + assert(ExternalAtoms.count(&A) && "A is not in the external atoms set"); + ExternalAtoms.erase(&A); + A.~Atom(); + } + + /// Remove the given absolute atom from the graph. + void removeAbsoluteAtom(Atom &A) { + assert(A.isAbsolute() && "A is not an absolute atom"); + assert(AbsoluteAtoms.count(&A) && "A is not in the absolute atoms set"); + AbsoluteAtoms.erase(&A); + A.~Atom(); + } + + /// Remove the given defined atom from the graph. + void removeDefinedAtom(DefinedAtom &DA) { + if (AddrToAtomCache) { + assert(AddrToAtomCache->count(DA.getAddress()) && + "Cache exists, but does not contain atom"); + AddrToAtomCache->erase(DA.getAddress()); + } + if (DA.hasName()) { + assert(NamedAtoms.count(DA.getName()) && "Named atom not in map"); + NamedAtoms.erase(DA.getName()); + } + DA.getSection().removeAtom(DA); + DA.~DefinedAtom(); + } + + /// Invalidate the atom-to-address map. + void invalidateAddrToAtomMap() { AddrToAtomCache = None; } + + /// Dump the graph. + /// + /// If supplied, the EdgeKindToName function will be used to name edge + /// kinds in the debug output. Otherwise raw edge kind numbers will be + /// displayed. + void dump(raw_ostream &OS, + std::function EdegKindToName = + std::function()); + +private: + AddressToAtomMap &getAddrToAtomMap() { + refreshAddrToAtomCache(); + return *AddrToAtomCache; + } + + const AddressToAtomMap &getAddrToAtomMap() const { + refreshAddrToAtomCache(); + return *AddrToAtomCache; + } + + void refreshAddrToAtomCache() const { + if (!AddrToAtomCache) { + AddrToAtomCache = AddressToAtomMap(); + for (auto *DA : defined_atoms()) + (*AddrToAtomCache)[DA->getAddress()] = const_cast(DA); + } + } + + // Put the BumpPtrAllocator first so that we don't free any of the atoms in + // it until all of their destructors have been run. + BumpPtrAllocator AtomAllocator; + + std::string Name; + unsigned PointerSize; + support::endianness Endianness; + SectionList Sections; + NamedAtomMap NamedAtoms; + ExternalAtomSet ExternalAtoms; + ExternalAtomSet AbsoluteAtoms; + mutable Optional AddrToAtomCache; +}; + +/// A function for mutating AtomGraphs. +using AtomGraphPassFunction = std::function; + +/// A list of atom graph passes. +using AtomGraphPassList = std::vector; + +/// An atom graph pass configuration, consisting of a list of pre-prune, +/// post-prune, and post-fixup passes. +struct PassConfiguration { + + /// Pre-prune passes. + /// + /// These passes are called on the graph after it is built, and before any + /// atoms have been pruned. + /// + /// Notable use cases: Marking atoms live or should-discard. + AtomGraphPassList PrePrunePasses; + + /// Post-prune passes. + /// + /// These passes are called on the graph after dead and should-discard atoms + /// have been removed, but before fixups are applied. + /// + /// Notable use cases: Building GOT, stub, and TLV atoms. + AtomGraphPassList PostPrunePasses; + + /// Post-fixup passes. + /// + /// These passes are called on the graph after atom contents has been copied + /// to working memory, and fixups applied. + /// + /// Notable use cases: Testing and validation. + AtomGraphPassList PostFixupPasses; +}; + +/// A map of symbol names to resolved addresses. +using AsyncLookupResult = DenseMap; + +/// A function to call with a resolved symbol map (See AsyncLookupResult) or an +/// error if resolution failed. +using JITLinkAsyncLookupContinuation = + std::function LR)>; + +/// An asynchronous symbol lookup. Performs a search (possibly asynchronously) +/// for the given symbols, calling the given continuation with either the result +/// (if the lookup succeeds), or an error (if the lookup fails). +using JITLinkAsyncLookupFunction = + std::function &Symbols, + JITLinkAsyncLookupContinuation LookupContinuation)>; + +/// Holds context for a single jitLink invocation. +class JITLinkContext { +public: + /// Destroy a JITLinkContext. + virtual ~JITLinkContext(); + + /// Return the MemoryManager to be used for this link. + virtual JITLinkMemoryManager &getMemoryManager() = 0; + + /// Returns a StringRef for the object buffer. + /// This method can not be called once takeObjectBuffer has been called. + virtual MemoryBufferRef getObjectBuffer() const = 0; + + /// Notify this context that linking failed. + /// Called by JITLink if linking cannot be completed. + virtual void notifyFailed(Error Err) = 0; + + /// Called by JITLink to resolve external symbols. This method is passed a + /// lookup continutation which it must call with a result to continue the + /// linking process. + virtual void lookup(const DenseSet &Symbols, + JITLinkAsyncLookupContinuation LookupContinuation) = 0; + + /// Called by JITLink once all defined atoms in the graph have been assigned + /// their final memory locations in the target process. At this point he + /// atom graph can be, inspected to build a symbol table however the atom + /// content will not generally have been copied to the target location yet. + virtual void notifyResolved(AtomGraph &G) = 0; + + /// Called by JITLink to notify the context that the object has been + /// finalized (i.e. emitted to memory and memory permissions set). If all of + /// this objects dependencies have also been finalized then the code is ready + /// to run. + virtual void + notifyFinalized(std::unique_ptr A) = 0; + + /// Called by JITLink prior to linking to determine whether default passes for + /// the target should be added. The default implementation returns true. + /// If subclasses override this method to return false for any target then + /// they are required to fully configure the pass pipeline for that target. + virtual bool shouldAddDefaultTargetPasses(const Triple &TT) const; + + /// Returns the mark-live pass to be used for this link. If no pass is + /// returned (the default) then the target-specific linker implementation will + /// choose a conservative default (usually marking all atoms live). + /// This function is only called if shouldAddDefaultTargetPasses returns true, + /// otherwise the JITContext is responsible for adding a mark-live pass in + /// modifyPassConfig. + virtual AtomGraphPassFunction getMarkLivePass(const Triple &TT) const; + + /// Called by JITLink to modify the pass pipeline prior to linking. + /// The default version performs no modification. + virtual Error modifyPassConfig(const Triple &TT, PassConfiguration &Config); +}; + +/// Marks all atoms in a graph live. This can be used as a default, conservative +/// mark-live implementation. +Error markAllAtomsLive(AtomGraph &G); + +/// Basic JITLink implementation. +/// +/// This function will use sensible defaults for GOT and Stub handling. +void jitLink(std::unique_ptr Ctx); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_JITLINK_H diff --git a/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h b/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h new file mode 100644 index 000000000000..9d0b37fe4a4d --- /dev/null +++ b/include/llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h @@ -0,0 +1,99 @@ +//===-- JITLinkMemoryManager.h - JITLink mem manager interface --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contains the JITLinkMemoryManager interface. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_JITLINKMEMORYMANAGER_H +#define LLVM_EXECUTIONENGINE_JITLINK_JITLINKMEMORYMANAGER_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/Memory.h" +#include + +namespace llvm { +namespace jitlink { + +/// Manages allocations of JIT memory. +/// +/// Instances of this class may be accessed concurrently from multiple threads +/// and their implemetations should include any necessary synchronization. +class JITLinkMemoryManager { +public: + using ProtectionFlags = sys::Memory::ProtectionFlags; + + class SegmentRequest { + public: + SegmentRequest() = default; + SegmentRequest(size_t ContentSize, unsigned ContentAlign, + uint64_t ZeroFillSize, unsigned ZeroFillAlign) + : ContentSize(ContentSize), ZeroFillSize(ZeroFillSize), + ContentAlign(ContentAlign), ZeroFillAlign(ZeroFillAlign) {} + size_t getContentSize() const { return ContentSize; } + unsigned getContentAlignment() const { return ContentAlign; } + uint64_t getZeroFillSize() const { return ZeroFillSize; } + unsigned getZeroFillAlignment() const { return ZeroFillAlign; } + + private: + size_t ContentSize = 0; + uint64_t ZeroFillSize = 0; + unsigned ContentAlign = 0; + unsigned ZeroFillAlign = 0; + }; + + using SegmentsRequestMap = DenseMap; + + /// Represents an allocation created by the memory manager. + /// + /// An allocation object is responsible for allocating and owning jit-linker + /// working and target memory, and for transfering from working to target + /// memory. + /// + class Allocation { + public: + using FinalizeContinuation = std::function; + + virtual ~Allocation(); + + /// Should return the address of linker working memory for the segment with + /// the given protection flags. + virtual MutableArrayRef getWorkingMemory(ProtectionFlags Seg) = 0; + + /// Should return the final address in the target process where the segment + /// will reside. + virtual JITTargetAddress getTargetMemory(ProtectionFlags Seg) = 0; + + /// Should transfer from working memory to target memory, and release + /// working memory. + virtual void finalizeAsync(FinalizeContinuation OnFinalize) = 0; + + /// Should deallocate target memory. + virtual Error deallocate() = 0; + }; + + virtual ~JITLinkMemoryManager(); + + /// Create an Allocation object. + virtual Expected> + allocate(const SegmentsRequestMap &Request) = 0; +}; + +/// A JITLinkMemoryManager that allocates in-process memory. +class InProcessMemoryManager : public JITLinkMemoryManager { +public: + Expected> + allocate(const SegmentsRequestMap &Request) override; +}; + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_JITLINK_H diff --git a/include/llvm/ExecutionEngine/JITLink/MachO.h b/include/llvm/ExecutionEngine/JITLink/MachO.h new file mode 100644 index 000000000000..7facb657a51c --- /dev/null +++ b/include/llvm/ExecutionEngine/JITLink/MachO.h @@ -0,0 +1,30 @@ +//===------- MachO.h - Generic JIT link function for MachO ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic jit-link functions for MachO. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_MACHO_H +#define LLVM_EXECUTIONENGINE_JITLINK_MACHO_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { + +/// jit-link the given ObjBuffer, which must be a MachO object file. +/// +/// Uses conservative defaults for GOT and stub handling based on the target +/// platform. +void jitLink_MachO(std::unique_ptr Ctx); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_MACHO_H diff --git a/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h b/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h new file mode 100644 index 000000000000..1d5b586afc32 --- /dev/null +++ b/include/llvm/ExecutionEngine/JITLink/MachO_x86_64.h @@ -0,0 +1,63 @@ +//===--- MachO_x86_64.h - JIT link functions for MachO/x86-64 ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// jit-link functions for MachO/x86-64. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_JITLINK_MACHO_X86_64_H +#define LLVM_EXECUTIONENGINE_JITLINK_MACHO_X86_64_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { + +namespace MachO_x86_64_Edges { + +enum MachOX86RelocationKind : Edge::Kind { + Branch32 = Edge::FirstRelocation, + Pointer64, + Pointer64Anon, + PCRel32, + PCRel32Minus1, + PCRel32Minus2, + PCRel32Minus4, + PCRel32Anon, + PCRel32Minus1Anon, + PCRel32Minus2Anon, + PCRel32Minus4Anon, + PCRel32GOTLoad, + PCRel32GOT, + PCRel32TLV, + Delta32, + Delta64, + NegDelta32, + NegDelta64, +}; + +} // namespace MachO_x86_64_Edges + +/// jit-link the given object buffer, which must be a MachO x86-64 object file. +/// +/// If PrePrunePasses is empty then a default mark-live pass will be inserted +/// that will mark all exported atoms live. If PrePrunePasses is not empty, the +/// caller is responsible for including a pass to mark atoms as live. +/// +/// If PostPrunePasses is empty then a default GOT-and-stubs insertion pass will +/// be inserted. If PostPrunePasses is not empty then the caller is responsible +/// for including a pass to insert GOT and stub edges. +void jitLink_MachO_x86_64(std::unique_ptr Ctx); + +/// Return the string name of the given MachO x86-64 edge kind. +StringRef getMachOX86RelocationKindName(Edge::Kind R); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_JITLINK_MACHO_X86_64_H diff --git a/include/llvm/ExecutionEngine/JITSymbol.h b/include/llvm/ExecutionEngine/JITSymbol.h index 05c9590726df..b14154c5b5e8 100644 --- a/include/llvm/ExecutionEngine/JITSymbol.h +++ b/include/llvm/ExecutionEngine/JITSymbol.h @@ -1,9 +1,8 @@ //===- JITSymbol.h - JIT symbol abstraction ---------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -56,7 +55,7 @@ template JITTargetAddress pointerToJITTargetAddress(T *Ptr) { class JITSymbolFlags { public: using UnderlyingType = uint8_t; - using TargetFlagsType = uint64_t; + using TargetFlagsType = uint8_t; enum FlagNames : UnderlyingType { None = 0, @@ -66,15 +65,9 @@ public: Absolute = 1U << 3, Exported = 1U << 4, Callable = 1U << 5, - Lazy = 1U << 6, - Materializing = 1U << 7, - LLVM_MARK_AS_BITMASK_ENUM(/* LargestValue = */ Materializing) + LLVM_MARK_AS_BITMASK_ENUM(/* LargestValue = */ Callable) }; - static JITSymbolFlags stripTransientFlags(JITSymbolFlags Orig) { - return static_cast(Orig.Flags & ~Lazy & ~Materializing); - } - /// Default-construct a JITSymbolFlags instance. JITSymbolFlags() = default; @@ -84,7 +77,7 @@ public: /// Construct a JITSymbolFlags instance from the given flags and target /// flags. JITSymbolFlags(FlagNames Flags, TargetFlagsType TargetFlags) - : Flags(Flags), TargetFlags(TargetFlags) {} + : TargetFlags(TargetFlags), Flags(Flags) {} /// Implicitly convert to bool. Returs true if any flag is set. explicit operator bool() const { return Flags != None || TargetFlags != 0; } @@ -111,19 +104,6 @@ public: return (Flags & HasError) == HasError; } - /// Returns true if this is a lazy symbol. - /// This flag is used internally by the JIT APIs to track - /// materialization states. - bool isLazy() const { return Flags & Lazy; } - - /// Returns true if this symbol is in the process of being - /// materialized. - bool isMaterializing() const { return Flags & Materializing; } - - /// Returns true if this symbol is fully materialized. - /// (i.e. neither lazy, nor materializing). - bool isMaterialized() const { return !(Flags & (Lazy | Materializing)); } - /// Returns true if the Weak flag is set. bool isWeak() const { return (Flags & Weak) == Weak; @@ -168,8 +148,8 @@ public: fromObjectSymbol(const object::SymbolRef &Symbol); private: - FlagNames Flags = None; TargetFlagsType TargetFlags = 0; + FlagNames Flags = None; }; inline JITSymbolFlags operator&(const JITSymbolFlags &LHS, diff --git a/include/llvm/ExecutionEngine/MCJIT.h b/include/llvm/ExecutionEngine/MCJIT.h index 66ddb7cdb875..8253bf98963b 100644 --- a/include/llvm/ExecutionEngine/MCJIT.h +++ b/include/llvm/ExecutionEngine/MCJIT.h @@ -1,9 +1,8 @@ //===-- MCJIT.h - MC-Based Just-In-Time Execution Engine --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ExecutionEngine/OProfileWrapper.h b/include/llvm/ExecutionEngine/OProfileWrapper.h index 05da594a94a8..b13d7f6e245b 100644 --- a/include/llvm/ExecutionEngine/OProfileWrapper.h +++ b/include/llvm/ExecutionEngine/OProfileWrapper.h @@ -1,9 +1,8 @@ //===-- OProfileWrapper.h - OProfile JIT API Wrapper ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This file defines a OProfileWrapper object that detects if the oprofile diff --git a/include/llvm/ExecutionEngine/ObjectCache.h b/include/llvm/ExecutionEngine/ObjectCache.h index 077044408e09..47e94f18a1c7 100644 --- a/include/llvm/ExecutionEngine/ObjectCache.h +++ b/include/llvm/ExecutionEngine/ObjectCache.h @@ -1,9 +1,8 @@ //===-- ObjectCache.h - Class definition for the ObjectCache ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h index 884878925cde..5f593a27cad6 100644 --- a/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h +++ b/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h @@ -1,9 +1,8 @@ //===- CompileOnDemandLayer.h - Compile each function on demand -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -265,13 +264,26 @@ public: std::function R)>; /// Construct a compile-on-demand layer instance. - LegacyCompileOnDemandLayer(ExecutionSession &ES, BaseLayerT &BaseLayer, - SymbolResolverGetter GetSymbolResolver, - SymbolResolverSetter SetSymbolResolver, - PartitioningFtor Partition, - CompileCallbackMgrT &CallbackMgr, - IndirectStubsManagerBuilderT CreateIndirectStubsManager, - bool CloneStubsIntoPartitions = true) + LLVM_ATTRIBUTE_DEPRECATED( + LegacyCompileOnDemandLayer( + ExecutionSession &ES, BaseLayerT &BaseLayer, + SymbolResolverGetter GetSymbolResolver, + SymbolResolverSetter SetSymbolResolver, PartitioningFtor Partition, + CompileCallbackMgrT &CallbackMgr, + IndirectStubsManagerBuilderT CreateIndirectStubsManager, + bool CloneStubsIntoPartitions = true), + "ORCv1 layers (layers with the 'Legacy' prefix) are deprecated. Please " + "use " + "the ORCv2 LegacyCompileOnDemandLayer instead"); + + /// Legacy layer constructor with deprecation acknowledgement. + LegacyCompileOnDemandLayer( + ORCv1DeprecationAcknowledgement, ExecutionSession &ES, + BaseLayerT &BaseLayer, SymbolResolverGetter GetSymbolResolver, + SymbolResolverSetter SetSymbolResolver, PartitioningFtor Partition, + CompileCallbackMgrT &CallbackMgr, + IndirectStubsManagerBuilderT CreateIndirectStubsManager, + bool CloneStubsIntoPartitions = true) : ES(ES), BaseLayer(BaseLayer), GetSymbolResolver(std::move(GetSymbolResolver)), SetSymbolResolver(std::move(SetSymbolResolver)), @@ -730,8 +742,24 @@ private: bool CloneStubsIntoPartitions; }; -} // end namespace orc +template +LegacyCompileOnDemandLayer:: + LegacyCompileOnDemandLayer( + ExecutionSession &ES, BaseLayerT &BaseLayer, + SymbolResolverGetter GetSymbolResolver, + SymbolResolverSetter SetSymbolResolver, PartitioningFtor Partition, + CompileCallbackMgrT &CallbackMgr, + IndirectStubsManagerBuilderT CreateIndirectStubsManager, + bool CloneStubsIntoPartitions) + : ES(ES), BaseLayer(BaseLayer), + GetSymbolResolver(std::move(GetSymbolResolver)), + SetSymbolResolver(std::move(SetSymbolResolver)), + Partition(std::move(Partition)), CompileCallbackMgr(CallbackMgr), + CreateIndirectStubsManager(std::move(CreateIndirectStubsManager)), + CloneStubsIntoPartitions(CloneStubsIntoPartitions) {} +} // end namespace orc } // end namespace llvm #endif // LLVM_EXECUTIONENGINE_ORC_COMPILEONDEMANDLAYER_H diff --git a/include/llvm/ExecutionEngine/Orc/CompileUtils.h b/include/llvm/ExecutionEngine/Orc/CompileUtils.h index f34f88311ba5..eb6d84e8cbb4 100644 --- a/include/llvm/ExecutionEngine/Orc/CompileUtils.h +++ b/include/llvm/ExecutionEngine/Orc/CompileUtils.h @@ -1,9 +1,8 @@ //===- CompileUtils.h - Utilities for compiling IR in the JIT ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,28 +13,21 @@ #ifndef LLVM_EXECUTIONENGINE_ORC_COMPILEUTILS_H #define LLVM_EXECUTIONENGINE_ORC_COMPILEUTILS_H -#include "llvm/ADT/SmallVector.h" -#include "llvm/ExecutionEngine/ObjectCache.h" #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" -#include "llvm/IR/LegacyPassManager.h" -#include "llvm/Object/Binary.h" -#include "llvm/Object/ObjectFile.h" -#include "llvm/Support/Error.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/MemoryBuffer.h" -#include "llvm/Support/SmallVectorMemoryBuffer.h" -#include "llvm/Support/raw_ostream.h" -#include "llvm/Target/TargetMachine.h" -#include #include namespace llvm { class MCContext; +class MemoryBuffer; class Module; +class ObjectCache; +class TargetMachine; namespace orc { +class JITTargetMachineBuilder; + /// Simple compile functor: Takes a single IR module and returns an ObjectFile. /// This compiler supports a single compilation thread and LLVMContext only. /// For multithreaded compilation, use ConcurrentIRCompiler below. @@ -51,56 +43,32 @@ public: void setObjectCache(ObjectCache *NewCache) { ObjCache = NewCache; } /// Compile a Module to an ObjectFile. - CompileResult operator()(Module &M) { - CompileResult CachedObject = tryToLoadFromObjectCache(M); - if (CachedObject) - return CachedObject; - - SmallVector ObjBufferSV; - - { - raw_svector_ostream ObjStream(ObjBufferSV); - - legacy::PassManager PM; - MCContext *Ctx; - if (TM.addPassesToEmitMC(PM, Ctx, ObjStream)) - llvm_unreachable("Target does not support MC emission."); - PM.run(M); - } - - auto ObjBuffer = - llvm::make_unique(std::move(ObjBufferSV)); - auto Obj = - object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef()); - - if (Obj) { - notifyObjectCompiled(M, *ObjBuffer); - return std::move(ObjBuffer); - } - - // TODO: Actually report errors helpfully. - consumeError(Obj.takeError()); - return nullptr; - } + CompileResult operator()(Module &M); private: - - CompileResult tryToLoadFromObjectCache(const Module &M) { - if (!ObjCache) - return CompileResult(); - - return ObjCache->getObject(&M); - } - - void notifyObjectCompiled(const Module &M, const MemoryBuffer &ObjBuffer) { - if (ObjCache) - ObjCache->notifyObjectCompiled(&M, ObjBuffer.getMemBufferRef()); - } + CompileResult tryToLoadFromObjectCache(const Module &M); + void notifyObjectCompiled(const Module &M, const MemoryBuffer &ObjBuffer); TargetMachine &TM; ObjectCache *ObjCache = nullptr; }; +/// A SimpleCompiler that owns its TargetMachine. +/// +/// This convenient for clients who don't want to own their TargetMachines, +/// e.g. LLJIT. +class TMOwningSimpleCompiler : public SimpleCompiler { +public: + TMOwningSimpleCompiler(std::unique_ptr TM, + ObjectCache *ObjCache = nullptr) + : SimpleCompiler(*TM, ObjCache), TM(std::move(TM)) {} + +private: + // FIXME: shared because std::functions (and consequently + // IRCompileLayer::CompileFunction) are not moveable. + std::shared_ptr TM; +}; + /// A thread-safe version of SimpleCompiler. /// /// This class creates a new TargetMachine and SimpleCompiler instance for each @@ -108,16 +76,11 @@ private: class ConcurrentIRCompiler { public: ConcurrentIRCompiler(JITTargetMachineBuilder JTMB, - ObjectCache *ObjCache = nullptr) - : JTMB(std::move(JTMB)), ObjCache(ObjCache) {} + ObjectCache *ObjCache = nullptr); void setObjectCache(ObjectCache *ObjCache) { this->ObjCache = ObjCache; } - std::unique_ptr operator()(Module &M) { - auto TM = cantFail(JTMB.createTargetMachine()); - SimpleCompiler C(*TM, ObjCache); - return C(M); - } + std::unique_ptr operator()(Module &M); private: JITTargetMachineBuilder JTMB; diff --git a/include/llvm/ExecutionEngine/Orc/Core.h b/include/llvm/ExecutionEngine/Orc/Core.h index 39d306e0bd4c..94a5618233e4 100644 --- a/include/llvm/ExecutionEngine/Orc/Core.h +++ b/include/llvm/ExecutionEngine/Orc/Core.h @@ -1,9 +1,8 @@ //===------ Core.h -- Core ORC APIs (Layer, JITDylib, etc.) -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -17,6 +16,7 @@ #include "llvm/ADT/BitmaskEnum.h" #include "llvm/ExecutionEngine/JITSymbol.h" #include "llvm/ExecutionEngine/Orc/SymbolStringPool.h" +#include "llvm/ExecutionEngine/OrcV1Deprecation.h" #include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" @@ -34,6 +34,7 @@ class ExecutionSession; class MaterializationUnit; class MaterializationResponsibility; class JITDylib; +enum class SymbolState : uint8_t; /// VModuleKey provides a unique identifier (allocated and managed by /// ExecutionSessions) for a module added to the JIT. @@ -57,6 +58,18 @@ using SymbolDependenceMap = DenseMap; /// A list of (JITDylib*, bool) pairs. using JITDylibSearchList = std::vector>; +struct SymbolAliasMapEntry { + SymbolAliasMapEntry() = default; + SymbolAliasMapEntry(SymbolStringPtr Aliasee, JITSymbolFlags AliasFlags) + : Aliasee(std::move(Aliasee)), AliasFlags(AliasFlags) {} + + SymbolStringPtr Aliasee; + JITSymbolFlags AliasFlags; +}; + +/// A map of Symbols to (Symbol, Flags) pairs. +using SymbolAliasMap = DenseMap; + /// Render a SymbolStringPtr. raw_ostream &operator<<(raw_ostream &OS, const SymbolStringPtr &Sym); @@ -88,12 +101,15 @@ raw_ostream &operator<<(raw_ostream &OS, const MaterializationUnit &MU); /// Render a JITDylibSearchList. raw_ostream &operator<<(raw_ostream &OS, const JITDylibSearchList &JDs); +/// Render a SymbolAliasMap. +raw_ostream &operator<<(raw_ostream &OS, const SymbolAliasMap &Aliases); + +/// Render a SymbolState. +raw_ostream &operator<<(raw_ostream &OS, const SymbolState &S); + /// Callback to notify client that symbols have been resolved. using SymbolsResolvedCallback = std::function)>; -/// Callback to notify client that symbols are ready for execution. -using SymbolsReadyCallback = std::function; - /// Callback to register the dependencies for a given query. using RegisterDependenciesFunction = std::function; @@ -175,7 +191,7 @@ public: /// Note: The returned flags may have transient flags (Lazy, Materializing) /// set. These should be stripped with JITSymbolFlags::stripTransientFlags /// before using. - const SymbolFlagsMap &getSymbols() { return SymbolFlags; } + const SymbolFlagsMap &getSymbols() const { return SymbolFlags; } /// Returns the names of any symbols covered by this /// MaterializationResponsibility object that have queries pending. This @@ -189,12 +205,12 @@ public: /// symbols must be ones covered by this MaterializationResponsibility /// instance. Individual calls to this method may resolve a subset of the /// symbols, but all symbols must have been resolved prior to calling emit. - void resolve(const SymbolMap &Symbols); + void notifyResolved(const SymbolMap &Symbols); /// Notifies the target JITDylib (and any pending queries on that JITDylib) /// that all symbols covered by this MaterializationResponsibility instance /// have been emitted. - void emit(); + void notifyEmitted(); /// Adds new symbols to the JITDylib and this responsibility instance. /// JITDylib entries start out in the materializing state. @@ -334,18 +350,6 @@ absoluteSymbols(SymbolMap Symbols, VModuleKey K = VModuleKey()) { std::move(Symbols), std::move(K)); } -struct SymbolAliasMapEntry { - SymbolAliasMapEntry() = default; - SymbolAliasMapEntry(SymbolStringPtr Aliasee, JITSymbolFlags AliasFlags) - : Aliasee(std::move(Aliasee)), AliasFlags(AliasFlags) {} - - SymbolStringPtr Aliasee; - JITSymbolFlags AliasFlags; -}; - -/// A map of Symbols to (Symbol, Flags) pairs. -using SymbolAliasMap = DenseMap; - /// A materialization unit for symbol aliases. Allows existing symbols to be /// aliased with alternate flags. class ReExportsMaterializationUnit : public MaterializationUnit { @@ -419,7 +423,7 @@ public: ReexportsGenerator(JITDylib &SourceJD, bool MatchNonExported = false, SymbolPredicate Allow = SymbolPredicate()); - SymbolNameSet operator()(JITDylib &JD, const SymbolNameSet &Names); + Expected operator()(JITDylib &JD, const SymbolNameSet &Names); private: JITDylib &SourceJD; @@ -427,6 +431,15 @@ private: SymbolPredicate Allow; }; +/// Represents the state that a symbol has reached during materialization. +enum class SymbolState : uint8_t { + Invalid, /// No symbol should be in this state. + NeverSearched, /// Added to the symbol table, never queried. + Materializing, /// Queried, materialization begun. + Resolved, /// Assigned address, still materializing. + Ready = 0x3f /// Ready and safe for clients to access. +}; + /// A symbol query that returns results via a callback when results are /// ready. /// @@ -437,38 +450,30 @@ class AsynchronousSymbolQuery { friend class JITSymbolResolverAdapter; public: - - /// Create a query for the given symbols, notify-resolved and - /// notify-ready callbacks. + /// Create a query for the given symbols. The NotifyComplete + /// callback will be called once all queried symbols reach the given + /// minimum state. AsynchronousSymbolQuery(const SymbolNameSet &Symbols, - SymbolsResolvedCallback NotifySymbolsResolved, - SymbolsReadyCallback NotifySymbolsReady); + SymbolState RequiredState, + SymbolsResolvedCallback NotifyComplete); - /// Set the resolved symbol information for the given symbol name. - void resolve(const SymbolStringPtr &Name, JITEvaluatedSymbol Sym); + /// Notify the query that a requested symbol has reached the required state. + void notifySymbolMetRequiredState(const SymbolStringPtr &Name, + JITEvaluatedSymbol Sym); /// Returns true if all symbols covered by this query have been /// resolved. - bool isFullyResolved() const { return NotYetResolvedCount == 0; } + bool isComplete() const { return OutstandingSymbolsCount == 0; } - /// Call the NotifySymbolsResolved callback. + /// Call the NotifyComplete callback. /// - /// This should only be called if all symbols covered by the query have been - /// resolved. - void handleFullyResolved(); - - /// Notify the query that a requested symbol is ready for execution. - void notifySymbolReady(); - - /// Returns true if all symbols covered by this query are ready. - bool isFullyReady() const { return NotYetReadyCount == 0; } - - /// Calls the NotifySymbolsReady callback. - /// - /// This should only be called if all symbols covered by this query are ready. - void handleFullyReady(); + /// This should only be called if all symbols covered by the query have + /// reached the specified state. + void handleComplete(); private: + SymbolState getRequiredState() { return RequiredState; } + void addQueryDependence(JITDylib &JD, SymbolStringPtr Name); void removeQueryDependence(JITDylib &JD, const SymbolStringPtr &Name); @@ -479,12 +484,11 @@ private: void detach(); - SymbolsResolvedCallback NotifySymbolsResolved; - SymbolsReadyCallback NotifySymbolsReady; + SymbolsResolvedCallback NotifyComplete; SymbolDependenceMap QueryRegistrations; SymbolMap ResolvedSymbols; - size_t NotYetResolvedCount; - size_t NotYetReadyCount; + size_t OutstandingSymbolsCount; + SymbolState RequiredState; }; /// A symbol table that supports asynchoronous symbol queries. @@ -498,7 +502,7 @@ class JITDylib { friend class ExecutionSession; friend class MaterializationResponsibility; public: - using GeneratorFunction = std::function( JITDylib &Parent, const SymbolNameSet &Names)>; using AsynchronousSymbolQuerySet = @@ -596,7 +600,7 @@ public: /// Search the given JITDylib for the symbols in Symbols. If found, store /// the flags for each symbol in Flags. Returns any unresolved symbols. - SymbolFlagsMap lookupFlags(const SymbolNameSet &Names); + Expected lookupFlags(const SymbolNameSet &Names); /// Dump current JITDylib state to OS. void dump(raw_ostream &OS); @@ -609,8 +613,8 @@ public: /// and the query will not be applied. The Query is not failed and can be /// re-used in a subsequent lookup once the symbols have been added, or /// manually failed. - SymbolNameSet legacyLookup(std::shared_ptr Q, - SymbolNameSet Names); + Expected + legacyLookup(std::shared_ptr Q, SymbolNameSet Names); private: using AsynchronousSymbolQueryList = @@ -627,40 +631,92 @@ private: DenseMap>; struct MaterializingInfo { - AsynchronousSymbolQueryList PendingQueries; SymbolDependenceMap Dependants; SymbolDependenceMap UnemittedDependencies; bool IsEmitted = false; + + void addQuery(std::shared_ptr Q); + void removeQuery(const AsynchronousSymbolQuery &Q); + AsynchronousSymbolQueryList takeQueriesMeeting(SymbolState RequiredState); + AsynchronousSymbolQueryList takeAllQueries(); + bool hasQueriesPending() const { return !PendingQueries.empty(); } + const AsynchronousSymbolQueryList &pendingQueries() const { + return PendingQueries; + } + + private: + AsynchronousSymbolQueryList PendingQueries; }; using MaterializingInfosMap = DenseMap; - using LookupImplActionFlags = enum { - None = 0, - NotifyFullyResolved = 1 << 0U, - NotifyFullyReady = 1 << 1U, - LLVM_MARK_AS_BITMASK_ENUM(NotifyFullyReady) + class SymbolTableEntry { + public: + SymbolTableEntry() = default; + SymbolTableEntry(JITSymbolFlags Flags) + : Flags(Flags), State(static_cast(SymbolState::NeverSearched)), + MaterializerAttached(false), PendingRemoval(false) {} + + JITTargetAddress getAddress() const { return Addr; } + JITSymbolFlags getFlags() const { return Flags; } + SymbolState getState() const { return static_cast(State); } + + bool isInMaterializationPhase() const { + return getState() == SymbolState::Materializing || + getState() == SymbolState::Resolved; + } + + bool hasMaterializerAttached() const { return MaterializerAttached; } + bool isPendingRemoval() const { return PendingRemoval; } + + void setAddress(JITTargetAddress Addr) { this->Addr = Addr; } + void setFlags(JITSymbolFlags Flags) { this->Flags = Flags; } + void setState(SymbolState State) { + assert(static_cast(State) < (1 << 6) && + "State does not fit in bitfield"); + this->State = static_cast(State); + } + + void setMaterializerAttached(bool MaterializerAttached) { + this->MaterializerAttached = MaterializerAttached; + } + + void setPendingRemoval(bool PendingRemoval) { + this->PendingRemoval = PendingRemoval; + } + + JITEvaluatedSymbol getSymbol() const { + return JITEvaluatedSymbol(Addr, Flags); + } + + private: + JITTargetAddress Addr = 0; + JITSymbolFlags Flags; + uint8_t State : 6; + uint8_t MaterializerAttached : 1; + uint8_t PendingRemoval : 1; }; + using SymbolTable = DenseMap; + JITDylib(ExecutionSession &ES, std::string Name); Error defineImpl(MaterializationUnit &MU); - SymbolNameSet lookupFlagsImpl(SymbolFlagsMap &Flags, - const SymbolNameSet &Names); + Expected lookupFlagsImpl(SymbolFlagsMap &Flags, + const SymbolNameSet &Names); - void lodgeQuery(std::shared_ptr &Q, - SymbolNameSet &Unresolved, bool MatchNonExported, - MaterializationUnitList &MUs); + Error lodgeQuery(std::shared_ptr &Q, + SymbolNameSet &Unresolved, bool MatchNonExported, + MaterializationUnitList &MUs); void lodgeQueryImpl(std::shared_ptr &Q, SymbolNameSet &Unresolved, bool MatchNonExported, MaterializationUnitList &MUs); - LookupImplActionFlags - lookupImpl(std::shared_ptr &Q, - std::vector> &MUs, - SymbolNameSet &Unresolved); + bool lookupImpl(std::shared_ptr &Q, + std::vector> &MUs, + SymbolNameSet &Unresolved); void detachQueryHelper(AsynchronousSymbolQuery &Q, const SymbolNameSet &QuerySymbols); @@ -686,7 +742,7 @@ private: ExecutionSession &ES; std::string JITDylibName; - SymbolMap Symbols; + SymbolTable Symbols; UnmaterializedInfosMap UnmaterializedInfos; MaterializingInfosMap MaterializingInfos; GeneratorFunction DefGenerator; @@ -727,7 +783,15 @@ public: /// the ExecutionSession. JITDylib &getMainJITDylib(); + /// Return a pointer to the "name" JITDylib. + /// Ownership of JITDylib remains within Execution Session + JITDylib *getJITDylibByName(StringRef Name); + /// Add a new JITDylib to this ExecutionSession. + /// + /// The JITDylib Name is required to be unique. Clients should verify that + /// names are not being re-used (e.g. by calling getJITDylibByName) if names + /// are based on user input. JITDylib &createJITDylib(std::string Name, bool AddToMainDylibSearchOrder = true); @@ -769,7 +833,7 @@ public: /// Do not use -- this will be removed soon. Expected legacyLookup(LegacyAsyncLookupFunction AsyncLookup, SymbolNameSet Names, - bool WaiUntilReady, + SymbolState RequiredState, RegisterDependenciesFunction RegisterDependencies); /// Search the given JITDylib list for the given symbols. @@ -779,11 +843,8 @@ public: /// (hidden visibility) symbols in that dylib (true means match against /// non-exported symbols, false means do not match). /// - /// The OnResolve callback will be called once all requested symbols are - /// resolved, or if an error occurs prior to resolution. - /// - /// The OnReady callback will be called once all requested symbols are ready, - /// or if an error occurs after resolution but before all symbols are ready. + /// The NotifyComplete callback will be called once all requested symbols + /// reach the required state. /// /// If all symbols are found, the RegisterDependencies function will be called /// while the session lock is held. This gives clients a chance to register @@ -795,7 +856,7 @@ public: /// client to get an address to call) then the value NoDependenciesToRegister /// can be used. void lookup(const JITDylibSearchList &SearchOrder, SymbolNameSet Symbols, - SymbolsResolvedCallback OnResolve, SymbolsReadyCallback OnReady, + SymbolState RequiredState, SymbolsResolvedCallback NotifyComplete, RegisterDependenciesFunction RegisterDependencies); /// Blocking version of lookup above. Returns the resolved symbol map. @@ -807,9 +868,9 @@ public: /// error will be reported via reportErrors. Expected lookup(const JITDylibSearchList &SearchOrder, const SymbolNameSet &Symbols, + SymbolState RequiredState = SymbolState::Ready, RegisterDependenciesFunction RegisterDependencies = - NoDependenciesToRegister, - bool WaitUntilReady = true); + NoDependenciesToRegister); /// Convenience version of blocking lookup. /// Searches each of the JITDylibs in the search order in turn for the given @@ -832,10 +893,11 @@ public: /// Materialize the given unit. void dispatchMaterialization(JITDylib &JD, std::unique_ptr MU) { - LLVM_DEBUG(runSessionLocked([&]() { - dbgs() << "Compiling, for " << JD.getName() << ", " << *MU - << "\n"; - });); + LLVM_DEBUG({ + runSessionLocked([&]() { + dbgs() << "Dispatching " << *MU << " for " << JD.getName() << "\n"; + }); + }); DispatchMaterialization(JD, std::move(MU)); } diff --git a/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h b/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h index 88559f822e5d..75865920c741 100644 --- a/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h +++ b/include/llvm/ExecutionEngine/Orc/ExecutionUtils.h @@ -1,9 +1,8 @@ //===- ExecutionUtils.h - Utilities for executing code in Orc ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -98,7 +97,14 @@ class LegacyCtorDtorRunner { public: /// Construct a CtorDtorRunner for the given range using the given /// name mangling function. - LegacyCtorDtorRunner(std::vector CtorDtorNames, VModuleKey K) + LLVM_ATTRIBUTE_DEPRECATED( + LegacyCtorDtorRunner(std::vector CtorDtorNames, + VModuleKey K), + "ORCv1 utilities (utilities with the 'Legacy' prefix) are deprecated. " + "Please use the ORCv2 CtorDtorRunner utility instead"); + + LegacyCtorDtorRunner(ORCv1DeprecationAcknowledgement, + std::vector CtorDtorNames, VModuleKey K) : CtorDtorNames(std::move(CtorDtorNames)), K(K) {} /// Run the recorded constructors/destructors through the given JIT @@ -129,6 +135,11 @@ private: orc::VModuleKey K; }; +template +LegacyCtorDtorRunner::LegacyCtorDtorRunner( + std::vector CtorDtorNames, VModuleKey K) + : CtorDtorNames(std::move(CtorDtorNames)), K(K) {} + class CtorDtorRunner { public: CtorDtorRunner(JITDylib &JD) : JD(JD) {} @@ -181,7 +192,14 @@ class LegacyLocalCXXRuntimeOverrides : public LocalCXXRuntimeOverridesBase { public: /// Create a runtime-overrides class. template - LegacyLocalCXXRuntimeOverrides(const MangleFtorT &Mangle) { + LLVM_ATTRIBUTE_DEPRECATED( + LegacyLocalCXXRuntimeOverrides(const MangleFtorT &Mangle), + "ORCv1 utilities (utilities with the 'Legacy' prefix) are deprecated. " + "Please use the ORCv2 LocalCXXRuntimeOverrides utility instead"); + + template + LegacyLocalCXXRuntimeOverrides(ORCv1DeprecationAcknowledgement, + const MangleFtorT &Mangle) { addOverride(Mangle("__dso_handle"), toTargetAddress(&DSOHandleOverride)); addOverride(Mangle("__cxa_atexit"), toTargetAddress(&CXAAtExitOverride)); } @@ -202,6 +220,13 @@ private: StringMap CXXRuntimeOverrides; }; +template +LegacyLocalCXXRuntimeOverrides::LegacyLocalCXXRuntimeOverrides( + const MangleFtorT &Mangle) { + addOverride(Mangle("__dso_handle"), toTargetAddress(&DSOHandleOverride)); + addOverride(Mangle("__cxa_atexit"), toTargetAddress(&CXAAtExitOverride)); +} + class LocalCXXRuntimeOverrides : public LocalCXXRuntimeOverridesBase { public: Error enable(JITDylib &JD, MangleAndInterner &Mangler); @@ -218,28 +243,29 @@ public: /// Create a DynamicLibrarySearchGenerator that searches for symbols in the /// given sys::DynamicLibrary. + /// /// If the Allow predicate is given then only symbols matching the predicate - /// will be searched for in the DynamicLibrary. If the predicate is not given - /// then all symbols will be searched for. - DynamicLibrarySearchGenerator(sys::DynamicLibrary Dylib, const DataLayout &DL, + /// will be searched for. If the predicate is not given then all symbols will + /// be searched for. + DynamicLibrarySearchGenerator(sys::DynamicLibrary Dylib, char GlobalPrefix, SymbolPredicate Allow = SymbolPredicate()); /// Permanently loads the library at the given path and, on success, returns /// a DynamicLibrarySearchGenerator that will search it for symbol definitions /// in the library. On failure returns the reason the library failed to load. static Expected - Load(const char *FileName, const DataLayout &DL, + Load(const char *FileName, char GlobalPrefix, SymbolPredicate Allow = SymbolPredicate()); /// Creates a DynamicLibrarySearchGenerator that searches for symbols in /// the current process. static Expected - GetForCurrentProcess(const DataLayout &DL, + GetForCurrentProcess(char GlobalPrefix, SymbolPredicate Allow = SymbolPredicate()) { - return Load(nullptr, DL, std::move(Allow)); + return Load(nullptr, GlobalPrefix, std::move(Allow)); } - SymbolNameSet operator()(JITDylib &JD, const SymbolNameSet &Names); + Expected operator()(JITDylib &JD, const SymbolNameSet &Names); private: sys::DynamicLibrary Dylib; diff --git a/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h b/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h index a8a88d7cb2d2..a4e43d4e1c9c 100644 --- a/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h +++ b/include/llvm/ExecutionEngine/Orc/GlobalMappingLayer.h @@ -1,9 +1,8 @@ //===- GlobalMappingLayer.h - Run all IR through a functor ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h b/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h index 30d71e69cd70..52223a83ad42 100644 --- a/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h +++ b/include/llvm/ExecutionEngine/Orc/IRCompileLayer.h @@ -1,9 +1,8 @@ //===- IRCompileLayer.h -- Eagerly compile IR for JIT -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -64,8 +63,18 @@ public: /// Construct an LegacyIRCompileLayer with the given BaseLayer, which must /// implement the ObjectLayer concept. + LLVM_ATTRIBUTE_DEPRECATED( + LegacyIRCompileLayer( + BaseLayerT &BaseLayer, CompileFtor Compile, + NotifyCompiledCallback NotifyCompiled = NotifyCompiledCallback()), + "ORCv1 layers (layers with the 'Legacy' prefix) are deprecated. Please " + "use " + "the ORCv2 IRCompileLayer instead"); + + /// Legacy layer constructor with deprecation acknowledgement. LegacyIRCompileLayer( - BaseLayerT &BaseLayer, CompileFtor Compile, + ORCv1DeprecationAcknowledgement, BaseLayerT &BaseLayer, + CompileFtor Compile, NotifyCompiledCallback NotifyCompiled = NotifyCompiledCallback()) : BaseLayer(BaseLayer), Compile(std::move(Compile)), NotifyCompiled(std::move(NotifyCompiled)) {} @@ -123,8 +132,14 @@ private: NotifyCompiledCallback NotifyCompiled; }; -} // end namespace orc +template +LegacyIRCompileLayer::LegacyIRCompileLayer( + BaseLayerT &BaseLayer, CompileFtor Compile, + NotifyCompiledCallback NotifyCompiled) + : BaseLayer(BaseLayer), Compile(std::move(Compile)), + NotifyCompiled(std::move(NotifyCompiled)) {} +} // end namespace orc } // end namespace llvm #endif // LLVM_EXECUTIONENGINE_ORC_IRCOMPILINGLAYER_H diff --git a/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h b/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h index 49e65b9f2a80..1b4c8b6cd95f 100644 --- a/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h +++ b/include/llvm/ExecutionEngine/Orc/IRTransformLayer.h @@ -1,9 +1,8 @@ //===- IRTransformLayer.h - Run all IR through a functor --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -57,9 +56,17 @@ class LegacyIRTransformLayer { public: /// Construct an LegacyIRTransformLayer with the given BaseLayer - LegacyIRTransformLayer(BaseLayerT &BaseLayer, - TransformFtor Transform = TransformFtor()) - : BaseLayer(BaseLayer), Transform(std::move(Transform)) {} + LLVM_ATTRIBUTE_DEPRECATED( + LegacyIRTransformLayer(BaseLayerT &BaseLayer, + TransformFtor Transform = TransformFtor()), + "ORCv1 layers (layers with the 'Legacy' prefix) are deprecated. Please " + "use " + "the ORCv2 IRTransformLayer instead"); + + /// Legacy layer constructor with deprecation acknowledgement. + LegacyIRTransformLayer(ORCv1DeprecationAcknowledgement, BaseLayerT &BaseLayer, + TransformFtor Transform = TransformFtor()) + : BaseLayer(BaseLayer), Transform(std::move(Transform)) {} /// Apply the transform functor to the module, then add the module to /// the layer below, along with the memory manager and symbol resolver. @@ -109,6 +116,11 @@ private: TransformFtor Transform; }; +template +LegacyIRTransformLayer::LegacyIRTransformLayer( + BaseLayerT &BaseLayer, TransformFtor Transform) + : BaseLayer(BaseLayer), Transform(std::move(Transform)) {} + } // end namespace orc } // end namespace llvm diff --git a/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h index c2527802f6a7..a7ed5372d1e4 100644 --- a/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h +++ b/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h @@ -1,9 +1,8 @@ //===- IndirectionUtils.h - Utilities for adding indirections ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -147,13 +146,13 @@ private: std::error_code EC; auto TrampolineBlock = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( - sys::Process::getPageSize(), nullptr, + sys::Process::getPageSizeEstimate(), nullptr, sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); if (EC) return errorCodeToError(EC); unsigned NumTrampolines = - (sys::Process::getPageSize() - ORCABI::PointerSize) / + (sys::Process::getPageSizeEstimate() - ORCABI::PointerSize) / ORCABI::TrampolineSize; uint8_t *TrampolineMem = static_cast(TrampolineBlock.base()); diff --git a/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h b/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h index eb9b6bf2dea6..bcbd72e68f15 100644 --- a/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h +++ b/include/llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h @@ -1,9 +1,8 @@ //===- JITTargetMachineBuilder.h - Build TargetMachines for JIT -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ExecutionEngine/Orc/LLJIT.h b/include/llvm/ExecutionEngine/Orc/LLJIT.h index ce3e5d519c73..0aac1916423f 100644 --- a/include/llvm/ExecutionEngine/Orc/LLJIT.h +++ b/include/llvm/ExecutionEngine/Orc/LLJIT.h @@ -1,9 +1,8 @@ //===----- LLJIT.h -- An ORC-based JIT for compiling LLVM IR ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for 3Bdetails. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -21,35 +20,49 @@ #include "llvm/ExecutionEngine/Orc/IRTransformLayer.h" #include "llvm/ExecutionEngine/Orc/JITTargetMachineBuilder.h" #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h" -#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" #include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h" #include "llvm/Support/ThreadPool.h" namespace llvm { namespace orc { +class LLJITBuilderState; +class LLLazyJITBuilderState; + /// A pre-fabricated ORC JIT stack that can serve as an alternative to MCJIT. +/// +/// Create instances using LLJITBuilder. class LLJIT { + template friend class LLJITBuilderSetters; + public: + static Expected> Create(LLJITBuilderState &S); /// Destruct this instance. If a multi-threaded instance, waits for all /// compile threads to complete. ~LLJIT(); - /// Create an LLJIT instance. - /// If NumCompileThreads is not equal to zero, creates a multi-threaded - /// LLJIT with the given number of compile threads. - static Expected> - Create(JITTargetMachineBuilder JTMB, DataLayout DL, - unsigned NumCompileThreads = 0); - /// Returns the ExecutionSession for this instance. ExecutionSession &getExecutionSession() { return *ES; } + /// Returns a reference to the DataLayout for this instance. + const DataLayout &getDataLayout() const { return DL; } + /// Returns a reference to the JITDylib representing the JIT'd main program. JITDylib &getMainJITDylib() { return Main; } + /// Returns the JITDylib with the given name, or nullptr if no JITDylib with + /// that name exists. + JITDylib *getJITDylibByName(StringRef Name) { + return ES->getJITDylibByName(Name); + } + /// Create a new JITDylib with the given name and return a reference to it. + /// + /// JITDylib names must be unique. If the given name is derived from user + /// input or elsewhere in the environment then the client should check + /// (e.g. by calling getJITDylibByName) that the given name is not already in + /// use. JITDylib &createJITDylib(std::string Name) { return ES->createJITDylib(std::move(Name)); } @@ -57,8 +70,6 @@ public: /// Convenience method for defining an absolute symbol. Error defineAbsolute(StringRef Name, JITEvaluatedSymbol Address); - /// Convenience method for defining an - /// Adds an IR module to the given JITDylib. Error addIRModule(JITDylib &JD, ThreadSafeModule TSM); @@ -104,17 +115,17 @@ public: Error runDestructors() { return DtorRunner.run(); } /// Returns a reference to the ObjLinkingLayer - RTDyldObjectLinkingLayer &getObjLinkingLayer() { return ObjLinkingLayer; } + ObjectLayer &getObjLinkingLayer() { return *ObjLinkingLayer; } protected: + static std::unique_ptr + createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES); - /// Create an LLJIT instance with a single compile thread. - LLJIT(std::unique_ptr ES, std::unique_ptr TM, - DataLayout DL); + static Expected + createCompileFunction(LLJITBuilderState &S, JITTargetMachineBuilder JTMB); - /// Create an LLJIT instance with multiple compile threads. - LLJIT(std::unique_ptr ES, JITTargetMachineBuilder JTMB, - DataLayout DL, unsigned NumCompileThreads); + /// Create an LLJIT instance with a single compile thread. + LLJIT(LLJITBuilderState &S, Error &Err); std::string mangle(StringRef UnmangledName); @@ -128,8 +139,8 @@ protected: DataLayout DL; std::unique_ptr CompileThreads; - RTDyldObjectLinkingLayer ObjLinkingLayer; - IRCompileLayer CompileLayer; + std::unique_ptr ObjLinkingLayer; + std::unique_ptr CompileLayer; CtorDtorRunner CtorRunner, DtorRunner; }; @@ -137,25 +148,20 @@ protected: /// An extended version of LLJIT that supports lazy function-at-a-time /// compilation of LLVM IR. class LLLazyJIT : public LLJIT { -public: + template friend class LLJITBuilderSetters; - /// Create an LLLazyJIT instance. - /// If NumCompileThreads is not equal to zero, creates a multi-threaded - /// LLLazyJIT with the given number of compile threads. - static Expected> - Create(JITTargetMachineBuilder JTMB, DataLayout DL, - JITTargetAddress ErrorAddr, unsigned NumCompileThreads = 0); +public: /// Set an IR transform (e.g. pass manager pipeline) to run on each function /// when it is compiled. void setLazyCompileTransform(IRTransformLayer::TransformFunction Transform) { - TransformLayer.setTransform(std::move(Transform)); + TransformLayer->setTransform(std::move(Transform)); } /// Sets the partition function. void setPartitionFunction(CompileOnDemandLayer::PartitionFunction Partition) { - CODLayer.setPartitionFunction(std::move(Partition)); + CODLayer->setPartitionFunction(std::move(Partition)); } /// Add a module to be lazily compiled to JITDylib JD. @@ -169,24 +175,160 @@ public: private: // Create a single-threaded LLLazyJIT instance. - LLLazyJIT(std::unique_ptr ES, - std::unique_ptr TM, DataLayout DL, - std::unique_ptr LCTMgr, - std::function()> ISMBuilder); + LLLazyJIT(LLLazyJITBuilderState &S, Error &Err); + + std::unique_ptr LCTMgr; + std::unique_ptr TransformLayer; + std::unique_ptr CODLayer; +}; + +class LLJITBuilderState { +public: + using ObjectLinkingLayerCreator = + std::function(ExecutionSession &)>; + + using CompileFunctionCreator = + std::function( + JITTargetMachineBuilder JTMB)>; + + std::unique_ptr ES; + Optional JTMB; + ObjectLinkingLayerCreator CreateObjectLinkingLayer; + CompileFunctionCreator CreateCompileFunction; + unsigned NumCompileThreads = 0; - // Create a multi-threaded LLLazyJIT instance. - LLLazyJIT(std::unique_ptr ES, JITTargetMachineBuilder JTMB, - DataLayout DL, unsigned NumCompileThreads, - std::unique_ptr LCTMgr, - std::function()> ISMBuilder); + /// Called prior to JIT class construcion to fix up defaults. + Error prepareForConstruction(); +}; + +template +class LLJITBuilderSetters { +public: + /// Set the JITTargetMachineBuilder for this instance. + /// + /// If this method is not called, JITTargetMachineBuilder::detectHost will be + /// used to construct a default target machine builder for the host platform. + SetterImpl &setJITTargetMachineBuilder(JITTargetMachineBuilder JTMB) { + impl().JTMB = std::move(JTMB); + return impl(); + } + + /// Return a reference to the JITTargetMachineBuilder. + /// + Optional &getJITTargetMachineBuilder() { + return impl().JTMB; + } + /// Set an ObjectLinkingLayer creation function. + /// + /// If this method is not called, a default creation function will be used + /// that will construct an RTDyldObjectLinkingLayer. + SetterImpl &setObjectLinkingLayerCreator( + LLJITBuilderState::ObjectLinkingLayerCreator CreateObjectLinkingLayer) { + impl().CreateObjectLinkingLayer = std::move(CreateObjectLinkingLayer); + return impl(); + } + + /// Set a CompileFunctionCreator. + /// + /// If this method is not called, a default creation function wil be used + /// that will construct a basic IR compile function that is compatible with + /// the selected number of threads (SimpleCompiler for '0' compile threads, + /// ConcurrentIRCompiler otherwise). + SetterImpl &setCompileFunctionCreator( + LLJITBuilderState::CompileFunctionCreator CreateCompileFunction) { + impl().CreateCompileFunction = std::move(CreateCompileFunction); + return impl(); + } + + /// Set the number of compile threads to use. + /// + /// If set to zero, compilation will be performed on the execution thread when + /// JITing in-process. If set to any other number N, a thread pool of N + /// threads will be created for compilation. + /// + /// If this method is not called, behavior will be as if it were called with + /// a zero argument. + SetterImpl &setNumCompileThreads(unsigned NumCompileThreads) { + impl().NumCompileThreads = NumCompileThreads; + return impl(); + } + + /// Create an instance of the JIT. + Expected> create() { + if (auto Err = impl().prepareForConstruction()) + return std::move(Err); + + Error Err = Error::success(); + std::unique_ptr J(new JITType(impl(), Err)); + if (Err) + return std::move(Err); + return std::move(J); + } + +protected: + SetterImpl &impl() { return static_cast(*this); } +}; + +/// Constructs LLJIT instances. +class LLJITBuilder + : public LLJITBuilderState, + public LLJITBuilderSetters {}; + +class LLLazyJITBuilderState : public LLJITBuilderState { + friend class LLLazyJIT; + +public: + using IndirectStubsManagerBuilderFunction = + std::function()>; + + Triple TT; + JITTargetAddress LazyCompileFailureAddr = 0; std::unique_ptr LCTMgr; - std::function()> ISMBuilder; + IndirectStubsManagerBuilderFunction ISMBuilder; + + Error prepareForConstruction(); +}; + +template +class LLLazyJITBuilderSetters + : public LLJITBuilderSetters { +public: + /// Set the address in the target address to call if a lazy compile fails. + /// + /// If this method is not called then the value will default to 0. + SetterImpl &setLazyCompileFailureAddr(JITTargetAddress Addr) { + this->impl().LazyCompileFailureAddr = Addr; + return this->impl(); + } + + /// Set the lazy-callthrough manager. + /// + /// If this method is not called then a default, in-process lazy callthrough + /// manager for the host platform will be used. + SetterImpl & + setLazyCallthroughManager(std::unique_ptr LCTMgr) { + this->impl().LCTMgr = std::move(LCTMgr); + return this->impl(); + } - IRTransformLayer TransformLayer; - CompileOnDemandLayer CODLayer; + /// Set the IndirectStubsManager builder function. + /// + /// If this method is not called then a default, in-process + /// IndirectStubsManager builder for the host platform will be used. + SetterImpl &setIndirectStubsManagerBuilder( + LLLazyJITBuilderState::IndirectStubsManagerBuilderFunction ISMBuilder) { + this->impl().ISMBuilder = std::move(ISMBuilder); + return this->impl(); + } }; +/// Constructs LLLazyJIT instances. +class LLLazyJITBuilder + : public LLLazyJITBuilderState, + public LLLazyJITBuilderSetters {}; + } // End namespace orc } // End namespace llvm diff --git a/include/llvm/ExecutionEngine/Orc/LambdaResolver.h b/include/llvm/ExecutionEngine/Orc/LambdaResolver.h index 7b6f3d2f92ab..855e31b33549 100644 --- a/include/llvm/ExecutionEngine/Orc/LambdaResolver.h +++ b/include/llvm/ExecutionEngine/Orc/LambdaResolver.h @@ -1,9 +1,8 @@ //===- LambdaResolverMM - Redirect symbol lookup via a functor --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,7 +24,15 @@ namespace orc { template class LambdaResolver : public LegacyJITSymbolResolver { public: - LambdaResolver(DylibLookupFtorT DylibLookupFtor, + LLVM_ATTRIBUTE_DEPRECATED( + LambdaResolver(DylibLookupFtorT DylibLookupFtor, + ExternalLookupFtorT ExternalLookupFtor), + "ORCv1 utilities (including resolvers) are deprecated and will be " + "removed " + "in the next release. Please use ORCv2 (see docs/ORCv2.rst)"); + + LambdaResolver(ORCv1DeprecationAcknowledgement, + DylibLookupFtorT DylibLookupFtor, ExternalLookupFtorT ExternalLookupFtor) : DylibLookupFtor(DylibLookupFtor), ExternalLookupFtor(ExternalLookupFtor) {} @@ -43,6 +50,12 @@ private: ExternalLookupFtorT ExternalLookupFtor; }; +template +LambdaResolver::LambdaResolver( + DylibLookupFtorT DylibLookupFtor, ExternalLookupFtorT ExternalLookupFtor) + : DylibLookupFtor(DylibLookupFtor), ExternalLookupFtor(ExternalLookupFtor) { +} + template std::shared_ptr> @@ -53,6 +66,17 @@ createLambdaResolver(DylibLookupFtorT DylibLookupFtor, std::move(ExternalLookupFtor)); } +template +std::shared_ptr> +createLambdaResolver(ORCv1DeprecationAcknowledgement, + DylibLookupFtorT DylibLookupFtor, + ExternalLookupFtorT ExternalLookupFtor) { + using LR = LambdaResolver; + return make_unique(AcknowledgeORCv1Deprecation, + std::move(DylibLookupFtor), + std::move(ExternalLookupFtor)); +} + } // end namespace orc } // end namespace llvm diff --git a/include/llvm/ExecutionEngine/Orc/Layer.h b/include/llvm/ExecutionEngine/Orc/Layer.h index cd797445a2e6..8f9bd704395e 100644 --- a/include/llvm/ExecutionEngine/Orc/Layer.h +++ b/include/llvm/ExecutionEngine/Orc/Layer.h @@ -1,9 +1,8 @@ //===---------------- Layer.h -- Layer interfaces --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h b/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h index 46761b0ca7e1..16202d89f861 100644 --- a/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h +++ b/include/llvm/ExecutionEngine/Orc/LazyEmittingLayer.h @@ -1,9 +1,8 @@ //===- LazyEmittingLayer.h - Lazily emit IR to lower JIT layers -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -35,8 +34,8 @@ namespace orc { /// Lazy-emitting IR layer. /// -/// This layer accepts LLVM IR Modules (via addModule), but does not -/// immediately emit them the layer below. Instead, emissing to the base layer +/// This layer accepts LLVM IR Modules (via addModule) but does not +/// immediately emit them the layer below. Instead, emission to the base layer /// is deferred until the first time the client requests the address (via /// JITSymbol::getAddress) for a symbol contained in this layer. template class LazyEmittingLayer { @@ -197,7 +196,14 @@ private: public: /// Construct a lazy emitting layer. - LazyEmittingLayer(BaseLayerT &BaseLayer) : BaseLayer(BaseLayer) {} + LLVM_ATTRIBUTE_DEPRECATED( + LazyEmittingLayer(BaseLayerT &BaseLayer), + "ORCv1 layers (including LazyEmittingLayer) are deprecated. Please use " + "ORCv2, where lazy emission is the default"); + + /// Construct a lazy emitting layer. + LazyEmittingLayer(ORCv1DeprecationAcknowledgement, BaseLayerT &BaseLayer) + : BaseLayer(BaseLayer) {} /// Add the given module to the lazy emitting layer. Error addModule(VModuleKey K, std::unique_ptr M) { @@ -255,6 +261,10 @@ public: } }; +template +LazyEmittingLayer::LazyEmittingLayer(BaseLayerT &BaseLayer) + : BaseLayer(BaseLayer) {} + } // end namespace orc } // end namespace llvm diff --git a/include/llvm/ExecutionEngine/Orc/LazyReexports.h b/include/llvm/ExecutionEngine/Orc/LazyReexports.h index b5041325bce2..9fdd1d15f782 100644 --- a/include/llvm/ExecutionEngine/Orc/LazyReexports.h +++ b/include/llvm/ExecutionEngine/Orc/LazyReexports.h @@ -1,9 +1,8 @@ //===------ LazyReexports.h -- Utilities for lazy reexports -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ExecutionEngine/Orc/Legacy.h b/include/llvm/ExecutionEngine/Orc/Legacy.h index 4c6162ac4b8b..f9cbbf6ff180 100644 --- a/include/llvm/ExecutionEngine/Orc/Legacy.h +++ b/include/llvm/ExecutionEngine/Orc/Legacy.h @@ -1,9 +1,8 @@ //===--- Legacy.h -- Adapters for ExecutionEngine API interop ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -149,8 +148,8 @@ lookupWithLegacyFn(ExecutionSession &ES, AsynchronousSymbolQuery &Query, for (auto &S : Symbols) { if (JITSymbol Sym = FindSymbol(*S)) { if (auto Addr = Sym.getAddress()) { - Query.resolve(S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); - Query.notifySymbolReady(); + Query.notifySymbolMetRequiredState( + S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); NewSymbolsResolved = true; } else { ES.legacyFailQuery(Query, Addr.takeError()); @@ -163,11 +162,8 @@ lookupWithLegacyFn(ExecutionSession &ES, AsynchronousSymbolQuery &Query, SymbolsNotFound.insert(S); } - if (NewSymbolsResolved && Query.isFullyResolved()) - Query.handleFullyResolved(); - - if (NewSymbolsResolved && Query.isFullyReady()) - Query.handleFullyReady(); + if (NewSymbolsResolved && Query.isComplete()) + Query.handleComplete(); return SymbolsNotFound; } diff --git a/include/llvm/ExecutionEngine/Orc/NullResolver.h b/include/llvm/ExecutionEngine/Orc/NullResolver.h index 03fefb69a928..ffa37a13d064 100644 --- a/include/llvm/ExecutionEngine/Orc/NullResolver.h +++ b/include/llvm/ExecutionEngine/Orc/NullResolver.h @@ -1,9 +1,8 @@ //===------ NullResolver.h - Reject symbol lookup requests ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h b/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h new file mode 100644 index 000000000000..c1e7d27f446e --- /dev/null +++ b/include/llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h @@ -0,0 +1,165 @@ +//===-- ObjectLinkingLayer.h - JITLink-based jit linking layer --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Contains the definition for an JITLink-based, in-process object linking +// layer. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORC_OBJECTLINKINGLAYER_H +#define LLVM_EXECUTIONENGINE_ORC_OBJECTLINKINGLAYER_H + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/Layer.h" +#include "llvm/Support/Error.h" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace llvm { + +namespace jitlink { +class EHFrameRegistrar; +} // namespace jitlink + +namespace object { +class ObjectFile; +} // namespace object + +namespace orc { + +class ObjectLinkingLayerJITLinkContext; + +/// An ObjectLayer implementation built on JITLink. +/// +/// Clients can use this class to add relocatable object files to an +/// ExecutionSession, and it typically serves as the base layer (underneath +/// a compiling layer like IRCompileLayer) for the rest of the JIT. +class ObjectLinkingLayer : public ObjectLayer { + friend class ObjectLinkingLayerJITLinkContext; + +public: + /// Plugin instances can be added to the ObjectLinkingLayer to receive + /// callbacks when code is loaded or emitted, and when JITLink is being + /// configured. + class Plugin { + public: + virtual ~Plugin(); + virtual void modifyPassConfig(MaterializationResponsibility &MR, + const Triple &TT, + jitlink::PassConfiguration &Config) {} + virtual void notifyLoaded(MaterializationResponsibility &MR) {} + virtual Error notifyEmitted(MaterializationResponsibility &MR) { + return Error::success(); + } + virtual Error notifyRemovingModule(VModuleKey K) { + return Error::success(); + } + virtual Error notifyRemovingAllModules() { return Error::success(); } + }; + + /// Construct an ObjectLinkingLayer with the given NotifyLoaded, + /// and NotifyEmitted functors. + ObjectLinkingLayer(ExecutionSession &ES, + jitlink::JITLinkMemoryManager &MemMgr); + + /// Destruct an ObjectLinkingLayer. + ~ObjectLinkingLayer(); + + /// Add a pass-config modifier. + ObjectLinkingLayer &addPlugin(std::unique_ptr P) { + std::lock_guard Lock(LayerMutex); + Plugins.push_back(std::move(P)); + return *this; + } + + /// Emit the object. + void emit(MaterializationResponsibility R, + std::unique_ptr O) override; + + /// Instructs this ObjectLinkingLayer instance to override the symbol flags + /// found in the AtomGraph with the flags supplied by the + /// MaterializationResponsibility instance. This is a workaround to support + /// symbol visibility in COFF, which does not use the libObject's + /// SF_Exported flag. Use only when generating / adding COFF object files. + /// + /// FIXME: We should be able to remove this if/when COFF properly tracks + /// exported symbols. + ObjectLinkingLayer & + setOverrideObjectFlagsWithResponsibilityFlags(bool OverrideObjectFlags) { + this->OverrideObjectFlags = OverrideObjectFlags; + return *this; + } + + /// If set, this ObjectLinkingLayer instance will claim responsibility + /// for any symbols provided by a given object file that were not already in + /// the MaterializationResponsibility instance. Setting this flag allows + /// higher-level program representations (e.g. LLVM IR) to be added based on + /// only a subset of the symbols they provide, without having to write + /// intervening layers to scan and add the additional symbols. This trades + /// diagnostic quality for convenience however: If all symbols are enumerated + /// up-front then clashes can be detected and reported early (and usually + /// deterministically). If this option is set, clashes for the additional + /// symbols may not be detected until late, and detection may depend on + /// the flow of control through JIT'd code. Use with care. + ObjectLinkingLayer & + setAutoClaimResponsibilityForObjectSymbols(bool AutoClaimObjectSymbols) { + this->AutoClaimObjectSymbols = AutoClaimObjectSymbols; + return *this; + } + +private: + using AllocPtr = std::unique_ptr; + + void modifyPassConfig(MaterializationResponsibility &MR, const Triple &TT, + jitlink::PassConfiguration &PassConfig); + void notifyLoaded(MaterializationResponsibility &MR); + Error notifyEmitted(MaterializationResponsibility &MR, AllocPtr Alloc); + + Error removeModule(VModuleKey K); + Error removeAllModules(); + + mutable std::mutex LayerMutex; + jitlink::JITLinkMemoryManager &MemMgr; + bool OverrideObjectFlags = false; + bool AutoClaimObjectSymbols = false; + DenseMap TrackedAllocs; + std::vector UntrackedAllocs; + std::vector> Plugins; +}; + +class EHFrameRegistrationPlugin : public ObjectLinkingLayer::Plugin { +public: + EHFrameRegistrationPlugin(jitlink::EHFrameRegistrar &Registrar); + Error notifyEmitted(MaterializationResponsibility &MR) override; + void modifyPassConfig(MaterializationResponsibility &MR, const Triple &TT, + jitlink::PassConfiguration &PassConfig) override; + Error notifyRemovingModule(VModuleKey K) override; + Error notifyRemovingAllModules() override; + +private: + jitlink::EHFrameRegistrar &Registrar; + DenseMap InProcessLinks; + DenseMap TrackedEHFrameAddrs; + std::vector UntrackedEHFrameAddrs; +}; + +} // end namespace orc +} // end namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORC_OBJECTLINKINGLAYER_H diff --git a/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h b/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h index 44d6b490e19d..eac1cc3e097a 100644 --- a/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h +++ b/include/llvm/ExecutionEngine/Orc/ObjectTransformLayer.h @@ -1,9 +1,8 @@ //===- ObjectTransformLayer.h - Run all objects through functor -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -49,7 +48,16 @@ template class LegacyObjectTransformLayer { public: /// Construct an ObjectTransformLayer with the given BaseLayer - LegacyObjectTransformLayer(BaseLayerT &BaseLayer, + LLVM_ATTRIBUTE_DEPRECATED( + LegacyObjectTransformLayer(BaseLayerT &BaseLayer, + TransformFtor Transform = TransformFtor()), + "ORCv1 layers (layers with the 'Legacy' prefix) are deprecated. Please " + "use " + "the ORCv2 ObjectTransformLayer instead"); + + /// Legacy layer constructor with deprecation acknowledgement. + LegacyObjectTransformLayer(ORCv1DeprecationAcknowledgement, + BaseLayerT &BaseLayer, TransformFtor Transform = TransformFtor()) : BaseLayer(BaseLayer), Transform(std::move(Transform)) {} @@ -108,6 +116,11 @@ private: TransformFtor Transform; }; +template +LegacyObjectTransformLayer:: + LegacyObjectTransformLayer(BaseLayerT &BaseLayer, TransformFtor Transform) + : BaseLayer(BaseLayer), Transform(std::move(Transform)) {} + } // end namespace orc } // end namespace llvm diff --git a/include/llvm/ExecutionEngine/Orc/OrcABISupport.h b/include/llvm/ExecutionEngine/Orc/OrcABISupport.h index a70fc373713d..38246bc480b6 100644 --- a/include/llvm/ExecutionEngine/Orc/OrcABISupport.h +++ b/include/llvm/ExecutionEngine/Orc/OrcABISupport.h @@ -1,9 +1,8 @@ //===- OrcABISupport.h - ABI support code -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ExecutionEngine/Orc/OrcError.h b/include/llvm/ExecutionEngine/Orc/OrcError.h index dc60e8d74e97..e5d6a3eca85f 100644 --- a/include/llvm/ExecutionEngine/Orc/OrcError.h +++ b/include/llvm/ExecutionEngine/Orc/OrcError.h @@ -1,9 +1,8 @@ //===------ OrcError.h - Reject symbol lookup requests ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h index 3e07f5cf3742..8b875b7906e1 100644 --- a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h +++ b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h @@ -1,9 +1,8 @@ //===- OrcRemoteTargetClient.h - Orc Remote-target Client -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h index 8db9e317a18a..e7b598d8f812 100644 --- a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h +++ b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h @@ -1,9 +1,8 @@ //===- OrcRemoteTargetRPCAPI.h - Orc Remote-target RPC API ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h index acbc1682fa5d..4c8e2ea1a7be 100644 --- a/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h +++ b/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetServer.h @@ -1,9 +1,8 @@ //===- OrcRemoteTargetServer.h - Orc Remote-target Server -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -300,13 +299,13 @@ private: std::error_code EC; auto TrampolineBlock = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( - sys::Process::getPageSize(), nullptr, + sys::Process::getPageSizeEstimate(), nullptr, sys::Memory::MF_READ | sys::Memory::MF_WRITE, EC)); if (EC) return errorCodeToError(EC); uint32_t NumTrampolines = - (sys::Process::getPageSize() - TargetT::PointerSize) / + (sys::Process::getPageSizeEstimate() - TargetT::PointerSize) / TargetT::TrampolineSize; uint8_t *TrampolineMem = static_cast(TrampolineBlock.base()); @@ -336,7 +335,7 @@ private: handleGetRemoteInfo() { std::string ProcessTriple = sys::getProcessTriple(); uint32_t PointerSize = TargetT::PointerSize; - uint32_t PageSize = sys::Process::getPageSize(); + uint32_t PageSize = sys::Process::getPageSizeEstimate(); uint32_t TrampolineSize = TargetT::TrampolineSize; uint32_t IndirectStubSize = TargetT::IndirectStubsInfo::StubSize; LLVM_DEBUG(dbgs() << " Remote info:\n" diff --git a/include/llvm/ExecutionEngine/Orc/RPCSerialization.h b/include/llvm/ExecutionEngine/Orc/RPCSerialization.h index 1e5f6ced597a..07c7471afc6a 100644 --- a/include/llvm/ExecutionEngine/Orc/RPCSerialization.h +++ b/include/llvm/ExecutionEngine/Orc/RPCSerialization.h @@ -1,9 +1,8 @@ //===- llvm/ExecutionEngine/Orc/RPCSerialization.h --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -128,123 +127,85 @@ template class RPCTypeName> { public: static const char* getName() { - std::lock_guard Lock(NameMutex); - if (Name.empty()) + static std::string Name = [] { + std::string Name; raw_string_ostream(Name) << "Expected<" << RPCTypeNameSequence() << ">"; + return Name; + }(); return Name.data(); } - -private: - static std::mutex NameMutex; - static std::string Name; }; -template -std::mutex RPCTypeName>::NameMutex; - -template -std::string RPCTypeName>::Name; - template class RPCTypeName> { public: static const char* getName() { - std::lock_guard Lock(NameMutex); - if (Name.empty()) + static std::string Name = [] { + std::string Name; raw_string_ostream(Name) << "std::pair<" << RPCTypeNameSequence() << ">"; + return Name; + }(); return Name.data(); } -private: - static std::mutex NameMutex; - static std::string Name; }; -template -std::mutex RPCTypeName>::NameMutex; -template -std::string RPCTypeName>::Name; - template class RPCTypeName> { public: static const char* getName() { - std::lock_guard Lock(NameMutex); - if (Name.empty()) + static std::string Name = [] { + std::string Name; raw_string_ostream(Name) << "std::tuple<" << RPCTypeNameSequence() << ">"; + return Name; + }(); return Name.data(); } -private: - static std::mutex NameMutex; - static std::string Name; }; -template -std::mutex RPCTypeName>::NameMutex; -template -std::string RPCTypeName>::Name; - template class RPCTypeName> { public: static const char*getName() { - std::lock_guard Lock(NameMutex); - if (Name.empty()) + static std::string Name = [] { + std::string Name; raw_string_ostream(Name) << "std::vector<" << RPCTypeName::getName() << ">"; + return Name; + }(); return Name.data(); } - -private: - static std::mutex NameMutex; - static std::string Name; }; -template -std::mutex RPCTypeName>::NameMutex; -template -std::string RPCTypeName>::Name; - template class RPCTypeName> { public: static const char *getName() { - std::lock_guard Lock(NameMutex); - if (Name.empty()) + static std::string Name = [] { + std::string Name; raw_string_ostream(Name) << "std::set<" << RPCTypeName::getName() << ">"; + return Name; + }(); return Name.data(); } - -private: - static std::mutex NameMutex; - static std::string Name; }; -template std::mutex RPCTypeName>::NameMutex; -template std::string RPCTypeName>::Name; - template class RPCTypeName> { public: static const char *getName() { - std::lock_guard Lock(NameMutex); - if (Name.empty()) + static std::string Name = [] { + std::string Name; raw_string_ostream(Name) << "std::map<" << RPCTypeNameSequence() << ">"; + return Name; + }(); return Name.data(); } - -private: - static std::mutex NameMutex; - static std::string Name; }; -template -std::mutex RPCTypeName>::NameMutex; -template std::string RPCTypeName>::Name; - /// The SerializationTraits class describes how to serialize and /// deserialize an instance of type T to/from an abstract channel of type /// ChannelT. It also provides a representation of the type's name via the diff --git a/include/llvm/ExecutionEngine/Orc/RPCUtils.h b/include/llvm/ExecutionEngine/Orc/RPCUtils.h index 953b73e10e43..3b11e1b283de 100644 --- a/include/llvm/ExecutionEngine/Orc/RPCUtils.h +++ b/include/llvm/ExecutionEngine/Orc/RPCUtils.h @@ -1,9 +1,8 @@ -//===------- RPCUTils.h - Utilities for building RPC APIs -------*- C++ -*-===// +//===- RPCUtils.h - Utilities for building RPC APIs -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -152,25 +151,17 @@ public: /// Returns the full function prototype as a string. static const char *getPrototype() { - std::lock_guard Lock(NameMutex); - if (Name.empty()) + static std::string Name = [] { + std::string Name; raw_string_ostream(Name) << RPCTypeName::getName() << " " << DerivedFunc::getName() << "(" << llvm::orc::rpc::RPCTypeNameSequence() << ")"; + return Name; + }(); return Name.data(); } - -private: - static std::mutex NameMutex; - static std::string Name; }; -template -std::mutex Function::NameMutex; - -template -std::string Function::Name; - /// Allocates RPC function ids during autonegotiation. /// Specializations of this class must provide four members: /// diff --git a/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h b/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h index 6f90f0380d95..d9535ce5f21f 100644 --- a/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h +++ b/include/llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h @@ -1,9 +1,8 @@ //===- RTDyldObjectLinkingLayer.h - RTDyld-based jit linking ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -44,22 +43,34 @@ public: const RuntimeDyld::LoadedObjectInfo &)>; /// Functor for receiving finalization notifications. - using NotifyEmittedFunction = std::function; + using NotifyEmittedFunction = + std::function)>; using GetMemoryManagerFunction = std::function()>; /// Construct an ObjectLinkingLayer with the given NotifyLoaded, /// and NotifyEmitted functors. - RTDyldObjectLinkingLayer( - ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager, - NotifyLoadedFunction NotifyLoaded = NotifyLoadedFunction(), - NotifyEmittedFunction NotifyEmitted = NotifyEmittedFunction()); + RTDyldObjectLinkingLayer(ExecutionSession &ES, + GetMemoryManagerFunction GetMemoryManager); /// Emit the object. void emit(MaterializationResponsibility R, std::unique_ptr O) override; + /// Set the NotifyLoaded callback. + RTDyldObjectLinkingLayer &setNotifyLoaded(NotifyLoadedFunction NotifyLoaded) { + this->NotifyLoaded = std::move(NotifyLoaded); + return *this; + } + + /// Set the NotifyEmitted callback. + RTDyldObjectLinkingLayer & + setNotifyEmitted(NotifyEmittedFunction NotifyEmitted) { + this->NotifyEmitted = std::move(NotifyEmitted); + return *this; + } + /// Set the 'ProcessAllSections' flag. /// /// If set to true, all sections in each object file will be allocated using @@ -109,7 +120,8 @@ private: std::map Resolved, std::set &InternalSymbols); - void onObjEmit(VModuleKey K, MaterializationResponsibility &R, Error Err); + void onObjEmit(VModuleKey K, std::unique_ptr ObjBuffer, + MaterializationResponsibility &R, Error Err); mutable std::mutex RTDyldLayerMutex; GetMemoryManagerFunction GetMemoryManager; @@ -341,17 +353,27 @@ public: /// Construct an ObjectLinkingLayer with the given NotifyLoaded, /// and NotifyFinalized functors. + LLVM_ATTRIBUTE_DEPRECATED( + LegacyRTDyldObjectLinkingLayer( + ExecutionSession &ES, ResourcesGetter GetResources, + NotifyLoadedFtor NotifyLoaded = NotifyLoadedFtor(), + NotifyFinalizedFtor NotifyFinalized = NotifyFinalizedFtor(), + NotifyFreedFtor NotifyFreed = NotifyFreedFtor()), + "ORCv1 layers (layers with the 'Legacy' prefix) are deprecated. Please " + "use " + "ORCv2 (see docs/ORCv2.rst)"); + + // Legacy layer constructor with deprecation acknowledgement. LegacyRTDyldObjectLinkingLayer( - ExecutionSession &ES, ResourcesGetter GetResources, + ORCv1DeprecationAcknowledgement, ExecutionSession &ES, + ResourcesGetter GetResources, NotifyLoadedFtor NotifyLoaded = NotifyLoadedFtor(), NotifyFinalizedFtor NotifyFinalized = NotifyFinalizedFtor(), NotifyFreedFtor NotifyFreed = NotifyFreedFtor()) : ES(ES), GetResources(std::move(GetResources)), NotifyLoaded(std::move(NotifyLoaded)), NotifyFinalized(std::move(NotifyFinalized)), - NotifyFreed(std::move(NotifyFreed)), - ProcessAllSections(false) { - } + NotifyFreed(std::move(NotifyFreed)), ProcessAllSections(false) {} /// Set the 'ProcessAllSections' flag. /// diff --git a/include/llvm/ExecutionEngine/Orc/RawByteChannel.h b/include/llvm/ExecutionEngine/Orc/RawByteChannel.h index db810f4ef2e5..46b7c59450e6 100644 --- a/include/llvm/ExecutionEngine/Orc/RawByteChannel.h +++ b/include/llvm/ExecutionEngine/Orc/RawByteChannel.h @@ -1,9 +1,8 @@ //===- llvm/ExecutionEngine/Orc/RawByteChannel.h ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h b/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h index 955e77607a18..b87cf697a81e 100644 --- a/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h +++ b/include/llvm/ExecutionEngine/Orc/RemoteObjectLayer.h @@ -1,9 +1,8 @@ //===------ RemoteObjectLayer.h - Forwards objs to a remote -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,9 +13,10 @@ #ifndef LLVM_EXECUTIONENGINE_ORC_REMOTEOBJECTLAYER_H #define LLVM_EXECUTIONENGINE_ORC_REMOTEOBJECTLAYER_H +#include "llvm/ExecutionEngine/Orc/Core.h" +#include "llvm/ExecutionEngine/Orc/LambdaResolver.h" #include "llvm/ExecutionEngine/Orc/OrcRemoteTargetRPCAPI.h" #include "llvm/Object/ObjectFile.h" -#include "llvm/ExecutionEngine/Orc/LambdaResolver.h" #include namespace llvm { @@ -313,7 +313,14 @@ public: /// /// The ReportError functor can be used locally log errors that are intended /// to be sent sent - RemoteObjectClientLayer(RPCEndpoint &Remote, + LLVM_ATTRIBUTE_DEPRECATED( + RemoteObjectClientLayer(RPCEndpoint &Remote, + std::function ReportError), + "ORCv1 layers (including RemoteObjectClientLayer) are deprecated. Please " + "use " + "ORCv2 (see docs/ORCv2.rst)"); + + RemoteObjectClientLayer(ORCv1DeprecationAcknowledgement, RPCEndpoint &Remote, std::function ReportError) : RemoteObjectLayer(Remote, std::move(ReportError)) { using ThisT = RemoteObjectClientLayer; @@ -418,11 +425,18 @@ public: /// Create a RemoteObjectServerLayer with the given base layer (which must be /// an object layer), RPC endpoint, and error reporter function. - RemoteObjectServerLayer(BaseLayerT &BaseLayer, - RPCEndpoint &Remote, + LLVM_ATTRIBUTE_DEPRECATED( + RemoteObjectServerLayer(BaseLayerT &BaseLayer, RPCEndpoint &Remote, + std::function ReportError), + "ORCv1 layers (including RemoteObjectServerLayer) are deprecated. Please " + "use " + "ORCv2 (see docs/ORCv2.rst)"); + + RemoteObjectServerLayer(ORCv1DeprecationAcknowledgement, + BaseLayerT &BaseLayer, RPCEndpoint &Remote, std::function ReportError) - : RemoteObjectLayer(Remote, std::move(ReportError)), - BaseLayer(BaseLayer), HandleIdMgr(1) { + : RemoteObjectLayer(Remote, std::move(ReportError)), + BaseLayer(BaseLayer), HandleIdMgr(1) { using ThisT = RemoteObjectServerLayer; Remote.template addHandler(*this, &ThisT::addObject); @@ -463,6 +477,7 @@ private: assert(!BaseLayerHandles.count(Id) && "Id already in use?"); auto Resolver = createLambdaResolver( + AcknowledgeORCv1Deprecation, [this, Id](const std::string &Name) { return lookup(Id, Name); }, [this, Id](const std::string &Name) { return lookupInLogicalDylib(Id, Name); @@ -523,6 +538,31 @@ private: std::map BaseLayerHandles; }; +template +RemoteObjectClientLayer::RemoteObjectClientLayer( + RPCEndpoint &Remote, std::function ReportError) + : RemoteObjectLayer(Remote, std::move(ReportError)) { + using ThisT = RemoteObjectClientLayer; + Remote.template addHandler(*this, &ThisT::lookup); + Remote.template addHandler( + *this, &ThisT::lookupInLogicalDylib); +} + +template +RemoteObjectServerLayer::RemoteObjectServerLayer( + BaseLayerT &BaseLayer, RPCEndpoint &Remote, + std::function ReportError) + : RemoteObjectLayer(Remote, std::move(ReportError)), + BaseLayer(BaseLayer), HandleIdMgr(1) { + using ThisT = RemoteObjectServerLayer; + + Remote.template addHandler(*this, &ThisT::addObject); + Remote.template addHandler(*this, &ThisT::removeObject); + Remote.template addHandler(*this, &ThisT::findSymbol); + Remote.template addHandler(*this, &ThisT::findSymbolIn); + Remote.template addHandler(*this, &ThisT::emitAndFinalize); +} + } // end namespace orc } // end namespace llvm diff --git a/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h b/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h index 717076e25609..c354f6c3559c 100644 --- a/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h +++ b/include/llvm/ExecutionEngine/Orc/SymbolStringPool.h @@ -1,9 +1,8 @@ //===- SymbolStringPool.h - Multi-threaded pool for JIT symbols -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -51,25 +50,20 @@ private: class SymbolStringPtr { friend class SymbolStringPool; friend struct DenseMapInfo; - friend bool operator==(const SymbolStringPtr &LHS, - const SymbolStringPtr &RHS); - friend bool operator<(const SymbolStringPtr &LHS, const SymbolStringPtr &RHS); - - static SymbolStringPool::PoolMapEntry Tombstone; public: SymbolStringPtr() = default; SymbolStringPtr(const SymbolStringPtr &Other) : S(Other.S) { - if (S) + if (isRealPoolEntry(S)) ++S->getValue(); } SymbolStringPtr& operator=(const SymbolStringPtr &Other) { - if (S) + if (isRealPoolEntry(S)) --S->getValue(); S = Other.S; - if (S) + if (isRealPoolEntry(S)) ++S->getValue(); return *this; } @@ -79,7 +73,7 @@ public: } SymbolStringPtr& operator=(SymbolStringPtr &&Other) { - if (S) + if (isRealPoolEntry(S)) --S->getValue(); S = nullptr; std::swap(S, Other.S); @@ -87,34 +81,64 @@ public: } ~SymbolStringPtr() { - if (S) + if (isRealPoolEntry(S)) --S->getValue(); } StringRef operator*() const { return S->first(); } + friend bool operator==(const SymbolStringPtr &LHS, + const SymbolStringPtr &RHS) { + return LHS.S == RHS.S; + } + + friend bool operator!=(const SymbolStringPtr &LHS, + const SymbolStringPtr &RHS) { + return !(LHS == RHS); + } + + friend bool operator<(const SymbolStringPtr &LHS, + const SymbolStringPtr &RHS) { + return LHS.S < RHS.S; + } + private: + using PoolEntryPtr = SymbolStringPool::PoolMapEntry *; SymbolStringPtr(SymbolStringPool::PoolMapEntry *S) : S(S) { - if (S) + if (isRealPoolEntry(S)) ++S->getValue(); } - SymbolStringPool::PoolMapEntry *S = nullptr; -}; + // Returns false for null, empty, and tombstone values, true otherwise. + bool isRealPoolEntry(PoolEntryPtr P) { + return ((reinterpret_cast(P) - 1) & InvalidPtrMask) != + InvalidPtrMask; + } -inline bool operator==(const SymbolStringPtr &LHS, const SymbolStringPtr &RHS) { - return LHS.S == RHS.S; -} + static SymbolStringPtr getEmptyVal() { + return SymbolStringPtr(reinterpret_cast(EmptyBitPattern)); + } -inline bool operator!=(const SymbolStringPtr &LHS, const SymbolStringPtr &RHS) { - return !(LHS == RHS); -} + static SymbolStringPtr getTombstoneVal() { + return SymbolStringPtr(reinterpret_cast(TombstoneBitPattern)); + } -inline bool operator<(const SymbolStringPtr &LHS, const SymbolStringPtr &RHS) { - return LHS.S < RHS.S; -} + constexpr static uintptr_t EmptyBitPattern = + std::numeric_limits::max() + << PointerLikeTypeTraits::NumLowBitsAvailable; + + constexpr static uintptr_t TombstoneBitPattern = + (std::numeric_limits::max() - 1) + << PointerLikeTypeTraits::NumLowBitsAvailable; + + constexpr static uintptr_t InvalidPtrMask = + (std::numeric_limits::max() - 3) + << PointerLikeTypeTraits::NumLowBitsAvailable; + + PoolEntryPtr S = nullptr; +}; inline SymbolStringPool::~SymbolStringPool() { #ifndef NDEBUG @@ -151,16 +175,15 @@ template <> struct DenseMapInfo { static orc::SymbolStringPtr getEmptyKey() { - return orc::SymbolStringPtr(); + return orc::SymbolStringPtr::getEmptyVal(); } static orc::SymbolStringPtr getTombstoneKey() { - return orc::SymbolStringPtr(&orc::SymbolStringPtr::Tombstone); + return orc::SymbolStringPtr::getTombstoneVal(); } - static unsigned getHashValue(orc::SymbolStringPtr V) { - uintptr_t IV = reinterpret_cast(V.S); - return unsigned(IV) ^ unsigned(IV >> 9); + static unsigned getHashValue(const orc::SymbolStringPtr &V) { + return DenseMapInfo::getHashValue(V.S); } static bool isEqual(const orc::SymbolStringPtr &LHS, diff --git a/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h b/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h index bf946de532d3..5787500387c4 100644 --- a/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h +++ b/include/llvm/ExecutionEngine/Orc/ThreadSafeModule.h @@ -1,9 +1,8 @@ //===----------- ThreadSafeModule.h -- Layer interfaces ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ExecutionEngine/OrcMCJITReplacement.h b/include/llvm/ExecutionEngine/OrcMCJITReplacement.h index 4cd5648b2fc2..6cca1933f39f 100644 --- a/include/llvm/ExecutionEngine/OrcMCJITReplacement.h +++ b/include/llvm/ExecutionEngine/OrcMCJITReplacement.h @@ -1,9 +1,8 @@ //===---- OrcMCJITReplacement.h - Orc-based MCJIT replacement ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ExecutionEngine/OrcV1Deprecation.h b/include/llvm/ExecutionEngine/OrcV1Deprecation.h new file mode 100644 index 000000000000..7ed254b3ee04 --- /dev/null +++ b/include/llvm/ExecutionEngine/OrcV1Deprecation.h @@ -0,0 +1,22 @@ +//===------ OrcV1Deprecation.h - Memory manager for MC-JIT ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Tag for suppressing ORCv1 deprecation warnings. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_EXECUTIONENGINE_ORCV1DEPRECATION_H +#define LLVM_EXECUTIONENGINE_ORCV1DEPRECATION_H + +namespace llvm { + +enum ORCv1DeprecationAcknowledgement { AcknowledgeORCv1Deprecation }; + +} // namespace llvm + +#endif // LLVM_EXECUTIONENGINE_ORCV1DEPRECATION_H diff --git a/include/llvm/ExecutionEngine/RTDyldMemoryManager.h b/include/llvm/ExecutionEngine/RTDyldMemoryManager.h index 23d651f6d1b6..c7c87ecdfa09 100644 --- a/include/llvm/ExecutionEngine/RTDyldMemoryManager.h +++ b/include/llvm/ExecutionEngine/RTDyldMemoryManager.h @@ -1,9 +1,8 @@ //===-- RTDyldMemoryManager.cpp - Memory manager for MC-JIT -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ExecutionEngine/RuntimeDyld.h b/include/llvm/ExecutionEngine/RuntimeDyld.h index e419ee05e566..b2b4eba47074 100644 --- a/include/llvm/ExecutionEngine/RuntimeDyld.h +++ b/include/llvm/ExecutionEngine/RuntimeDyld.h @@ -1,9 +1,8 @@ //===- RuntimeDyld.h - Run-time dynamic linker for MC-JIT -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -53,18 +52,19 @@ private: std::string ErrMsg; }; -class RuntimeDyldCheckerImpl; class RuntimeDyldImpl; class RuntimeDyld { - friend class RuntimeDyldCheckerImpl; - protected: // Change the address associated with a section when resolving relocations. // Any relocations already associated with the symbol will be re-resolved. void reassignSectionAddress(unsigned SectionID, uint64_t Addr); public: + using NotifyStubEmittedFunction = std::function; + /// Information about the loaded object. class LoadedObjectInfo : public llvm::LoadedObjectInfo { friend class RuntimeDyldImpl; @@ -185,6 +185,9 @@ public: /// and resolve relocatons based on where they put it). void *getSymbolLocalAddress(StringRef Name) const; + /// Get the section ID for the section containing the given symbol. + unsigned getSymbolSectionID(StringRef Name) const; + /// Get the target address and flags for the named symbol. /// This address is the one used for relocation. JITEvaluatedSymbol getSymbol(StringRef Name) const; @@ -205,6 +208,19 @@ public: /// This is the address which will be used for relocation resolution. void mapSectionAddress(const void *LocalAddress, uint64_t TargetAddress); + /// Returns the section's working memory. + StringRef getSectionContent(unsigned SectionID) const; + + /// If the section was loaded, return the section's load address, + /// otherwise return None. + uint64_t getSectionLoadAddress(unsigned SectionID) const; + + /// Set the NotifyStubEmitted callback. This is used for debugging + /// purposes. A callback is made for each stub that is generated. + void setNotifyStubEmitted(NotifyStubEmittedFunction NotifyStubEmitted) { + this->NotifyStubEmitted = std::move(NotifyStubEmitted); + } + /// Register any EH frame sections that have been loaded but not previously /// registered with the memory manager. Note, RuntimeDyld is responsible /// for identifying the EH frame and calling the memory manager with the @@ -266,7 +282,7 @@ private: MemoryManager &MemMgr; JITSymbolResolver &Resolver; bool ProcessAllSections; - RuntimeDyldCheckerImpl *Checker; + NotifyStubEmittedFunction NotifyStubEmitted; }; // Asynchronous JIT link for ORC. diff --git a/include/llvm/ExecutionEngine/RuntimeDyldChecker.h b/include/llvm/ExecutionEngine/RuntimeDyldChecker.h index 13fc5fd5a3e7..93ea09107bd1 100644 --- a/include/llvm/ExecutionEngine/RuntimeDyldChecker.h +++ b/include/llvm/ExecutionEngine/RuntimeDyldChecker.h @@ -1,16 +1,18 @@ //===---- RuntimeDyldChecker.h - RuntimeDyld tester framework -----*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef LLVM_EXECUTIONENGINE_RUNTIMEDYLDCHECKER_H #define LLVM_EXECUTIONENGINE_RUNTIMEDYLDCHECKER_H +#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/Optional.h" +#include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/Support/Endian.h" #include #include @@ -58,7 +60,8 @@ class raw_ostream; /// /// ident_expr = 'decode_operand' '(' symbol ',' operand-index ')' /// | 'next_pc' '(' symbol ')' -/// | 'stub_addr' '(' file-name ',' section-name ',' symbol ')' +/// | 'stub_addr' '(' stub-container-name ',' symbol ')' +/// | 'got_addr' '(' stub-container-name ',' symbol ')' /// | symbol /// /// binary_expr = expr '+' expr @@ -70,15 +73,84 @@ class raw_ostream; /// class RuntimeDyldChecker { public: - RuntimeDyldChecker(RuntimeDyld &RTDyld, MCDisassembler *Disassembler, - MCInstPrinter *InstPrinter, raw_ostream &ErrStream); - ~RuntimeDyldChecker(); + class MemoryRegionInfo { + public: + MemoryRegionInfo() = default; + + /// Constructor for symbols/sections with content. + MemoryRegionInfo(StringRef Content, JITTargetAddress TargetAddress) + : ContentPtr(Content.data()), Size(Content.size()), + TargetAddress(TargetAddress) {} + + /// Constructor for zero-fill symbols/sections. + MemoryRegionInfo(uint64_t Size, JITTargetAddress TargetAddress) + : Size(Size), TargetAddress(TargetAddress) {} + + /// Returns true if this is a zero-fill symbol/section. + bool isZeroFill() const { + assert(Size && "setContent/setZeroFill must be called first"); + return !ContentPtr; + } + + /// Set the content for this memory region. + void setContent(StringRef Content) { + assert(!ContentPtr && !Size && "Content/zero-fill already set"); + ContentPtr = Content.data(); + Size = Content.size(); + } + + /// Set a zero-fill length for this memory region. + void setZeroFill(uint64_t Size) { + assert(!ContentPtr && !this->Size && "Content/zero-fill already set"); + this->Size = Size; + } - // Get the associated RTDyld instance. - RuntimeDyld& getRTDyld(); + /// Returns the content for this section if there is any. + StringRef getContent() const { + assert(!isZeroFill() && "Can't get content for a zero-fill section"); + return StringRef(ContentPtr, static_cast(Size)); + } - // Get the associated RTDyld instance. - const RuntimeDyld& getRTDyld() const; + /// Returns the zero-fill length for this section. + uint64_t getZeroFillLength() const { + assert(isZeroFill() && "Can't get zero-fill length for content section"); + return Size; + } + + /// Set the target address for this region. + void setTargetAddress(JITTargetAddress TargetAddress) { + assert(!this->TargetAddress && "TargetAddress already set"); + this->TargetAddress = TargetAddress; + } + + /// Return the target address for this region. + JITTargetAddress getTargetAddress() const { return TargetAddress; } + + private: + const char *ContentPtr = 0; + uint64_t Size = 0; + JITTargetAddress TargetAddress = 0; + }; + + using IsSymbolValidFunction = std::function; + using GetSymbolInfoFunction = + std::function(StringRef SymbolName)>; + using GetSectionInfoFunction = std::function( + StringRef FileName, StringRef SectionName)>; + using GetStubInfoFunction = std::function( + StringRef StubContainer, StringRef TargetName)>; + using GetGOTInfoFunction = std::function( + StringRef GOTContainer, StringRef TargetName)>; + + RuntimeDyldChecker(IsSymbolValidFunction IsSymbolValid, + GetSymbolInfoFunction GetSymbolInfo, + GetSectionInfoFunction GetSectionInfo, + GetStubInfoFunction GetStubInfo, + GetGOTInfoFunction GetGOTInfo, + support::endianness Endianness, + MCDisassembler *Disassembler, MCInstPrinter *InstPrinter, + raw_ostream &ErrStream); + ~RuntimeDyldChecker(); /// Check a single expression against the attached RuntimeDyld /// instance. @@ -100,7 +172,7 @@ public: bool LocalAddress); /// If there is a section at the given local address, return its load - /// address, otherwise return none. + /// address, otherwise return none. Optional getSectionLoadAddress(void *LocalAddress) const; private: diff --git a/include/llvm/ExecutionEngine/SectionMemoryManager.h b/include/llvm/ExecutionEngine/SectionMemoryManager.h index 3cf131c27778..d7316425da2f 100644 --- a/include/llvm/ExecutionEngine/SectionMemoryManager.h +++ b/include/llvm/ExecutionEngine/SectionMemoryManager.h @@ -1,9 +1,8 @@ //===- SectionMemoryManager.h - Memory manager for MCJIT/RtDyld -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/FuzzMutate/FuzzerCLI.h b/include/llvm/FuzzMutate/FuzzerCLI.h index 3333e96db166..2a16e43a6ab3 100644 --- a/include/llvm/FuzzMutate/FuzzerCLI.h +++ b/include/llvm/FuzzMutate/FuzzerCLI.h @@ -1,9 +1,8 @@ //===-- FuzzerCLI.h - Common logic for CLIs of fuzzers ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/FuzzMutate/IRMutator.h b/include/llvm/FuzzMutate/IRMutator.h index 9aa9d6d6a4bc..40a1ce8aeec9 100644 --- a/include/llvm/FuzzMutate/IRMutator.h +++ b/include/llvm/FuzzMutate/IRMutator.h @@ -1,9 +1,8 @@ //===-- IRMutator.h - Mutation engine for fuzzing IR ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/FuzzMutate/OpDescriptor.h b/include/llvm/FuzzMutate/OpDescriptor.h index dd30fda99bea..d6c98cd949a2 100644 --- a/include/llvm/FuzzMutate/OpDescriptor.h +++ b/include/llvm/FuzzMutate/OpDescriptor.h @@ -1,9 +1,8 @@ //===-- OpDescriptor.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/FuzzMutate/Operations.h b/include/llvm/FuzzMutate/Operations.h index 668bd952ebb2..2eb4c38c2aeb 100644 --- a/include/llvm/FuzzMutate/Operations.h +++ b/include/llvm/FuzzMutate/Operations.h @@ -1,9 +1,8 @@ //===-- Operations.h - ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/FuzzMutate/Random.h b/include/llvm/FuzzMutate/Random.h index 3a5f46a07554..615b15f04ceb 100644 --- a/include/llvm/FuzzMutate/Random.h +++ b/include/llvm/FuzzMutate/Random.h @@ -1,9 +1,8 @@ //===--- Random.h - Utilities for random sampling -------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/FuzzMutate/RandomIRBuilder.h b/include/llvm/FuzzMutate/RandomIRBuilder.h index 5cf3f0b22709..f3b609702e9d 100644 --- a/include/llvm/FuzzMutate/RandomIRBuilder.h +++ b/include/llvm/FuzzMutate/RandomIRBuilder.h @@ -1,9 +1,8 @@ -//===-- Mutator.h - Utils for randomly mutation IR --------------*- C++ -*-===// +//===- RandomIRBuilder.h - Utils for randomly mutation IR -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/Argument.h b/include/llvm/IR/Argument.h index 497dca44547c..5f514b9c47d2 100644 --- a/include/llvm/IR/Argument.h +++ b/include/llvm/IR/Argument.h @@ -1,9 +1,8 @@ //===-- llvm/Argument.h - Definition of the Argument class ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -79,6 +78,9 @@ public: /// If this is a byval or inalloca argument, return its alignment. unsigned getParamAlignment() const; + /// If this is a byval argument, return its type. + Type *getParamByValType() const; + /// Return true if this argument has the nest attribute. bool hasNestAttr() const; @@ -91,6 +93,9 @@ public: /// Return true if this argument has the sret attribute. bool hasStructRetAttr() const; + /// Return true if this argument has the inreg attribute. + bool hasInRegAttr() const; + /// Return true if this argument has the returned attribute. bool hasReturnedAttr() const; @@ -119,6 +124,8 @@ public: /// Check if an argument has a given attribute. bool hasAttribute(Attribute::AttrKind Kind) const; + Attribute getAttribute(Attribute::AttrKind Kind) const; + /// Method for support type inquiry through isa, cast, and dyn_cast. static bool classof(const Value *V) { return V->getValueID() == ArgumentVal; diff --git a/include/llvm/IR/AssemblyAnnotationWriter.h b/include/llvm/IR/AssemblyAnnotationWriter.h index 6e1f5c43e12e..3fd3c57a6796 100644 --- a/include/llvm/IR/AssemblyAnnotationWriter.h +++ b/include/llvm/IR/AssemblyAnnotationWriter.h @@ -1,9 +1,8 @@ //===-- AssemblyAnnotationWriter.h - Annotation .ll files -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/Attributes.h b/include/llvm/IR/Attributes.h index 9fc4614af010..06cc09e1cfc7 100644 --- a/include/llvm/IR/Attributes.h +++ b/include/llvm/IR/Attributes.h @@ -1,9 +1,8 @@ //===- llvm/Attributes.h - Container for Attributes -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -91,6 +90,7 @@ public: static Attribute get(LLVMContext &Context, AttrKind Kind, uint64_t Val = 0); static Attribute get(LLVMContext &Context, StringRef Kind, StringRef Val = StringRef()); + static Attribute get(LLVMContext &Context, AttrKind Kind, Type *Ty); /// Return a uniquified Attribute object that has the specific /// alignment set. @@ -103,6 +103,7 @@ public: static Attribute getWithAllocSizeArgs(LLVMContext &Context, unsigned ElemSizeArg, const Optional &NumElemsArg); + static Attribute getWithByValType(LLVMContext &Context, Type *Ty); //===--------------------------------------------------------------------===// // Attribute Accessors @@ -118,6 +119,9 @@ public: /// attribute. bool isStringAttribute() const; + /// Return true if the attribute is a type attribute. + bool isTypeAttribute() const; + /// Return true if the attribute is present. bool hasAttribute(AttrKind Val) const; @@ -140,6 +144,10 @@ public: /// attribute to be a string attribute. StringRef getValueAsString() const; + /// Return the attribute's value as a Type. This requires the attribute to be + /// a type attribute. + Type *getValueAsType() const; + /// Returns the alignment field of an attribute as a byte alignment /// value. unsigned getAlignment() const; @@ -280,6 +288,7 @@ public: unsigned getStackAlignment() const; uint64_t getDereferenceableBytes() const; uint64_t getDereferenceableOrNullBytes() const; + Type *getByValType() const; std::pair> getAllocSizeArgs() const; std::string getAsString(bool InAttrGrp = false) const; @@ -599,6 +608,9 @@ public: /// Return the alignment for the specified function parameter. unsigned getParamAlignment(unsigned ArgNo) const; + /// Return the byval type for the specified function parameter. + Type *getParamByValType(unsigned ArgNo) const; + /// Get the stack alignment. unsigned getStackAlignment(unsigned Index) const; @@ -698,6 +710,7 @@ class AttrBuilder { uint64_t DerefBytes = 0; uint64_t DerefOrNullBytes = 0; uint64_t AllocSizeArgs = 0; + Type *ByValType = nullptr; public: AttrBuilder() = default; @@ -773,6 +786,9 @@ public: /// dereferenceable_or_null attribute exists (zero is returned otherwise). uint64_t getDereferenceableOrNullBytes() const { return DerefOrNullBytes; } + /// Retrieve the byval type. + Type *getByValType() const { return ByValType; } + /// Retrieve the allocsize args, if the allocsize attribute exists. If it /// doesn't exist, pair(0, 0) is returned. std::pair> getAllocSizeArgs() const; @@ -797,6 +813,9 @@ public: AttrBuilder &addAllocSizeAttr(unsigned ElemSizeArg, const Optional &NumElemsArg); + /// This turns a byval type into the form used internally in Attribute. + AttrBuilder &addByValAttr(Type *Ty); + /// Add an allocsize attribute, using the representation returned by /// Attribute.getIntValue(). AttrBuilder &addAllocSizeAttrFromRawRepr(uint64_t RawAllocSizeRepr); diff --git a/include/llvm/IR/Attributes.td b/include/llvm/IR/Attributes.td index e786d85d05a8..153046d2311c 100644 --- a/include/llvm/IR/Attributes.td +++ b/include/llvm/IR/Attributes.td @@ -85,6 +85,9 @@ def NoCapture : EnumAttr<"nocapture">; /// Call cannot be duplicated. def NoDuplicate : EnumAttr<"noduplicate">; +/// Function does not deallocate memory. +def NoFree : EnumAttr<"nofree">; + /// Disable implicit floating point insts. def NoImplicitFloat : EnumAttr<"noimplicitfloat">; @@ -106,6 +109,9 @@ def NoRedZone : EnumAttr<"noredzone">; /// Mark the function as not returning. def NoReturn : EnumAttr<"noreturn">; +/// Function does not synchronize. +def NoSync : EnumAttr<"nosync">; + /// Disable Indirect Branch Tracking. def NoCfCheck : EnumAttr<"nocf_check">; @@ -130,6 +136,9 @@ def ReadOnly : EnumAttr<"readonly">; /// Return value is always equal to this argument. def Returned : EnumAttr<"returned">; +/// Parameter is required to be a trivial constant. +def ImmArg : EnumAttr<"immarg">; + /// Function can return twice. def ReturnsTwice : EnumAttr<"returns_twice">; @@ -176,6 +185,9 @@ def SanitizeMemory : EnumAttr<"sanitize_memory">; /// HWAddressSanitizer is on. def SanitizeHWAddress : EnumAttr<"sanitize_hwaddress">; +/// MemTagSanitizer is on. +def SanitizeMemTag : EnumAttr<"sanitize_memtag">; + /// Speculative Load Hardening is enabled. /// /// Note that this uses the default compatibility (always compatible during @@ -193,6 +205,9 @@ def SwiftSelf : EnumAttr<"swiftself">; /// Function must be in a unwind table. def UWTable : EnumAttr<"uwtable">; +/// Function always comes back to callsite. +def WillReturn : EnumAttr<"willreturn">; + /// Function only writes to memory. def WriteOnly : EnumAttr<"writeonly">; @@ -221,6 +236,7 @@ def : CompatRule<"isEqual">; def : CompatRule<"isEqual">; def : CompatRule<"isEqual">; def : CompatRule<"isEqual">; +def : CompatRule<"isEqual">; def : CompatRule<"isEqual">; def : CompatRule<"isEqual">; diff --git a/include/llvm/IR/AutoUpgrade.h b/include/llvm/IR/AutoUpgrade.h index 8cf574c6a138..017ad93d8a2a 100644 --- a/include/llvm/IR/AutoUpgrade.h +++ b/include/llvm/IR/AutoUpgrade.h @@ -1,9 +1,8 @@ //===- AutoUpgrade.h - AutoUpgrade Helpers ----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -47,9 +46,9 @@ namespace llvm { /// so that it can update all calls to the old function. void UpgradeCallsToIntrinsic(Function* F); - /// This checks for global variables which should be upgraded. It returns true - /// if it requires upgrading. - bool UpgradeGlobalVariable(GlobalVariable *GV); + /// This checks for global variables which should be upgraded. It it requires + /// upgrading, returns a pointer to the upgraded variable. + GlobalVariable *UpgradeGlobalVariable(GlobalVariable *GV); /// This checks for module flags which should be upgraded. It returns true if /// module is modified. diff --git a/include/llvm/IR/BasicBlock.h b/include/llvm/IR/BasicBlock.h index 99eac33f742e..69555af50e1f 100644 --- a/include/llvm/IR/BasicBlock.h +++ b/include/llvm/IR/BasicBlock.h @@ -1,9 +1,8 @@ //===- llvm/BasicBlock.h - Represent a basic block in the VM ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -363,7 +362,7 @@ public: /// This is actually not used to update the Predecessor list, but is actually /// used to update the PHI nodes that reside in the block. Note that this /// should be called while the predecessor still refers to this block. - void removePredecessor(BasicBlock *Pred, bool DontDeleteUselessPHIs = false); + void removePredecessor(BasicBlock *Pred, bool KeepOneInputPHIs = false); bool canSplitPredecessors() const; @@ -391,6 +390,14 @@ public: /// direct branches, switches, etc. to it. bool hasAddressTaken() const { return getSubclassDataFromValue() != 0; } + /// Update all phi nodes in this basic block to refer to basic block \p New + /// instead of basic block \p Old. + void replacePhiUsesWith(BasicBlock *Old, BasicBlock *New); + + /// Update all phi nodes in this basic block's successors to refer to basic + /// block \p New instead of basic block \p Old. + void replaceSuccessorsPhiUsesWith(BasicBlock *Old, BasicBlock *New); + /// Update all phi nodes in this basic block's successors to refer to basic /// block \p New instead of to it. void replaceSuccessorsPhiUsesWith(BasicBlock *New); diff --git a/include/llvm/IR/CFG.h b/include/llvm/IR/CFG.h index 8385c4647e12..55aff7137e86 100644 --- a/include/llvm/IR/CFG.h +++ b/include/llvm/IR/CFG.h @@ -1,9 +1,8 @@ //===- CFG.h ----------------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -238,10 +237,6 @@ public: } }; -template struct isPodLike> { - static const bool value = isPodLike::value; -}; - using succ_iterator = SuccIterator; using succ_const_iterator = SuccIterator; using succ_range = iterator_range; diff --git a/include/llvm/IR/CFGDiff.h b/include/llvm/IR/CFGDiff.h index da4373f7bce2..57b62dd66a47 100644 --- a/include/llvm/IR/CFGDiff.h +++ b/include/llvm/IR/CFGDiff.h @@ -1,9 +1,8 @@ //===- CFGDiff.h - Define a CFG snapshot. -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/CallSite.h b/include/llvm/IR/CallSite.h index a3e78049f4be..b47a96c5d5fa 100644 --- a/include/llvm/IR/CallSite.h +++ b/include/llvm/IR/CallSite.h @@ -1,15 +1,14 @@ //===- CallSite.h - Abstract Call & Invoke instrs ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file defines the CallSite class, which is a handy wrapper for code that -// wants to treat Call and Invoke instructions in a generic way. When in non- -// mutation context (e.g. an analysis) ImmutableCallSite should be used. +// wants to treat Call, Invoke and CallBr instructions in a generic way. When +// in non-mutation context (e.g. an analysis) ImmutableCallSite should be used. // Finally, when some degree of customization is necessary between these two // extremes, CallSiteBase<> can be supplied with fine-tuned parameters. // @@ -18,7 +17,7 @@ // They are efficiently copyable, assignable and constructable, with cost // equivalent to copying a pointer (notice that they have only a single data // member). The internal representation carries a flag which indicates which of -// the two variants is enclosed. This allows for cheaper checks when various +// the three variants is enclosed. This allows for cheaper checks when various // accessors of CallSite are employed. // //===----------------------------------------------------------------------===// @@ -49,45 +48,50 @@ namespace Intrinsic { enum ID : unsigned; } -template class CallSiteBase { protected: - PointerIntPair I; + PointerIntPair I; CallSiteBase() = default; - CallSiteBase(CallTy *CI) : I(CI, true) { assert(CI); } - CallSiteBase(InvokeTy *II) : I(II, false) { assert(II); } + CallSiteBase(CallTy *CI) : I(CI, 1) { assert(CI); } + CallSiteBase(InvokeTy *II) : I(II, 0) { assert(II); } + CallSiteBase(CallBrTy *CBI) : I(CBI, 2) { assert(CBI); } explicit CallSiteBase(ValTy *II) { *this = get(II); } private: /// This static method is like a constructor. It will create an appropriate - /// call site for a Call or Invoke instruction, but it can also create a null - /// initialized CallSiteBase object for something which is NOT a call site. + /// call site for a Call, Invoke or CallBr instruction, but it can also create + /// a null initialized CallSiteBase object for something which is NOT a call + /// site. static CallSiteBase get(ValTy *V) { if (InstrTy *II = dyn_cast(V)) { if (II->getOpcode() == Instruction::Call) return CallSiteBase(static_cast(II)); - else if (II->getOpcode() == Instruction::Invoke) + if (II->getOpcode() == Instruction::Invoke) return CallSiteBase(static_cast(II)); + if (II->getOpcode() == Instruction::CallBr) + return CallSiteBase(static_cast(II)); } return CallSiteBase(); } public: - /// Return true if a CallInst is enclosed. Note that !isCall() does not mean - /// an InvokeInst is enclosed. It may also signify a NULL instruction pointer. - bool isCall() const { return I.getInt(); } + /// Return true if a CallInst is enclosed. + bool isCall() const { return I.getInt() == 1; } + + /// Return true if a InvokeInst is enclosed. !I.getInt() may also signify a + /// NULL instruction pointer, so check that. + bool isInvoke() const { return getInstruction() && I.getInt() == 0; } - /// Return true if a InvokeInst is enclosed. - bool isInvoke() const { return getInstruction() && !I.getInt(); } + /// Return true if a CallBrInst is enclosed. + bool isCallBr() const { return I.getInt() == 2; } InstrTy *getInstruction() const { return I.getPointer(); } InstrTy *operator->() const { return I.getPointer(); } @@ -98,7 +102,7 @@ public: /// Return the pointer to function that is being called. ValTy *getCalledValue() const { - assert(getInstruction() && "Not a call or invoke instruction!"); + assert(getInstruction() && "Not a call, invoke or callbr instruction!"); return *getCallee(); } @@ -115,16 +119,19 @@ public: return false; if (isa(V) || isa(V)) return false; - if (const CallInst *CI = dyn_cast(getInstruction())) { - if (CI->isInlineAsm()) + if (const CallBase *CB = dyn_cast(getInstruction())) + if (CB->isInlineAsm()) return false; - } return true; } - /// Set the callee to the specified value. + /// Set the callee to the specified value. Unlike the function of the same + /// name on CallBase, does not modify the type! void setCalledFunction(Value *V) { - assert(getInstruction() && "Not a call or invoke instruction!"); + assert(getInstruction() && "Not a call, callbr, or invoke instruction!"); + assert(cast(V->getType())->getElementType() == + cast(getInstruction())->getFunctionType() && + "New callee type does not match FunctionType on call"); *getCallee() = V; } @@ -189,7 +196,7 @@ public: } void setArgument(unsigned ArgNo, Value* newVal) { - assert(getInstruction() && "Not a call or invoke instruction!"); + assert(getInstruction() && "Not a call, invoke or callbr instruction!"); assert(arg_begin() + ArgNo < arg_end() && "Argument # out of range!"); getInstruction()->setOperand(ArgNo, newVal); } @@ -203,7 +210,7 @@ public: /// Given a use for an argument, get the argument number that corresponds to /// it. unsigned getArgumentNo(const Use *U) const { - assert(getInstruction() && "Not a call or invoke instruction!"); + assert(getInstruction() && "Not a call, invoke or callbr instruction!"); assert(isArgOperand(U) && "Argument # out of range!"); return U - arg_begin(); } @@ -227,7 +234,7 @@ public: /// Given a use for a data operand, get the data operand number that /// corresponds to it. unsigned getDataOperandNo(const Use *U) const { - assert(getInstruction() && "Not a call or invoke instruction!"); + assert(getInstruction() && "Not a call, invoke or callbr instruction!"); assert(isDataOperand(U) && "Data operand # out of range!"); return U - data_operands_begin(); } @@ -237,18 +244,19 @@ public: using data_operand_iterator = IterTy; /// data_operands_begin/data_operands_end - Return iterators iterating over - /// the call / invoke argument list and bundle operands. For invokes, this is - /// the set of instruction operands except the invoke target and the two - /// successor blocks; and for calls this is the set of instruction operands - /// except the call target. + /// the call / invoke / callbr argument list and bundle operands. For invokes, + /// this is the set of instruction operands except the invoke target and the + /// two successor blocks; for calls this is the set of instruction operands + /// except the call target; for callbrs the number of labels to skip must be + /// determined first. IterTy data_operands_begin() const { assert(getInstruction() && "Not a call or invoke instruction!"); - return (*this)->op_begin(); + return cast(getInstruction())->data_operands_begin(); } IterTy data_operands_end() const { assert(getInstruction() && "Not a call or invoke instruction!"); - return (*this)->op_end() - (isCall() ? 1 : 3); + return cast(getInstruction())->data_operands_end(); } iterator_range data_ops() const { return make_range(data_operands_begin(), data_operands_end()); @@ -277,17 +285,19 @@ public: return isCall() && cast(getInstruction())->isTailCall(); } -#define CALLSITE_DELEGATE_GETTER(METHOD) \ - InstrTy *II = getInstruction(); \ - return isCall() \ - ? cast(II)->METHOD \ - : cast(II)->METHOD +#define CALLSITE_DELEGATE_GETTER(METHOD) \ + InstrTy *II = getInstruction(); \ + return isCall() ? cast(II)->METHOD \ + : isCallBr() ? cast(II)->METHOD \ + : cast(II)->METHOD -#define CALLSITE_DELEGATE_SETTER(METHOD) \ - InstrTy *II = getInstruction(); \ - if (isCall()) \ - cast(II)->METHOD; \ - else \ +#define CALLSITE_DELEGATE_SETTER(METHOD) \ + InstrTy *II = getInstruction(); \ + if (isCall()) \ + cast(II)->METHOD; \ + else if (isCallBr()) \ + cast(II)->METHOD; \ + else \ cast(II)->METHOD unsigned getNumArgOperands() const { @@ -303,9 +313,7 @@ public: } bool isInlineAsm() const { - if (isCall()) - return cast(getInstruction())->isInlineAsm(); - return false; + return cast(getInstruction())->isInlineAsm(); } /// Get the calling convention of the call. @@ -389,10 +397,10 @@ public: /// Return true if the data operand at index \p i directly or indirectly has /// the attribute \p A. /// - /// Normal call or invoke arguments have per operand attributes, as specified - /// in the attribute set attached to this instruction, while operand bundle - /// operands may have some attributes implied by the type of its containing - /// operand bundle. + /// Normal call, invoke or callbr arguments have per operand attributes, as + /// specified in the attribute set attached to this instruction, while operand + /// bundle operands may have some attributes implied by the type of its + /// containing operand bundle. bool dataOperandHasImpliedAttr(unsigned i, Attribute::AttrKind Kind) const { CALLSITE_DELEGATE_GETTER(dataOperandHasImpliedAttr(i, Kind)); } @@ -407,6 +415,11 @@ public: CALLSITE_DELEGATE_GETTER(getParamAlignment(ArgNo)); } + /// Extract the byval type for a call or parameter (nullptr=unknown). + Type *getParamByValType(unsigned ArgNo) const { + CALLSITE_DELEGATE_GETTER(getParamByValType(ArgNo)); + } + /// Extract the number of dereferenceable bytes for a call or parameter /// (0=unknown). uint64_t getDereferenceableBytes(unsigned i) const { @@ -580,13 +593,9 @@ public: #undef CALLSITE_DELEGATE_SETTER void getOperandBundlesAsDefs(SmallVectorImpl &Defs) const { - const Instruction *II = getInstruction(); // Since this is actually a getter that "looks like" a setter, don't use the // above macros to avoid confusion. - if (isCall()) - cast(II)->getOperandBundlesAsDefs(Defs); - else - cast(II)->getOperandBundlesAsDefs(Defs); + cast(getInstruction())->getOperandBundlesAsDefs(Defs); } /// Determine whether this data operand is not captured. @@ -662,12 +671,13 @@ private: class CallSite : public CallSiteBase { + CallBrInst, User::op_iterator> { public: CallSite() = default; CallSite(CallSiteBase B) : CallSiteBase(B) {} CallSite(CallInst *CI) : CallSiteBase(CI) {} CallSite(InvokeInst *II) : CallSiteBase(II) {} + CallSite(CallBrInst *CBI) : CallSiteBase(CBI) {} explicit CallSite(Instruction *II) : CallSiteBase(II) {} explicit CallSite(Value *V) : CallSiteBase(V) {} @@ -683,6 +693,182 @@ private: User::op_iterator getCallee() const; }; +/// AbstractCallSite +/// +/// An abstract call site is a wrapper that allows to treat direct, +/// indirect, and callback calls the same. If an abstract call site +/// represents a direct or indirect call site it behaves like a stripped +/// down version of a normal call site object. The abstract call site can +/// also represent a callback call, thus the fact that the initially +/// called function (=broker) may invoke a third one (=callback callee). +/// In this case, the abstract call site hides the middle man, hence the +/// broker function. The result is a representation of the callback call, +/// inside the broker, but in the context of the original call to the broker. +/// +/// There are up to three functions involved when we talk about callback call +/// sites. The caller (1), which invokes the broker function. The broker +/// function (2), that will invoke the callee zero or more times. And finally +/// the callee (3), which is the target of the callback call. +/// +/// The abstract call site will handle the mapping from parameters to arguments +/// depending on the semantic of the broker function. However, it is important +/// to note that the mapping is often partial. Thus, some arguments of the +/// call/invoke instruction are mapped to parameters of the callee while others +/// are not. +class AbstractCallSite { +public: + + /// The encoding of a callback with regards to the underlying instruction. + struct CallbackInfo { + + /// For direct/indirect calls the parameter encoding is empty. If it is not, + /// the abstract call site represents a callback. In that case, the first + /// element of the encoding vector represents which argument of the call + /// site CS is the callback callee. The remaining elements map parameters + /// (identified by their position) to the arguments that will be passed + /// through (also identified by position but in the call site instruction). + /// + /// NOTE that we use LLVM argument numbers (starting at 0) and not + /// clang/source argument numbers (starting at 1). The -1 entries represent + /// unknown values that are passed to the callee. + using ParameterEncodingTy = SmallVector; + ParameterEncodingTy ParameterEncoding; + + }; + +private: + + /// The underlying call site: + /// caller -> callee, if this is a direct or indirect call site + /// caller -> broker function, if this is a callback call site + CallSite CS; + + /// The encoding of a callback with regards to the underlying instruction. + CallbackInfo CI; + +public: + /// Sole constructor for abstract call sites (ACS). + /// + /// An abstract call site can only be constructed through a llvm::Use because + /// each operand (=use) of an instruction could potentially be a different + /// abstract call site. Furthermore, even if the value of the llvm::Use is the + /// same, and the user is as well, the abstract call sites might not be. + /// + /// If a use is not associated with an abstract call site the constructed ACS + /// will evaluate to false if converted to a boolean. + /// + /// If the use is the callee use of a call or invoke instruction, the + /// constructed abstract call site will behave as a llvm::CallSite would. + /// + /// If the use is not a callee use of a call or invoke instruction, the + /// callback metadata is used to determine the argument <-> parameter mapping + /// as well as the callee of the abstract call site. + AbstractCallSite(const Use *U); + + /// Conversion operator to conveniently check for a valid/initialized ACS. + explicit operator bool() const { return (bool)CS; } + + /// Return the underlying instruction. + Instruction *getInstruction() const { return CS.getInstruction(); } + + /// Return the call site abstraction for the underlying instruction. + CallSite getCallSite() const { return CS; } + + /// Return true if this ACS represents a direct call. + bool isDirectCall() const { + return !isCallbackCall() && !CS.isIndirectCall(); + } + + /// Return true if this ACS represents an indirect call. + bool isIndirectCall() const { + return !isCallbackCall() && CS.isIndirectCall(); + } + + /// Return true if this ACS represents a callback call. + bool isCallbackCall() const { + // For a callback call site the callee is ALWAYS stored first in the + // transitive values vector. Thus, a non-empty vector indicates a callback. + return !CI.ParameterEncoding.empty(); + } + + /// Return true if @p UI is the use that defines the callee of this ACS. + bool isCallee(Value::const_user_iterator UI) const { + return isCallee(&UI.getUse()); + } + + /// Return true if @p U is the use that defines the callee of this ACS. + bool isCallee(const Use *U) const { + if (isDirectCall()) + return CS.isCallee(U); + + assert(!CI.ParameterEncoding.empty() && + "Callback without parameter encoding!"); + + return (int)CS.getArgumentNo(U) == CI.ParameterEncoding[0]; + } + + /// Return the number of parameters of the callee. + unsigned getNumArgOperands() const { + if (isDirectCall()) + return CS.getNumArgOperands(); + // Subtract 1 for the callee encoding. + return CI.ParameterEncoding.size() - 1; + } + + /// Return the operand index of the underlying instruction associated with @p + /// Arg. + int getCallArgOperandNo(Argument &Arg) const { + return getCallArgOperandNo(Arg.getArgNo()); + } + + /// Return the operand index of the underlying instruction associated with + /// the function parameter number @p ArgNo or -1 if there is none. + int getCallArgOperandNo(unsigned ArgNo) const { + if (isDirectCall()) + return ArgNo; + // Add 1 for the callee encoding. + return CI.ParameterEncoding[ArgNo + 1]; + } + + /// Return the operand of the underlying instruction associated with @p Arg. + Value *getCallArgOperand(Argument &Arg) const { + return getCallArgOperand(Arg.getArgNo()); + } + + /// Return the operand of the underlying instruction associated with the + /// function parameter number @p ArgNo or nullptr if there is none. + Value *getCallArgOperand(unsigned ArgNo) const { + if (isDirectCall()) + return CS.getArgOperand(ArgNo); + // Add 1 for the callee encoding. + return CI.ParameterEncoding[ArgNo + 1] >= 0 + ? CS.getArgOperand(CI.ParameterEncoding[ArgNo + 1]) + : nullptr; + } + + /// Return the operand index of the underlying instruction associated with the + /// callee of this ACS. Only valid for callback calls! + int getCallArgOperandNoForCallee() const { + assert(isCallbackCall()); + assert(CI.ParameterEncoding.size() && CI.ParameterEncoding[0] > 0); + return CI.ParameterEncoding[0]; + } + + /// Return the pointer to function that is being called. + Value *getCalledValue() const { + if (isDirectCall()) + return CS.getCalledValue(); + return CS.getArgOperand(getCallArgOperandNoForCallee()); + } + + /// Return the function being called if this is a direct call, otherwise + /// return null (if it's an indirect call). + Function *getCalledFunction() const { + Value *V = getCalledValue(); + return V ? dyn_cast(V->stripPointerCasts()) : nullptr; + } +}; + template <> struct DenseMapInfo { using BaseInfo = DenseMapInfo; @@ -713,6 +899,7 @@ public: ImmutableCallSite() = default; ImmutableCallSite(const CallInst *CI) : CallSiteBase(CI) {} ImmutableCallSite(const InvokeInst *II) : CallSiteBase(II) {} + ImmutableCallSite(const CallBrInst *CBI) : CallSiteBase(CBI) {} explicit ImmutableCallSite(const Instruction *II) : CallSiteBase(II) {} explicit ImmutableCallSite(const Value *V) : CallSiteBase(V) {} ImmutableCallSite(CallSite CS) : CallSiteBase(CS.getInstruction()) {} diff --git a/include/llvm/IR/CallingConv.h b/include/llvm/IR/CallingConv.h index 49c3be960373..399c6ad521fa 100644 --- a/include/llvm/IR/CallingConv.h +++ b/include/llvm/IR/CallingConv.h @@ -1,9 +1,8 @@ //===- llvm/CallingConv.h - LLVM Calling Conventions ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/Comdat.h b/include/llvm/IR/Comdat.h index 555121e928f7..f712a16dd318 100644 --- a/include/llvm/IR/Comdat.h +++ b/include/llvm/IR/Comdat.h @@ -1,9 +1,8 @@ //===- llvm/IR/Comdat.h - Comdat definitions --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/Constant.h b/include/llvm/IR/Constant.h index 98437f8eff1f..931576651224 100644 --- a/include/llvm/IR/Constant.h +++ b/include/llvm/IR/Constant.h @@ -1,9 +1,8 @@ //===-- llvm/Constant.h - Constant class definition -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -91,6 +90,10 @@ public: /// elements. bool containsUndefElement() const; + /// Return true if this is a vector constant that includes any constant + /// expressions. + bool containsConstantExpression() const; + /// Return true if evaluation of this constant could trap. This is true for /// things like constant expressions that could divide by zero. bool canTrap() const; diff --git a/include/llvm/IR/ConstantFolder.h b/include/llvm/IR/ConstantFolder.h index da5bba7ba141..5a5cabfd0206 100644 --- a/include/llvm/IR/ConstantFolder.h +++ b/include/llvm/IR/ConstantFolder.h @@ -1,9 +1,8 @@ //===- ConstantFolder.h - Constant folding helper ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -135,6 +134,10 @@ public: return ConstantExpr::getNot(C); } + Constant *CreateUnOp(Instruction::UnaryOps Opc, Constant *C) const { + return ConstantExpr::get(Opc, C); + } + //===--------------------------------------------------------------------===// // Memory Instructions //===--------------------------------------------------------------------===// diff --git a/include/llvm/IR/ConstantRange.h b/include/llvm/IR/ConstantRange.h index 1adda3269abc..91f3f31abe17 100644 --- a/include/llvm/IR/ConstantRange.h +++ b/include/llvm/IR/ConstantRange.h @@ -1,9 +1,8 @@ //===- ConstantRange.h - Represent a range ----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -42,14 +41,25 @@ namespace llvm { class MDNode; class raw_ostream; +struct KnownBits; /// This class represents a range of values. class LLVM_NODISCARD ConstantRange { APInt Lower, Upper; + /// Create empty constant range with same bitwidth. + ConstantRange getEmpty() const { + return ConstantRange(getBitWidth(), false); + } + + /// Create full constant range with same bitwidth. + ConstantRange getFull() const { + return ConstantRange(getBitWidth(), true); + } + public: - /// Initialize a full (the default) or empty set for the specified bit width. - explicit ConstantRange(uint32_t BitWidth, bool isFullSet = true); + /// Initialize a full or empty set for the specified bit width. + explicit ConstantRange(uint32_t BitWidth, bool isFullSet); /// Initialize a range to hold the single specified value. ConstantRange(APInt Value); @@ -59,6 +69,29 @@ public: /// assert out if the two APInt's are not the same bit width. ConstantRange(APInt Lower, APInt Upper); + /// Create empty constant range with the given bit width. + static ConstantRange getEmpty(uint32_t BitWidth) { + return ConstantRange(BitWidth, false); + } + + /// Create full constant range with the given bit width. + static ConstantRange getFull(uint32_t BitWidth) { + return ConstantRange(BitWidth, true); + } + + /// Create non-empty constant range with the given bounds. If Lower and + /// Upper are the same, a full range is returned. + static ConstantRange getNonEmpty(APInt Lower, APInt Upper) { + if (Lower == Upper) + return getFull(Lower.getBitWidth()); + return ConstantRange(std::move(Lower), std::move(Upper)); + } + + /// Initialize a range based on a known bits constraint. The IsSigned flag + /// indicates whether the constant range should not wrap in the signed or + /// unsigned domain. + static ConstantRange fromKnownBits(const KnownBits &Known, bool IsSigned); + /// Produce the smallest range such that all values that may satisfy the given /// predicate with any value contained within Other is contained in the /// returned range. Formally, this returns a superset of @@ -91,14 +124,12 @@ public: static ConstantRange makeExactICmpRegion(CmpInst::Predicate Pred, const APInt &Other); - /// Return the largest range containing all X such that "X BinOpC Y" is - /// guaranteed not to wrap (overflow) for all Y in Other. + /// Produce the largest range containing all X such that "X BinOp Y" is + /// guaranteed not to wrap (overflow) for *all* Y in Other. However, there may + /// be *some* Y in Other for which additional X not contained in the result + /// also do not overflow. /// - /// NB! The returned set does *not* contain **all** possible values of X for - /// which "X BinOpC Y" does not wrap -- some viable values of X may be - /// missing, so you cannot use this to constrain X's range. E.g. in the - /// fourth example, "(-2) + 1" is both nsw and nuw (so the "X" could be -2), - /// but (-2) is not in the set returned. + /// NoWrapKind must be one of OBO::NoUnsignedWrap or OBO::NoSignedWrap. /// /// Examples: /// typedef OverflowingBinaryOperator OBO; @@ -106,17 +137,19 @@ public: /// MGNR(Add, [i8 1, 2), OBO::NoSignedWrap) == [-128, 127) /// MGNR(Add, [i8 1, 2), OBO::NoUnsignedWrap) == [0, -1) /// MGNR(Add, [i8 0, 1), OBO::NoUnsignedWrap) == Full Set - /// MGNR(Add, [i8 1, 2), OBO::NoUnsignedWrap | OBO::NoSignedWrap) - /// == [0,INT_MAX) /// MGNR(Add, [i8 -1, 6), OBO::NoSignedWrap) == [INT_MIN+1, INT_MAX-4) /// MGNR(Sub, [i8 1, 2), OBO::NoSignedWrap) == [-127, 128) /// MGNR(Sub, [i8 1, 2), OBO::NoUnsignedWrap) == [1, 0) - /// MGNR(Sub, [i8 1, 2), OBO::NoUnsignedWrap | OBO::NoSignedWrap) - /// == [1,INT_MAX) static ConstantRange makeGuaranteedNoWrapRegion(Instruction::BinaryOps BinOp, const ConstantRange &Other, unsigned NoWrapKind); + /// Produce the range that contains X if and only if "X BinOp Other" does + /// not wrap. + static ConstantRange makeExactNoWrapRegion(Instruction::BinaryOps BinOp, + const APInt &Other, + unsigned NoWrapKind); + /// Set up \p Pred and \p RHS such that /// ConstantRange::makeExactICmpRegion(Pred, RHS) == *this. Return true if /// successful. @@ -138,14 +171,32 @@ public: /// Return true if this set contains no members. bool isEmptySet() const; - /// Return true if this set wraps around the top of the range. - /// For example: [100, 8). + /// Return true if this set wraps around the unsigned domain. Special cases: + /// * Empty set: Not wrapped. + /// * Full set: Not wrapped. + /// * [X, 0) == [X, Max]: Not wrapped. bool isWrappedSet() const; - /// Return true if this set wraps around the INT_MIN of - /// its bitwidth. For example: i8 [120, 140). + /// Return true if the exclusive upper bound wraps around the unsigned + /// domain. Special cases: + /// * Empty set: Not wrapped. + /// * Full set: Not wrapped. + /// * [X, 0): Wrapped. + bool isUpperWrapped() const; + + /// Return true if this set wraps around the signed domain. Special cases: + /// * Empty set: Not wrapped. + /// * Full set: Not wrapped. + /// * [X, SignedMin) == [X, SignedMax]: Not wrapped. bool isSignWrappedSet() const; + /// Return true if the (exclusive) upper bound wraps around the signed + /// domain. Special cases: + /// * Empty set: Not wrapped. + /// * Full set: Not wrapped. + /// * [X, SignedMin): Wrapped. + bool isUpperSignWrapped() const; + /// Return true if the specified value is in the set. bool contains(const APInt &Val) const; @@ -170,15 +221,18 @@ public: /// Return true if this set contains exactly one member. bool isSingleElement() const { return getSingleElement() != nullptr; } - /// Return the number of elements in this set. - APInt getSetSize() const; - /// Compare set size of this range with the range CR. bool isSizeStrictlySmallerThan(const ConstantRange &CR) const; - // Compare set size of this range with Value. + /// Compare set size of this range with Value. bool isSizeLargerThan(uint64_t MaxSize) const; + /// Return true if all values in this range are negative. + bool isAllNegative() const; + + /// Return true if all values in this range are non-negative. + bool isAllNonNegative() const; + /// Return the largest unsigned value contained in the ConstantRange. APInt getUnsignedMax() const; @@ -206,20 +260,30 @@ public: /// the sets). ConstantRange difference(const ConstantRange &CR) const; - /// Return the range that results from the intersection of - /// this range with another range. The resultant range is guaranteed to - /// include all elements contained in both input ranges, and to have the - /// smallest possible set size that does so. Because there may be two - /// intersections with the same set size, A.intersectWith(B) might not - /// be equal to B.intersectWith(A). - ConstantRange intersectWith(const ConstantRange &CR) const; + /// If represented precisely, the result of some range operations may consist + /// of multiple disjoint ranges. As only a single range may be returned, any + /// range covering these disjoint ranges constitutes a valid result, but some + /// may be more useful than others depending on context. The preferred range + /// type specifies whether a range that is non-wrapping in the unsigned or + /// signed domain, or has the smallest size, is preferred. If a signedness is + /// preferred but all ranges are non-wrapping or all wrapping, then the + /// smallest set size is preferred. If there are multiple smallest sets, any + /// one of them may be returned. + enum PreferredRangeType { Smallest, Unsigned, Signed }; + + /// Return the range that results from the intersection of this range with + /// another range. If the intersection is disjoint, such that two results + /// are possible, the preferred range is determined by the PreferredRangeType. + ConstantRange intersectWith(const ConstantRange &CR, + PreferredRangeType Type = Smallest) const; /// Return the range that results from the union of this range /// with another range. The resultant range is guaranteed to include the /// elements of both sets, but may contain more. For example, [3, 9) union /// [12,15) is [3, 15), which includes 9, 10, and 11, which were not included /// in either set before. - ConstantRange unionWith(const ConstantRange &CR) const; + ConstantRange unionWith(const ConstantRange &CR, + PreferredRangeType Type = Smallest) const; /// Return a new range representing the possible values resulting /// from an application of the specified cast operator to this range. \p @@ -300,6 +364,23 @@ public: /// \p Other. ConstantRange udiv(const ConstantRange &Other) const; + /// Return a new range representing the possible values resulting + /// from a signed division of a value in this range and a value in + /// \p Other. Division by zero and division of SignedMin by -1 are considered + /// undefined behavior, in line with IR, and do not contribute towards the + /// result. + ConstantRange sdiv(const ConstantRange &Other) const; + + /// Return a new range representing the possible values resulting + /// from an unsigned remainder operation of a value in this range and a + /// value in \p Other. + ConstantRange urem(const ConstantRange &Other) const; + + /// Return a new range representing the possible values resulting + /// from a signed remainder operation of a value in this range and a + /// value in \p Other. + ConstantRange srem(const ConstantRange &Other) const; + /// Return a new range representing the possible values resulting /// from a binary-and of a value in this range by a value in \p Other. ConstantRange binaryAnd(const ConstantRange &Other) const; @@ -321,9 +402,53 @@ public: /// arithmetic right shift of a value in this range and a value in \p Other. ConstantRange ashr(const ConstantRange &Other) const; + /// Perform an unsigned saturating addition of two constant ranges. + ConstantRange uadd_sat(const ConstantRange &Other) const; + + /// Perform a signed saturating addition of two constant ranges. + ConstantRange sadd_sat(const ConstantRange &Other) const; + + /// Perform an unsigned saturating subtraction of two constant ranges. + ConstantRange usub_sat(const ConstantRange &Other) const; + + /// Perform a signed saturating subtraction of two constant ranges. + ConstantRange ssub_sat(const ConstantRange &Other) const; + /// Return a new range that is the logical not of the current set. ConstantRange inverse() const; + /// Calculate absolute value range. If the original range contains signed + /// min, then the resulting range will also contain signed min. + ConstantRange abs() const; + + /// Represents whether an operation on the given constant range is known to + /// always or never overflow. + enum class OverflowResult { + /// Always overflows in the direction of signed/unsigned min value. + AlwaysOverflowsLow, + /// Always overflows in the direction of signed/unsigned max value. + AlwaysOverflowsHigh, + /// May or may not overflow. + MayOverflow, + /// Never overflows. + NeverOverflows, + }; + + /// Return whether unsigned add of the two ranges always/never overflows. + OverflowResult unsignedAddMayOverflow(const ConstantRange &Other) const; + + /// Return whether signed add of the two ranges always/never overflows. + OverflowResult signedAddMayOverflow(const ConstantRange &Other) const; + + /// Return whether unsigned sub of the two ranges always/never overflows. + OverflowResult unsignedSubMayOverflow(const ConstantRange &Other) const; + + /// Return whether signed sub of the two ranges always/never overflows. + OverflowResult signedSubMayOverflow(const ConstantRange &Other) const; + + /// Return whether unsigned mul of the two ranges always/never overflows. + OverflowResult unsignedMulMayOverflow(const ConstantRange &Other) const; + /// Print out the bounds to a stream. void print(raw_ostream &OS) const; diff --git a/include/llvm/IR/Constants.h b/include/llvm/IR/Constants.h index afc93cd61d47..ca56e8b9328c 100644 --- a/include/llvm/IR/Constants.h +++ b/include/llvm/IR/Constants.h @@ -1,9 +1,8 @@ //===-- llvm/Constants.h - Constant class subclass definitions --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/DIBuilder.h b/include/llvm/IR/DIBuilder.h index 443332b1b23c..ad9a35b55414 100644 --- a/include/llvm/IR/DIBuilder.h +++ b/include/llvm/IR/DIBuilder.h @@ -1,9 +1,8 @@ //===- DIBuilder.h - Debug Information Builder ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -707,6 +706,16 @@ namespace llvm { DITemplateParameterArray TParams = nullptr, DITypeArray ThrownTypes = nullptr); + /// Create common block entry for a Fortran common block. + /// \param Scope Scope of this common block. + /// \param decl Global variable declaration. + /// \param Name The name of this common block. + /// \param File The file this common block is defined. + /// \param LineNo Line number. + DICommonBlock *createCommonBlock(DIScope *Scope, DIGlobalVariable *decl, + StringRef Name, DIFile *File, + unsigned LineNo); + /// This creates new descriptor for a namespace with the specified /// parent scope. /// \param Scope Namespace scope diff --git a/include/llvm/IR/DataLayout.h b/include/llvm/IR/DataLayout.h index c144d1c13c34..ac9770a15120 100644 --- a/include/llvm/IR/DataLayout.h +++ b/include/llvm/IR/DataLayout.h @@ -1,9 +1,8 @@ //===- llvm/DataLayout.h - Data size & alignment info -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -109,6 +108,13 @@ struct PointerAlignElem { /// generating LLVM IR is required to generate the right target data for the /// target being codegen'd to. class DataLayout { +public: + enum class FunctionPtrAlignType { + /// The function pointer alignment is independent of the function alignment. + Independent, + /// The function pointer alignment is a multiple of the function alignment. + MultipleOfFunctionAlign, + }; private: /// Defaults to false. bool BigEndian; @@ -117,6 +123,9 @@ private: unsigned StackNaturalAlign; unsigned ProgramAddrSpace; + unsigned FunctionPtrAlign; + FunctionPtrAlignType TheFunctionPtrAlignType; + enum ManglingModeT { MM_None, MM_ELF, @@ -200,6 +209,8 @@ public: BigEndian = DL.isBigEndian(); AllocaAddrSpace = DL.AllocaAddrSpace; StackNaturalAlign = DL.StackNaturalAlign; + FunctionPtrAlign = DL.FunctionPtrAlign; + TheFunctionPtrAlignType = DL.TheFunctionPtrAlignType; ProgramAddrSpace = DL.ProgramAddrSpace; ManglingMode = DL.ManglingMode; LegalIntWidths = DL.LegalIntWidths; @@ -257,6 +268,17 @@ public: unsigned getStackAlignment() const { return StackNaturalAlign; } unsigned getAllocaAddrSpace() const { return AllocaAddrSpace; } + /// Returns the alignment of function pointers, which may or may not be + /// related to the alignment of functions. + /// \see getFunctionPtrAlignType + unsigned getFunctionPtrAlign() const { return FunctionPtrAlign; } + + /// Return the type of function pointer alignment. + /// \see getFunctionPtrAlign + FunctionPtrAlignType getFunctionPtrAlignType() const { + return TheFunctionPtrAlignType; + } + unsigned getProgramAddressSpace() const { return ProgramAddrSpace; } bool hasMicrosoftFastStdCallMangling() const { @@ -346,10 +368,13 @@ public: return NonIntegralAddressSpaces; } - bool isNonIntegralPointerType(PointerType *PT) const { + bool isNonIntegralAddressSpace(unsigned AddrSpace) const { ArrayRef NonIntegralSpaces = getNonIntegralAddressSpaces(); - return find(NonIntegralSpaces, PT->getAddressSpace()) != - NonIntegralSpaces.end(); + return find(NonIntegralSpaces, AddrSpace) != NonIntegralSpaces.end(); + } + + bool isNonIntegralPointerType(PointerType *PT) const { + return isNonIntegralAddressSpace(PT->getAddressSpace()); } bool isNonIntegralPointerType(Type *Ty) const { @@ -428,6 +453,14 @@ public: return 8 * getTypeStoreSize(Ty); } + /// Returns true if no extra padding bits are needed when storing the + /// specified type. + /// + /// For example, returns false for i19 that has a 24-bit store size. + bool typeSizeEqualsStoreSize(Type *Ty) const { + return getTypeSizeInBits(Ty) == getTypeStoreSizeInBits(Ty); + } + /// Returns the offset in bytes between successive objects of the /// specified type, including alignment padding. /// diff --git a/include/llvm/IR/DebugInfo.h b/include/llvm/IR/DebugInfo.h index 01178af3c9ff..171e1621889f 100644 --- a/include/llvm/IR/DebugInfo.h +++ b/include/llvm/IR/DebugInfo.h @@ -1,9 +1,8 @@ //===- DebugInfo.h - Debug Information Helpers ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/DebugInfoFlags.def b/include/llvm/IR/DebugInfoFlags.def index ce117aa452aa..07e3d6bdc9e5 100644 --- a/include/llvm/IR/DebugInfoFlags.def +++ b/include/llvm/IR/DebugInfoFlags.def @@ -1,9 +1,8 @@ //===- llvm/IR/DebugInfoFlags.def - Debug info flag definitions -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -51,12 +50,12 @@ HANDLE_DI_FLAG((3 << 16), VirtualInheritance) HANDLE_DI_FLAG((1 << 18), IntroducedVirtual) HANDLE_DI_FLAG((1 << 19), BitField) HANDLE_DI_FLAG((1 << 20), NoReturn) -HANDLE_DI_FLAG((1 << 21), MainSubprogram) +HANDLE_DI_FLAG((1 << 21), ArgumentNotModified) HANDLE_DI_FLAG((1 << 22), TypePassByValue) HANDLE_DI_FLAG((1 << 23), TypePassByReference) HANDLE_DI_FLAG((1 << 24), EnumClass) HANDLE_DI_FLAG((1 << 25), Thunk) -HANDLE_DI_FLAG((1 << 26), Trivial) +HANDLE_DI_FLAG((1 << 26), NonTrivial) HANDLE_DI_FLAG((1 << 27), BigEndian) HANDLE_DI_FLAG((1 << 28), LittleEndian) HANDLE_DI_FLAG((1 << 29), AllCallsDescribed) @@ -85,11 +84,15 @@ HANDLE_DISP_FLAG(2u, PureVirtual) HANDLE_DISP_FLAG((1u << 2), LocalToUnit) HANDLE_DISP_FLAG((1u << 3), Definition) HANDLE_DISP_FLAG((1u << 4), Optimized) +HANDLE_DISP_FLAG((1u << 5), Pure) +HANDLE_DISP_FLAG((1u << 6), Elemental) +HANDLE_DISP_FLAG((1u << 7), Recursive) +HANDLE_DISP_FLAG((1u << 8), MainSubprogram) #ifdef DISP_FLAG_LARGEST_NEEDED // Intended to be used with ADT/BitmaskEnum.h. // NOTE: Always must be equal to largest flag, check this when adding new flags. -HANDLE_DISP_FLAG((1 << 4), Largest) +HANDLE_DISP_FLAG((1 << 8), Largest) #undef DISP_FLAG_LARGEST_NEEDED #endif diff --git a/include/llvm/IR/DebugInfoMetadata.h b/include/llvm/IR/DebugInfoMetadata.h index a461d1bd4fe8..9dc6dfbb0f68 100644 --- a/include/llvm/IR/DebugInfoMetadata.h +++ b/include/llvm/IR/DebugInfoMetadata.h @@ -1,9 +1,8 @@ //===- llvm/IR/DebugInfoMetadata.h - Debug info metadata --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -61,44 +60,6 @@ namespace llvm { -/// Holds a subclass of DINode. -/// -/// FIXME: This class doesn't currently make much sense. Previously it was a -/// union beteen MDString (for ODR-uniqued types) and things like DIType. To -/// support CodeView work, it wasn't deleted outright when MDString-based type -/// references were deleted; we'll soon need a similar concept for CodeView -/// DITypeIndex. -template class TypedDINodeRef { - const Metadata *MD = nullptr; - -public: - TypedDINodeRef() = default; - TypedDINodeRef(std::nullptr_t) {} - TypedDINodeRef(const T *MD) : MD(MD) {} - - explicit TypedDINodeRef(const Metadata *MD) : MD(MD) { - assert((!MD || isa(MD)) && "Expected valid type ref"); - } - - template - TypedDINodeRef( - const TypedDINodeRef &X, - typename std::enable_if::value>::type * = - nullptr) - : MD(X) {} - - operator Metadata *() const { return const_cast(MD); } - - T *resolve() const { return const_cast(cast_or_null(MD)); } - - bool operator==(const TypedDINodeRef &X) const { return MD == X.MD; } - bool operator!=(const TypedDINodeRef &X) const { return MD != X.MD; } -}; - -using DINodeRef = TypedDINodeRef; -using DIScopeRef = TypedDINodeRef; -using DITypeRef = TypedDINodeRef; - class DITypeRefArray { const MDTuple *N = nullptr; @@ -115,17 +76,19 @@ public: // FIXME: Fix callers and remove condition on N. unsigned size() const { return N ? N->getNumOperands() : 0u; } - DITypeRef operator[](unsigned I) const { return DITypeRef(N->getOperand(I)); } + DIType *operator[](unsigned I) const { + return cast_or_null(N->getOperand(I)); + } - class iterator : std::iterator { + class iterator : std::iterator { MDNode::op_iterator I = nullptr; public: iterator() = default; explicit iterator(MDNode::op_iterator I) : I(I) {} - DITypeRef operator*() const { return DITypeRef(*I); } + DIType *operator*() const { return cast_or_null(*I); } iterator &operator++() { ++I; @@ -228,6 +191,7 @@ public: case DILexicalBlockKind: case DILexicalBlockFileKind: case DINamespaceKind: + case DICommonBlockKind: case DITemplateTypeParameterKind: case DITemplateValueParameterKind: case DIGlobalVariableKind: @@ -241,18 +205,6 @@ public: } }; -template struct simplify_type> { - using SimpleType = Metadata *; - - static SimpleType getSimplifiedValue(const TypedDINodeRef &MD) { - return MD; - } -}; - -template -struct simplify_type> - : simplify_type> {}; - /// Generic tagged DWARF-like metadata node. /// /// An un-specialized DWARF-like metadata node. The first operand is a @@ -459,7 +411,7 @@ public: inline Optional getSource() const; StringRef getName() const; - DIScopeRef getScope() const; + DIScope *getScope() const; /// Return the raw underlying file. /// @@ -486,6 +438,7 @@ public: case DILexicalBlockKind: case DILexicalBlockFileKind: case DINamespaceKind: + case DICommonBlockKind: case DIModuleKind: return true; } @@ -672,7 +625,7 @@ public: uint64_t getOffsetInBits() const { return OffsetInBits; } DIFlags getFlags() const { return Flags; } - DIScopeRef getScope() const { return DIScopeRef(getRawScope()); } + DIScope *getScope() const { return cast_or_null(getRawScope()); } StringRef getName() const { return getStringOperand(2); } @@ -817,14 +770,12 @@ class DIDerivedType : public DIType { DWARFAddressSpace(DWARFAddressSpace) {} ~DIDerivedType() = default; - static DIDerivedType *getImpl(LLVMContext &Context, unsigned Tag, - StringRef Name, DIFile *File, unsigned Line, - DIScopeRef Scope, DITypeRef BaseType, - uint64_t SizeInBits, uint32_t AlignInBits, - uint64_t OffsetInBits, - Optional DWARFAddressSpace, - DIFlags Flags, Metadata *ExtraData, - StorageType Storage, bool ShouldCreate = true) { + static DIDerivedType * + getImpl(LLVMContext &Context, unsigned Tag, StringRef Name, DIFile *File, + unsigned Line, DIScope *Scope, DIType *BaseType, uint64_t SizeInBits, + uint32_t AlignInBits, uint64_t OffsetInBits, + Optional DWARFAddressSpace, DIFlags Flags, + Metadata *ExtraData, StorageType Storage, bool ShouldCreate = true) { return getImpl(Context, Tag, getCanonicalMDString(Context, Name), File, Line, Scope, BaseType, SizeInBits, AlignInBits, OffsetInBits, DWARFAddressSpace, Flags, ExtraData, Storage, ShouldCreate); @@ -858,7 +809,7 @@ public: ExtraData)) DEFINE_MDNODE_GET(DIDerivedType, (unsigned Tag, StringRef Name, DIFile *File, unsigned Line, - DIScopeRef Scope, DITypeRef BaseType, uint64_t SizeInBits, + DIScope *Scope, DIType *BaseType, uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits, Optional DWARFAddressSpace, DIFlags Flags, Metadata *ExtraData = nullptr), @@ -869,7 +820,7 @@ public: TempDIDerivedType clone() const { return cloneImpl(); } /// Get the base type this is derived from. - DITypeRef getBaseType() const { return DITypeRef(getRawBaseType()); } + DIType *getBaseType() const { return cast_or_null(getRawBaseType()); } Metadata *getRawBaseType() const { return getOperand(3); } /// \returns The DWARF address space of the memory pointed to or referenced by @@ -889,9 +840,9 @@ public: /// Get casted version of extra data. /// @{ - DITypeRef getClassType() const { + DIType *getClassType() const { assert(getTag() == dwarf::DW_TAG_ptr_to_member_type); - return DITypeRef(getExtraData()); + return cast_or_null(getExtraData()); } DIObjCProperty *getObjCProperty() const { @@ -963,12 +914,12 @@ class DICompositeType : public DIType { static DICompositeType * getImpl(LLVMContext &Context, unsigned Tag, StringRef Name, Metadata *File, - unsigned Line, DIScopeRef Scope, DITypeRef BaseType, - uint64_t SizeInBits, uint32_t AlignInBits, uint64_t OffsetInBits, - DIFlags Flags, DINodeArray Elements, unsigned RuntimeLang, - DITypeRef VTableHolder, DITemplateParameterArray TemplateParams, - StringRef Identifier, DIDerivedType *Discriminator, - StorageType Storage, bool ShouldCreate = true) { + unsigned Line, DIScope *Scope, DIType *BaseType, uint64_t SizeInBits, + uint32_t AlignInBits, uint64_t OffsetInBits, DIFlags Flags, + DINodeArray Elements, unsigned RuntimeLang, DIType *VTableHolder, + DITemplateParameterArray TemplateParams, StringRef Identifier, + DIDerivedType *Discriminator, StorageType Storage, + bool ShouldCreate = true) { return getImpl( Context, Tag, getCanonicalMDString(Context, Name), File, Line, Scope, BaseType, SizeInBits, AlignInBits, OffsetInBits, Flags, Elements.get(), @@ -995,12 +946,13 @@ class DICompositeType : public DIType { public: DEFINE_MDNODE_GET(DICompositeType, (unsigned Tag, StringRef Name, DIFile *File, unsigned Line, - DIScopeRef Scope, DITypeRef BaseType, uint64_t SizeInBits, - uint32_t AlignInBits, uint64_t OffsetInBits, - DIFlags Flags, DINodeArray Elements, unsigned RuntimeLang, - DITypeRef VTableHolder, + DIScope *Scope, DIType *BaseType, uint64_t SizeInBits, + uint32_t AlignInBits, uint64_t OffsetInBits, DIFlags Flags, + DINodeArray Elements, unsigned RuntimeLang, + DIType *VTableHolder, DITemplateParameterArray TemplateParams = nullptr, - StringRef Identifier = "", DIDerivedType *Discriminator = nullptr), + StringRef Identifier = "", + DIDerivedType *Discriminator = nullptr), (Tag, Name, File, Line, Scope, BaseType, SizeInBits, AlignInBits, OffsetInBits, Flags, Elements, RuntimeLang, VTableHolder, TemplateParams, Identifier, Discriminator)) @@ -1053,11 +1005,13 @@ public: unsigned RuntimeLang, Metadata *VTableHolder, Metadata *TemplateParams, Metadata *Discriminator); - DITypeRef getBaseType() const { return DITypeRef(getRawBaseType()); } + DIType *getBaseType() const { return cast_or_null(getRawBaseType()); } DINodeArray getElements() const { return cast_or_null(getRawElements()); } - DITypeRef getVTableHolder() const { return DITypeRef(getRawVTableHolder()); } + DIType *getVTableHolder() const { + return cast_or_null(getRawVTableHolder()); + } DITemplateParameterArray getTemplateParams() const { return cast_or_null(getRawTemplateParams()); } @@ -1087,7 +1041,7 @@ public: replaceOperandWith(4, Elements.get()); } - void replaceVTableHolder(DITypeRef VTableHolder) { + void replaceVTableHolder(DIType *VTableHolder) { replaceOperandWith(5, VTableHolder); } @@ -1541,9 +1495,6 @@ public: /// /// For precise control over the data being encoded in the discriminator, /// use encodeDiscriminator/decodeDiscriminator. - /// - /// Use {get|set}BaseDiscriminator and cloneWithDuplicationFactor after reading - /// their documentation, as their behavior has side-effects. inline unsigned getDiscriminator() const; @@ -1554,7 +1505,7 @@ public: /// base discriminator is set in the new DILocation, the other encoded values /// are elided. /// If the discriminator cannot be encoded, the function returns None. - inline Optional setBaseDiscriminator(unsigned BD) const; + inline Optional cloneWithBaseDiscriminator(unsigned BD) const; /// Returns the duplication factor stored in the discriminator, or 1 if no /// duplication factor (or 0) is encoded. @@ -1570,7 +1521,7 @@ public: /// duplication factor encoded in the discriminator. The current duplication /// factor is as defined by getDuplicationFactor(). /// Returns None if encoding failed. - inline Optional cloneWithDuplicationFactor(unsigned DF) const; + inline Optional cloneByMultiplyingDuplicationFactor(unsigned DF) const; /// When two instructions are combined into a single instruction we also /// need to combine the original locations into a single location. @@ -1594,10 +1545,11 @@ public: return getUnsignedFromPrefixEncoding(D); } - /// Raw encoding of the discriminator. APIs such as setBaseDiscriminator or - /// cloneWithDuplicationFactor have certain side-effects. This API, in - /// conjunction with cloneWithDiscriminator, may be used to encode precisely - /// the values provided. \p BD: base discriminator \p DF: duplication factor + /// Raw encoding of the discriminator. APIs such as cloneWithDuplicationFactor + /// have certain special case behavior (e.g. treating empty duplication factor + /// as the value '1'). + /// This API, in conjunction with cloneWithDiscriminator, may be used to encode + /// the raw values provided. \p BD: base discriminator \p DF: duplication factor /// \p CI: copy index /// The return is None if the values cannot be encoded in 32 bits - for /// example, values for BD or DF larger than 12 bits. Otherwise, the return @@ -1638,9 +1590,6 @@ public: }; /// Subprogram description. -/// -/// TODO: Remove DisplayName. It's always equal to Name. -/// TODO: Split up flags. class DISubprogram : public DILocalScope { friend class LLVMContextImpl; friend class MDNode; @@ -1678,7 +1627,8 @@ public: // Helper for converting old bitfields to new flags word. static DISPFlags toSPFlags(bool IsLocalToUnit, bool IsDefinition, bool IsOptimized, - unsigned Virtuality = SPFlagNonvirtual) { + unsigned Virtuality = SPFlagNonvirtual, + bool IsMainSubprogram = false) { // We're assuming virtuality is the low-order field. static_assert( int(SPFlagVirtual) == int(dwarf::DW_VIRTUALITY_virtual) && @@ -1688,7 +1638,8 @@ public: (Virtuality & SPFlagVirtuality) | (IsLocalToUnit ? SPFlagLocalToUnit : SPFlagZero) | (IsDefinition ? SPFlagDefinition : SPFlagZero) | - (IsOptimized ? SPFlagOptimized : SPFlagZero)); + (IsOptimized ? SPFlagOptimized : SPFlagZero) | + (IsMainSubprogram ? SPFlagMainSubprogram : SPFlagZero)); } private: @@ -1707,9 +1658,9 @@ private: ~DISubprogram() = default; static DISubprogram * - getImpl(LLVMContext &Context, DIScopeRef Scope, StringRef Name, + getImpl(LLVMContext &Context, DIScope *Scope, StringRef Name, StringRef LinkageName, DIFile *File, unsigned Line, - DISubroutineType *Type, unsigned ScopeLine, DITypeRef ContainingType, + DISubroutineType *Type, unsigned ScopeLine, DIType *ContainingType, unsigned VirtualIndex, int ThisAdjustment, DIFlags Flags, DISPFlags SPFlags, DICompileUnit *Unit, DITemplateParameterArray TemplateParams, DISubprogram *Declaration, @@ -1744,9 +1695,9 @@ private: public: DEFINE_MDNODE_GET( DISubprogram, - (DIScopeRef Scope, StringRef Name, StringRef LinkageName, DIFile *File, + (DIScope * Scope, StringRef Name, StringRef LinkageName, DIFile *File, unsigned Line, DISubroutineType *Type, unsigned ScopeLine, - DITypeRef ContainingType, unsigned VirtualIndex, int ThisAdjustment, + DIType *ContainingType, unsigned VirtualIndex, int ThisAdjustment, DIFlags Flags, DISPFlags SPFlags, DICompileUnit *Unit, DITemplateParameterArray TemplateParams = nullptr, DISubprogram *Declaration = nullptr, DINodeArray RetainedNodes = nullptr, @@ -1787,6 +1738,7 @@ public: bool isLocalToUnit() const { return getSPFlags() & SPFlagLocalToUnit; } bool isDefinition() const { return getSPFlags() & SPFlagDefinition; } bool isOptimized() const { return getSPFlags() & SPFlagOptimized; } + bool isMainSubprogram() const { return getSPFlags() & SPFlagMainSubprogram; } bool isArtificial() const { return getFlags() & FlagArtificial; } bool isPrivate() const { @@ -1803,7 +1755,9 @@ public: bool areAllCallsDescribed() const { return getFlags() & FlagAllCallsDescribed; } - bool isMainSubprogram() const { return getFlags() & FlagMainSubprogram; } + bool isPure() const { return getSPFlags() & SPFlagPure; } + bool isElemental() const { return getSPFlags() & SPFlagElemental; } + bool isRecursive() const { return getSPFlags() & SPFlagRecursive; } /// Check if this is reference-qualified. /// @@ -1827,7 +1781,7 @@ public: // Returns true if this subprogram is a thunk generated by the compiler. bool isThunk() const { return getFlags() & FlagThunk; } - DIScopeRef getScope() const { return DIScopeRef(getRawScope()); } + DIScope *getScope() const { return cast_or_null(getRawScope()); } StringRef getName() const { return getStringOperand(2); } StringRef getLinkageName() const { return getStringOperand(3); } @@ -1835,8 +1789,8 @@ public: DISubroutineType *getType() const { return cast_or_null(getRawType()); } - DITypeRef getContainingType() const { - return DITypeRef(getRawContainingType()); + DIType *getContainingType() const { + return cast_or_null(getRawContainingType()); } DICompileUnit *getUnit() const { @@ -2039,15 +1993,17 @@ unsigned DILocation::getCopyIdentifier() const { return getCopyIdentifierFromDiscriminator(getDiscriminator()); } -Optional DILocation::setBaseDiscriminator(unsigned D) const { - if (D == 0) +Optional DILocation::cloneWithBaseDiscriminator(unsigned D) const { + unsigned BD, DF, CI; + decodeDiscriminator(getDiscriminator(), BD, DF, CI); + if (D == BD) return this; - if (D > 0xfff) - return None; - return cloneWithDiscriminator(encodeComponent(D)); + if (Optional Encoded = encodeDiscriminator(D, DF, CI)) + return cloneWithDiscriminator(*Encoded); + return None; } -Optional DILocation::cloneWithDuplicationFactor(unsigned DF) const { +Optional DILocation::cloneByMultiplyingDuplicationFactor(unsigned DF) const { DF *= getDuplicationFactor(); if (DF <= 1) return this; @@ -2179,7 +2135,7 @@ protected: public: StringRef getName() const { return getStringOperand(0); } - DITypeRef getType() const { return DITypeRef(getRawType()); } + DIType *getType() const { return cast_or_null(getRawType()); } MDString *getRawName() const { return getOperandAs(0); } Metadata *getRawType() const { return getOperand(1); } @@ -2201,7 +2157,7 @@ class DITemplateTypeParameter : public DITemplateParameter { ~DITemplateTypeParameter() = default; static DITemplateTypeParameter *getImpl(LLVMContext &Context, StringRef Name, - DITypeRef Type, StorageType Storage, + DIType *Type, StorageType Storage, bool ShouldCreate = true) { return getImpl(Context, getCanonicalMDString(Context, Name), Type, Storage, ShouldCreate); @@ -2215,7 +2171,7 @@ class DITemplateTypeParameter : public DITemplateParameter { } public: - DEFINE_MDNODE_GET(DITemplateTypeParameter, (StringRef Name, DITypeRef Type), + DEFINE_MDNODE_GET(DITemplateTypeParameter, (StringRef Name, DIType *Type), (Name, Type)) DEFINE_MDNODE_GET(DITemplateTypeParameter, (MDString * Name, Metadata *Type), (Name, Type)) @@ -2238,7 +2194,7 @@ class DITemplateValueParameter : public DITemplateParameter { ~DITemplateValueParameter() = default; static DITemplateValueParameter *getImpl(LLVMContext &Context, unsigned Tag, - StringRef Name, DITypeRef Type, + StringRef Name, DIType *Type, Metadata *Value, StorageType Storage, bool ShouldCreate = true) { return getImpl(Context, Tag, getCanonicalMDString(Context, Name), Type, @@ -2255,8 +2211,9 @@ class DITemplateValueParameter : public DITemplateParameter { } public: - DEFINE_MDNODE_GET(DITemplateValueParameter, (unsigned Tag, StringRef Name, - DITypeRef Type, Metadata *Value), + DEFINE_MDNODE_GET(DITemplateValueParameter, + (unsigned Tag, StringRef Name, DIType *Type, + Metadata *Value), (Tag, Name, Type, Value)) DEFINE_MDNODE_GET(DITemplateValueParameter, (unsigned Tag, MDString *Name, Metadata *Type, Metadata *Value), @@ -2288,7 +2245,7 @@ public: DIScope *getScope() const { return cast_or_null(getRawScope()); } StringRef getName() const { return getStringOperand(1); } DIFile *getFile() const { return cast_or_null(getRawFile()); } - DITypeRef getType() const { return DITypeRef(getRawType()); } + DIType *getType() const { return cast_or_null(getRawType()); } uint32_t getAlignInBits() const { return AlignInBits; } uint32_t getAlignInBytes() const { return getAlignInBits() / CHAR_BIT; } /// Determines the size of the variable's type. @@ -2297,7 +2254,7 @@ public: /// Return the signedness of this variable's type, or None if this type is /// neither signed nor unsigned. Optional getSignedness() const { - if (auto *BT = dyn_cast(getType().resolve())) + if (auto *BT = dyn_cast(getType())) return BT->getSignedness(); return None; } @@ -2504,6 +2461,13 @@ public: /// Return whether this is a piece of an aggregate variable. bool isFragment() const { return getFragmentInfo().hasValue(); } + /// Return whether this is an implicit location description. + bool isImplicit() const; + + /// Return whether the location is computed on the expression stack, meaning + /// it cannot be a simple register location. + bool isComplex() const; + /// Append \p Ops with operations to apply the \p Offset. static void appendOffset(SmallVectorImpl &Ops, int64_t Offset); @@ -2511,20 +2475,32 @@ public: /// return true with an offset of zero. bool extractIfOffset(int64_t &Offset) const; - /// Constants for DIExpression::prepend. - enum { NoDeref = false, WithDeref = true, WithStackValue = true }; + /// Checks if the last 4 elements of the expression are DW_OP_constu DW_OP_swap DW_OP_xderef and extracts the . + static const DIExpression *extractAddressClass(const DIExpression *Expr, + unsigned &AddrClass); + + /// Used for DIExpression::prepend. + enum PrependOps : uint8_t { + ApplyOffset = 0, + DerefBefore = 1 << 0, + DerefAfter = 1 << 1, + StackValue = 1 << 2, + EntryValue = 1 << 3 + }; /// Prepend \p DIExpr with a deref and offset operation and optionally turn it - /// into a stack value. - static DIExpression *prepend(const DIExpression *Expr, bool DerefBefore, - int64_t Offset = 0, bool DerefAfter = false, - bool StackValue = false); + /// into a stack value or/and an entry value. + static DIExpression *prepend(const DIExpression *Expr, uint8_t Flags, + int64_t Offset = 0); /// Prepend \p DIExpr with the given opcodes and optionally turn it into a /// stack value. static DIExpression *prependOpcodes(const DIExpression *Expr, SmallVectorImpl &Ops, - bool StackValue = false); + bool StackValue = false, + bool EntryValue = false); /// Append the opcodes \p Ops to \p DIExpr. Unlike \ref appendToStack, the /// returned expression is a stack value only if \p DIExpr is a stack value. @@ -2553,17 +2529,14 @@ public: createFragmentExpression(const DIExpression *Expr, unsigned OffsetInBits, unsigned SizeInBits); - /// Determine the relative position of the fragments described by this - /// DIExpression and \p Other. + /// Determine the relative position of the fragments passed in. /// Returns -1 if this is entirely before Other, 0 if this and Other overlap, /// 1 if this is entirely after Other. - int fragmentCmp(const DIExpression *Other) const { - auto Fragment1 = *getFragmentInfo(); - auto Fragment2 = *Other->getFragmentInfo(); - unsigned l1 = Fragment1.OffsetInBits; - unsigned l2 = Fragment2.OffsetInBits; - unsigned r1 = l1 + Fragment1.SizeInBits; - unsigned r2 = l2 + Fragment2.SizeInBits; + static int fragmentCmp(const FragmentInfo &A, const FragmentInfo &B) { + uint64_t l1 = A.OffsetInBits; + uint64_t l2 = B.OffsetInBits; + uint64_t r1 = l1 + A.SizeInBits; + uint64_t r2 = l2 + B.SizeInBits; if (r1 <= l2) return -1; else if (r2 <= l1) @@ -2572,12 +2545,59 @@ public: return 0; } + /// Check if fragments overlap between a pair of FragmentInfos. + static bool fragmentsOverlap(const FragmentInfo &A, const FragmentInfo &B) { + return fragmentCmp(A, B) == 0; + } + + /// Determine the relative position of the fragments described by this + /// DIExpression and \p Other. Calls static fragmentCmp implementation. + int fragmentCmp(const DIExpression *Other) const { + auto Fragment1 = *getFragmentInfo(); + auto Fragment2 = *Other->getFragmentInfo(); + return fragmentCmp(Fragment1, Fragment2); + } + /// Check if fragments overlap between this DIExpression and \p Other. bool fragmentsOverlap(const DIExpression *Other) const { if (!isFragment() || !Other->isFragment()) return true; return fragmentCmp(Other) == 0; } + + /// Check if the expression consists of exactly one entry value operand. + /// (This is the only configuration of entry values that is supported.) + bool isEntryValue() const { + return getNumElements() > 0 && + getElement(0) == dwarf::DW_OP_entry_value; + } +}; + +inline bool operator==(const DIExpression::FragmentInfo &A, + const DIExpression::FragmentInfo &B) { + return std::tie(A.SizeInBits, A.OffsetInBits) == + std::tie(B.SizeInBits, B.OffsetInBits); +} + +inline bool operator<(const DIExpression::FragmentInfo &A, + const DIExpression::FragmentInfo &B) { + return std::tie(A.SizeInBits, A.OffsetInBits) < + std::tie(B.SizeInBits, B.OffsetInBits); +} + +template <> struct DenseMapInfo { + using FragInfo = DIExpression::FragmentInfo; + static const uint64_t MaxVal = std::numeric_limits::max(); + + static inline FragInfo getEmptyKey() { return {MaxVal, MaxVal}; } + + static inline FragInfo getTombstoneKey() { return {MaxVal - 1, MaxVal - 1}; } + + static unsigned getHashValue(const FragInfo &Frag) { + return (Frag.SizeInBits & 0xffff) << 16 | (Frag.OffsetInBits & 0xffff); + } + + static bool isEqual(const FragInfo &A, const FragInfo &B) { return A == B; } }; /// Global variables. @@ -2599,7 +2619,7 @@ class DIGlobalVariable : public DIVariable { static DIGlobalVariable * getImpl(LLVMContext &Context, DIScope *Scope, StringRef Name, - StringRef LinkageName, DIFile *File, unsigned Line, DITypeRef Type, + StringRef LinkageName, DIFile *File, unsigned Line, DIType *Type, bool IsLocalToUnit, bool IsDefinition, DIDerivedType *StaticDataMemberDeclaration, MDTuple *TemplateParams, uint32_t AlignInBits, StorageType Storage, bool ShouldCreate = true) { @@ -2626,7 +2646,7 @@ class DIGlobalVariable : public DIVariable { public: DEFINE_MDNODE_GET(DIGlobalVariable, (DIScope * Scope, StringRef Name, StringRef LinkageName, - DIFile *File, unsigned Line, DITypeRef Type, + DIFile *File, unsigned Line, DIType *Type, bool IsLocalToUnit, bool IsDefinition, DIDerivedType *StaticDataMemberDeclaration, MDTuple *TemplateParams, uint32_t AlignInBits), @@ -2663,6 +2683,65 @@ public: } }; +class DICommonBlock : public DIScope { + unsigned LineNo; + + friend class LLVMContextImpl; + friend class MDNode; + + DICommonBlock(LLVMContext &Context, StorageType Storage, unsigned LineNo, + ArrayRef Ops) + : DIScope(Context, DICommonBlockKind, Storage, dwarf::DW_TAG_common_block, + Ops), LineNo(LineNo) {} + + static DICommonBlock *getImpl(LLVMContext &Context, DIScope *Scope, + DIGlobalVariable *Decl, StringRef Name, + DIFile *File, unsigned LineNo, + StorageType Storage, + bool ShouldCreate = true) { + return getImpl(Context, Scope, Decl, getCanonicalMDString(Context, Name), + File, LineNo, Storage, ShouldCreate); + } + static DICommonBlock *getImpl(LLVMContext &Context, Metadata *Scope, + Metadata *Decl, MDString *Name, Metadata *File, + unsigned LineNo, + StorageType Storage, bool ShouldCreate = true); + + TempDICommonBlock cloneImpl() const { + return getTemporary(getContext(), getScope(), getDecl(), getName(), + getFile(), getLineNo()); + } + +public: + DEFINE_MDNODE_GET(DICommonBlock, + (DIScope *Scope, DIGlobalVariable *Decl, StringRef Name, + DIFile *File, unsigned LineNo), + (Scope, Decl, Name, File, LineNo)) + DEFINE_MDNODE_GET(DICommonBlock, + (Metadata *Scope, Metadata *Decl, MDString *Name, + Metadata *File, unsigned LineNo), + (Scope, Decl, Name, File, LineNo)) + + TempDICommonBlock clone() const { return cloneImpl(); } + + DIScope *getScope() const { return cast_or_null(getRawScope()); } + DIGlobalVariable *getDecl() const { + return cast_or_null(getRawDecl()); + } + StringRef getName() const { return getStringOperand(2); } + DIFile *getFile() const { return cast_or_null(getRawFile()); } + unsigned getLineNo() const { return LineNo; } + + Metadata *getRawScope() const { return getOperand(0); } + Metadata *getRawDecl() const { return getOperand(1); } + MDString *getRawName() const { return getOperandAs(2); } + Metadata *getRawFile() const { return getOperand(3); } + + static bool classof(const Metadata *MD) { + return MD->getMetadataID() == DICommonBlockKind; + } +}; + /// Local variable. /// /// TODO: Split up flags. @@ -2684,7 +2763,7 @@ class DILocalVariable : public DIVariable { static DILocalVariable *getImpl(LLVMContext &Context, DIScope *Scope, StringRef Name, DIFile *File, unsigned Line, - DITypeRef Type, unsigned Arg, DIFlags Flags, + DIType *Type, unsigned Arg, DIFlags Flags, uint32_t AlignInBits, StorageType Storage, bool ShouldCreate = true) { return getImpl(Context, Scope, getCanonicalMDString(Context, Name), File, @@ -2705,8 +2784,8 @@ class DILocalVariable : public DIVariable { public: DEFINE_MDNODE_GET(DILocalVariable, (DILocalScope * Scope, StringRef Name, DIFile *File, - unsigned Line, DITypeRef Type, unsigned Arg, - DIFlags Flags, uint32_t AlignInBits), + unsigned Line, DIType *Type, unsigned Arg, DIFlags Flags, + uint32_t AlignInBits), (Scope, Name, File, Line, Type, Arg, Flags, AlignInBits)) DEFINE_MDNODE_GET(DILocalVariable, (Metadata * Scope, MDString *Name, Metadata *File, @@ -2730,6 +2809,11 @@ public: bool isArtificial() const { return getFlags() & FlagArtificial; } bool isObjectPointer() const { return getFlags() & FlagObjectPointer; } + /// Check that an argument is unmodified. + bool isNotModified() const { return getFlags() & FlagArgumentNotModified; } + /// Set the flag if an argument is unmodified. + void setIsNotModified() { Flags |= FlagArgumentNotModified; } + /// Check that a location is valid for this variable. /// /// Check that \c DL exists, is in the same subprogram, and has the same @@ -2831,7 +2915,7 @@ class DIObjCProperty : public DINode { static DIObjCProperty * getImpl(LLVMContext &Context, StringRef Name, DIFile *File, unsigned Line, StringRef GetterName, StringRef SetterName, unsigned Attributes, - DITypeRef Type, StorageType Storage, bool ShouldCreate = true) { + DIType *Type, StorageType Storage, bool ShouldCreate = true) { return getImpl(Context, getCanonicalMDString(Context, Name), File, Line, getCanonicalMDString(Context, GetterName), getCanonicalMDString(Context, SetterName), Attributes, Type, @@ -2853,7 +2937,7 @@ public: DEFINE_MDNODE_GET(DIObjCProperty, (StringRef Name, DIFile *File, unsigned Line, StringRef GetterName, StringRef SetterName, - unsigned Attributes, DITypeRef Type), + unsigned Attributes, DIType *Type), (Name, File, Line, GetterName, SetterName, Attributes, Type)) DEFINE_MDNODE_GET(DIObjCProperty, @@ -2871,7 +2955,7 @@ public: DIFile *getFile() const { return cast_or_null(getRawFile()); } StringRef getGetterName() const { return getStringOperand(2); } StringRef getSetterName() const { return getStringOperand(3); } - DITypeRef getType() const { return DITypeRef(getRawType()); } + DIType *getType() const { return cast_or_null(getRawType()); } StringRef getFilename() const { if (auto *F = getFile()) @@ -2915,8 +2999,8 @@ class DIImportedEntity : public DINode { ~DIImportedEntity() = default; static DIImportedEntity *getImpl(LLVMContext &Context, unsigned Tag, - DIScope *Scope, DINodeRef Entity, - DIFile *File, unsigned Line, StringRef Name, + DIScope *Scope, DINode *Entity, DIFile *File, + unsigned Line, StringRef Name, StorageType Storage, bool ShouldCreate = true) { return getImpl(Context, Tag, Scope, Entity, File, Line, @@ -2935,8 +3019,8 @@ class DIImportedEntity : public DINode { public: DEFINE_MDNODE_GET(DIImportedEntity, - (unsigned Tag, DIScope *Scope, DINodeRef Entity, - DIFile *File, unsigned Line, StringRef Name = ""), + (unsigned Tag, DIScope *Scope, DINode *Entity, DIFile *File, + unsigned Line, StringRef Name = ""), (Tag, Scope, Entity, File, Line, Name)) DEFINE_MDNODE_GET(DIImportedEntity, (unsigned Tag, Metadata *Scope, Metadata *Entity, @@ -2947,7 +3031,7 @@ public: unsigned getLine() const { return Line; } DIScope *getScope() const { return cast_or_null(getRawScope()); } - DINodeRef getEntity() const { return DINodeRef(getRawEntity()); } + DINode *getEntity() const { return cast_or_null(getRawEntity()); } StringRef getName() const { return getStringOperand(2); } DIFile *getFile() const { return cast_or_null(getRawFile()); } diff --git a/include/llvm/IR/DebugLoc.h b/include/llvm/IR/DebugLoc.h index 4f0d7f51b5f9..780d17a33661 100644 --- a/include/llvm/IR/DebugLoc.h +++ b/include/llvm/IR/DebugLoc.h @@ -1,9 +1,8 @@ //===- DebugLoc.h - Debug Location Information ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/DerivedTypes.h b/include/llvm/IR/DerivedTypes.h index 9526d6287d2f..3c1d4278905f 100644 --- a/include/llvm/IR/DerivedTypes.h +++ b/include/llvm/IR/DerivedTypes.h @@ -1,9 +1,8 @@ //===- llvm/DerivedTypes.h - Classes for handling data types ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -24,6 +23,7 @@ #include "llvm/IR/Type.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Compiler.h" +#include "llvm/Support/ScalableSize.h" #include #include @@ -158,6 +158,38 @@ unsigned Type::getFunctionNumParams() const { return cast(this)->getNumParams(); } +/// A handy container for a FunctionType+Callee-pointer pair, which can be +/// passed around as a single entity. This assists in replacing the use of +/// PointerType::getElementType() to access the function's type, since that's +/// slated for removal as part of the [opaque pointer types] project. +class FunctionCallee { +public: + // Allow implicit conversion from types which have a getFunctionType member + // (e.g. Function and InlineAsm). + template + FunctionCallee(T *Fn) + : FnTy(Fn ? Fn->getFunctionType() : nullptr), Callee(Fn) {} + + FunctionCallee(FunctionType *FnTy, Value *Callee) + : FnTy(FnTy), Callee(Callee) { + assert((FnTy == nullptr) == (Callee == nullptr)); + } + + FunctionCallee(std::nullptr_t) {} + + FunctionCallee() = default; + + FunctionType *getFunctionType() { return FnTy; } + + Value *getCallee() { return Callee; } + + explicit operator bool() { return Callee; } + +private: + FunctionType *FnTy = nullptr; + Value *Callee = nullptr; +}; + /// Common super class of ArrayType, StructType and VectorType. class CompositeType : public Type { protected: @@ -356,6 +388,8 @@ public: SequentialType(const SequentialType &) = delete; SequentialType &operator=(const SequentialType &) = delete; + /// For scalable vectors, this will return the minimum number of elements + /// in the vector. uint64_t getNumElements() const { return NumElements; } Type *getElementType() const { return ContainedType; } @@ -391,14 +425,37 @@ uint64_t Type::getArrayNumElements() const { /// Class to represent vector types. class VectorType : public SequentialType { - VectorType(Type *ElType, unsigned NumEl); + /// A fully specified VectorType is of the form . 'n' is the + /// minimum number of elements of type Ty contained within the vector, and + /// 'vscale x' indicates that the total element count is an integer multiple + /// of 'n', where the multiple is either guaranteed to be one, or is + /// statically unknown at compile time. + /// + /// If the multiple is known to be 1, then the extra term is discarded in + /// textual IR: + /// + /// <4 x i32> - a vector containing 4 i32s + /// - a vector containing an unknown integer multiple + /// of 4 i32s + + VectorType(Type *ElType, unsigned NumEl, bool Scalable = false); + VectorType(Type *ElType, ElementCount EC); + + // If true, the total number of elements is an unknown multiple of the + // minimum 'NumElements' from SequentialType. Otherwise the total number + // of elements is exactly equal to 'NumElements'. + bool Scalable; public: VectorType(const VectorType &) = delete; VectorType &operator=(const VectorType &) = delete; /// This static method is the primary way to construct an VectorType. - static VectorType *get(Type *ElementType, unsigned NumElements); + static VectorType *get(Type *ElementType, ElementCount EC); + static VectorType *get(Type *ElementType, unsigned NumElements, + bool Scalable = false) { + return VectorType::get(ElementType, {NumElements, Scalable}); + } /// This static method gets a VectorType with the same number of elements as /// the input type, and the element type is an integer type of the same width @@ -407,7 +464,7 @@ public: unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); assert(EltBits && "Element size must be of a non-zero size"); Type *EltTy = IntegerType::get(VTy->getContext(), EltBits); - return VectorType::get(EltTy, VTy->getNumElements()); + return VectorType::get(EltTy, VTy->getElementCount()); } /// This static method is like getInteger except that the element types are @@ -415,7 +472,7 @@ public: static VectorType *getExtendedElementVectorType(VectorType *VTy) { unsigned EltBits = VTy->getElementType()->getPrimitiveSizeInBits(); Type *EltTy = IntegerType::get(VTy->getContext(), EltBits * 2); - return VectorType::get(EltTy, VTy->getNumElements()); + return VectorType::get(EltTy, VTy->getElementCount()); } /// This static method is like getInteger except that the element types are @@ -425,29 +482,45 @@ public: assert((EltBits & 1) == 0 && "Cannot truncate vector element with odd bit-width"); Type *EltTy = IntegerType::get(VTy->getContext(), EltBits / 2); - return VectorType::get(EltTy, VTy->getNumElements()); + return VectorType::get(EltTy, VTy->getElementCount()); } /// This static method returns a VectorType with half as many elements as the /// input type and the same element type. static VectorType *getHalfElementsVectorType(VectorType *VTy) { - unsigned NumElts = VTy->getNumElements(); - assert ((NumElts & 1) == 0 && + auto EltCnt = VTy->getElementCount(); + assert ((EltCnt.Min & 1) == 0 && "Cannot halve vector with odd number of elements."); - return VectorType::get(VTy->getElementType(), NumElts/2); + return VectorType::get(VTy->getElementType(), EltCnt/2); } /// This static method returns a VectorType with twice as many elements as the /// input type and the same element type. static VectorType *getDoubleElementsVectorType(VectorType *VTy) { - unsigned NumElts = VTy->getNumElements(); - return VectorType::get(VTy->getElementType(), NumElts*2); + auto EltCnt = VTy->getElementCount(); + assert((VTy->getNumElements() * 2ull) <= UINT_MAX && + "Too many elements in vector"); + return VectorType::get(VTy->getElementType(), EltCnt*2); } /// Return true if the specified type is valid as a element type. static bool isValidElementType(Type *ElemTy); - /// Return the number of bits in the Vector type. + /// Return an ElementCount instance to represent the (possibly scalable) + /// number of elements in the vector. + ElementCount getElementCount() const { + uint64_t MinimumEltCnt = getNumElements(); + assert(MinimumEltCnt <= UINT_MAX && "Too many elements in vector"); + return { (unsigned)MinimumEltCnt, Scalable }; + } + + /// Returns whether or not this is a scalable vector (meaning the total + /// element count is a multiple of the minimum). + bool isScalable() const { + return Scalable; + } + + /// Return the minimum number of bits in the Vector type. /// Returns zero when the vector is a vector of pointers. unsigned getBitWidth() const { return getNumElements() * getElementType()->getPrimitiveSizeInBits(); @@ -463,6 +536,10 @@ unsigned Type::getVectorNumElements() const { return cast(this)->getNumElements(); } +bool Type::getVectorIsScalable() const { + return cast(this)->isScalable(); +} + /// Class to represent pointers. class PointerType : public Type { explicit PointerType(Type *ElType, unsigned AddrSpace); diff --git a/include/llvm/IR/DerivedUser.h b/include/llvm/IR/DerivedUser.h index 67c483d3c497..a25d316c2d60 100644 --- a/include/llvm/IR/DerivedUser.h +++ b/include/llvm/IR/DerivedUser.h @@ -1,9 +1,8 @@ //===- DerivedUser.h - Base for non-IR Users --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/IR/DiagnosticHandler.h b/include/llvm/IR/DiagnosticHandler.h index 51873bea3d41..55e5e5975808 100644 --- a/include/llvm/IR/DiagnosticHandler.h +++ b/include/llvm/IR/DiagnosticHandler.h @@ -1,9 +1,8 @@ -//===- DiagnosticHandler.h - DiagnosticHandler class for LLVM -*- C++ ---*-===// +//===- DiagnosticHandler.h - DiagnosticHandler class for LLVM ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Base DiagnosticHandler class declaration. Derive from this class to provide diff --git a/include/llvm/IR/DiagnosticInfo.h b/include/llvm/IR/DiagnosticInfo.h index 3a55a7dca7f4..373663289dbd 100644 --- a/include/llvm/IR/DiagnosticInfo.h +++ b/include/llvm/IR/DiagnosticInfo.h @@ -1,9 +1,8 @@ //===- llvm/IR/DiagnosticInfo.h - Diagnostic Declaration --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -466,12 +465,15 @@ public: virtual bool isEnabled() const = 0; StringRef getPassName() const { return PassName; } + StringRef getRemarkName() const { return RemarkName; } std::string getMsg() const; Optional getHotness() const { return Hotness; } void setHotness(Optional H) { Hotness = H; } bool isVerbose() const { return IsVerbose; } + ArrayRef getArgs() const { return Args; } + static bool classof(const DiagnosticInfo *DI) { return (DI->getKind() >= DK_FirstRemark && DI->getKind() <= DK_LastRemark) || @@ -501,7 +503,7 @@ protected: const char *PassName; /// Textual identifier for the remark (single-word, camel-case). Can be used - /// by external tools reading the YAML output file for optimization remarks to + /// by external tools reading the output file for optimization remarks to /// identify the remark. StringRef RemarkName; @@ -519,8 +521,6 @@ protected: /// the optimization records and not in the remark printed in the compiler /// output. int FirstExtraArgIndex = -1; - - friend struct yaml::MappingTraits; }; /// Allow the insertion operator to return the actual remark type rather than a @@ -1002,12 +1002,6 @@ public: void print(DiagnosticPrinter &DP) const override; }; -namespace yaml { -template <> struct MappingTraits { - static void mapping(IO &io, DiagnosticInfoOptimizationBase *&OptDiag); -}; -} // namespace yaml - } // end namespace llvm #endif // LLVM_IR_DIAGNOSTICINFO_H diff --git a/include/llvm/IR/DiagnosticPrinter.h b/include/llvm/IR/DiagnosticPrinter.h index 25c47cdd1a12..102932ceefa5 100644 --- a/include/llvm/IR/DiagnosticPrinter.h +++ b/include/llvm/IR/DiagnosticPrinter.h @@ -1,9 +1,8 @@ //===- llvm/Support/DiagnosticPrinter.h - Diagnostic Printer ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/DomTreeUpdater.h b/include/llvm/IR/DomTreeUpdater.h deleted file mode 100644 index e5bb092d21ca..000000000000 --- a/include/llvm/IR/DomTreeUpdater.h +++ /dev/null @@ -1,257 +0,0 @@ -//===- DomTreeUpdater.h - DomTree/Post DomTree Updater ----------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file defines the DomTreeUpdater class, which provides a uniform way to -// update dominator tree related data structures. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_DOMTREEUPDATER_H -#define LLVM_DOMTREEUPDATER_H - -#include "llvm/Analysis/PostDominators.h" -#include "llvm/IR/Dominators.h" -#include "llvm/IR/Instructions.h" -#include "llvm/IR/ValueHandle.h" -#include "llvm/Support/GenericDomTree.h" -#include -#include - -namespace llvm { -class DomTreeUpdater { -public: - enum class UpdateStrategy : unsigned char { Eager = 0, Lazy = 1 }; - - explicit DomTreeUpdater(UpdateStrategy Strategy_) : Strategy(Strategy_) {} - DomTreeUpdater(DominatorTree &DT_, UpdateStrategy Strategy_) - : DT(&DT_), Strategy(Strategy_) {} - DomTreeUpdater(DominatorTree *DT_, UpdateStrategy Strategy_) - : DT(DT_), Strategy(Strategy_) {} - DomTreeUpdater(PostDominatorTree &PDT_, UpdateStrategy Strategy_) - : PDT(&PDT_), Strategy(Strategy_) {} - DomTreeUpdater(PostDominatorTree *PDT_, UpdateStrategy Strategy_) - : PDT(PDT_), Strategy(Strategy_) {} - DomTreeUpdater(DominatorTree &DT_, PostDominatorTree &PDT_, - UpdateStrategy Strategy_) - : DT(&DT_), PDT(&PDT_), Strategy(Strategy_) {} - DomTreeUpdater(DominatorTree *DT_, PostDominatorTree *PDT_, - UpdateStrategy Strategy_) - : DT(DT_), PDT(PDT_), Strategy(Strategy_) {} - - ~DomTreeUpdater() { flush(); } - - /// Returns true if the current strategy is Lazy. - bool isLazy() const { return Strategy == UpdateStrategy::Lazy; }; - - /// Returns true if the current strategy is Eager. - bool isEager() const { return Strategy == UpdateStrategy::Eager; }; - - /// Returns true if it holds a DominatorTree. - bool hasDomTree() const { return DT != nullptr; } - - /// Returns true if it holds a PostDominatorTree. - bool hasPostDomTree() const { return PDT != nullptr; } - - /// Returns true if there is BasicBlock awaiting deletion. - /// The deletion will only happen until a flush event and - /// all available trees are up-to-date. - /// Returns false under Eager UpdateStrategy. - bool hasPendingDeletedBB() const { return !DeletedBBs.empty(); } - - /// Returns true if DelBB is awaiting deletion. - /// Returns false under Eager UpdateStrategy. - bool isBBPendingDeletion(BasicBlock *DelBB) const; - - /// Returns true if either of DT or PDT is valid and the tree has at - /// least one update pending. If DT or PDT is nullptr it is treated - /// as having no pending updates. This function does not check - /// whether there is BasicBlock awaiting deletion. - /// Returns false under Eager UpdateStrategy. - bool hasPendingUpdates() const; - - /// Returns true if there are DominatorTree updates queued. - /// Returns false under Eager UpdateStrategy or DT is nullptr. - bool hasPendingDomTreeUpdates() const; - - /// Returns true if there are PostDominatorTree updates queued. - /// Returns false under Eager UpdateStrategy or PDT is nullptr. - bool hasPendingPostDomTreeUpdates() const; - - /// Apply updates on all available trees. Under Eager UpdateStrategy with - /// ForceRemoveDuplicates enabled or under Lazy UpdateStrategy, it will - /// discard duplicated updates and self-dominance updates. If both DT and PDT - /// are nullptrs, this function discards all updates. The Eager Strategy - /// applies the updates immediately while the Lazy Strategy queues the - /// updates. It is required for the state of the LLVM IR to be updated - /// *before* applying the Updates because the internal update routine will - /// analyze the current state of the relationship between a pair of (From, To) - /// BasicBlocks to determine whether a single update needs to be discarded. - void applyUpdates(ArrayRef Updates, - bool ForceRemoveDuplicates = false); - - /// Notify all available trees on an edge insertion. If both DT and PDT are - /// nullptrs, this function discards the update. Under either Strategy, - /// self-dominance update will be removed. The Eager Strategy applies - /// the update immediately while the Lazy Strategy queues the update. - /// It is recommended to only use this method when you have exactly one - /// insertion (and no deletions). It is recommended to use applyUpdates() in - /// all other cases. This function has to be called *after* making the update - /// on the actual CFG. An internal functions checks if the edge exists in the - /// CFG in DEBUG mode. - void insertEdge(BasicBlock *From, BasicBlock *To); - - /// Notify all available trees on an edge insertion. - /// Under either Strategy, the following updates will be discard silently - /// 1. Invalid - Inserting an edge that does not exist in the CFG. - /// 2. Self-dominance update. - /// 3. Both DT and PDT are nullptrs. - /// The Eager Strategy applies the update immediately while the Lazy Strategy - /// queues the update. It is recommended to only use this method when you have - /// exactly one insertion (and no deletions) and want to discard an invalid - /// update. - void insertEdgeRelaxed(BasicBlock *From, BasicBlock *To); - - /// Notify all available trees on an edge deletion. If both DT and PDT are - /// nullptrs, this function discards the update. Under either Strategy, - /// self-dominance update will be removed. The Eager Strategy applies - /// the update immediately while the Lazy Strategy queues the update. - /// It is recommended to only use this method when you have exactly one - /// deletion (and no insertions). It is recommended to use applyUpdates() in - /// all other cases. This function has to be called *after* making the update - /// on the actual CFG. An internal functions checks if the edge doesn't exist - /// in the CFG in DEBUG mode. - void deleteEdge(BasicBlock *From, BasicBlock *To); - - /// Notify all available trees on an edge deletion. - /// Under either Strategy, the following updates will be discard silently - /// 1. Invalid - Deleting an edge that still exists in the CFG. - /// 2. Self-dominance update. - /// 3. Both DT and PDT are nullptrs. - /// The Eager Strategy applies the update immediately while the Lazy Strategy - /// queues the update. It is recommended to only use this method when you have - /// exactly one deletion (and no insertions) and want to discard an invalid - /// update. - void deleteEdgeRelaxed(BasicBlock *From, BasicBlock *To); - - /// Delete DelBB. DelBB will be removed from its Parent and - /// erased from available trees if it exists and finally get deleted. - /// Under Eager UpdateStrategy, DelBB will be processed immediately. - /// Under Lazy UpdateStrategy, DelBB will be queued until a flush event and - /// all available trees are up-to-date. Assert if any instruction of DelBB is - /// modified while awaiting deletion. When both DT and PDT are nullptrs, DelBB - /// will be queued until flush() is called. - void deleteBB(BasicBlock *DelBB); - - /// Delete DelBB. DelBB will be removed from its Parent and - /// erased from available trees if it exists. Then the callback will - /// be called. Finally, DelBB will be deleted. - /// Under Eager UpdateStrategy, DelBB will be processed immediately. - /// Under Lazy UpdateStrategy, DelBB will be queued until a flush event and - /// all available trees are up-to-date. Assert if any instruction of DelBB is - /// modified while awaiting deletion. Multiple callbacks can be queued for one - /// DelBB under Lazy UpdateStrategy. - void callbackDeleteBB(BasicBlock *DelBB, - std::function Callback); - - /// Recalculate all available trees and flush all BasicBlocks - /// awaiting deletion immediately. - void recalculate(Function &F); - - /// Flush DomTree updates and return DomTree. - /// It also flush out of date updates applied by all available trees - /// and flush Deleted BBs if both trees are up-to-date. - /// It must only be called when it has a DomTree. - DominatorTree &getDomTree(); - - /// Flush PostDomTree updates and return PostDomTree. - /// It also flush out of date updates applied by all available trees - /// and flush Deleted BBs if both trees are up-to-date. - /// It must only be called when it has a PostDomTree. - PostDominatorTree &getPostDomTree(); - - /// Apply all pending updates to available trees and flush all BasicBlocks - /// awaiting deletion. - /// Does nothing under Eager UpdateStrategy. - void flush(); - - /// Debug method to help view the internal state of this class. - LLVM_DUMP_METHOD void dump() const; - -private: - class CallBackOnDeletion final : public CallbackVH { - public: - CallBackOnDeletion(BasicBlock *V, - std::function Callback) - : CallbackVH(V), DelBB(V), Callback_(Callback) {} - - private: - BasicBlock *DelBB = nullptr; - std::function Callback_; - - void deleted() override { - Callback_(DelBB); - CallbackVH::deleted(); - } - }; - - SmallVector PendUpdates; - size_t PendDTUpdateIndex = 0; - size_t PendPDTUpdateIndex = 0; - DominatorTree *DT = nullptr; - PostDominatorTree *PDT = nullptr; - const UpdateStrategy Strategy; - SmallPtrSet DeletedBBs; - std::vector Callbacks; - bool IsRecalculatingDomTree = false; - bool IsRecalculatingPostDomTree = false; - - /// First remove all the instructions of DelBB and then make sure DelBB has a - /// valid terminator instruction which is necessary to have when DelBB still - /// has to be inside of its parent Function while awaiting deletion under Lazy - /// UpdateStrategy to prevent other routines from asserting the state of the - /// IR is inconsistent. Assert if DelBB is nullptr or has predecessors. - void validateDeleteBB(BasicBlock *DelBB); - - /// Returns true if at least one BasicBlock is deleted. - bool forceFlushDeletedBB(); - - /// Deduplicate and remove unnecessary updates (no-ops) when using Lazy - /// UpdateStrategy. Returns true if the update is queued for update. - bool applyLazyUpdate(DominatorTree::UpdateKind Kind, BasicBlock *From, - BasicBlock *To); - - /// Helper function to apply all pending DomTree updates. - void applyDomTreeUpdates(); - - /// Helper function to apply all pending PostDomTree updates. - void applyPostDomTreeUpdates(); - - /// Helper function to flush deleted BasicBlocks if all available - /// trees are up-to-date. - void tryFlushDeletedBB(); - - /// Drop all updates applied by all available trees and delete BasicBlocks if - /// all available trees are up-to-date. - void dropOutOfDateUpdates(); - - /// Erase Basic Block node that has been unlinked from Function - /// in the DomTree and PostDomTree. - void eraseDelBBNode(BasicBlock *DelBB); - - /// Returns true if the update appears in the LLVM IR. - /// It is used to check whether an update is valid in - /// insertEdge/deleteEdge or is unnecessary in the batch update. - bool isUpdateValid(DominatorTree::UpdateType Update) const; - - /// Returns true if the update is self dominance. - bool isSelfDominance(DominatorTree::UpdateType Update) const; -}; -} // namespace llvm - -#endif // LLVM_DOMTREEUPDATER_H diff --git a/include/llvm/IR/Dominators.h b/include/llvm/IR/Dominators.h index f7da47d07663..fef1c6abf8c2 100644 --- a/include/llvm/IR/Dominators.h +++ b/include/llvm/IR/Dominators.h @@ -1,9 +1,8 @@ //===- Dominators.h - Dominator Info Calculation ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/Function.h b/include/llvm/IR/Function.h index 630f47e8bb57..7fa61e12f431 100644 --- a/include/llvm/IR/Function.h +++ b/include/llvm/IR/Function.h @@ -1,9 +1,8 @@ //===- llvm/Function.h - Class to represent a single function ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -297,15 +296,18 @@ public: /// Get the entry count for this function. /// - /// Entry count is the number of times the function was executed based on - /// pgo data. - ProfileCount getEntryCount() const; + /// Entry count is the number of times the function was executed. + /// When AllowSynthetic is false, only pgo_data will be returned. + ProfileCount getEntryCount(bool AllowSynthetic = false) const; /// Return true if the function is annotated with profile data. /// /// Presence of entry counts from a profile run implies the function has - /// profile annotations. - bool hasProfileData() const { return getEntryCount().hasValue(); } + /// profile annotations. If IncludeSynthetic is false, only return true + /// when the profile data is real. + bool hasProfileData(bool IncludeSynthetic = false) const { + return getEntryCount(IncludeSynthetic).hasValue(); + } /// Returns the set of GUIDs that needs to be imported to the function for /// sample PGO, to enable the same inlines as the profiled optimized binary. @@ -399,6 +401,11 @@ public: return getAttributes().hasParamAttribute(ArgNo, Kind); } + /// gets the specified attribute from the list of attributes. + Attribute getParamAttribute(unsigned ArgNo, Attribute::AttrKind Kind) const { + return getAttributes().getParamAttr(ArgNo, Kind); + } + /// gets the attribute from the list of attributes. Attribute getAttribute(unsigned i, Attribute::AttrKind Kind) const { return AttributeSets.getAttribute(i, Kind); @@ -429,6 +436,12 @@ public: return AttributeSets.getParamAlignment(ArgNo); } + /// Extract the byval type for a parameter. + Type *getParamByValType(unsigned ArgNo) const { + Type *Ty = AttributeSets.getParamByValType(ArgNo); + return Ty ? Ty : (arg_begin() + ArgNo)->getType()->getPointerElementType(); + } + /// Extract the number of dereferenceable bytes for a call or /// parameter (0=unknown). /// @param i AttributeList index, referring to a return value or argument. @@ -551,6 +564,14 @@ public: addFnAttr(Attribute::Speculatable); } + /// Determine if the call might deallocate memory. + bool doesNotFreeMemory() const { + return onlyReadsMemory() || hasFnAttribute(Attribute::NoFree); + } + void setDoesNotFreeMemory() { + addFnAttr(Attribute::NoFree); + } + /// Determine if the function is known not to recurse, directly or /// indirectly. bool doesNotRecurse() const { @@ -591,12 +612,15 @@ public: addAttribute(AttributeList::ReturnIndex, Attribute::NoAlias); } + /// Do not optimize this function (-O0). + bool hasOptNone() const { return hasFnAttribute(Attribute::OptimizeNone); } + /// Optimize this function for minimum size (-Oz). - bool optForMinSize() const { return hasFnAttribute(Attribute::MinSize); } + bool hasMinSize() const { return hasFnAttribute(Attribute::MinSize); } /// Optimize this function for size (-Os) or minimum size (-Oz). - bool optForSize() const { - return hasFnAttribute(Attribute::OptimizeForSize) || optForMinSize(); + bool hasOptSize() const { + return hasFnAttribute(Attribute::OptimizeForSize) || hasMinSize(); } /// copyAttributesFrom - copy all additional attributes (those not needed to diff --git a/include/llvm/IR/GVMaterializer.h b/include/llvm/IR/GVMaterializer.h index 675abeb6ec3a..d62da41ebc29 100644 --- a/include/llvm/IR/GVMaterializer.h +++ b/include/llvm/IR/GVMaterializer.h @@ -1,9 +1,8 @@ //===- GVMaterializer.h - Interface for GV materializers --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/GetElementPtrTypeIterator.h b/include/llvm/IR/GetElementPtrTypeIterator.h index 3c143ea5f703..9b257abc7c1f 100644 --- a/include/llvm/IR/GetElementPtrTypeIterator.h +++ b/include/llvm/IR/GetElementPtrTypeIterator.h @@ -1,9 +1,8 @@ //===- GetElementPtrTypeIterator.h ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/GlobalAlias.h b/include/llvm/IR/GlobalAlias.h index 450583baaa3c..3cd405701300 100644 --- a/include/llvm/IR/GlobalAlias.h +++ b/include/llvm/IR/GlobalAlias.h @@ -1,9 +1,8 @@ //===-------- llvm/GlobalAlias.h - GlobalAlias class ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/GlobalIFunc.h b/include/llvm/IR/GlobalIFunc.h index ef51315a6f5d..bc0d3c053cce 100644 --- a/include/llvm/IR/GlobalIFunc.h +++ b/include/llvm/IR/GlobalIFunc.h @@ -1,9 +1,8 @@ //===-------- llvm/GlobalIFunc.h - GlobalIFunc class ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/include/llvm/IR/GlobalIndirectSymbol.h b/include/llvm/IR/GlobalIndirectSymbol.h index 22c00686c549..8bc3f90b94aa 100644 --- a/include/llvm/IR/GlobalIndirectSymbol.h +++ b/include/llvm/IR/GlobalIndirectSymbol.h @@ -1,9 +1,8 @@ //===- llvm/GlobalIndirectSymbol.h - GlobalIndirectSymbol class -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/GlobalObject.h b/include/llvm/IR/GlobalObject.h index 1fd3568100c2..b8ab6140ebe7 100644 --- a/include/llvm/IR/GlobalObject.h +++ b/include/llvm/IR/GlobalObject.h @@ -1,9 +1,8 @@ //===-- llvm/GlobalObject.h - Class to represent global objects -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/GlobalValue.h b/include/llvm/IR/GlobalValue.h index c07d4051c803..2209881dbda6 100644 --- a/include/llvm/IR/GlobalValue.h +++ b/include/llvm/IR/GlobalValue.h @@ -1,9 +1,8 @@ //===-- llvm/GlobalValue.h - Class to represent a global value --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -80,15 +79,15 @@ protected: ValueType(Ty), Visibility(DefaultVisibility), UnnamedAddrVal(unsigned(UnnamedAddr::None)), DllStorageClass(DefaultStorageClass), ThreadLocal(NotThreadLocal), - HasLLVMReservedName(false), IsDSOLocal(false), IntID((Intrinsic::ID)0U), - Parent(nullptr) { + HasLLVMReservedName(false), IsDSOLocal(false), HasPartition(false), + IntID((Intrinsic::ID)0U), Parent(nullptr) { setLinkage(Linkage); setName(Name); } Type *ValueType; - static const unsigned GlobalValueSubClassDataBits = 17; + static const unsigned GlobalValueSubClassDataBits = 16; // All bitfields use unsigned as the underlying type so that MSVC will pack // them. @@ -109,9 +108,13 @@ protected: /// definition cannot be runtime preempted. unsigned IsDSOLocal : 1; + /// True if this symbol has a partition name assigned (see + /// https://lld.llvm.org/Partitions.html). + unsigned HasPartition : 1; + private: // Give subclasses access to what otherwise would be wasted padding. - // (17 + 4 + 2 + 2 + 2 + 3 + 1 + 1) == 32. + // (16 + 4 + 2 + 2 + 2 + 3 + 1 + 1 + 1) == 32. unsigned SubClassData : GlobalValueSubClassDataBits; friend class Constant; @@ -281,6 +284,12 @@ public: return IsDSOLocal; } + bool hasPartition() const { + return HasPartition; + } + StringRef getPartition() const; + void setPartition(StringRef Part); + static LinkageTypes getLinkOnceLinkage(bool ODR) { return ODR ? LinkOnceODRLinkage : LinkOnceAnyLinkage; } diff --git a/include/llvm/IR/GlobalVariable.h b/include/llvm/IR/GlobalVariable.h index 03b9ec46ebb4..2e2c8c477913 100644 --- a/include/llvm/IR/GlobalVariable.h +++ b/include/llvm/IR/GlobalVariable.h @@ -1,9 +1,8 @@ //===-- llvm/GlobalVariable.h - GlobalVariable class ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/IRBuilder.h b/include/llvm/IR/IRBuilder.h index fac2ff46c453..a74364dffb2e 100644 --- a/include/llvm/IR/IRBuilder.h +++ b/include/llvm/IR/IRBuilder.h @@ -1,9 +1,8 @@ //===- llvm/IRBuilder.h - Builder for LLVM Instructions ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -32,7 +31,7 @@ #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" -#include "llvm/IR/Intrinsics.h" +#include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" @@ -97,12 +96,18 @@ protected: MDNode *DefaultFPMathTag; FastMathFlags FMF; + bool IsFPConstrained; + ConstrainedFPIntrinsic::ExceptionBehavior DefaultConstrainedExcept; + ConstrainedFPIntrinsic::RoundingMode DefaultConstrainedRounding; + ArrayRef DefaultOperandBundles; public: IRBuilderBase(LLVMContext &context, MDNode *FPMathTag = nullptr, ArrayRef OpBundles = None) - : Context(context), DefaultFPMathTag(FPMathTag), + : Context(context), DefaultFPMathTag(FPMathTag), IsFPConstrained(false), + DefaultConstrainedExcept(ConstrainedFPIntrinsic::ebStrict), + DefaultConstrainedRounding(ConstrainedFPIntrinsic::rmDynamic), DefaultOperandBundles(OpBundles) { ClearInsertionPoint(); } @@ -219,6 +224,37 @@ public: /// Set the fast-math flags to be used with generated fp-math operators void setFastMathFlags(FastMathFlags NewFMF) { FMF = NewFMF; } + /// Enable/Disable use of constrained floating point math. When + /// enabled the CreateF() calls instead create constrained + /// floating point intrinsic calls. Fast math flags are unaffected + /// by this setting. + void setIsFPConstrained(bool IsCon) { IsFPConstrained = IsCon; } + + /// Query for the use of constrained floating point math + bool getIsFPConstrained() { return IsFPConstrained; } + + /// Set the exception handling to be used with constrained floating point + void setDefaultConstrainedExcept( + ConstrainedFPIntrinsic::ExceptionBehavior NewExcept) { + DefaultConstrainedExcept = NewExcept; + } + + /// Set the rounding mode handling to be used with constrained floating point + void setDefaultConstrainedRounding( + ConstrainedFPIntrinsic::RoundingMode NewRounding) { + DefaultConstrainedRounding = NewRounding; + } + + /// Get the exception handling used with constrained floating point + ConstrainedFPIntrinsic::ExceptionBehavior getDefaultConstrainedExcept() { + return DefaultConstrainedExcept; + } + + /// Get the rounding mode handling used with constrained floating point + ConstrainedFPIntrinsic::RoundingMode getDefaultConstrainedRounding() { + return DefaultConstrainedRounding; + } + //===--------------------------------------------------------------------===// // RAII helpers. //===--------------------------------------------------------------------===// @@ -906,20 +942,20 @@ public: Name); } - InvokeInst *CreateInvoke(Function *Callee, BasicBlock *NormalDest, + InvokeInst *CreateInvoke(FunctionCallee Callee, BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef Args, ArrayRef OpBundles, const Twine &Name = "") { - return CreateInvoke(Callee->getFunctionType(), Callee, NormalDest, - UnwindDest, Args, OpBundles, Name); + return CreateInvoke(Callee.getFunctionType(), Callee.getCallee(), + NormalDest, UnwindDest, Args, OpBundles, Name); } - InvokeInst *CreateInvoke(Function *Callee, BasicBlock *NormalDest, + InvokeInst *CreateInvoke(FunctionCallee Callee, BasicBlock *NormalDest, BasicBlock *UnwindDest, ArrayRef Args = None, const Twine &Name = "") { - return CreateInvoke(Callee->getFunctionType(), Callee, NormalDest, - UnwindDest, Args, Name); + return CreateInvoke(Callee.getFunctionType(), Callee.getCallee(), + NormalDest, UnwindDest, Args, Name); } // Deprecated [opaque pointer types] @@ -944,6 +980,42 @@ public: Callee, NormalDest, UnwindDest, Args, Name); } + /// \brief Create a callbr instruction. + CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee, + BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args = None, + const Twine &Name = "") { + return Insert(CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests, + Args), Name); + } + CallBrInst *CreateCallBr(FunctionType *Ty, Value *Callee, + BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef OpBundles, + const Twine &Name = "") { + return Insert( + CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests, Args, + OpBundles), Name); + } + + CallBrInst *CreateCallBr(FunctionCallee Callee, BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args = None, + const Twine &Name = "") { + return CreateCallBr(Callee.getFunctionType(), Callee.getCallee(), + DefaultDest, IndirectDests, Args, Name); + } + CallBrInst *CreateCallBr(FunctionCallee Callee, BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef OpBundles, + const Twine &Name = "") { + return CreateCallBr(Callee.getFunctionType(), Callee.getCallee(), + DefaultDest, IndirectDests, Args, Name); + } + ResumeInst *CreateResume(Value *Exn) { return Insert(ResumeInst::Create(Exn)); } @@ -1004,12 +1076,44 @@ private: } Value *foldConstant(Instruction::BinaryOps Opc, Value *L, - Value *R, const Twine &Name = nullptr) const { + Value *R, const Twine &Name) const { auto *LC = dyn_cast(L); auto *RC = dyn_cast(R); return (LC && RC) ? Insert(Folder.CreateBinOp(Opc, LC, RC), Name) : nullptr; } + Value *getConstrainedFPRounding( + Optional Rounding) { + ConstrainedFPIntrinsic::RoundingMode UseRounding = + DefaultConstrainedRounding; + + if (Rounding.hasValue()) + UseRounding = Rounding.getValue(); + + Optional RoundingStr = + ConstrainedFPIntrinsic::RoundingModeToStr(UseRounding); + assert(RoundingStr.hasValue() && "Garbage strict rounding mode!"); + auto *RoundingMDS = MDString::get(Context, RoundingStr.getValue()); + + return MetadataAsValue::get(Context, RoundingMDS); + } + + Value *getConstrainedFPExcept( + Optional Except) { + ConstrainedFPIntrinsic::ExceptionBehavior UseExcept = + DefaultConstrainedExcept; + + if (Except.hasValue()) + UseExcept = Except.getValue(); + + Optional ExceptStr = + ConstrainedFPIntrinsic::ExceptionBehaviorToStr(UseExcept); + assert(ExceptStr.hasValue() && "Garbage strict exception behavior!"); + auto *ExceptMDS = MDString::get(Context, ExceptStr.getValue()); + + return MetadataAsValue::get(Context, ExceptMDS); + } + public: Value *CreateAdd(Value *LHS, Value *RHS, const Twine &Name = "", bool HasNUW = false, bool HasNSW = false) { @@ -1179,6 +1283,14 @@ public: return CreateAnd(LHS, ConstantInt::get(LHS->getType(), RHS), Name); } + Value *CreateAnd(ArrayRef Ops) { + assert(!Ops.empty()); + Value *Accum = Ops[0]; + for (unsigned i = 1; i < Ops.size(); i++) + Accum = CreateAnd(Accum, Ops[i]); + return Accum; + } + Value *CreateOr(Value *LHS, Value *RHS, const Twine &Name = "") { if (auto *RC = dyn_cast(RHS)) { if (RC->isNullValue()) @@ -1197,6 +1309,14 @@ public: return CreateOr(LHS, ConstantInt::get(LHS->getType(), RHS), Name); } + Value *CreateOr(ArrayRef Ops) { + assert(!Ops.empty()); + Value *Accum = Ops[0]; + for (unsigned i = 1; i < Ops.size(); i++) + Accum = CreateOr(Accum, Ops[i]); + return Accum; + } + Value *CreateXor(Value *LHS, Value *RHS, const Twine &Name = "") { if (Value *V = foldConstant(Instruction::Xor, LHS, RHS, Name)) return V; return Insert(BinaryOperator::CreateXor(LHS, RHS), Name); @@ -1212,6 +1332,10 @@ public: Value *CreateFAdd(Value *L, Value *R, const Twine &Name = "", MDNode *FPMD = nullptr) { + if (IsFPConstrained) + return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fadd, + L, R, nullptr, Name, FPMD); + if (Value *V = foldConstant(Instruction::FAdd, L, R, Name)) return V; Instruction *I = setFPAttrs(BinaryOperator::CreateFAdd(L, R), FPMD, FMF); return Insert(I, Name); @@ -1221,6 +1345,10 @@ public: /// default FMF. Value *CreateFAddFMF(Value *L, Value *R, Instruction *FMFSource, const Twine &Name = "") { + if (IsFPConstrained) + return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fadd, + L, R, FMFSource, Name); + if (Value *V = foldConstant(Instruction::FAdd, L, R, Name)) return V; Instruction *I = setFPAttrs(BinaryOperator::CreateFAdd(L, R), nullptr, FMFSource->getFastMathFlags()); @@ -1229,6 +1357,10 @@ public: Value *CreateFSub(Value *L, Value *R, const Twine &Name = "", MDNode *FPMD = nullptr) { + if (IsFPConstrained) + return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fsub, + L, R, nullptr, Name, FPMD); + if (Value *V = foldConstant(Instruction::FSub, L, R, Name)) return V; Instruction *I = setFPAttrs(BinaryOperator::CreateFSub(L, R), FPMD, FMF); return Insert(I, Name); @@ -1238,6 +1370,10 @@ public: /// default FMF. Value *CreateFSubFMF(Value *L, Value *R, Instruction *FMFSource, const Twine &Name = "") { + if (IsFPConstrained) + return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fsub, + L, R, FMFSource, Name); + if (Value *V = foldConstant(Instruction::FSub, L, R, Name)) return V; Instruction *I = setFPAttrs(BinaryOperator::CreateFSub(L, R), nullptr, FMFSource->getFastMathFlags()); @@ -1246,6 +1382,10 @@ public: Value *CreateFMul(Value *L, Value *R, const Twine &Name = "", MDNode *FPMD = nullptr) { + if (IsFPConstrained) + return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fmul, + L, R, nullptr, Name, FPMD); + if (Value *V = foldConstant(Instruction::FMul, L, R, Name)) return V; Instruction *I = setFPAttrs(BinaryOperator::CreateFMul(L, R), FPMD, FMF); return Insert(I, Name); @@ -1255,6 +1395,10 @@ public: /// default FMF. Value *CreateFMulFMF(Value *L, Value *R, Instruction *FMFSource, const Twine &Name = "") { + if (IsFPConstrained) + return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fmul, + L, R, FMFSource, Name); + if (Value *V = foldConstant(Instruction::FMul, L, R, Name)) return V; Instruction *I = setFPAttrs(BinaryOperator::CreateFMul(L, R), nullptr, FMFSource->getFastMathFlags()); @@ -1263,6 +1407,10 @@ public: Value *CreateFDiv(Value *L, Value *R, const Twine &Name = "", MDNode *FPMD = nullptr) { + if (IsFPConstrained) + return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fdiv, + L, R, nullptr, Name, FPMD); + if (Value *V = foldConstant(Instruction::FDiv, L, R, Name)) return V; Instruction *I = setFPAttrs(BinaryOperator::CreateFDiv(L, R), FPMD, FMF); return Insert(I, Name); @@ -1272,6 +1420,10 @@ public: /// default FMF. Value *CreateFDivFMF(Value *L, Value *R, Instruction *FMFSource, const Twine &Name = "") { + if (IsFPConstrained) + return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_fdiv, + L, R, FMFSource, Name); + if (Value *V = foldConstant(Instruction::FDiv, L, R, Name)) return V; Instruction *I = setFPAttrs(BinaryOperator::CreateFDiv(L, R), nullptr, FMFSource->getFastMathFlags()); @@ -1280,6 +1432,10 @@ public: Value *CreateFRem(Value *L, Value *R, const Twine &Name = "", MDNode *FPMD = nullptr) { + if (IsFPConstrained) + return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_frem, + L, R, nullptr, Name, FPMD); + if (Value *V = foldConstant(Instruction::FRem, L, R, Name)) return V; Instruction *I = setFPAttrs(BinaryOperator::CreateFRem(L, R), FPMD, FMF); return Insert(I, Name); @@ -1289,6 +1445,10 @@ public: /// default FMF. Value *CreateFRemFMF(Value *L, Value *R, Instruction *FMFSource, const Twine &Name = "") { + if (IsFPConstrained) + return CreateConstrainedFPBinOp(Intrinsic::experimental_constrained_frem, + L, R, FMFSource, Name); + if (Value *V = foldConstant(Instruction::FRem, L, R, Name)) return V; Instruction *I = setFPAttrs(BinaryOperator::CreateFRem(L, R), nullptr, FMFSource->getFastMathFlags()); @@ -1305,6 +1465,23 @@ public: return Insert(BinOp, Name); } + CallInst *CreateConstrainedFPBinOp( + Intrinsic::ID ID, Value *L, Value *R, Instruction *FMFSource = nullptr, + const Twine &Name = "", MDNode *FPMathTag = nullptr, + Optional Rounding = None, + Optional Except = None) { + Value *RoundingV = getConstrainedFPRounding(Rounding); + Value *ExceptV = getConstrainedFPExcept(Except); + + FastMathFlags UseFMF = FMF; + if (FMFSource) + UseFMF = FMFSource->getFastMathFlags(); + + CallInst *C = CreateIntrinsic(ID, {L->getType()}, + {L, R, RoundingV, ExceptV}, nullptr, Name); + return cast(setFPAttrs(C, FPMathTag, UseFMF)); + } + Value *CreateNeg(Value *V, const Twine &Name = "", bool HasNUW = false, bool HasNSW = false) { if (auto *VC = dyn_cast(V)) @@ -1331,12 +1508,54 @@ public: Name); } + /// Copy fast-math-flags from an instruction rather than using the builder's + /// default FMF. + Value *CreateFNegFMF(Value *V, Instruction *FMFSource, + const Twine &Name = "") { + if (auto *VC = dyn_cast(V)) + return Insert(Folder.CreateFNeg(VC), Name); + // TODO: This should return UnaryOperator::CreateFNeg(...) once we are + // confident that they are optimized sufficiently. + return Insert(setFPAttrs(BinaryOperator::CreateFNeg(V), nullptr, + FMFSource->getFastMathFlags()), + Name); + } + Value *CreateNot(Value *V, const Twine &Name = "") { if (auto *VC = dyn_cast(V)) return Insert(Folder.CreateNot(VC), Name); return Insert(BinaryOperator::CreateNot(V), Name); } + Value *CreateUnOp(Instruction::UnaryOps Opc, + Value *V, const Twine &Name = "", + MDNode *FPMathTag = nullptr) { + if (auto *VC = dyn_cast(V)) + return Insert(Folder.CreateUnOp(Opc, VC), Name); + Instruction *UnOp = UnaryOperator::Create(Opc, V); + if (isa(UnOp)) + UnOp = setFPAttrs(UnOp, FPMathTag, FMF); + return Insert(UnOp, Name); + } + + /// Create either a UnaryOperator or BinaryOperator depending on \p Opc. + /// Correct number of operands must be passed accordingly. + Value *CreateNAryOp(unsigned Opc, ArrayRef Ops, + const Twine &Name = "", + MDNode *FPMathTag = nullptr) { + if (Instruction::isBinaryOp(Opc)) { + assert(Ops.size() == 2 && "Invalid number of operands!"); + return CreateBinOp(static_cast(Opc), + Ops[0], Ops[1], Name, FPMathTag); + } + if (Instruction::isUnaryOp(Opc)) { + assert(Ops.size() == 1 && "Invalid number of operands!"); + return CreateUnOp(static_cast(Opc), + Ops[0], Name, FPMathTag); + } + llvm_unreachable("Unexpected opcode!"); + } + //===--------------------------------------------------------------------===// // Instruction creation methods: Memory Instructions //===--------------------------------------------------------------------===// @@ -1989,16 +2208,17 @@ public: return Insert(CI, Name); } - CallInst *CreateCall(Function *Callee, ArrayRef Args = None, + CallInst *CreateCall(FunctionCallee Callee, ArrayRef Args = None, const Twine &Name = "", MDNode *FPMathTag = nullptr) { - return CreateCall(Callee->getFunctionType(), Callee, Args, Name, FPMathTag); + return CreateCall(Callee.getFunctionType(), Callee.getCallee(), Args, Name, + FPMathTag); } - CallInst *CreateCall(Function *Callee, ArrayRef Args, + CallInst *CreateCall(FunctionCallee Callee, ArrayRef Args, ArrayRef OpBundles, const Twine &Name = "", MDNode *FPMathTag = nullptr) { - return CreateCall(Callee->getFunctionType(), Callee, Args, OpBundles, Name, - FPMathTag); + return CreateCall(Callee.getFunctionType(), Callee.getCallee(), Args, + OpBundles, Name, FPMathTag); } // Deprecated [opaque pointer types] @@ -2031,6 +2251,8 @@ public: MDNode *Unpred = MDFrom->getMetadata(LLVMContext::MD_unpredictable); Sel = addBranchMetadata(Sel, Prof, Unpred); } + if (isa(Sel)) + Sel = cast(setFPAttrs(Sel, nullptr /* MDNode* */, FMF)); return Insert(Sel, Name); } @@ -2231,6 +2453,74 @@ public: return V; } + Value *CreatePreserveArrayAccessIndex(Value *Base, unsigned Dimension, + unsigned LastIndex) { + assert(isa(Base->getType()) && + "Invalid Base ptr type for preserve.array.access.index."); + auto *BaseType = Base->getType(); + + Value *LastIndexV = getInt32(LastIndex); + Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0); + SmallVector IdxList; + for (unsigned I = 0; I < Dimension; ++I) + IdxList.push_back(Zero); + IdxList.push_back(LastIndexV); + + Type *ResultType = + GetElementPtrInst::getGEPReturnType(Base, IdxList); + + Module *M = BB->getParent()->getParent(); + Function *FnPreserveArrayAccessIndex = Intrinsic::getDeclaration( + M, Intrinsic::preserve_array_access_index, {ResultType, BaseType}); + + Value *DimV = getInt32(Dimension); + CallInst *Fn = + CreateCall(FnPreserveArrayAccessIndex, {Base, DimV, LastIndexV}); + + return Fn; + } + + Value *CreatePreserveUnionAccessIndex(Value *Base, unsigned FieldIndex, + MDNode *DbgInfo) { + assert(isa(Base->getType()) && + "Invalid Base ptr type for preserve.union.access.index."); + auto *BaseType = Base->getType(); + + Module *M = BB->getParent()->getParent(); + Function *FnPreserveUnionAccessIndex = Intrinsic::getDeclaration( + M, Intrinsic::preserve_union_access_index, {BaseType, BaseType}); + + Value *DIIndex = getInt32(FieldIndex); + CallInst *Fn = + CreateCall(FnPreserveUnionAccessIndex, {Base, DIIndex}); + Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); + + return Fn; + } + + Value *CreatePreserveStructAccessIndex(Value *Base, unsigned Index, + unsigned FieldIndex, MDNode *DbgInfo) { + assert(isa(Base->getType()) && + "Invalid Base ptr type for preserve.struct.access.index."); + auto *BaseType = Base->getType(); + + Value *GEPIndex = getInt32(Index); + Constant *Zero = ConstantInt::get(Type::getInt32Ty(Context), 0); + Type *ResultType = + GetElementPtrInst::getGEPReturnType(Base, {Zero, GEPIndex}); + + Module *M = BB->getParent()->getParent(); + Function *FnPreserveStructAccessIndex = Intrinsic::getDeclaration( + M, Intrinsic::preserve_struct_access_index, {ResultType, BaseType}); + + Value *DIIndex = getInt32(FieldIndex); + CallInst *Fn = CreateCall(FnPreserveStructAccessIndex, + {Base, GEPIndex, DIIndex}); + Fn->setMetadata(LLVMContext::MD_preserve_access_index, DbgInfo); + + return Fn; + } + private: /// Helper function that creates an assume intrinsic call that /// represents an alignment assumption on the provided Ptr, Mask, Type @@ -2280,10 +2570,11 @@ public: Value **TheCheck = nullptr) { assert(isa(PtrValue->getType()) && "trying to create an alignment assumption on a non-pointer?"); + assert(Alignment != 0 && "Invalid Alignment"); auto *PtrTy = cast(PtrValue->getType()); Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace()); - Value *Mask = ConstantInt::get(IntPtrTy, Alignment > 0 ? Alignment - 1 : 0); + Value *Mask = ConstantInt::get(IntPtrTy, Alignment - 1); return CreateAlignmentAssumptionHelper(DL, PtrValue, Mask, IntPtrTy, OffsetValue, TheCheck); } @@ -2310,15 +2601,10 @@ public: Type *IntPtrTy = getIntPtrTy(DL, PtrTy->getAddressSpace()); if (Alignment->getType() != IntPtrTy) - Alignment = CreateIntCast(Alignment, IntPtrTy, /*isSigned*/ true, + Alignment = CreateIntCast(Alignment, IntPtrTy, /*isSigned*/ false, "alignmentcast"); - Value *IsPositive = - CreateICmp(CmpInst::ICMP_SGT, Alignment, - ConstantInt::get(Alignment->getType(), 0), "ispositive"); - Value *PositiveMask = - CreateSub(Alignment, ConstantInt::get(IntPtrTy, 1), "positivemask"); - Value *Mask = CreateSelect(IsPositive, PositiveMask, - ConstantInt::get(IntPtrTy, 0), "mask"); + + Value *Mask = CreateSub(Alignment, ConstantInt::get(IntPtrTy, 1), "mask"); return CreateAlignmentAssumptionHelper(DL, PtrValue, Mask, IntPtrTy, OffsetValue, TheCheck); diff --git a/include/llvm/IR/IRPrintingPasses.h b/include/llvm/IR/IRPrintingPasses.h index 75f80567dbd5..3be9449c1a93 100644 --- a/include/llvm/IR/IRPrintingPasses.h +++ b/include/llvm/IR/IRPrintingPasses.h @@ -1,9 +1,8 @@ //===- IRPrintingPasses.h - Passes to print out IR constructs ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/IR/InlineAsm.h b/include/llvm/IR/InlineAsm.h index 1519a45d59e9..2aac807623a9 100644 --- a/include/llvm/IR/InlineAsm.h +++ b/include/llvm/IR/InlineAsm.h @@ -1,9 +1,8 @@ //===- llvm/InlineAsm.h - Class to represent inline asm strings -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/InstIterator.h b/include/llvm/IR/InstIterator.h index 2988fc935dd5..054fe4e9cbe9 100644 --- a/include/llvm/IR/InstIterator.h +++ b/include/llvm/IR/InstIterator.h @@ -1,9 +1,8 @@ //===- InstIterator.h - Classes for inst iteration --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/InstVisitor.h b/include/llvm/IR/InstVisitor.h index c5b4c6f71d7d..fbeb2caf14e6 100644 --- a/include/llvm/IR/InstVisitor.h +++ b/include/llvm/IR/InstVisitor.h @@ -1,9 +1,8 @@ //===- InstVisitor.h - Instruction visitor templates ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -218,14 +217,17 @@ public: RetTy visitVACopyInst(VACopyInst &I) { DELEGATE(IntrinsicInst); } RetTy visitIntrinsicInst(IntrinsicInst &I) { DELEGATE(CallInst); } - // Call and Invoke are slightly different as they delegate first through - // a generic CallSite visitor. + // Call, Invoke and CallBr are slightly different as they delegate first + // through a generic CallSite visitor. RetTy visitCallInst(CallInst &I) { return static_cast(this)->visitCallSite(&I); } RetTy visitInvokeInst(InvokeInst &I) { return static_cast(this)->visitCallSite(&I); } + RetTy visitCallBrInst(CallBrInst &I) { + return static_cast(this)->visitCallSite(&I); + } // While terminators don't have a distinct type modeling them, we support // intercepting them with dedicated a visitor callback. @@ -271,14 +273,14 @@ public: // The next level delegation for `CallBase` is slightly more complex in order // to support visiting cases where the call is also a terminator. RetTy visitCallBase(CallBase &I) { - if (isa(I)) + if (isa(I) || isa(I)) return static_cast(this)->visitTerminator(I); DELEGATE(Instruction); } - // Provide a legacy visitor for a 'callsite' that visits both calls and - // invokes. + // Provide a legacy visitor for a 'callsite' that visits calls, invokes, + // and calbrs. // // Prefer overriding the type system based `CallBase` instead. RetTy visitCallSite(CallSite CS) { diff --git a/include/llvm/IR/InstrTypes.h b/include/llvm/IR/InstrTypes.h index 3f384a6ee40c..ca419b50da6b 100644 --- a/include/llvm/IR/InstrTypes.h +++ b/include/llvm/IR/InstrTypes.h @@ -1,9 +1,8 @@ //===- llvm/InstrTypes.h - Important Instruction subclasses -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -28,6 +27,7 @@ #include "llvm/IR/CallingConv.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DerivedTypes.h" +#include "llvm/IR/Function.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/OperandTraits.h" @@ -77,7 +77,8 @@ public: // Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const Instruction *I) { - return I->getOpcode() == Instruction::Alloca || + return I->isUnaryOp() || + I->getOpcode() == Instruction::Alloca || I->getOpcode() == Instruction::Load || I->getOpcode() == Instruction::VAArg || I->getOpcode() == Instruction::ExtractValue || @@ -95,6 +96,91 @@ struct OperandTraits : DEFINE_TRANSPARENT_OPERAND_ACCESSORS(UnaryInstruction, Value) +//===----------------------------------------------------------------------===// +// UnaryOperator Class +//===----------------------------------------------------------------------===// + +class UnaryOperator : public UnaryInstruction { + void AssertOK(); + +protected: + UnaryOperator(UnaryOps iType, Value *S, Type *Ty, + const Twine &Name, Instruction *InsertBefore); + UnaryOperator(UnaryOps iType, Value *S, Type *Ty, + const Twine &Name, BasicBlock *InsertAtEnd); + + // Note: Instruction needs to be a friend here to call cloneImpl. + friend class Instruction; + + UnaryOperator *cloneImpl() const; + +public: + + /// Construct a unary instruction, given the opcode and an operand. + /// Optionally (if InstBefore is specified) insert the instruction + /// into a BasicBlock right before the specified instruction. The specified + /// Instruction is allowed to be a dereferenced end iterator. + /// + static UnaryOperator *Create(UnaryOps Op, Value *S, + const Twine &Name = Twine(), + Instruction *InsertBefore = nullptr); + + /// Construct a unary instruction, given the opcode and an operand. + /// Also automatically insert this instruction to the end of the + /// BasicBlock specified. + /// + static UnaryOperator *Create(UnaryOps Op, Value *S, + const Twine &Name, + BasicBlock *InsertAtEnd); + + /// These methods just forward to Create, and are useful when you + /// statically know what type of instruction you're going to create. These + /// helpers just save some typing. +#define HANDLE_UNARY_INST(N, OPC, CLASS) \ + static UnaryOperator *Create##OPC(Value *V, const Twine &Name = "") {\ + return Create(Instruction::OPC, V, Name);\ + } +#include "llvm/IR/Instruction.def" +#define HANDLE_UNARY_INST(N, OPC, CLASS) \ + static UnaryOperator *Create##OPC(Value *V, const Twine &Name, \ + BasicBlock *BB) {\ + return Create(Instruction::OPC, V, Name, BB);\ + } +#include "llvm/IR/Instruction.def" +#define HANDLE_UNARY_INST(N, OPC, CLASS) \ + static UnaryOperator *Create##OPC(Value *V, const Twine &Name, \ + Instruction *I) {\ + return Create(Instruction::OPC, V, Name, I);\ + } +#include "llvm/IR/Instruction.def" + + static UnaryOperator *CreateWithCopiedFlags(UnaryOps Opc, + Value *V, + Instruction *CopyO, + const Twine &Name = "") { + UnaryOperator *UO = Create(Opc, V, Name); + UO->copyIRFlags(CopyO); + return UO; + } + + static UnaryOperator *CreateFNegFMF(Value *Op, Instruction *FMFSource, + const Twine &Name = "") { + return CreateWithCopiedFlags(Instruction::FNeg, Op, FMFSource, Name); + } + + UnaryOps getOpcode() const { + return static_cast(Instruction::getOpcode()); + } + + // Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const Instruction *I) { + return I->isUnaryOp(); + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } +}; + //===----------------------------------------------------------------------===// // BinaryOperator Class //===----------------------------------------------------------------------===// @@ -162,42 +248,42 @@ public: static BinaryOperator *CreateWithCopiedFlags(BinaryOps Opc, Value *V1, Value *V2, - BinaryOperator *CopyBO, + Instruction *CopyO, const Twine &Name = "") { BinaryOperator *BO = Create(Opc, V1, V2, Name); - BO->copyIRFlags(CopyBO); + BO->copyIRFlags(CopyO); return BO; } static BinaryOperator *CreateFAddFMF(Value *V1, Value *V2, - BinaryOperator *FMFSource, + Instruction *FMFSource, const Twine &Name = "") { return CreateWithCopiedFlags(Instruction::FAdd, V1, V2, FMFSource, Name); } static BinaryOperator *CreateFSubFMF(Value *V1, Value *V2, - BinaryOperator *FMFSource, + Instruction *FMFSource, const Twine &Name = "") { return CreateWithCopiedFlags(Instruction::FSub, V1, V2, FMFSource, Name); } static BinaryOperator *CreateFMulFMF(Value *V1, Value *V2, - BinaryOperator *FMFSource, + Instruction *FMFSource, const Twine &Name = "") { return CreateWithCopiedFlags(Instruction::FMul, V1, V2, FMFSource, Name); } static BinaryOperator *CreateFDivFMF(Value *V1, Value *V2, - BinaryOperator *FMFSource, + Instruction *FMFSource, const Twine &Name = "") { return CreateWithCopiedFlags(Instruction::FDiv, V1, V2, FMFSource, Name); } static BinaryOperator *CreateFRemFMF(Value *V1, Value *V2, - BinaryOperator *FMFSource, + Instruction *FMFSource, const Twine &Name = "") { return CreateWithCopiedFlags(Instruction::FRem, V1, V2, FMFSource, Name); } - static BinaryOperator *CreateFNegFMF(Value *Op, BinaryOperator *FMFSource, + static BinaryOperator *CreateFNegFMF(Value *Op, Instruction *FMFSource, const Twine &Name = "") { Value *Zero = ConstantFP::getNegativeZero(Op->getType()); - return CreateWithCopiedFlags(Instruction::FSub, Zero, Op, FMFSource); + return CreateWithCopiedFlags(Instruction::FSub, Zero, Op, FMFSource, Name); } static BinaryOperator *CreateNSW(BinaryOps Opc, Value *V1, Value *V2, @@ -1033,16 +1119,23 @@ protected: return 0; case Instruction::Invoke: return 2; + case Instruction::CallBr: + return getNumSubclassExtraOperandsDynamic(); } llvm_unreachable("Invalid opcode!"); } + /// Get the number of extra operands for instructions that don't have a fixed + /// number of extra operands. + unsigned getNumSubclassExtraOperandsDynamic() const; + public: using Instruction::getContext; static bool classof(const Instruction *I) { return I->getOpcode() == Instruction::Call || - I->getOpcode() == Instruction::Invoke; + I->getOpcode() == Instruction::Invoke || + I->getOpcode() == Instruction::CallBr; } static bool classof(const Value *V) { return isa(V) && classof(cast(V)); @@ -1096,6 +1189,19 @@ public: return isDataOperand(&UI.getUse()); } + /// Given a value use iterator, return the data operand corresponding to it. + /// Iterator must actually correspond to a data operand. + unsigned getDataOperandNo(Value::const_user_iterator UI) const { + return getDataOperandNo(&UI.getUse()); + } + + /// Given a use for a data operand, get the data operand number that + /// corresponds to it. + unsigned getDataOperandNo(const Use *U) const { + assert(isDataOperand(U) && "Data operand # out of range!"); + return U - data_operands_begin(); + } + /// Return the iterator pointing to the beginning of the argument list. User::op_iterator arg_begin() { return op_begin(); } User::const_op_iterator arg_begin() const { @@ -1199,6 +1305,13 @@ public: return const_cast(this)->getCaller(); } + /// Tests if this call site must be tail call optimized. Only a CallInst can + /// be tail call optimized. + bool isMustTailCall() const; + + /// Tests if this call site is marked as a tail call. + bool isTailCall() const; + /// Returns the intrinsic ID of the intrinsic called or /// Intrinsic::not_intrinsic if the called function is not an intrinsic, or if /// this is an indirect call. @@ -1207,10 +1320,13 @@ public: void setCalledOperand(Value *V) { Op() = V; } /// Sets the function called, including updating the function type. - void setCalledFunction(Value *Fn) { - setCalledFunction( - cast(cast(Fn->getType())->getElementType()), - Fn); + void setCalledFunction(Function *Fn) { + setCalledFunction(Fn->getFunctionType(), Fn); + } + + /// Sets the function called, including updating the function type. + void setCalledFunction(FunctionCallee Fn) { + setCalledFunction(Fn.getFunctionType(), Fn.getCallee()); } /// Sets the function called, including updating to the specified function @@ -1219,6 +1335,9 @@ public: this->FTy = FTy; assert(FTy == cast( cast(Fn->getType())->getElementType())); + // This function doesn't mutate the return type, only the function + // type. Seems broken, but I'm just gonna stick an assert in for now. + assert(getType() == FTy->getReturnType()); setCalledOperand(Fn); } @@ -1233,6 +1352,9 @@ public: (ID << 2)); } + /// Check if this call is an inline asm statement. + bool isInlineAsm() const { return isa(getCalledOperand()); } + /// \name Attribute API /// /// These methods access and modify attributes on this call (including @@ -1452,6 +1574,12 @@ public: return Attrs.getParamAlignment(ArgNo); } + /// Extract the byval type for a call or parameter. + Type *getParamByValType(unsigned ArgNo) const { + Type *Ty = Attrs.getParamByValType(ArgNo); + return Ty ? Ty : getArgOperand(ArgNo)->getType()->getPointerElementType(); + } + /// Extract the number of dereferenceable bytes for a call or /// parameter (0=unknown). uint64_t getDereferenceableBytes(unsigned i) const { diff --git a/include/llvm/IR/Instruction.def b/include/llvm/IR/Instruction.def index 58e4e2e1d6cc..41cdf613ad64 100644 --- a/include/llvm/IR/Instruction.def +++ b/include/llvm/IR/Instruction.def @@ -1,9 +1,8 @@ //===-- llvm/Instruction.def - File that describes Instructions -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -135,89 +134,90 @@ HANDLE_TERM_INST ( 7, Unreachable , UnreachableInst) HANDLE_TERM_INST ( 8, CleanupRet , CleanupReturnInst) HANDLE_TERM_INST ( 9, CatchRet , CatchReturnInst) HANDLE_TERM_INST (10, CatchSwitch , CatchSwitchInst) - LAST_TERM_INST (10) +HANDLE_TERM_INST (11, CallBr , CallBrInst) // A call-site terminator + LAST_TERM_INST (11) // Standard unary operators... - FIRST_UNARY_INST(11) -HANDLE_UNARY_INST(11, FNeg , UnaryOperator) - LAST_UNARY_INST(11) + FIRST_UNARY_INST(12) +HANDLE_UNARY_INST(12, FNeg , UnaryOperator) + LAST_UNARY_INST(12) // Standard binary operators... - FIRST_BINARY_INST(12) -HANDLE_BINARY_INST(12, Add , BinaryOperator) -HANDLE_BINARY_INST(13, FAdd , BinaryOperator) -HANDLE_BINARY_INST(14, Sub , BinaryOperator) -HANDLE_BINARY_INST(15, FSub , BinaryOperator) -HANDLE_BINARY_INST(16, Mul , BinaryOperator) -HANDLE_BINARY_INST(17, FMul , BinaryOperator) -HANDLE_BINARY_INST(18, UDiv , BinaryOperator) -HANDLE_BINARY_INST(19, SDiv , BinaryOperator) -HANDLE_BINARY_INST(20, FDiv , BinaryOperator) -HANDLE_BINARY_INST(21, URem , BinaryOperator) -HANDLE_BINARY_INST(22, SRem , BinaryOperator) -HANDLE_BINARY_INST(23, FRem , BinaryOperator) + FIRST_BINARY_INST(13) +HANDLE_BINARY_INST(13, Add , BinaryOperator) +HANDLE_BINARY_INST(14, FAdd , BinaryOperator) +HANDLE_BINARY_INST(15, Sub , BinaryOperator) +HANDLE_BINARY_INST(16, FSub , BinaryOperator) +HANDLE_BINARY_INST(17, Mul , BinaryOperator) +HANDLE_BINARY_INST(18, FMul , BinaryOperator) +HANDLE_BINARY_INST(19, UDiv , BinaryOperator) +HANDLE_BINARY_INST(20, SDiv , BinaryOperator) +HANDLE_BINARY_INST(21, FDiv , BinaryOperator) +HANDLE_BINARY_INST(22, URem , BinaryOperator) +HANDLE_BINARY_INST(23, SRem , BinaryOperator) +HANDLE_BINARY_INST(24, FRem , BinaryOperator) // Logical operators (integer operands) -HANDLE_BINARY_INST(24, Shl , BinaryOperator) // Shift left (logical) -HANDLE_BINARY_INST(25, LShr , BinaryOperator) // Shift right (logical) -HANDLE_BINARY_INST(26, AShr , BinaryOperator) // Shift right (arithmetic) -HANDLE_BINARY_INST(27, And , BinaryOperator) -HANDLE_BINARY_INST(28, Or , BinaryOperator) -HANDLE_BINARY_INST(29, Xor , BinaryOperator) - LAST_BINARY_INST(29) +HANDLE_BINARY_INST(25, Shl , BinaryOperator) // Shift left (logical) +HANDLE_BINARY_INST(26, LShr , BinaryOperator) // Shift right (logical) +HANDLE_BINARY_INST(27, AShr , BinaryOperator) // Shift right (arithmetic) +HANDLE_BINARY_INST(28, And , BinaryOperator) +HANDLE_BINARY_INST(29, Or , BinaryOperator) +HANDLE_BINARY_INST(30, Xor , BinaryOperator) + LAST_BINARY_INST(30) // Memory operators... - FIRST_MEMORY_INST(30) -HANDLE_MEMORY_INST(30, Alloca, AllocaInst) // Stack management -HANDLE_MEMORY_INST(31, Load , LoadInst ) // Memory manipulation instrs -HANDLE_MEMORY_INST(32, Store , StoreInst ) -HANDLE_MEMORY_INST(33, GetElementPtr, GetElementPtrInst) -HANDLE_MEMORY_INST(34, Fence , FenceInst ) -HANDLE_MEMORY_INST(35, AtomicCmpXchg , AtomicCmpXchgInst ) -HANDLE_MEMORY_INST(36, AtomicRMW , AtomicRMWInst ) - LAST_MEMORY_INST(36) + FIRST_MEMORY_INST(31) +HANDLE_MEMORY_INST(31, Alloca, AllocaInst) // Stack management +HANDLE_MEMORY_INST(32, Load , LoadInst ) // Memory manipulation instrs +HANDLE_MEMORY_INST(33, Store , StoreInst ) +HANDLE_MEMORY_INST(34, GetElementPtr, GetElementPtrInst) +HANDLE_MEMORY_INST(35, Fence , FenceInst ) +HANDLE_MEMORY_INST(36, AtomicCmpXchg , AtomicCmpXchgInst ) +HANDLE_MEMORY_INST(37, AtomicRMW , AtomicRMWInst ) + LAST_MEMORY_INST(37) // Cast operators ... // NOTE: The order matters here because CastInst::isEliminableCastPair // NOTE: (see Instructions.cpp) encodes a table based on this ordering. - FIRST_CAST_INST(37) -HANDLE_CAST_INST(37, Trunc , TruncInst ) // Truncate integers -HANDLE_CAST_INST(38, ZExt , ZExtInst ) // Zero extend integers -HANDLE_CAST_INST(39, SExt , SExtInst ) // Sign extend integers -HANDLE_CAST_INST(40, FPToUI , FPToUIInst ) // floating point -> UInt -HANDLE_CAST_INST(41, FPToSI , FPToSIInst ) // floating point -> SInt -HANDLE_CAST_INST(42, UIToFP , UIToFPInst ) // UInt -> floating point -HANDLE_CAST_INST(43, SIToFP , SIToFPInst ) // SInt -> floating point -HANDLE_CAST_INST(44, FPTrunc , FPTruncInst ) // Truncate floating point -HANDLE_CAST_INST(45, FPExt , FPExtInst ) // Extend floating point -HANDLE_CAST_INST(46, PtrToInt, PtrToIntInst) // Pointer -> Integer -HANDLE_CAST_INST(47, IntToPtr, IntToPtrInst) // Integer -> Pointer -HANDLE_CAST_INST(48, BitCast , BitCastInst ) // Type cast -HANDLE_CAST_INST(49, AddrSpaceCast, AddrSpaceCastInst) // addrspace cast - LAST_CAST_INST(49) - - FIRST_FUNCLETPAD_INST(50) -HANDLE_FUNCLETPAD_INST(50, CleanupPad, CleanupPadInst) -HANDLE_FUNCLETPAD_INST(51, CatchPad , CatchPadInst) - LAST_FUNCLETPAD_INST(51) + FIRST_CAST_INST(38) +HANDLE_CAST_INST(38, Trunc , TruncInst ) // Truncate integers +HANDLE_CAST_INST(39, ZExt , ZExtInst ) // Zero extend integers +HANDLE_CAST_INST(40, SExt , SExtInst ) // Sign extend integers +HANDLE_CAST_INST(41, FPToUI , FPToUIInst ) // floating point -> UInt +HANDLE_CAST_INST(42, FPToSI , FPToSIInst ) // floating point -> SInt +HANDLE_CAST_INST(43, UIToFP , UIToFPInst ) // UInt -> floating point +HANDLE_CAST_INST(44, SIToFP , SIToFPInst ) // SInt -> floating point +HANDLE_CAST_INST(45, FPTrunc , FPTruncInst ) // Truncate floating point +HANDLE_CAST_INST(46, FPExt , FPExtInst ) // Extend floating point +HANDLE_CAST_INST(47, PtrToInt, PtrToIntInst) // Pointer -> Integer +HANDLE_CAST_INST(48, IntToPtr, IntToPtrInst) // Integer -> Pointer +HANDLE_CAST_INST(49, BitCast , BitCastInst ) // Type cast +HANDLE_CAST_INST(50, AddrSpaceCast, AddrSpaceCastInst) // addrspace cast + LAST_CAST_INST(50) + + FIRST_FUNCLETPAD_INST(51) +HANDLE_FUNCLETPAD_INST(51, CleanupPad, CleanupPadInst) +HANDLE_FUNCLETPAD_INST(52, CatchPad , CatchPadInst) + LAST_FUNCLETPAD_INST(52) // Other operators... - FIRST_OTHER_INST(52) -HANDLE_OTHER_INST(52, ICmp , ICmpInst ) // Integer comparison instruction -HANDLE_OTHER_INST(53, FCmp , FCmpInst ) // Floating point comparison instr. -HANDLE_OTHER_INST(54, PHI , PHINode ) // PHI node instruction -HANDLE_OTHER_INST(55, Call , CallInst ) // Call a function -HANDLE_OTHER_INST(56, Select , SelectInst ) // select instruction -HANDLE_USER_INST (57, UserOp1, Instruction) // May be used internally in a pass -HANDLE_USER_INST (58, UserOp2, Instruction) // Internal to passes only -HANDLE_OTHER_INST(59, VAArg , VAArgInst ) // vaarg instruction -HANDLE_OTHER_INST(60, ExtractElement, ExtractElementInst)// extract from vector -HANDLE_OTHER_INST(61, InsertElement, InsertElementInst) // insert into vector -HANDLE_OTHER_INST(62, ShuffleVector, ShuffleVectorInst) // shuffle two vectors. -HANDLE_OTHER_INST(63, ExtractValue, ExtractValueInst)// extract from aggregate -HANDLE_OTHER_INST(64, InsertValue, InsertValueInst) // insert into aggregate -HANDLE_OTHER_INST(65, LandingPad, LandingPadInst) // Landing pad instruction. - LAST_OTHER_INST(65) + FIRST_OTHER_INST(53) +HANDLE_OTHER_INST(53, ICmp , ICmpInst ) // Integer comparison instruction +HANDLE_OTHER_INST(54, FCmp , FCmpInst ) // Floating point comparison instr. +HANDLE_OTHER_INST(55, PHI , PHINode ) // PHI node instruction +HANDLE_OTHER_INST(56, Call , CallInst ) // Call a function +HANDLE_OTHER_INST(57, Select , SelectInst ) // select instruction +HANDLE_USER_INST (58, UserOp1, Instruction) // May be used internally in a pass +HANDLE_USER_INST (59, UserOp2, Instruction) // Internal to passes only +HANDLE_OTHER_INST(60, VAArg , VAArgInst ) // vaarg instruction +HANDLE_OTHER_INST(61, ExtractElement, ExtractElementInst)// extract from vector +HANDLE_OTHER_INST(62, InsertElement, InsertElementInst) // insert into vector +HANDLE_OTHER_INST(63, ShuffleVector, ShuffleVectorInst) // shuffle two vectors. +HANDLE_OTHER_INST(64, ExtractValue, ExtractValueInst)// extract from aggregate +HANDLE_OTHER_INST(65, InsertValue, InsertValueInst) // insert into aggregate +HANDLE_OTHER_INST(66, LandingPad, LandingPadInst) // Landing pad instruction. + LAST_OTHER_INST(66) #undef FIRST_TERM_INST #undef HANDLE_TERM_INST diff --git a/include/llvm/IR/Instruction.h b/include/llvm/IR/Instruction.h index 5e78cb1edf02..6a9a74bd16f0 100644 --- a/include/llvm/IR/Instruction.h +++ b/include/llvm/IR/Instruction.h @@ -1,9 +1,8 @@ //===-- llvm/Instruction.h - Instruction class definition -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -136,6 +135,9 @@ public: bool isExceptionalTerminator() const { return isExceptionalTerminator(getOpcode()); } + bool isIndirectTerminator() const { + return isIndirectTerminator(getOpcode()); + } static const char* getOpcodeName(unsigned OpCode); @@ -203,6 +205,17 @@ public: } } + /// Returns true if the OpCode is a terminator with indirect targets. + static inline bool isIndirectTerminator(unsigned OpCode) { + switch (OpCode) { + case Instruction::IndirectBr: + case Instruction::CallBr: + return true; + default: + return false; + } + } + //===--------------------------------------------------------------------===// // Metadata manipulation. //===--------------------------------------------------------------------===// @@ -298,9 +311,6 @@ public: /// Returns false if no metadata was found. bool extractProfTotalWeight(uint64_t &TotalVal) const; - /// Updates branch_weights metadata by scaling it by \p S / \p T. - void updateProfWeight(uint64_t S, uint64_t T); - /// Sets the branch_weights metadata to \p W for CallInst. void setProfWeight(uint64_t W); @@ -655,6 +665,10 @@ public: /// instruction must be a terminator. void setSuccessor(unsigned Idx, BasicBlock *BB); + /// Replace specified successor OldBB to point at the provided block. + /// This instruction must be a terminator. + void replaceSuccessorWith(BasicBlock *OldBB, BasicBlock *NewBB); + /// Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const Value *V) { return V->getValueID() >= Value::InstructionVal; diff --git a/include/llvm/IR/Instructions.h b/include/llvm/IR/Instructions.h index 0ff8f56f213a..215ce45c7b75 100644 --- a/include/llvm/IR/Instructions.h +++ b/include/llvm/IR/Instructions.h @@ -1,9 +1,8 @@ //===- llvm/Instructions.h - Instruction subclass definitions ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -522,9 +521,11 @@ private: // AtomicCmpXchgInst Class //===----------------------------------------------------------------------===// -/// an instruction that atomically checks whether a +/// An instruction that atomically checks whether a /// specified value is in a memory location, and, if it is, stores a new value -/// there. Returns the value that was loaded. +/// there. The value returned by this instruction is a pair containing the +/// original value as first element, and an i1 indicating success (true) or +/// failure (false) as second element. /// class AtomicCmpXchgInst : public Instruction { void Init(Value *Ptr, Value *Cmp, Value *NewVal, @@ -725,8 +726,14 @@ public: /// *p = old getType()->getPointerAddressSpace(); } + bool isFloatingPointOperation() const { + return isFPOperation(getOperation()); + } + // Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const Instruction *I) { return I->getOpcode() == Instruction::AtomicRMW; @@ -1114,71 +1135,6 @@ GetElementPtrInst::GetElementPtrInst(Type *PointeeType, Value *Ptr, DEFINE_TRANSPARENT_OPERAND_ACCESSORS(GetElementPtrInst, Value) -//===----------------------------------------------------------------------===// -// UnaryOperator Class -//===----------------------------------------------------------------------===// - -/// a unary instruction -class UnaryOperator : public UnaryInstruction { - void AssertOK(); - -protected: - UnaryOperator(UnaryOps iType, Value *S, Type *Ty, - const Twine &Name, Instruction *InsertBefore); - UnaryOperator(UnaryOps iType, Value *S, Type *Ty, - const Twine &Name, BasicBlock *InsertAtEnd); - - // Note: Instruction needs to be a friend here to call cloneImpl. - friend class Instruction; - - UnaryOperator *cloneImpl() const; - -public: - - /// Construct a unary instruction, given the opcode and an operand. - /// Optionally (if InstBefore is specified) insert the instruction - /// into a BasicBlock right before the specified instruction. The specified - /// Instruction is allowed to be a dereferenced end iterator. - /// - static UnaryOperator *Create(UnaryOps Op, Value *S, - const Twine &Name = Twine(), - Instruction *InsertBefore = nullptr); - - /// Construct a unary instruction, given the opcode and an operand. - /// Also automatically insert this instruction to the end of the - /// BasicBlock specified. - /// - static UnaryOperator *Create(UnaryOps Op, Value *S, - const Twine &Name, - BasicBlock *InsertAtEnd); - - /// These methods just forward to Create, and are useful when you - /// statically know what type of instruction you're going to create. These - /// helpers just save some typing. -#define HANDLE_UNARY_INST(N, OPC, CLASS) \ - static UnaryInstruction *Create##OPC(Value *V, \ - const Twine &Name = "") {\ - return Create(Instruction::OPC, V, Name);\ - } -#include "llvm/IR/Instruction.def" -#define HANDLE_UNARY_INST(N, OPC, CLASS) \ - static UnaryInstruction *Create##OPC(Value *V, \ - const Twine &Name, BasicBlock *BB) {\ - return Create(Instruction::OPC, V, Name, BB);\ - } -#include "llvm/IR/Instruction.def" -#define HANDLE_UNARY_INST(N, OPC, CLASS) \ - static UnaryInstruction *Create##OPC(Value *V, \ - const Twine &Name, Instruction *I) {\ - return Create(Instruction::OPC, V, Name, I);\ - } -#include "llvm/IR/Instruction.def" - - UnaryOps getOpcode() const { - return static_cast(Instruction::getOpcode()); - } -}; - //===----------------------------------------------------------------------===// // ICmpInst Class //===----------------------------------------------------------------------===// @@ -1524,25 +1480,44 @@ public: CallInst(Ty, Func, Args, Bundles, NameStr, InsertAtEnd); } - static CallInst *Create(Function *Func, const Twine &NameStr = "", + static CallInst *Create(FunctionCallee Func, const Twine &NameStr = "", Instruction *InsertBefore = nullptr) { - return Create(Func->getFunctionType(), Func, NameStr, InsertBefore); + return Create(Func.getFunctionType(), Func.getCallee(), NameStr, + InsertBefore); } - static CallInst *Create(Function *Func, ArrayRef Args, + static CallInst *Create(FunctionCallee Func, ArrayRef Args, + ArrayRef Bundles = None, const Twine &NameStr = "", Instruction *InsertBefore = nullptr) { - return Create(Func->getFunctionType(), Func, Args, NameStr, InsertBefore); + return Create(Func.getFunctionType(), Func.getCallee(), Args, Bundles, + NameStr, InsertBefore); } - static CallInst *Create(Function *Func, const Twine &NameStr, + static CallInst *Create(FunctionCallee Func, ArrayRef Args, + const Twine &NameStr, + Instruction *InsertBefore = nullptr) { + return Create(Func.getFunctionType(), Func.getCallee(), Args, NameStr, + InsertBefore); + } + + static CallInst *Create(FunctionCallee Func, const Twine &NameStr, BasicBlock *InsertAtEnd) { - return Create(Func->getFunctionType(), Func, NameStr, InsertAtEnd); + return Create(Func.getFunctionType(), Func.getCallee(), NameStr, + InsertAtEnd); } - static CallInst *Create(Function *Func, ArrayRef Args, + static CallInst *Create(FunctionCallee Func, ArrayRef Args, const Twine &NameStr, BasicBlock *InsertAtEnd) { - return Create(Func->getFunctionType(), Func, Args, NameStr, InsertAtEnd); + return Create(Func.getFunctionType(), Func.getCallee(), Args, NameStr, + InsertAtEnd); + } + + static CallInst *Create(FunctionCallee Func, ArrayRef Args, + ArrayRef Bundles, + const Twine &NameStr, BasicBlock *InsertAtEnd) { + return Create(Func.getFunctionType(), Func.getCallee(), Args, Bundles, + NameStr, InsertAtEnd); } // Deprecated [opaque pointer types] @@ -1684,9 +1659,6 @@ public: addAttribute(AttributeList::FunctionIndex, Attribute::ReturnsTwice); } - /// Check if this call is an inline asm statement. - bool isInlineAsm() const { return isa(getCalledOperand()); } - // Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const Instruction *I) { return I->getOpcode() == Instruction::Call; @@ -1695,6 +1667,9 @@ public: return isa(V) && classof(cast(V)); } + /// Updates profile metadata by scaling it by \p S / \p T. + void updateProfWeight(uint64_t S, uint64_t T); + private: // Shadow Instruction::setInstructionSubclassData with a private forwarding // method so that subclasses cannot accidentally use it. @@ -2008,6 +1983,10 @@ public: return User::operator new(s, 3); } + /// Swap the first 2 operands and adjust the mask to preserve the semantics + /// of the instruction. + void commute(); + /// Return true if a shufflevector instruction can be /// formed with the specified operands. static bool isValidOperands(const Value *V1, const Value *V2, @@ -2696,6 +2675,14 @@ public: block_begin()[i] = BB; } + /// Replace every incoming basic block \p Old to basic block \p New. + void replaceIncomingBlockWith(const BasicBlock *Old, BasicBlock *New) { + assert(New && Old && "PHI node got a null basic block!"); + for (unsigned Op = 0, NumOps = getNumOperands(); Op != NumOps; ++Op) + if (getIncomingBlock(Op) == Old) + setIncomingBlock(Op, New); + } + /// Add an incoming value to the end of the PHI list /// void addIncoming(Value *V, BasicBlock *BB) { @@ -2739,6 +2726,19 @@ public: return getIncomingValue(Idx); } + /// Set every incoming value(s) for block \p BB to \p V. + void setIncomingValueForBlock(const BasicBlock *BB, Value *V) { + assert(BB && "PHI node got a null basic block!"); + bool Found = false; + for (unsigned Op = 0, NumOps = getNumOperands(); Op != NumOps; ++Op) + if (getIncomingBlock(Op) == BB) { + Found = true; + setIncomingValue(Op, V); + } + (void)Found; + assert(Found && "Invalid basic block argument to set!"); + } + /// If the specified PHI node always merges together the /// same value, return the value, otherwise return null. Value *hasConstantValue() const; @@ -3450,6 +3450,60 @@ public: } }; +/// A wrapper class to simplify modification of SwitchInst cases along with +/// their prof branch_weights metadata. +class SwitchInstProfUpdateWrapper { + SwitchInst &SI; + Optional > Weights = None; + + // Sticky invalid state is needed to safely ignore operations with prof data + // in cases where SwitchInstProfUpdateWrapper is created from SwitchInst + // with inconsistent prof data. TODO: once we fix all prof data + // inconsistencies we can turn invalid state to assertions. + enum { + Invalid, + Initialized, + Changed + } State = Invalid; + +protected: + static MDNode *getProfBranchWeightsMD(const SwitchInst &SI); + + MDNode *buildProfBranchWeightsMD(); + + void init(); + +public: + using CaseWeightOpt = Optional; + SwitchInst *operator->() { return &SI; } + SwitchInst &operator*() { return SI; } + operator SwitchInst *() { return &SI; } + + SwitchInstProfUpdateWrapper(SwitchInst &SI) : SI(SI) { init(); } + + ~SwitchInstProfUpdateWrapper() { + if (State == Changed) + SI.setMetadata(LLVMContext::MD_prof, buildProfBranchWeightsMD()); + } + + /// Delegate the call to the underlying SwitchInst::removeCase() and remove + /// correspondent branch weight. + SwitchInst::CaseIt removeCase(SwitchInst::CaseIt I); + + /// Delegate the call to the underlying SwitchInst::addCase() and set the + /// specified branch weight for the added case. + void addCase(ConstantInt *OnVal, BasicBlock *Dest, CaseWeightOpt W); + + /// Delegate the call to the underlying SwitchInst::eraseFromParent() and mark + /// this object to not touch the underlying SwitchInst in destructor. + SymbolTableList::iterator eraseFromParent(); + + void setSuccessorWeight(unsigned idx, CaseWeightOpt W); + CaseWeightOpt getSuccessorWeight(unsigned idx); + + static CaseWeightOpt getSuccessorWeight(const SwitchInst &SI, unsigned idx); +}; + template <> struct OperandTraits : public HungoffOperandTraits<2> { }; @@ -3688,36 +3742,36 @@ public: NameStr, InsertAtEnd); } - static InvokeInst *Create(Function *Func, BasicBlock *IfNormal, + static InvokeInst *Create(FunctionCallee Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef Args, const Twine &NameStr, Instruction *InsertBefore = nullptr) { - return Create(Func->getFunctionType(), Func, IfNormal, IfException, Args, - None, NameStr, InsertBefore); + return Create(Func.getFunctionType(), Func.getCallee(), IfNormal, + IfException, Args, None, NameStr, InsertBefore); } - static InvokeInst *Create(Function *Func, BasicBlock *IfNormal, + static InvokeInst *Create(FunctionCallee Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef Args, ArrayRef Bundles = None, const Twine &NameStr = "", Instruction *InsertBefore = nullptr) { - return Create(Func->getFunctionType(), Func, IfNormal, IfException, Args, - Bundles, NameStr, InsertBefore); + return Create(Func.getFunctionType(), Func.getCallee(), IfNormal, + IfException, Args, Bundles, NameStr, InsertBefore); } - static InvokeInst *Create(Function *Func, BasicBlock *IfNormal, + static InvokeInst *Create(FunctionCallee Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef Args, const Twine &NameStr, BasicBlock *InsertAtEnd) { - return Create(Func->getFunctionType(), Func, IfNormal, IfException, Args, - NameStr, InsertAtEnd); + return Create(Func.getFunctionType(), Func.getCallee(), IfNormal, + IfException, Args, NameStr, InsertAtEnd); } - static InvokeInst *Create(Function *Func, BasicBlock *IfNormal, + static InvokeInst *Create(FunctionCallee Func, BasicBlock *IfNormal, BasicBlock *IfException, ArrayRef Args, ArrayRef Bundles, const Twine &NameStr, BasicBlock *InsertAtEnd) { - return Create(Func->getFunctionType(), Func, IfNormal, IfException, Args, - Bundles, NameStr, InsertAtEnd); + return Create(Func.getFunctionType(), Func.getCallee(), IfNormal, + IfException, Args, Bundles, NameStr, InsertAtEnd); } // Deprecated [opaque pointer types] @@ -3851,6 +3905,249 @@ InvokeInst::InvokeInst(FunctionType *Ty, Value *Func, BasicBlock *IfNormal, init(Ty, Func, IfNormal, IfException, Args, Bundles, NameStr); } +//===----------------------------------------------------------------------===// +// CallBrInst Class +//===----------------------------------------------------------------------===// + +/// CallBr instruction, tracking function calls that may not return control but +/// instead transfer it to a third location. The SubclassData field is used to +/// hold the calling convention of the call. +/// +class CallBrInst : public CallBase { + + unsigned NumIndirectDests; + + CallBrInst(const CallBrInst &BI); + + /// Construct a CallBrInst given a range of arguments. + /// + /// Construct a CallBrInst from a range of arguments + inline CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles, int NumOperands, + const Twine &NameStr, Instruction *InsertBefore); + + inline CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles, int NumOperands, + const Twine &NameStr, BasicBlock *InsertAtEnd); + + void init(FunctionType *FTy, Value *Func, BasicBlock *DefaultDest, + ArrayRef IndirectDests, ArrayRef Args, + ArrayRef Bundles, const Twine &NameStr); + + /// Compute the number of operands to allocate. + static int ComputeNumOperands(int NumArgs, int NumIndirectDests, + int NumBundleInputs = 0) { + // We need one operand for the called function, plus our extra operands and + // the input operand counts provided. + return 2 + NumIndirectDests + NumArgs + NumBundleInputs; + } + +protected: + // Note: Instruction needs to be a friend here to call cloneImpl. + friend class Instruction; + + CallBrInst *cloneImpl() const; + +public: + static CallBrInst *Create(FunctionType *Ty, Value *Func, + BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, const Twine &NameStr, + Instruction *InsertBefore = nullptr) { + int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size()); + return new (NumOperands) + CallBrInst(Ty, Func, DefaultDest, IndirectDests, Args, None, + NumOperands, NameStr, InsertBefore); + } + + static CallBrInst *Create(FunctionType *Ty, Value *Func, + BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles = None, + const Twine &NameStr = "", + Instruction *InsertBefore = nullptr) { + int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size(), + CountBundleInputs(Bundles)); + unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo); + + return new (NumOperands, DescriptorBytes) + CallBrInst(Ty, Func, DefaultDest, IndirectDests, Args, Bundles, + NumOperands, NameStr, InsertBefore); + } + + static CallBrInst *Create(FunctionType *Ty, Value *Func, + BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, const Twine &NameStr, + BasicBlock *InsertAtEnd) { + int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size()); + return new (NumOperands) + CallBrInst(Ty, Func, DefaultDest, IndirectDests, Args, None, + NumOperands, NameStr, InsertAtEnd); + } + + static CallBrInst *Create(FunctionType *Ty, Value *Func, + BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles, + const Twine &NameStr, BasicBlock *InsertAtEnd) { + int NumOperands = ComputeNumOperands(Args.size(), IndirectDests.size(), + CountBundleInputs(Bundles)); + unsigned DescriptorBytes = Bundles.size() * sizeof(BundleOpInfo); + + return new (NumOperands, DescriptorBytes) + CallBrInst(Ty, Func, DefaultDest, IndirectDests, Args, Bundles, + NumOperands, NameStr, InsertAtEnd); + } + + static CallBrInst *Create(FunctionCallee Func, BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, const Twine &NameStr, + Instruction *InsertBefore = nullptr) { + return Create(Func.getFunctionType(), Func.getCallee(), DefaultDest, + IndirectDests, Args, NameStr, InsertBefore); + } + + static CallBrInst *Create(FunctionCallee Func, BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles = None, + const Twine &NameStr = "", + Instruction *InsertBefore = nullptr) { + return Create(Func.getFunctionType(), Func.getCallee(), DefaultDest, + IndirectDests, Args, Bundles, NameStr, InsertBefore); + } + + static CallBrInst *Create(FunctionCallee Func, BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, const Twine &NameStr, + BasicBlock *InsertAtEnd) { + return Create(Func.getFunctionType(), Func.getCallee(), DefaultDest, + IndirectDests, Args, NameStr, InsertAtEnd); + } + + static CallBrInst *Create(FunctionCallee Func, + BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles, + const Twine &NameStr, BasicBlock *InsertAtEnd) { + return Create(Func.getFunctionType(), Func.getCallee(), DefaultDest, + IndirectDests, Args, Bundles, NameStr, InsertAtEnd); + } + + /// Create a clone of \p CBI with a different set of operand bundles and + /// insert it before \p InsertPt. + /// + /// The returned callbr instruction is identical to \p CBI in every way + /// except that the operand bundles for the new instruction are set to the + /// operand bundles in \p Bundles. + static CallBrInst *Create(CallBrInst *CBI, + ArrayRef Bundles, + Instruction *InsertPt = nullptr); + + /// Return the number of callbr indirect dest labels. + /// + unsigned getNumIndirectDests() const { return NumIndirectDests; } + + /// getIndirectDestLabel - Return the i-th indirect dest label. + /// + Value *getIndirectDestLabel(unsigned i) const { + assert(i < getNumIndirectDests() && "Out of bounds!"); + return getOperand(i + getNumArgOperands() + getNumTotalBundleOperands() + + 1); + } + + Value *getIndirectDestLabelUse(unsigned i) const { + assert(i < getNumIndirectDests() && "Out of bounds!"); + return getOperandUse(i + getNumArgOperands() + getNumTotalBundleOperands() + + 1); + } + + // Return the destination basic blocks... + BasicBlock *getDefaultDest() const { + return cast(*(&Op<-1>() - getNumIndirectDests() - 1)); + } + BasicBlock *getIndirectDest(unsigned i) const { + return cast(*(&Op<-1>() - getNumIndirectDests() + i)); + } + SmallVector getIndirectDests() const { + SmallVector IndirectDests; + for (unsigned i = 0, e = getNumIndirectDests(); i < e; ++i) + IndirectDests.push_back(getIndirectDest(i)); + return IndirectDests; + } + void setDefaultDest(BasicBlock *B) { + *(&Op<-1>() - getNumIndirectDests() - 1) = reinterpret_cast(B); + } + void setIndirectDest(unsigned i, BasicBlock *B) { + *(&Op<-1>() - getNumIndirectDests() + i) = reinterpret_cast(B); + } + + BasicBlock *getSuccessor(unsigned i) const { + assert(i < getNumSuccessors() + 1 && + "Successor # out of range for callbr!"); + return i == 0 ? getDefaultDest() : getIndirectDest(i - 1); + } + + void setSuccessor(unsigned idx, BasicBlock *NewSucc) { + assert(idx < getNumIndirectDests() + 1 && + "Successor # out of range for callbr!"); + *(&Op<-1>() - getNumIndirectDests() -1 + idx) = + reinterpret_cast(NewSucc); + } + + unsigned getNumSuccessors() const { return getNumIndirectDests() + 1; } + + // Methods for support type inquiry through isa, cast, and dyn_cast: + static bool classof(const Instruction *I) { + return (I->getOpcode() == Instruction::CallBr); + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } + +private: + + // Shadow Instruction::setInstructionSubclassData with a private forwarding + // method so that subclasses cannot accidentally use it. + void setInstructionSubclassData(unsigned short D) { + Instruction::setInstructionSubclassData(D); + } +}; + +CallBrInst::CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles, int NumOperands, + const Twine &NameStr, Instruction *InsertBefore) + : CallBase(Ty->getReturnType(), Instruction::CallBr, + OperandTraits::op_end(this) - NumOperands, NumOperands, + InsertBefore) { + init(Ty, Func, DefaultDest, IndirectDests, Args, Bundles, NameStr); +} + +CallBrInst::CallBrInst(FunctionType *Ty, Value *Func, BasicBlock *DefaultDest, + ArrayRef IndirectDests, + ArrayRef Args, + ArrayRef Bundles, int NumOperands, + const Twine &NameStr, BasicBlock *InsertAtEnd) + : CallBase( + cast( + cast(Func->getType())->getElementType()) + ->getReturnType(), + Instruction::CallBr, + OperandTraits::op_end(this) - NumOperands, NumOperands, + InsertAtEnd) { + init(Ty, Func, DefaultDest, IndirectDests, Args, Bundles, NameStr); +} + //===----------------------------------------------------------------------===// // ResumeInst Class //===----------------------------------------------------------------------===// diff --git a/include/llvm/IR/IntrinsicInst.h b/include/llvm/IR/IntrinsicInst.h index 80a7a7052574..438bdb29b706 100644 --- a/include/llvm/IR/IntrinsicInst.h +++ b/include/llvm/IR/IntrinsicInst.h @@ -1,9 +1,8 @@ //===-- llvm/IntrinsicInst.h - Intrinsic Instruction Wrappers ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -209,26 +208,47 @@ namespace llvm { /// This is the common base class for constrained floating point intrinsics. class ConstrainedFPIntrinsic : public IntrinsicInst { public: - enum RoundingMode { - rmInvalid, - rmDynamic, - rmToNearest, - rmDownward, - rmUpward, - rmTowardZero + /// Specifies the rounding mode to be assumed. This is only used when + /// when constrained floating point is enabled. See the LLVM Language + /// Reference Manual for details. + enum RoundingMode : uint8_t { + rmDynamic, ///< This corresponds to "fpround.dynamic". + rmToNearest, ///< This corresponds to "fpround.tonearest". + rmDownward, ///< This corresponds to "fpround.downward". + rmUpward, ///< This corresponds to "fpround.upward". + rmTowardZero ///< This corresponds to "fpround.tozero". }; - enum ExceptionBehavior { - ebInvalid, - ebIgnore, - ebMayTrap, - ebStrict + /// Specifies the required exception behavior. This is only used when + /// when constrained floating point is used. See the LLVM Language + /// Reference Manual for details. + enum ExceptionBehavior : uint8_t { + ebIgnore, ///< This corresponds to "fpexcept.ignore". + ebMayTrap, ///< This corresponds to "fpexcept.maytrap". + ebStrict ///< This corresponds to "fpexcept.strict". }; bool isUnaryOp() const; bool isTernaryOp() const; - RoundingMode getRoundingMode() const; - ExceptionBehavior getExceptionBehavior() const; + Optional getRoundingMode() const; + Optional getExceptionBehavior() const; + + /// Returns a valid RoundingMode enumerator when given a string + /// that is valid as input in constrained intrinsic rounding mode + /// metadata. + static Optional StrToRoundingMode(StringRef); + + /// For any RoundingMode enumerator, returns a string valid as input in + /// constrained intrinsic rounding mode metadata. + static Optional RoundingModeToStr(RoundingMode); + + /// Returns a valid ExceptionBehavior enumerator when given a string + /// valid as input in constrained intrinsic exception behavior metadata. + static Optional StrToExceptionBehavior(StringRef); + + /// For any ExceptionBehavior enumerator, returns a string valid as + /// input in constrained intrinsic exception behavior metadata. + static Optional ExceptionBehaviorToStr(ExceptionBehavior); // Methods for support type inquiry through isa, cast, and dyn_cast: static bool classof(const IntrinsicInst *I) { @@ -239,6 +259,8 @@ namespace llvm { case Intrinsic::experimental_constrained_fdiv: case Intrinsic::experimental_constrained_frem: case Intrinsic::experimental_constrained_fma: + case Intrinsic::experimental_constrained_fptrunc: + case Intrinsic::experimental_constrained_fpext: case Intrinsic::experimental_constrained_sqrt: case Intrinsic::experimental_constrained_pow: case Intrinsic::experimental_constrained_powi: @@ -266,6 +288,84 @@ namespace llvm { } }; + /// This class represents an intrinsic that is based on a binary operation. + /// This includes op.with.overflow and saturating add/sub intrinsics. + class BinaryOpIntrinsic : public IntrinsicInst { + public: + static bool classof(const IntrinsicInst *I) { + switch (I->getIntrinsicID()) { + case Intrinsic::uadd_with_overflow: + case Intrinsic::sadd_with_overflow: + case Intrinsic::usub_with_overflow: + case Intrinsic::ssub_with_overflow: + case Intrinsic::umul_with_overflow: + case Intrinsic::smul_with_overflow: + case Intrinsic::uadd_sat: + case Intrinsic::sadd_sat: + case Intrinsic::usub_sat: + case Intrinsic::ssub_sat: + return true; + default: + return false; + } + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } + + Value *getLHS() const { return const_cast(getArgOperand(0)); } + Value *getRHS() const { return const_cast(getArgOperand(1)); } + + /// Returns the binary operation underlying the intrinsic. + Instruction::BinaryOps getBinaryOp() const; + + /// Whether the intrinsic is signed or unsigned. + bool isSigned() const; + + /// Returns one of OBO::NoSignedWrap or OBO::NoUnsignedWrap. + unsigned getNoWrapKind() const; + }; + + /// Represents an op.with.overflow intrinsic. + class WithOverflowInst : public BinaryOpIntrinsic { + public: + static bool classof(const IntrinsicInst *I) { + switch (I->getIntrinsicID()) { + case Intrinsic::uadd_with_overflow: + case Intrinsic::sadd_with_overflow: + case Intrinsic::usub_with_overflow: + case Intrinsic::ssub_with_overflow: + case Intrinsic::umul_with_overflow: + case Intrinsic::smul_with_overflow: + return true; + default: + return false; + } + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } + }; + + /// Represents a saturating add/sub intrinsic. + class SaturatingInst : public BinaryOpIntrinsic { + public: + static bool classof(const IntrinsicInst *I) { + switch (I->getIntrinsicID()) { + case Intrinsic::uadd_sat: + case Intrinsic::sadd_sat: + case Intrinsic::usub_sat: + case Intrinsic::ssub_sat: + return true; + default: + return false; + } + } + static bool classof(const Value *V) { + return isa(V) && classof(cast(V)); + } + }; + /// Common base class for all memory intrinsics. Simply provides /// common methods. /// Written as CRTP to avoid a common base class amongst the diff --git a/include/llvm/IR/Intrinsics.h b/include/llvm/IR/Intrinsics.h index e1e17f983ff8..f38f92022d21 100644 --- a/include/llvm/IR/Intrinsics.h +++ b/include/llvm/IR/Intrinsics.h @@ -1,9 +1,8 @@ -//===-- llvm/Instrinsics.h - LLVM Intrinsic Function Handling ---*- C++ -*-===// +//===- Intrinsics.h - LLVM Intrinsic Function Handling ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -100,7 +99,8 @@ namespace Intrinsic { Void, VarArg, MMX, Token, Metadata, Half, Float, Double, Quad, Integer, Vector, Pointer, Struct, Argument, ExtendArgument, TruncArgument, HalfVecArgument, - SameVecWidthArgument, PtrToArgument, PtrToElt, VecOfAnyPtrsToElt + SameVecWidthArgument, PtrToArgument, PtrToElt, VecOfAnyPtrsToElt, + VecElementArgument } Kind; union { @@ -117,20 +117,22 @@ namespace Intrinsic { AK_AnyInteger, AK_AnyFloat, AK_AnyVector, - AK_AnyPointer + AK_AnyPointer, + AK_MatchType = 7 }; unsigned getArgumentNumber() const { assert(Kind == Argument || Kind == ExtendArgument || Kind == TruncArgument || Kind == HalfVecArgument || Kind == SameVecWidthArgument || Kind == PtrToArgument || - Kind == PtrToElt); + Kind == PtrToElt || Kind == VecElementArgument); return Argument_Info >> 3; } ArgKind getArgumentKind() const { assert(Kind == Argument || Kind == ExtendArgument || Kind == TruncArgument || Kind == HalfVecArgument || - Kind == SameVecWidthArgument || Kind == PtrToArgument); + Kind == SameVecWidthArgument || Kind == PtrToArgument || + Kind == VecElementArgument); return (ArgKind)(Argument_Info & 7); } @@ -162,14 +164,21 @@ namespace Intrinsic { /// of IITDescriptors. void getIntrinsicInfoTableEntries(ID id, SmallVectorImpl &T); - /// Match the specified type (which comes from an intrinsic argument or return - /// value) with the type constraints specified by the .td file. If the given - /// type is an overloaded type it is pushed to the ArgTys vector. + enum MatchIntrinsicTypesResult { + MatchIntrinsicTypes_Match = 0, + MatchIntrinsicTypes_NoMatchRet = 1, + MatchIntrinsicTypes_NoMatchArg = 2, + }; + + /// Match the specified function type with the type constraints specified by + /// the .td file. If the given type is an overloaded type it is pushed to the + /// ArgTys vector. /// /// Returns false if the given type matches with the constraints, true /// otherwise. - bool matchIntrinsicType(Type *Ty, ArrayRef &Infos, - SmallVectorImpl &ArgTys); + MatchIntrinsicTypesResult + matchIntrinsicSignature(FunctionType *FTy, ArrayRef &Infos, + SmallVectorImpl &ArgTys); /// Verify if the intrinsic has variable arguments. This method is intended to /// be called after all the fixed arguments have been matched first. diff --git a/include/llvm/IR/Intrinsics.td b/include/llvm/IR/Intrinsics.td index 64603d8ea030..d660f8278437 100644 --- a/include/llvm/IR/Intrinsics.td +++ b/include/llvm/IR/Intrinsics.td @@ -1,9 +1,8 @@ //===- Intrinsics.td - Defines all LLVM intrinsics ---------*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -70,6 +69,11 @@ class Returned : IntrinsicProperty { int ArgNo = argNo; } +// ImmArg - The specified argument must be an immediate. +class ImmArg : IntrinsicProperty { + int ArgNo = argNo; +} + // ReadOnly - The specified argument pointer is not written to through the // pointer by the intrinsic. class ReadOnly : IntrinsicProperty { @@ -90,6 +94,8 @@ class ReadNone : IntrinsicProperty { def IntrNoReturn : IntrinsicProperty; +def IntrWillReturn : IntrinsicProperty; + // IntrCold - Calls to this intrinsic are cold. // Parallels the cold attribute on LLVM IR functions. def IntrCold : IntrinsicProperty; @@ -157,13 +163,19 @@ class LLVMMatchType // the intrinsic is overloaded, so the matched type should be declared as iAny. class LLVMExtendedType : LLVMMatchType; class LLVMTruncatedType : LLVMMatchType; -class LLVMVectorSameWidth - : LLVMMatchType { + +// Match the scalar/vector of another intrinsic parameter but with a different +// element type. Either both are scalars or both are vectors with the same +// number of elements. +class LLVMScalarOrSameVectorWidth + : LLVMMatchType { ValueType ElTy = elty.VT; } + class LLVMPointerTo : LLVMMatchType; class LLVMPointerToElt : LLVMMatchType; class LLVMVectorOfAnyPointersToElt : LLVMMatchType; +class LLVMVectorElementType : LLVMMatchType; // Match the type of another intrinsic parameter that is expected to be a // vector type, but change the element count to be half as many @@ -251,6 +263,7 @@ def llvm_v2f32_ty : LLVMType; // 2 x float def llvm_v4f32_ty : LLVMType; // 4 x float def llvm_v8f32_ty : LLVMType; // 8 x float def llvm_v16f32_ty : LLVMType; // 16 x float +def llvm_v32f32_ty : LLVMType; // 32 x float def llvm_v1f64_ty : LLVMType; // 1 x double def llvm_v2f64_ty : LLVMType; // 2 x double def llvm_v4f64_ty : LLVMType; // 4 x double @@ -393,9 +406,9 @@ def int_objc_arc_annotation_bottomup_bbend : Intrinsic<[], //===--------------------- Code Generator Intrinsics ----------------------===// // -def int_returnaddress : Intrinsic<[llvm_ptr_ty], [llvm_i32_ty], [IntrNoMem]>; +def int_returnaddress : Intrinsic<[llvm_ptr_ty], [llvm_i32_ty], [IntrNoMem, ImmArg<0>]>; def int_addressofreturnaddress : Intrinsic<[llvm_ptr_ty], [], [IntrNoMem]>; -def int_frameaddress : Intrinsic<[llvm_ptr_ty], [llvm_i32_ty], [IntrNoMem]>; +def int_frameaddress : Intrinsic<[llvm_ptr_ty], [llvm_i32_ty], [IntrNoMem, ImmArg<0>]>; def int_sponentry : Intrinsic<[llvm_ptr_ty], [], [IntrNoMem]>; def int_read_register : Intrinsic<[llvm_anyint_ty], [llvm_metadata_ty], [IntrReadMem], "llvm.read_register">; @@ -413,7 +426,7 @@ def int_localescape : Intrinsic<[], [llvm_vararg_ty]>; // to an escaped allocation indicated by the index. def int_localrecover : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_ptr_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; // Given the frame pointer passed into an SEH filter function, returns a // pointer to the local variable area suitable for use with llvm.localrecover. @@ -439,7 +452,8 @@ def int_thread_pointer : Intrinsic<[llvm_ptr_ty], [], [IntrNoMem]>, // memory while not impeding optimization. def int_prefetch : Intrinsic<[], [ llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty ], - [ IntrInaccessibleMemOrArgMemOnly, ReadOnly<0>, NoCapture<0> ]>; + [ IntrInaccessibleMemOrArgMemOnly, ReadOnly<0>, NoCapture<0>, + ImmArg<1>, ImmArg<2>]>; def int_pcmarker : Intrinsic<[], [llvm_i32_ty]>; def int_readcyclecounter : Intrinsic<[llvm_i64_ty]>; @@ -480,16 +494,17 @@ def int_memcpy : Intrinsic<[], [llvm_anyptr_ty, llvm_anyptr_ty, llvm_anyint_ty, llvm_i1_ty], [IntrArgMemOnly, NoCapture<0>, NoCapture<1>, - WriteOnly<0>, ReadOnly<1>]>; + WriteOnly<0>, ReadOnly<1>, ImmArg<3>]>; def int_memmove : Intrinsic<[], [llvm_anyptr_ty, llvm_anyptr_ty, llvm_anyint_ty, llvm_i1_ty], [IntrArgMemOnly, NoCapture<0>, NoCapture<1>, - ReadOnly<1>]>; + ReadOnly<1>, ImmArg<3>]>; def int_memset : Intrinsic<[], [llvm_anyptr_ty, llvm_i8_ty, llvm_anyint_ty, llvm_i1_ty], - [IntrArgMemOnly, NoCapture<0>, WriteOnly<0>]>; + [IntrArgMemOnly, NoCapture<0>, WriteOnly<0>, + ImmArg<3>]>; // FIXME: Add version of these floating point intrinsics which allow non-default // rounding modes and FP exception handling. @@ -527,6 +542,11 @@ let IntrProperties = [IntrNoMem, IntrSpeculatable] in { def int_round : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>]>; def int_canonicalize : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>; + + def int_lround : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty]>; + def int_llround : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty]>; + def int_lrint : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty]>; + def int_llrint : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty]>; } def int_minnum : Intrinsic<[llvm_anyfloat_ty], @@ -554,8 +574,9 @@ def int_siglongjmp : Intrinsic<[], [llvm_ptr_ty, llvm_i32_ty], [IntrNoReturn]>; // Internal interface for object size checking def int_objectsize : Intrinsic<[llvm_anyint_ty], - [llvm_anyptr_ty, llvm_i1_ty, llvm_i1_ty], - [IntrNoMem, IntrSpeculatable]>, + [llvm_anyptr_ty, llvm_i1_ty, + llvm_i1_ty, llvm_i1_ty], + [IntrNoMem, IntrSpeculatable, ImmArg<1>, ImmArg<2>, ImmArg<3>]>, GCCBuiltin<"__builtin_object_size">; //===--------------- Constrained Floating Point Intrinsics ----------------===// @@ -595,6 +616,15 @@ let IntrProperties = [IntrInaccessibleMemOnly] in { llvm_metadata_ty, llvm_metadata_ty ]>; + def int_experimental_constrained_fptrunc : Intrinsic<[ llvm_anyfloat_ty ], + [ llvm_anyfloat_ty, + llvm_metadata_ty, + llvm_metadata_ty ]>; + + def int_experimental_constrained_fpext : Intrinsic<[ llvm_anyfloat_ty ], + [ llvm_anyfloat_ty, + llvm_metadata_ty ]>; + // These intrinsics are sensitive to the rounding mode so we need constrained // versions of each of them. When strict rounding and exception control are // not required the non-constrained versions of these intrinsics should be @@ -676,14 +706,12 @@ let IntrProperties = [IntrInaccessibleMemOnly] in { llvm_metadata_ty, llvm_metadata_ty ]>; } -// FIXME: Add intrinsics for fcmp, fptrunc, fpext, fptoui and fptosi. -// FIXME: Add intrinsics for fabs and copysign? - +// FIXME: Add intrinsics for fcmp, fptoui and fptosi. //===------------------------- Expect Intrinsics --------------------------===// // -def int_expect : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, - LLVMMatchType<0>], [IntrNoMem]>; +def int_expect : Intrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem]>; //===-------------------- Bit Manipulation Intrinsics ---------------------===// // @@ -692,8 +720,6 @@ def int_expect : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, let IntrProperties = [IntrNoMem, IntrSpeculatable] in { def int_bswap: Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>]>; def int_ctpop: Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>]>; - def int_ctlz : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, llvm_i1_ty]>; - def int_cttz : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, llvm_i1_ty]>; def int_bitreverse : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>]>; def int_fshl : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>; @@ -701,6 +727,11 @@ let IntrProperties = [IntrNoMem, IntrSpeculatable] in { [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>]>; } +let IntrProperties = [IntrNoMem, IntrSpeculatable, ImmArg<1>] in { + def int_ctlz : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, llvm_i1_ty]>; + def int_cttz : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, llvm_i1_ty]>; +} + //===------------------------ Debugger Intrinsics -------------------------===// // @@ -797,24 +828,30 @@ def int_adjust_trampoline : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty], // // Expose the carry flag from add operations on two integrals. -def int_sadd_with_overflow : Intrinsic<[llvm_anyint_ty, llvm_i1_ty], +def int_sadd_with_overflow : Intrinsic<[llvm_anyint_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable]>; -def int_uadd_with_overflow : Intrinsic<[llvm_anyint_ty, llvm_i1_ty], +def int_uadd_with_overflow : Intrinsic<[llvm_anyint_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable]>; -def int_ssub_with_overflow : Intrinsic<[llvm_anyint_ty, llvm_i1_ty], +def int_ssub_with_overflow : Intrinsic<[llvm_anyint_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable]>; -def int_usub_with_overflow : Intrinsic<[llvm_anyint_ty, llvm_i1_ty], +def int_usub_with_overflow : Intrinsic<[llvm_anyint_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable]>; -def int_smul_with_overflow : Intrinsic<[llvm_anyint_ty, llvm_i1_ty], +def int_smul_with_overflow : Intrinsic<[llvm_anyint_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable]>; -def int_umul_with_overflow : Intrinsic<[llvm_anyint_ty, llvm_i1_ty], +def int_umul_with_overflow : Intrinsic<[llvm_anyint_ty, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable]>; @@ -837,23 +874,33 @@ def int_usub_sat : Intrinsic<[llvm_anyint_ty], // def int_smul_fix : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty], - [IntrNoMem, IntrSpeculatable, Commutative]>; + [IntrNoMem, IntrSpeculatable, Commutative, ImmArg<2>]>; + +def int_umul_fix : Intrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty], + [IntrNoMem, IntrSpeculatable, Commutative, ImmArg<2>]>; + +//===------------------- Fixed Point Saturation Arithmetic Intrinsics ----------------===// +// +def int_smul_fix_sat : Intrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty], + [IntrNoMem, IntrSpeculatable, Commutative, ImmArg<2>]>; //===------------------------- Memory Use Markers -------------------------===// // def int_lifetime_start : Intrinsic<[], [llvm_i64_ty, llvm_anyptr_ty], - [IntrArgMemOnly, NoCapture<1>]>; + [IntrArgMemOnly, NoCapture<1>, ImmArg<0>]>; def int_lifetime_end : Intrinsic<[], [llvm_i64_ty, llvm_anyptr_ty], - [IntrArgMemOnly, NoCapture<1>]>; + [IntrArgMemOnly, NoCapture<1>, ImmArg<0>]>; def int_invariant_start : Intrinsic<[llvm_descriptor_ty], [llvm_i64_ty, llvm_anyptr_ty], - [IntrArgMemOnly, NoCapture<1>]>; + [IntrArgMemOnly, NoCapture<1>, ImmArg<0>]>; def int_invariant_end : Intrinsic<[], [llvm_descriptor_ty, llvm_i64_ty, llvm_anyptr_ty], - [IntrArgMemOnly, NoCapture<2>]>; + [IntrArgMemOnly, NoCapture<2>, ImmArg<1>]>; // launder.invariant.group can't be marked with 'readnone' (IntrNoMem), // because it would cause CSE of two barriers with the same argument. @@ -900,13 +947,13 @@ def int_experimental_gc_statepoint : Intrinsic<[llvm_token_ty], [llvm_i64_ty, llvm_i32_ty, llvm_anyptr_ty, llvm_i32_ty, llvm_i32_ty, llvm_vararg_ty], - [Throws]>; + [Throws, ImmArg<0>, ImmArg<1>, ImmArg<3>, ImmArg<4>]>; def int_experimental_gc_result : Intrinsic<[llvm_any_ty], [llvm_token_ty], [IntrReadMem]>; def int_experimental_gc_relocate : Intrinsic<[llvm_any_ty], [llvm_token_ty, llvm_i32_ty, llvm_i32_ty], - [IntrReadMem]>; + [IntrReadMem, ImmArg<1>, ImmArg<2>]>; //===------------------------ Coroutine Intrinsics ---------------===// // These are documented in docs/Coroutines.rst @@ -996,41 +1043,41 @@ def int_clear_cache : Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty], // Intrinsic to detect whether its argument is a constant. def int_is_constant : Intrinsic<[llvm_i1_ty], [llvm_any_ty], [IntrNoMem], "llvm.is.constant">; - //===-------------------------- Masked Intrinsics -------------------------===// // def int_masked_store : Intrinsic<[], [llvm_anyvector_ty, LLVMAnyPointerType>, llvm_i32_ty, - LLVMVectorSameWidth<0, llvm_i1_ty>], - [IntrArgMemOnly]>; + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], + [IntrArgMemOnly, ImmArg<2>]>; def int_masked_load : Intrinsic<[llvm_anyvector_ty], [LLVMAnyPointerType>, llvm_i32_ty, - LLVMVectorSameWidth<0, llvm_i1_ty>, LLVMMatchType<0>], - [IntrReadMem, IntrArgMemOnly]>; + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMMatchType<0>], + [IntrReadMem, IntrArgMemOnly, ImmArg<1>]>; def int_masked_gather: Intrinsic<[llvm_anyvector_ty], [LLVMVectorOfAnyPointersToElt<0>, llvm_i32_ty, - LLVMVectorSameWidth<0, llvm_i1_ty>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMMatchType<0>], - [IntrReadMem]>; + [IntrReadMem, ImmArg<1>]>; def int_masked_scatter: Intrinsic<[], [llvm_anyvector_ty, LLVMVectorOfAnyPointersToElt<0>, llvm_i32_ty, - LLVMVectorSameWidth<0, llvm_i1_ty>]>; + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], + [ImmArg<2>]>; def int_masked_expandload: Intrinsic<[llvm_anyvector_ty], [LLVMPointerToElt<0>, - LLVMVectorSameWidth<0, llvm_i1_ty>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMMatchType<0>], [IntrReadMem]>; def int_masked_compressstore: Intrinsic<[], [llvm_anyvector_ty, LLVMPointerToElt<0>, - LLVMVectorSameWidth<0, llvm_i1_ty>], + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], [IntrArgMemOnly]>; // Test whether a pointer is associated with a type metadata identifier. @@ -1049,6 +1096,9 @@ def int_icall_branch_funnel : Intrinsic<[], [llvm_vararg_ty], []>; def int_load_relative: Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_anyint_ty], [IntrReadMem, IntrArgMemOnly]>; +def int_hwasan_check_memaccess : + Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty, llvm_i32_ty], [IntrInaccessibleMemOnly, ImmArg<2>]>; + // Xray intrinsics //===----------------------------------------------------------------------===// // Custom event logging for x-ray. @@ -1072,7 +1122,7 @@ def int_memcpy_element_unordered_atomic ], [ IntrArgMemOnly, NoCapture<0>, NoCapture<1>, WriteOnly<0>, - ReadOnly<1> + ReadOnly<1>, ImmArg<3> ]>; // @llvm.memmove.element.unordered.atomic.*(dest, src, length, elementsize) @@ -1083,62 +1133,105 @@ def int_memmove_element_unordered_atomic ], [ IntrArgMemOnly, NoCapture<0>, NoCapture<1>, WriteOnly<0>, - ReadOnly<1> + ReadOnly<1>, ImmArg<3> ]>; // @llvm.memset.element.unordered.atomic.*(dest, value, length, elementsize) def int_memset_element_unordered_atomic : Intrinsic<[], [ llvm_anyptr_ty, llvm_i8_ty, llvm_anyint_ty, llvm_i32_ty ], - [ IntrArgMemOnly, NoCapture<0>, WriteOnly<0> ]>; + [ IntrArgMemOnly, NoCapture<0>, WriteOnly<0>, ImmArg<3> ]>; //===------------------------ Reduction Intrinsics ------------------------===// // -def int_experimental_vector_reduce_fadd : Intrinsic<[llvm_anyfloat_ty], - [llvm_anyfloat_ty, - llvm_anyvector_ty], - [IntrNoMem]>; -def int_experimental_vector_reduce_fmul : Intrinsic<[llvm_anyfloat_ty], - [llvm_anyfloat_ty, - llvm_anyvector_ty], - [IntrNoMem]>; -def int_experimental_vector_reduce_add : Intrinsic<[llvm_anyint_ty], +def int_experimental_vector_reduce_v2_fadd : Intrinsic<[llvm_anyfloat_ty], + [LLVMMatchType<0>, + llvm_anyvector_ty], + [IntrNoMem]>; +def int_experimental_vector_reduce_v2_fmul : Intrinsic<[llvm_anyfloat_ty], + [LLVMMatchType<0>, + llvm_anyvector_ty], + [IntrNoMem]>; +def int_experimental_vector_reduce_add : Intrinsic<[LLVMVectorElementType<0>], [llvm_anyvector_ty], [IntrNoMem]>; -def int_experimental_vector_reduce_mul : Intrinsic<[llvm_anyint_ty], +def int_experimental_vector_reduce_mul : Intrinsic<[LLVMVectorElementType<0>], [llvm_anyvector_ty], [IntrNoMem]>; -def int_experimental_vector_reduce_and : Intrinsic<[llvm_anyint_ty], +def int_experimental_vector_reduce_and : Intrinsic<[LLVMVectorElementType<0>], [llvm_anyvector_ty], [IntrNoMem]>; -def int_experimental_vector_reduce_or : Intrinsic<[llvm_anyint_ty], +def int_experimental_vector_reduce_or : Intrinsic<[LLVMVectorElementType<0>], [llvm_anyvector_ty], [IntrNoMem]>; -def int_experimental_vector_reduce_xor : Intrinsic<[llvm_anyint_ty], +def int_experimental_vector_reduce_xor : Intrinsic<[LLVMVectorElementType<0>], [llvm_anyvector_ty], [IntrNoMem]>; -def int_experimental_vector_reduce_smax : Intrinsic<[llvm_anyint_ty], +def int_experimental_vector_reduce_smax : Intrinsic<[LLVMVectorElementType<0>], [llvm_anyvector_ty], [IntrNoMem]>; -def int_experimental_vector_reduce_smin : Intrinsic<[llvm_anyint_ty], +def int_experimental_vector_reduce_smin : Intrinsic<[LLVMVectorElementType<0>], [llvm_anyvector_ty], [IntrNoMem]>; -def int_experimental_vector_reduce_umax : Intrinsic<[llvm_anyint_ty], +def int_experimental_vector_reduce_umax : Intrinsic<[LLVMVectorElementType<0>], [llvm_anyvector_ty], [IntrNoMem]>; -def int_experimental_vector_reduce_umin : Intrinsic<[llvm_anyint_ty], +def int_experimental_vector_reduce_umin : Intrinsic<[LLVMVectorElementType<0>], [llvm_anyvector_ty], [IntrNoMem]>; -def int_experimental_vector_reduce_fmax : Intrinsic<[llvm_anyfloat_ty], +def int_experimental_vector_reduce_fmax : Intrinsic<[LLVMVectorElementType<0>], [llvm_anyvector_ty], [IntrNoMem]>; -def int_experimental_vector_reduce_fmin : Intrinsic<[llvm_anyfloat_ty], +def int_experimental_vector_reduce_fmin : Intrinsic<[LLVMVectorElementType<0>], [llvm_anyvector_ty], [IntrNoMem]>; +//===---------- Intrinsics to control hardware supported loops ----------===// + +// Specify that the value given is the number of iterations that the next loop +// will execute. +def int_set_loop_iterations : + Intrinsic<[], [llvm_anyint_ty], [IntrNoDuplicate]>; + +// Specify that the value given is the number of iterations that the next loop +// will execute. Also test that the given count is not zero, allowing it to +// control entry to a 'while' loop. +def int_test_set_loop_iterations : + Intrinsic<[llvm_i1_ty], [llvm_anyint_ty], [IntrNoDuplicate]>; + +// Decrement loop counter by the given argument. Return false if the loop +// should exit. +def int_loop_decrement : + Intrinsic<[llvm_i1_ty], [llvm_anyint_ty], [IntrNoDuplicate]>; + +// Decrement the first operand (the loop counter) by the second operand (the +// maximum number of elements processed in an iteration). Return the remaining +// number of iterations still to be executed. This is effectively a sub which +// can be used with a phi, icmp and br to control the number of iterations +// executed, as usual. +def int_loop_decrement_reg : + Intrinsic<[llvm_anyint_ty], + [llvm_anyint_ty, llvm_anyint_ty], [IntrNoDuplicate]>; + //===----- Intrinsics that are used to provide predicate information -----===// def int_ssa_copy : Intrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrNoMem, Returned<0>]>; + +//===------- Intrinsics that are used to preserve debug information -------===// + +def int_preserve_array_access_index : Intrinsic<[llvm_anyptr_ty], + [llvm_anyptr_ty, llvm_i32_ty, + llvm_i32_ty], + [IntrNoMem, ImmArg<1>, ImmArg<2>]>; +def int_preserve_union_access_index : Intrinsic<[llvm_anyptr_ty], + [llvm_anyptr_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; +def int_preserve_struct_access_index : Intrinsic<[llvm_anyptr_ty], + [llvm_anyptr_ty, llvm_i32_ty, + llvm_i32_ty], + [IntrNoMem, ImmArg<1>, + ImmArg<2>]>; + //===----------------------------------------------------------------------===// // Target-specific intrinsics //===----------------------------------------------------------------------===// diff --git a/include/llvm/IR/IntrinsicsAArch64.td b/include/llvm/IR/IntrinsicsAArch64.td index ff25750fe399..832aca4fd30f 100644 --- a/include/llvm/IR/IntrinsicsAArch64.td +++ b/include/llvm/IR/IntrinsicsAArch64.td @@ -1,9 +1,8 @@ //===- IntrinsicsAARCH64.td - Defines AARCH64 intrinsics ---*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -32,6 +31,8 @@ def int_aarch64_sdiv : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, def int_aarch64_udiv : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem]>; +def int_aarch64_fjcvtzs : Intrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem]>; + //===----------------------------------------------------------------------===// // HINT @@ -290,6 +291,7 @@ let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in { // Pairwise Add def int_aarch64_neon_addp : AdvSIMD_2VectorArg_Intrinsic; + def int_aarch64_neon_faddp : AdvSIMD_2VectorArg_Intrinsic; // Long Pairwise Add // FIXME: In theory, we shouldn't need intrinsics for saddlp or @@ -462,12 +464,12 @@ let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". [IntrArgMemOnly, NoCapture<2>]>; class AdvSIMD_2Vec_Load_Intrinsic - : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], + : Intrinsic<[LLVMMatchType<0>, llvm_anyvector_ty], [LLVMAnyPointerType>], [IntrReadMem, IntrArgMemOnly]>; class AdvSIMD_2Vec_Load_Lane_Intrinsic - : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], - [LLVMMatchType<0>, LLVMMatchType<0>, + : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>], + [LLVMMatchType<0>, llvm_anyvector_ty, llvm_i64_ty, llvm_anyptr_ty], [IntrReadMem, IntrArgMemOnly]>; class AdvSIMD_2Vec_Store_Intrinsic @@ -480,12 +482,12 @@ let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". [IntrArgMemOnly, NoCapture<3>]>; class AdvSIMD_3Vec_Load_Intrinsic - : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>], + : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty], [LLVMAnyPointerType>], [IntrReadMem, IntrArgMemOnly]>; class AdvSIMD_3Vec_Load_Lane_Intrinsic - : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>], - [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, + : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty, llvm_i64_ty, llvm_anyptr_ty], [IntrReadMem, IntrArgMemOnly]>; class AdvSIMD_3Vec_Store_Intrinsic @@ -499,15 +501,15 @@ let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". [IntrArgMemOnly, NoCapture<4>]>; class AdvSIMD_4Vec_Load_Intrinsic - : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, - LLVMMatchType<0>, LLVMMatchType<0>], + : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, + LLVMMatchType<0>, llvm_anyvector_ty], [LLVMAnyPointerType>], [IntrReadMem, IntrArgMemOnly]>; class AdvSIMD_4Vec_Load_Lane_Intrinsic - : Intrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, + : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], [LLVMMatchType<0>, LLVMMatchType<0>, - LLVMMatchType<0>, LLVMMatchType<0>, + LLVMMatchType<0>, llvm_anyvector_ty, llvm_i64_ty, llvm_anyptr_ty], [IntrReadMem, IntrArgMemOnly]>; class AdvSIMD_4Vec_Store_Intrinsic @@ -684,3 +686,50 @@ def int_aarch64_crc32x : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty], def int_aarch64_crc32cx : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty], [IntrNoMem]>; } + +//===----------------------------------------------------------------------===// +// Memory Tagging Extensions (MTE) Intrinsics +let TargetPrefix = "aarch64" in { +def int_aarch64_irg : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty], + [IntrInaccessibleMemOnly]>; +def int_aarch64_addg : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty], + [IntrNoMem]>; +def int_aarch64_gmi : Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], + [IntrNoMem]>; +def int_aarch64_ldg : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_ptr_ty], + [IntrReadMem]>; +def int_aarch64_stg : Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty], + [IntrWriteMem]>; +def int_aarch64_subp : Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_ptr_ty], + [IntrNoMem]>; + +// The following are codegen-only intrinsics for stack instrumentation. + +// Generate a randomly tagged stack base pointer. +def int_aarch64_irg_sp : Intrinsic<[llvm_ptr_ty], [llvm_i64_ty], + [IntrInaccessibleMemOnly]>; + +// Transfer pointer tag with offset. +// ptr1 = tagp(ptr0, baseptr, tag_offset) returns a pointer where +// * address is the address in ptr0 +// * tag is a function of (tag in baseptr, tag_offset). +// Address bits in baseptr and tag bits in ptr0 are ignored. +// When offset between ptr0 and baseptr is a compile time constant, this can be emitted as +// ADDG ptr1, baseptr, (ptr0 - baseptr), tag_offset +// It is intended that ptr0 is an alloca address, and baseptr is the direct output of llvm.aarch64.irg.sp. +def int_aarch64_tagp : Intrinsic<[llvm_anyptr_ty], [LLVMMatchType<0>, llvm_ptr_ty, llvm_i64_ty], + [IntrNoMem, ImmArg<2>]>; + +// Update allocation tags for the memory range to match the tag in the pointer argument. +def int_aarch64_settag : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty], + [IntrWriteMem, IntrArgMemOnly, NoCapture<0>, WriteOnly<0>]>; + +// Update allocation tags for the memory range to match the tag in the pointer argument, +// and set memory contents to zero. +def int_aarch64_settag_zero : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty], + [IntrWriteMem, IntrArgMemOnly, NoCapture<0>, WriteOnly<0>]>; + +// Update allocation tags for 16-aligned, 16-sized memory region, and store a pair 8-byte values. +def int_aarch64_stgp : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty], + [IntrWriteMem, IntrArgMemOnly, NoCapture<0>, WriteOnly<0>]>; +} diff --git a/include/llvm/IR/IntrinsicsAMDGPU.td b/include/llvm/IR/IntrinsicsAMDGPU.td index 7913ce828fbc..3982444b5401 100644 --- a/include/llvm/IR/IntrinsicsAMDGPU.td +++ b/include/llvm/IR/IntrinsicsAMDGPU.td @@ -1,9 +1,8 @@ //===- IntrinsicsAMDGPU.td - Defines AMDGPU intrinsics -----*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -178,7 +177,7 @@ def int_amdgcn_implicit_buffer_ptr : // This is always moved to the beginning of the basic block. def int_amdgcn_init_exec : Intrinsic<[], [llvm_i64_ty], // 64-bit literal constant - [IntrConvergent]>; + [IntrConvergent, ImmArg<0>]>; // Set EXEC according to a thread count packed in an SGPR input: // thread_count = (input >> bitoffset) & 0x7f; @@ -188,6 +187,10 @@ def int_amdgcn_init_exec_from_input : Intrinsic<[], llvm_i32_ty], // bit offset of the thread count [IntrConvergent]>; +def int_amdgcn_wavefrontsize : + GCCBuiltin<"__builtin_amdgcn_wavefrontsize">, + Intrinsic<[llvm_i32_ty], [], [IntrNoMem, IntrSpeculatable]>; + //===----------------------------------------------------------------------===// // Instruction Intrinsics @@ -196,9 +199,9 @@ def int_amdgcn_init_exec_from_input : Intrinsic<[], // The first parameter is s_sendmsg immediate (i16), // the second one is copied to m0 def int_amdgcn_s_sendmsg : GCCBuiltin<"__builtin_amdgcn_s_sendmsg">, - Intrinsic <[], [llvm_i32_ty, llvm_i32_ty], []>; + Intrinsic <[], [llvm_i32_ty, llvm_i32_ty], [ImmArg<0>, IntrInaccessibleMemOnly]>; def int_amdgcn_s_sendmsghalt : GCCBuiltin<"__builtin_amdgcn_s_sendmsghalt">, - Intrinsic <[], [llvm_i32_ty, llvm_i32_ty], []>; + Intrinsic <[], [llvm_i32_ty, llvm_i32_ty], [ImmArg<0>, IntrInaccessibleMemOnly]>; def int_amdgcn_s_barrier : GCCBuiltin<"__builtin_amdgcn_s_barrier">, Intrinsic<[], [], [IntrConvergent]>; @@ -207,7 +210,7 @@ def int_amdgcn_wave_barrier : GCCBuiltin<"__builtin_amdgcn_wave_barrier">, Intrinsic<[], [], [IntrConvergent]>; def int_amdgcn_s_waitcnt : GCCBuiltin<"__builtin_amdgcn_s_waitcnt">, - Intrinsic<[], [llvm_i32_ty], []>; + Intrinsic<[], [llvm_i32_ty], [ImmArg<0>]>; def int_amdgcn_div_scale : Intrinsic< // 1st parameter: Numerator @@ -216,7 +219,7 @@ def int_amdgcn_div_scale : Intrinsic< // second. (0 = first, 1 = second). [llvm_anyfloat_ty, llvm_i1_ty], [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i1_ty], - [IntrNoMem, IntrSpeculatable] + [IntrNoMem, IntrSpeculatable, ImmArg<2>] >; def int_amdgcn_div_fmas : Intrinsic<[llvm_anyfloat_ty], @@ -293,29 +296,33 @@ def int_amdgcn_fract : Intrinsic< [llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable] >; -def int_amdgcn_cvt_pkrtz : Intrinsic< - [llvm_v2f16_ty], [llvm_float_ty, llvm_float_ty], - [IntrNoMem, IntrSpeculatable] +def int_amdgcn_cvt_pkrtz : GCCBuiltin<"__builtin_amdgcn_cvt_pkrtz">, + Intrinsic<[llvm_v2f16_ty], [llvm_float_ty, llvm_float_ty], + [IntrNoMem, IntrSpeculatable] >; -def int_amdgcn_cvt_pknorm_i16 : Intrinsic< - [llvm_v2i16_ty], [llvm_float_ty, llvm_float_ty], - [IntrNoMem, IntrSpeculatable] +def int_amdgcn_cvt_pknorm_i16 : + GCCBuiltin<"__builtin_amdgcn_cvt_pknorm_i16">, + Intrinsic<[llvm_v2i16_ty], [llvm_float_ty, llvm_float_ty], + [IntrNoMem, IntrSpeculatable] >; -def int_amdgcn_cvt_pknorm_u16 : Intrinsic< - [llvm_v2i16_ty], [llvm_float_ty, llvm_float_ty], - [IntrNoMem, IntrSpeculatable] +def int_amdgcn_cvt_pknorm_u16 : + GCCBuiltin<"__builtin_amdgcn_cvt_pknorm_u16">, + Intrinsic<[llvm_v2i16_ty], [llvm_float_ty, llvm_float_ty], + [IntrNoMem, IntrSpeculatable] >; -def int_amdgcn_cvt_pk_i16 : Intrinsic< +def int_amdgcn_cvt_pk_i16 : + GCCBuiltin<"__builtin_amdgcn_cvt_pk_i16">, + Intrinsic< [llvm_v2i16_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable] >; -def int_amdgcn_cvt_pk_u16 : Intrinsic< - [llvm_v2i16_ty], [llvm_i32_ty, llvm_i32_ty], - [IntrNoMem, IntrSpeculatable] +def int_amdgcn_cvt_pk_u16 : GCCBuiltin<"__builtin_amdgcn_cvt_pk_u16">, + Intrinsic<[llvm_v2i16_ty], [llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, IntrSpeculatable] >; def int_amdgcn_class : Intrinsic< @@ -374,7 +381,7 @@ class AMDGPUAtomicIncIntrin : Intrinsic<[llvm_anyint_ty], llvm_i32_ty, // ordering llvm_i32_ty, // scope llvm_i1_ty], // isVolatile - [IntrArgMemOnly, NoCapture<0>], "", + [IntrArgMemOnly, NoCapture<0>, ImmArg<2>, ImmArg<3>, ImmArg<4>], "", [SDNPMemOperand] >; @@ -389,9 +396,45 @@ class AMDGPULDSF32Intrin : llvm_i32_ty, // ordering llvm_i32_ty, // scope llvm_i1_ty], // isVolatile - [IntrArgMemOnly, NoCapture<0>] + [IntrArgMemOnly, NoCapture<0>, ImmArg<2>, ImmArg<3>, ImmArg<4>] +>; + +// FIXME: The m0 argument should be moved after the normal arguments +class AMDGPUDSOrderedIntrinsic : Intrinsic< + [llvm_i32_ty], + // M0 = {hi16:address, lo16:waveID}. Allow passing M0 as a pointer, so that + // the bit packing can be optimized at the IR level. + [LLVMQualPointerType, // IntToPtr(M0) + llvm_i32_ty, // value to add or swap + llvm_i32_ty, // ordering + llvm_i32_ty, // scope + llvm_i1_ty, // isVolatile + llvm_i32_ty, // ordered count index (OA index), also added to the address + // gfx10: bits 24-27 indicate the number of active threads/dwords + llvm_i1_ty, // wave release, usually set to 1 + llvm_i1_ty], // wave done, set to 1 for the last ordered instruction + [NoCapture<0>, + ImmArg<2>, ImmArg<3>, ImmArg<4>, + ImmArg<5>, ImmArg<6>, ImmArg<7> + ] +>; + +class AMDGPUDSAppendConsumedIntrinsic : Intrinsic< + [llvm_i32_ty], + [llvm_anyptr_ty, // LDS or GDS ptr + llvm_i1_ty], // isVolatile + [IntrConvergent, IntrArgMemOnly, NoCapture<0>, ImmArg<1>], + "", + [SDNPMemOperand] >; +def int_amdgcn_ds_ordered_add : AMDGPUDSOrderedIntrinsic; +def int_amdgcn_ds_ordered_swap : AMDGPUDSOrderedIntrinsic; + +// The pointer argument is assumed to be dynamically uniform if a VGPR. +def int_amdgcn_ds_append : AMDGPUDSAppendConsumedIntrinsic; +def int_amdgcn_ds_consume : AMDGPUDSAppendConsumedIntrinsic; + def int_amdgcn_ds_fadd : AMDGPULDSF32Intrin<"__builtin_amdgcn_ds_faddf">; def int_amdgcn_ds_fmin : AMDGPULDSF32Intrin<"__builtin_amdgcn_ds_fminf">; def int_amdgcn_ds_fmax : AMDGPULDSF32Intrin<"__builtin_amdgcn_ds_fmaxf">; @@ -442,9 +485,12 @@ class arglistconcat> arglists, int shift = 0> { } // Represent texture/image types / dimensionality. -class AMDGPUDimProps coord_names, list slice_names> { +class AMDGPUDimProps enc, string name, string asmsuffix, + list coord_names, list slice_names> { AMDGPUDimProps Dim = !cast(NAME); string Name = name; // e.g. "2darraymsaa" + string AsmSuffix = asmsuffix; // e.g. 2D_MSAA_ARRAY (used in assembly strings) + bits<3> Encoding = enc; bit DA = 0; // DA bit in MIMG encoding list CoordSliceArgs = @@ -460,17 +506,17 @@ class AMDGPUDimProps coord_names, list slice_n bits<8> NumGradients = !size(GradientArgs); } -def AMDGPUDim1D : AMDGPUDimProps<"1d", ["s"], []>; -def AMDGPUDim2D : AMDGPUDimProps<"2d", ["s", "t"], []>; -def AMDGPUDim3D : AMDGPUDimProps<"3d", ["s", "t", "r"], []>; +def AMDGPUDim1D : AMDGPUDimProps<0x0, "1d", "1D", ["s"], []>; +def AMDGPUDim2D : AMDGPUDimProps<0x1, "2d", "2D", ["s", "t"], []>; +def AMDGPUDim3D : AMDGPUDimProps<0x2, "3d", "3D", ["s", "t", "r"], []>; let DA = 1 in { - def AMDGPUDimCube : AMDGPUDimProps<"cube", ["s", "t"], ["face"]>; - def AMDGPUDim1DArray : AMDGPUDimProps<"1darray", ["s"], ["slice"]>; - def AMDGPUDim2DArray : AMDGPUDimProps<"2darray", ["s", "t"], ["slice"]>; + def AMDGPUDimCube : AMDGPUDimProps<0x3, "cube", "CUBE", ["s", "t"], ["face"]>; + def AMDGPUDim1DArray : AMDGPUDimProps<0x4, "1darray", "1D_ARRAY", ["s"], ["slice"]>; + def AMDGPUDim2DArray : AMDGPUDimProps<0x5, "2darray", "2D_ARRAY", ["s", "t"], ["slice"]>; } -def AMDGPUDim2DMsaa : AMDGPUDimProps<"2dmsaa", ["s", "t"], ["fragid"]>; +def AMDGPUDim2DMsaa : AMDGPUDimProps<0x6, "2dmsaa", "2D_MSAA", ["s", "t"], ["fragid"]>; let DA = 1 in { - def AMDGPUDim2DArrayMsaa : AMDGPUDimProps<"2darraymsaa", ["s", "t"], ["slice", "fragid"]>; + def AMDGPUDim2DArrayMsaa : AMDGPUDimProps<0x7, "2darraymsaa", "2D_MSAA_ARRAY", ["s", "t"], ["slice", "fragid"]>; } def AMDGPUDims { @@ -621,6 +667,19 @@ class AMDGPUDimGetResInfoProfile : AMDGPUDimProfile<"GET_RES let LodClampMip = "mip"; } +// Helper class for figuring out image intrinsic argument indexes. +class AMDGPUImageDimIntrinsicEval { + int NumDataArgs = !size(P_.DataArgs); + int NumDmaskArgs = !if(P_.IsAtomic, 0, 1); + int NumVAddrArgs = !size(P_.AddrArgs); + int NumRSrcArgs = 1; + int NumSampArgs = !if(P_.IsSample, 2, 0); + int DmaskArgIndex = NumDataArgs; + int UnormArgIndex = !add(NumDataArgs, NumDmaskArgs, NumVAddrArgs, NumRSrcArgs, 1); + int TexFailCtrlArgIndex = !add(NumDataArgs, NumDmaskArgs, NumVAddrArgs, NumRSrcArgs, NumSampArgs); + int CachePolicyArgIndex = !add(TexFailCtrlArgIndex, 1); +} + // All dimension-aware intrinsics are derived from this class. class AMDGPUImageDimIntrinsic props, @@ -634,8 +693,13 @@ class AMDGPUImageDimIntrinsic, + llvm_i32_ty]), // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc) + !listconcat(props, + !if(P_.IsAtomic, [], [ImmArg.DmaskArgIndex>]), + !if(P_.IsSample, [ImmArg.UnormArgIndex>], []), + [ImmArg.TexFailCtrlArgIndex>, + ImmArg.CachePolicyArgIndex>]), + "", sdnodeprops>, AMDGPURsrcIntrinsic { AMDGPUDimProfile P = P_; @@ -791,13 +855,13 @@ let TargetPrefix = "amdgcn" in { defset list AMDGPUBufferIntrinsics = { class AMDGPUBufferLoad : Intrinsic < - [llvm_anyfloat_ty], + [llvm_any_ty], [llvm_v4i32_ty, // rsrc(SGPR) llvm_i32_ty, // vindex(VGPR) llvm_i32_ty, // offset(SGPR/VGPR/imm) llvm_i1_ty, // glc(imm) llvm_i1_ty], // slc(imm) - [IntrReadMem], "", [SDNPMemOperand]>, + [IntrReadMem, ImmArg<3>, ImmArg<4>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<0>; def int_amdgcn_buffer_load_format : AMDGPUBufferLoad; def int_amdgcn_buffer_load : AMDGPUBufferLoad; @@ -805,20 +869,20 @@ def int_amdgcn_buffer_load : AMDGPUBufferLoad; def int_amdgcn_s_buffer_load : Intrinsic < [llvm_any_ty], [llvm_v4i32_ty, // rsrc(SGPR) - llvm_i32_ty, // byte offset(SGPR/VGPR/imm) - llvm_i32_ty], // cachepolicy(imm; bit 0 = glc) - [IntrNoMem]>, + llvm_i32_ty, // byte offset(SGPR/imm) + llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 2 = dlc) + [IntrNoMem, ImmArg<2>]>, AMDGPURsrcIntrinsic<0>; class AMDGPUBufferStore : Intrinsic < [], - [llvm_anyfloat_ty, // vdata(VGPR) -- can currently only select f32, v2f32, v4f32 + [llvm_any_ty, // vdata(VGPR) llvm_v4i32_ty, // rsrc(SGPR) llvm_i32_ty, // vindex(VGPR) llvm_i32_ty, // offset(SGPR/VGPR/imm) llvm_i1_ty, // glc(imm) llvm_i1_ty], // slc(imm) - [IntrWriteMem], "", [SDNPMemOperand]>, + [IntrWriteMem, ImmArg<4>, ImmArg<5>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<1>; def int_amdgcn_buffer_store_format : AMDGPUBufferStore; def int_amdgcn_buffer_store : AMDGPUBufferStore; @@ -835,8 +899,8 @@ class AMDGPURawBufferLoad : Intrinsic < [llvm_v4i32_ty, // rsrc(SGPR) llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling) llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling) - llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc) - [IntrReadMem], "", [SDNPMemOperand]>, + llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+) + [IntrReadMem, ImmArg<3>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<0>; def int_amdgcn_raw_buffer_load_format : AMDGPURawBufferLoad; def int_amdgcn_raw_buffer_load : AMDGPURawBufferLoad; @@ -847,8 +911,8 @@ class AMDGPUStructBufferLoad : Intrinsic < llvm_i32_ty, // vindex(VGPR) llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling) llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling) - llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc) - [IntrReadMem], "", [SDNPMemOperand]>, + llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+) + [IntrReadMem, ImmArg<4>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<0>; def int_amdgcn_struct_buffer_load_format : AMDGPUStructBufferLoad; def int_amdgcn_struct_buffer_load : AMDGPUStructBufferLoad; @@ -859,8 +923,8 @@ class AMDGPURawBufferStore : Intrinsic < llvm_v4i32_ty, // rsrc(SGPR) llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling) llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling) - llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc) - [IntrWriteMem], "", [SDNPMemOperand]>, + llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+) + [IntrWriteMem, ImmArg<4>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<1>; def int_amdgcn_raw_buffer_store_format : AMDGPURawBufferStore; def int_amdgcn_raw_buffer_store : AMDGPURawBufferStore; @@ -872,8 +936,8 @@ class AMDGPUStructBufferStore : Intrinsic < llvm_i32_ty, // vindex(VGPR) llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling) llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling) - llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc) - [IntrWriteMem], "", [SDNPMemOperand]>, + llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+) + [IntrWriteMem, ImmArg<5>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<1>; def int_amdgcn_struct_buffer_store_format : AMDGPUStructBufferStore; def int_amdgcn_struct_buffer_store : AMDGPUStructBufferStore; @@ -885,7 +949,7 @@ class AMDGPURawBufferAtomic : Intrinsic < llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling) llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling) llvm_i32_ty], // cachepolicy(imm; bit 1 = slc) - [], "", [SDNPMemOperand]>, + [ImmArg<4>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<1, 0>; def int_amdgcn_raw_buffer_atomic_swap : AMDGPURawBufferAtomic; def int_amdgcn_raw_buffer_atomic_add : AMDGPURawBufferAtomic; @@ -905,7 +969,7 @@ def int_amdgcn_raw_buffer_atomic_cmpswap : Intrinsic< llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling) llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling) llvm_i32_ty], // cachepolicy(imm; bit 1 = slc) - [], "", [SDNPMemOperand]>, + [ImmArg<5>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<2, 0>; class AMDGPUStructBufferAtomic : Intrinsic < @@ -916,7 +980,7 @@ class AMDGPUStructBufferAtomic : Intrinsic < llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling) llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling) llvm_i32_ty], // cachepolicy(imm; bit 1 = slc) - [], "", [SDNPMemOperand]>, + [ImmArg<5>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<1, 0>; def int_amdgcn_struct_buffer_atomic_swap : AMDGPUStructBufferAtomic; def int_amdgcn_struct_buffer_atomic_add : AMDGPUStructBufferAtomic; @@ -937,7 +1001,7 @@ def int_amdgcn_struct_buffer_atomic_cmpswap : Intrinsic< llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling) llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling) llvm_i32_ty], // cachepolicy(imm; bit 1 = slc) - [], "", [SDNPMemOperand]>, + [ImmArg<6>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<2, 0>; // Obsolescent tbuffer intrinsics. @@ -952,7 +1016,8 @@ def int_amdgcn_tbuffer_load : Intrinsic < llvm_i32_ty, // nfmt(imm) llvm_i1_ty, // glc(imm) llvm_i1_ty], // slc(imm) - [IntrReadMem], "", [SDNPMemOperand]>, + [IntrReadMem, ImmArg<4>, ImmArg<5>, ImmArg<6>, + ImmArg<7>, ImmArg<8>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<0>; def int_amdgcn_tbuffer_store : Intrinsic < @@ -967,7 +1032,8 @@ def int_amdgcn_tbuffer_store : Intrinsic < llvm_i32_ty, // nfmt(imm) llvm_i1_ty, // glc(imm) llvm_i1_ty], // slc(imm) - [IntrWriteMem], "", [SDNPMemOperand]>, + [IntrWriteMem, ImmArg<5>, ImmArg<6>, ImmArg<7>, + ImmArg<8>, ImmArg<9>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<1>; // New tbuffer intrinsics, with: @@ -980,8 +1046,8 @@ def int_amdgcn_raw_tbuffer_load : Intrinsic < llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling) llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling) llvm_i32_ty, // format(imm; bits 3..0 = dfmt, bits 6..4 = nfmt) - llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc) - [IntrReadMem], "", [SDNPMemOperand]>, + llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+) + [IntrReadMem, ImmArg<3>, ImmArg<4>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<0>; def int_amdgcn_raw_tbuffer_store : Intrinsic < @@ -991,8 +1057,8 @@ def int_amdgcn_raw_tbuffer_store : Intrinsic < llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling) llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling) llvm_i32_ty, // format(imm; bits 3..0 = dfmt, bits 6..4 = nfmt) - llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc) - [IntrWriteMem], "", [SDNPMemOperand]>, + llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+) + [IntrWriteMem, ImmArg<4>, ImmArg<5>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<1>; def int_amdgcn_struct_tbuffer_load : Intrinsic < @@ -1002,8 +1068,8 @@ def int_amdgcn_struct_tbuffer_load : Intrinsic < llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling) llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling) llvm_i32_ty, // format(imm; bits 3..0 = dfmt, bits 6..4 = nfmt) - llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc) - [IntrReadMem], "", [SDNPMemOperand]>, + llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+) + [IntrReadMem, ImmArg<4>, ImmArg<5>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<0>; def int_amdgcn_struct_tbuffer_store : Intrinsic < @@ -1014,18 +1080,18 @@ def int_amdgcn_struct_tbuffer_store : Intrinsic < llvm_i32_ty, // offset(VGPR/imm, included in bounds checking and swizzling) llvm_i32_ty, // soffset(SGPR/imm, excluded from bounds checking and swizzling) llvm_i32_ty, // format(imm; bits 3..0 = dfmt, bits 6..4 = nfmt) - llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc) - [IntrWriteMem], "", [SDNPMemOperand]>, + llvm_i32_ty], // cachepolicy(imm; bit 0 = glc, bit 1 = slc, bit 2 = dlc on gfx10+) + [IntrWriteMem, ImmArg<5>, ImmArg<6>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<1>; class AMDGPUBufferAtomic : Intrinsic < - [llvm_i32_ty], - [llvm_i32_ty, // vdata(VGPR) + [llvm_anyint_ty], + [LLVMMatchType<0>, // vdata(VGPR) llvm_v4i32_ty, // rsrc(SGPR) llvm_i32_ty, // vindex(VGPR) llvm_i32_ty, // offset(SGPR/VGPR/imm) llvm_i1_ty], // slc(imm) - [], "", [SDNPMemOperand]>, + [ImmArg<4>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<1, 0>; def int_amdgcn_buffer_atomic_swap : AMDGPUBufferAtomic; def int_amdgcn_buffer_atomic_add : AMDGPUBufferAtomic; @@ -1045,7 +1111,7 @@ def int_amdgcn_buffer_atomic_cmpswap : Intrinsic< llvm_i32_ty, // vindex(VGPR) llvm_i32_ty, // offset(SGPR/VGPR/imm) llvm_i1_ty], // slc(imm) - [], "", [SDNPMemOperand]>, + [ImmArg<5>], "", [SDNPMemOperand]>, AMDGPURsrcIntrinsic<2, 0>; } // defset AMDGPUBufferIntrinsics @@ -1062,7 +1128,7 @@ def int_amdgcn_exp : Intrinsic <[], [ llvm_i1_ty, // done llvm_i1_ty // vm ], - [] + [ImmArg<0>, ImmArg<1>, ImmArg<6>, ImmArg<7>, IntrInaccessibleMemOnly] >; // exp with compr bit set. @@ -1073,7 +1139,7 @@ def int_amdgcn_exp_compr : Intrinsic <[], [ LLVMMatchType<0>, // src1 llvm_i1_ty, // done llvm_i1_ty], // vm - [] + [ImmArg<0>, ImmArg<1>, ImmArg<4>, ImmArg<5>, IntrInaccessibleMemOnly] >; def int_amdgcn_buffer_wbinvl1_sc : @@ -1090,27 +1156,27 @@ def int_amdgcn_s_dcache_inv : def int_amdgcn_s_memtime : GCCBuiltin<"__builtin_amdgcn_s_memtime">, - Intrinsic<[llvm_i64_ty], [], [IntrReadMem]>; + Intrinsic<[llvm_i64_ty], []>; def int_amdgcn_s_sleep : GCCBuiltin<"__builtin_amdgcn_s_sleep">, - Intrinsic<[], [llvm_i32_ty], []> { + Intrinsic<[], [llvm_i32_ty], [ImmArg<0>]> { } def int_amdgcn_s_incperflevel : GCCBuiltin<"__builtin_amdgcn_s_incperflevel">, - Intrinsic<[], [llvm_i32_ty], []> { + Intrinsic<[], [llvm_i32_ty], [ImmArg<0>]> { } def int_amdgcn_s_decperflevel : GCCBuiltin<"__builtin_amdgcn_s_decperflevel">, - Intrinsic<[], [llvm_i32_ty], []> { + Intrinsic<[], [llvm_i32_ty], [ImmArg<0>]> { } def int_amdgcn_s_getreg : GCCBuiltin<"__builtin_amdgcn_s_getreg">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty], - [IntrReadMem, IntrSpeculatable] + [IntrInaccessibleMemOnly, IntrReadMem, IntrSpeculatable, ImmArg<0>] >; // int_amdgcn_s_getpc is provided to allow a specific style of position @@ -1129,7 +1195,7 @@ def int_amdgcn_interp_mov : GCCBuiltin<"__builtin_amdgcn_interp_mov">, Intrinsic<[llvm_float_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem, IntrSpeculatable]>; + [IntrNoMem, IntrSpeculatable, ImmArg<1>, ImmArg<2>]>; // __builtin_amdgcn_interp_p1 , , , // This intrinsic reads from lds, but the memory values are constant, @@ -1138,16 +1204,30 @@ def int_amdgcn_interp_p1 : GCCBuiltin<"__builtin_amdgcn_interp_p1">, Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem, IntrSpeculatable]>; + [IntrNoMem, IntrSpeculatable, ImmArg<1>, ImmArg<2>]>; // __builtin_amdgcn_interp_p2 , , , , def int_amdgcn_interp_p2 : GCCBuiltin<"__builtin_amdgcn_interp_p2">, Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem, IntrSpeculatable]>; + [IntrNoMem, IntrSpeculatable, ImmArg<2>, ImmArg<3>]>; // See int_amdgcn_v_interp_p1 for why this is IntrNoMem. +// __builtin_amdgcn_interp_p1_f16 , , , , +def int_amdgcn_interp_p1_f16 : + GCCBuiltin<"__builtin_amdgcn_interp_p1_f16">, + Intrinsic<[llvm_float_ty], + [llvm_float_ty, llvm_i32_ty, llvm_i32_ty, llvm_i1_ty, llvm_i32_ty], + [IntrNoMem, IntrSpeculatable, ImmArg<1>, ImmArg<2>, ImmArg<3>]>; + +// __builtin_amdgcn_interp_p2_f16 , , , , , +def int_amdgcn_interp_p2_f16 : + GCCBuiltin<"__builtin_amdgcn_interp_p2_f16">, + Intrinsic<[llvm_half_ty], + [llvm_float_ty, llvm_float_ty, llvm_i32_ty, llvm_i32_ty, llvm_i1_ty, llvm_i32_ty], + [IntrNoMem, IntrSpeculatable, ImmArg<2>, ImmArg<3>, ImmArg<4>]>; + // Pixel shaders only: whether the current pixel is live (i.e. not a helper // invocation for derivative computation). def int_amdgcn_ps_live : Intrinsic < @@ -1166,16 +1246,17 @@ def int_amdgcn_mbcnt_hi : // llvm.amdgcn.ds.swizzle src offset def int_amdgcn_ds_swizzle : GCCBuiltin<"__builtin_amdgcn_ds_swizzle">, - Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrConvergent]>; + Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, IntrConvergent, ImmArg<1>]>; def int_amdgcn_ubfe : Intrinsic<[llvm_anyint_ty], - [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem, IntrSpeculatable] + [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, IntrSpeculatable] >; def int_amdgcn_sbfe : Intrinsic<[llvm_anyint_ty], - [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem, IntrSpeculatable] + [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, IntrSpeculatable] >; def int_amdgcn_lerp : @@ -1233,12 +1314,12 @@ def int_amdgcn_cvt_pk_u8_f32 : >; def int_amdgcn_icmp : - Intrinsic<[llvm_i64_ty], [llvm_anyint_ty, LLVMMatchType<0>, llvm_i32_ty], - [IntrNoMem, IntrConvergent]>; + Intrinsic<[llvm_anyint_ty], [llvm_anyint_ty, LLVMMatchType<1>, llvm_i32_ty], + [IntrNoMem, IntrConvergent, ImmArg<2>]>; def int_amdgcn_fcmp : - Intrinsic<[llvm_i64_ty], [llvm_anyfloat_ty, LLVMMatchType<0>, llvm_i32_ty], - [IntrNoMem, IntrConvergent]>; + Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, LLVMMatchType<1>, llvm_i32_ty], + [IntrNoMem, IntrConvergent, ImmArg<2>]>; def int_amdgcn_readfirstlane : GCCBuiltin<"__builtin_amdgcn_readfirstlane">, @@ -1263,16 +1344,86 @@ def int_amdgcn_writelane : [IntrNoMem, IntrConvergent] >; -def int_amdgcn_alignbit : Intrinsic<[llvm_i32_ty], +def int_amdgcn_alignbit : + GCCBuiltin<"__builtin_amdgcn_alignbit">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable] >; -def int_amdgcn_alignbyte : Intrinsic<[llvm_i32_ty], - [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], +def int_amdgcn_alignbyte : GCCBuiltin<"__builtin_amdgcn_alignbyte">, + Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrSpeculatable] >; +def int_amdgcn_mul_i24 : Intrinsic<[llvm_i32_ty], + [llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, IntrSpeculatable] +>; + +def int_amdgcn_mul_u24 : Intrinsic<[llvm_i32_ty], + [llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, IntrSpeculatable] +>; + +// llvm.amdgcn.ds.gws.init(i32 bar_val, i32 resource_id) +// +// bar_val is the total number of waves that will wait on this +// barrier, minus 1. +def int_amdgcn_ds_gws_init : + GCCBuiltin<"__builtin_amdgcn_ds_gws_init">, + Intrinsic<[], + [llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrWriteMem, IntrInaccessibleMemOnly], "", + [SDNPMemOperand] +>; + +// llvm.amdgcn.ds.gws.barrier(i32 vsrc0, i32 resource_id) +// bar_val is the total number of waves that will wait on this +// barrier, minus 1. +def int_amdgcn_ds_gws_barrier : + GCCBuiltin<"__builtin_amdgcn_ds_gws_barrier">, + Intrinsic<[], + [llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrInaccessibleMemOnly], "", + [SDNPMemOperand] +>; + +// llvm.amdgcn.ds.gws.sema.v(i32 resource_id) +def int_amdgcn_ds_gws_sema_v : + GCCBuiltin<"__builtin_amdgcn_ds_gws_sema_v">, + Intrinsic<[], + [llvm_i32_ty], + [IntrConvergent, IntrInaccessibleMemOnly], "", + [SDNPMemOperand] +>; + +// llvm.amdgcn.ds.gws.sema.br(i32 vsrc, i32 resource_id) +def int_amdgcn_ds_gws_sema_br : + GCCBuiltin<"__builtin_amdgcn_ds_gws_sema_br">, + Intrinsic<[], + [llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrInaccessibleMemOnly], "", + [SDNPMemOperand] +>; + +// llvm.amdgcn.ds.gws.sema.p(i32 resource_id) +def int_amdgcn_ds_gws_sema_p : + GCCBuiltin<"__builtin_amdgcn_ds_gws_sema_p">, + Intrinsic<[], + [llvm_i32_ty], + [IntrConvergent, IntrInaccessibleMemOnly], "", + [SDNPMemOperand] +>; + +// llvm.amdgcn.ds.gws.sema.release.all(i32 resource_id) +def int_amdgcn_ds_gws_sema_release_all : + GCCBuiltin<"__builtin_amdgcn_ds_gws_sema_release_all">, + Intrinsic<[], + [llvm_i32_ty], + [IntrConvergent, IntrInaccessibleMemOnly], "", + [SDNPMemOperand] +>; + // Copies the source value to the destination value, with the guarantee that // the source value is computed as if the entire program were executed in WQM. @@ -1295,7 +1446,7 @@ def int_amdgcn_kill : Intrinsic<[], [llvm_i1_ty], []>; // enabled, with a few exceptions: - Phi nodes with require WWM return an // undefined value. def int_amdgcn_wwm : Intrinsic<[llvm_any_ty], - [LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable] + [LLVMMatchType<0>], [IntrNoMem, IntrSpeculatable, IntrConvergent] >; // Given a value, copies it while setting all the inactive lanes to a given @@ -1328,7 +1479,8 @@ def int_amdgcn_buffer_wbinvl1_vol : def int_amdgcn_mov_dpp : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, - llvm_i1_ty], [IntrNoMem, IntrConvergent]>; + llvm_i1_ty], [IntrNoMem, IntrConvergent, ImmArg<1>, + ImmArg<2>, ImmArg<3>, ImmArg<4>]>; // llvm.amdgcn.update.dpp.i32 // Should be equivalent to: @@ -1336,8 +1488,10 @@ def int_amdgcn_mov_dpp : // v_mov_b32 def int_amdgcn_update_dpp : Intrinsic<[llvm_anyint_ty], - [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty, llvm_i32_ty, - llvm_i32_ty, llvm_i1_ty], [IntrNoMem, IntrConvergent]>; + [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty, + llvm_i32_ty, llvm_i32_ty, llvm_i1_ty], + [IntrNoMem, IntrConvergent, + ImmArg<2>, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; def int_amdgcn_s_dcache_wb : GCCBuiltin<"__builtin_amdgcn_s_dcache_wb">, @@ -1349,7 +1503,7 @@ def int_amdgcn_s_dcache_wb_vol : def int_amdgcn_s_memrealtime : GCCBuiltin<"__builtin_amdgcn_s_memrealtime">, - Intrinsic<[llvm_i64_ty], [], [IntrReadMem]>; + Intrinsic<[llvm_i64_ty]>; // llvm.amdgcn.ds.permute def int_amdgcn_ds_permute : @@ -1361,6 +1515,34 @@ def int_amdgcn_ds_bpermute : GCCBuiltin<"__builtin_amdgcn_ds_bpermute">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrConvergent]>; +//===----------------------------------------------------------------------===// +// GFX10 Intrinsics +//===----------------------------------------------------------------------===// + +// llvm.amdgcn.permlane16 +def int_amdgcn_permlane16 : GCCBuiltin<"__builtin_amdgcn_permlane16">, + Intrinsic<[llvm_i32_ty], + [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i1_ty, llvm_i1_ty], + [IntrNoMem, IntrConvergent, ImmArg<4>, ImmArg<5>]>; + +// llvm.amdgcn.permlanex16 +def int_amdgcn_permlanex16 : GCCBuiltin<"__builtin_amdgcn_permlanex16">, + Intrinsic<[llvm_i32_ty], + [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i1_ty, llvm_i1_ty], + [IntrNoMem, IntrConvergent, ImmArg<4>, ImmArg<5>]>; + +// llvm.amdgcn.mov.dpp8.i32 +// is a 32-bit constant whose high 8 bits must be zero which selects +// the lanes to read from. +def int_amdgcn_mov_dpp8 : + Intrinsic<[llvm_anyint_ty], + [LLVMMatchType<0>, llvm_i32_ty], + [IntrNoMem, IntrConvergent, ImmArg<1>]>; + +def int_amdgcn_s_get_waveid_in_workgroup : + GCCBuiltin<"__builtin_amdgcn_s_get_waveid_in_workgroup">, + Intrinsic<[llvm_i32_ty], [], [IntrReadMem, IntrInaccessibleMemOnly]>; + //===----------------------------------------------------------------------===// // Deep learning intrinsics. //===----------------------------------------------------------------------===// @@ -1377,7 +1559,7 @@ def int_amdgcn_fdot2 : llvm_float_ty, // %c llvm_i1_ty // %clamp ], - [IntrNoMem, IntrSpeculatable] + [IntrNoMem, IntrSpeculatable, ImmArg<3>] >; // i32 %r = llvm.amdgcn.sdot2(v2i16 %a, v2i16 %b, i32 %c, i1 %clamp) @@ -1392,7 +1574,7 @@ def int_amdgcn_sdot2 : llvm_i32_ty, // %c llvm_i1_ty // %clamp ], - [IntrNoMem, IntrSpeculatable] + [IntrNoMem, IntrSpeculatable, ImmArg<3>] >; // u32 %r = llvm.amdgcn.udot2(v2u16 %a, v2u16 %b, u32 %c, i1 %clamp) @@ -1407,7 +1589,7 @@ def int_amdgcn_udot2 : llvm_i32_ty, // %c llvm_i1_ty // %clamp ], - [IntrNoMem, IntrSpeculatable] + [IntrNoMem, IntrSpeculatable, ImmArg<3>] >; // i32 %r = llvm.amdgcn.sdot4(v4i8 (as i32) %a, v4i8 (as i32) %b, i32 %c, i1 %clamp) @@ -1422,7 +1604,7 @@ def int_amdgcn_sdot4 : llvm_i32_ty, // %c llvm_i1_ty // %clamp ], - [IntrNoMem, IntrSpeculatable] + [IntrNoMem, IntrSpeculatable, ImmArg<3>] >; // u32 %r = llvm.amdgcn.udot4(v4u8 (as u32) %a, v4u8 (as u32) %b, u32 %c, i1 %clamp) @@ -1437,7 +1619,7 @@ def int_amdgcn_udot4 : llvm_i32_ty, // %c llvm_i1_ty // %clamp ], - [IntrNoMem, IntrSpeculatable] + [IntrNoMem, IntrSpeculatable, ImmArg<3>] >; // i32 %r = llvm.amdgcn.sdot8(v8i4 (as i32) %a, v8i4 (as i32) %b, i32 %c, i1 %clamp) @@ -1453,7 +1635,7 @@ def int_amdgcn_sdot8 : llvm_i32_ty, // %c llvm_i1_ty // %clamp ], - [IntrNoMem, IntrSpeculatable] + [IntrNoMem, IntrSpeculatable, ImmArg<3>] >; // u32 %r = llvm.amdgcn.udot8(v8u4 (as u32) %a, v8u4 (as u32) %b, u32 %c, i1 %clamp) @@ -1469,30 +1651,154 @@ def int_amdgcn_udot8 : llvm_i32_ty, // %c llvm_i1_ty // %clamp ], - [IntrNoMem, IntrSpeculatable] + [IntrNoMem, IntrSpeculatable, ImmArg<3>] >; +//===----------------------------------------------------------------------===// +// gfx908 intrinsics +// ===----------------------------------------------------------------------===// + +class AMDGPUBufferAtomicNoRtn : Intrinsic < + [], + [llvm_anyfloat_ty, // vdata(VGPR) + llvm_v4i32_ty, // rsrc(SGPR) + llvm_i32_ty, // vindex(VGPR) + llvm_i32_ty, // offset(SGPR/VGPR/imm) + llvm_i1_ty], // slc(imm) + [], "", [SDNPMemOperand]>, + AMDGPURsrcIntrinsic<1, 0>; + +class AMDGPUGlobalAtomicNoRtn : Intrinsic < + [], + [llvm_anyptr_ty, // vaddr + llvm_anyfloat_ty], // vdata(VGPR) + [IntrArgMemOnly, NoCapture<0>], "", [SDNPMemOperand]>; + +def int_amdgcn_buffer_atomic_fadd : AMDGPUBufferAtomicNoRtn; +def int_amdgcn_global_atomic_fadd : AMDGPUGlobalAtomicNoRtn; + +// llvm.amdgcn.mfma.f32.* vdst, srcA, srcB, srcC, cbsz, abid, blgp +def int_amdgcn_mfma_f32_32x32x1f32 : Intrinsic<[llvm_v32f32_ty], + [llvm_float_ty, llvm_float_ty, llvm_v32f32_ty, + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; + +def int_amdgcn_mfma_f32_16x16x1f32 : Intrinsic<[llvm_v16f32_ty], + [llvm_float_ty, llvm_float_ty, llvm_v16f32_ty, + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; + +def int_amdgcn_mfma_f32_4x4x1f32 : Intrinsic<[llvm_v4f32_ty], + [llvm_float_ty, llvm_float_ty, llvm_v4f32_ty, + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; + +def int_amdgcn_mfma_f32_32x32x2f32 : Intrinsic<[llvm_v16f32_ty], + [llvm_float_ty, llvm_float_ty, llvm_v16f32_ty, + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; + +def int_amdgcn_mfma_f32_16x16x4f32 : Intrinsic<[llvm_v4f32_ty], + [llvm_float_ty, llvm_float_ty, llvm_v4f32_ty, + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; + +def int_amdgcn_mfma_f32_32x32x4f16 : Intrinsic<[llvm_v32f32_ty], + [llvm_v4f16_ty, llvm_v4f16_ty, llvm_v32f32_ty, + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; + +def int_amdgcn_mfma_f32_16x16x4f16 : Intrinsic<[llvm_v16f32_ty], + [llvm_v4f16_ty, llvm_v4f16_ty, llvm_v16f32_ty, + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; + +def int_amdgcn_mfma_f32_4x4x4f16 : Intrinsic<[llvm_v4f32_ty], + [llvm_v4f16_ty, llvm_v4f16_ty, llvm_v4f32_ty, + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; + +def int_amdgcn_mfma_f32_32x32x8f16 : Intrinsic<[llvm_v16f32_ty], + [llvm_v4f16_ty, llvm_v4f16_ty, llvm_v16f32_ty, + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; + +def int_amdgcn_mfma_f32_16x16x16f16 : Intrinsic<[llvm_v4f32_ty], + [llvm_v4f16_ty, llvm_v4f16_ty, llvm_v4f32_ty, + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; + +def int_amdgcn_mfma_i32_32x32x4i8 : Intrinsic<[llvm_v32i32_ty], + [llvm_i32_ty, llvm_i32_ty, llvm_v32i32_ty, + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; + +def int_amdgcn_mfma_i32_16x16x4i8 : Intrinsic<[llvm_v16i32_ty], + [llvm_i32_ty, llvm_i32_ty, llvm_v16i32_ty, + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; + +def int_amdgcn_mfma_i32_4x4x4i8 : Intrinsic<[llvm_v4i32_ty], + [llvm_i32_ty, llvm_i32_ty, llvm_v4i32_ty, + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; + +def int_amdgcn_mfma_i32_32x32x8i8 : Intrinsic<[llvm_v16i32_ty], + [llvm_i32_ty, llvm_i32_ty, llvm_v16i32_ty, + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; + +def int_amdgcn_mfma_i32_16x16x16i8 : Intrinsic<[llvm_v4i32_ty], + [llvm_i32_ty, llvm_i32_ty, llvm_v4i32_ty, + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; + +def int_amdgcn_mfma_f32_32x32x2bf16 : Intrinsic<[llvm_v32f32_ty], + [llvm_v2i16_ty, llvm_v2i16_ty, llvm_v32f32_ty, + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; + +def int_amdgcn_mfma_f32_16x16x2bf16 : Intrinsic<[llvm_v16f32_ty], + [llvm_v2i16_ty, llvm_v2i16_ty, llvm_v16f32_ty, + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; + +def int_amdgcn_mfma_f32_4x4x2bf16 : Intrinsic<[llvm_v4f32_ty], + [llvm_v2i16_ty, llvm_v2i16_ty, llvm_v4f32_ty, + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; + +def int_amdgcn_mfma_f32_32x32x4bf16 : Intrinsic<[llvm_v16f32_ty], + [llvm_v2i16_ty, llvm_v2i16_ty, llvm_v16f32_ty, + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; + +def int_amdgcn_mfma_f32_16x16x8bf16 : Intrinsic<[llvm_v4f32_ty], + [llvm_v2i16_ty, llvm_v2i16_ty, llvm_v4f32_ty, + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrConvergent, IntrNoMem, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; + //===----------------------------------------------------------------------===// // Special Intrinsics for backend internal use only. No frontend // should emit calls to these. // ===----------------------------------------------------------------------===// -def int_amdgcn_if : Intrinsic<[llvm_i1_ty, llvm_i64_ty], +def int_amdgcn_if : Intrinsic<[llvm_i1_ty, llvm_anyint_ty], [llvm_i1_ty], [IntrConvergent] >; -def int_amdgcn_else : Intrinsic<[llvm_i1_ty, llvm_i64_ty], - [llvm_i64_ty], [IntrConvergent] +def int_amdgcn_else : Intrinsic<[llvm_i1_ty, llvm_anyint_ty], + [llvm_anyint_ty], [IntrConvergent] >; -def int_amdgcn_if_break : Intrinsic<[llvm_i64_ty], - [llvm_i1_ty, llvm_i64_ty], [IntrNoMem, IntrConvergent] +def int_amdgcn_if_break : Intrinsic<[llvm_anyint_ty], + [llvm_i1_ty, llvm_anyint_ty], [IntrNoMem, IntrConvergent] >; def int_amdgcn_loop : Intrinsic<[llvm_i1_ty], - [llvm_i64_ty], [IntrConvergent] + [llvm_anyint_ty], [IntrConvergent] >; -def int_amdgcn_end_cf : Intrinsic<[], [llvm_i64_ty], [IntrConvergent]>; +def int_amdgcn_end_cf : Intrinsic<[], [llvm_anyint_ty], [IntrConvergent]>; // Represent unreachable in a divergent region. def int_amdgcn_unreachable : Intrinsic<[], [], [IntrConvergent]>; diff --git a/include/llvm/IR/IntrinsicsARM.td b/include/llvm/IR/IntrinsicsARM.td index 4e11f9c29dd0..4792af097d95 100644 --- a/include/llvm/IR/IntrinsicsARM.td +++ b/include/llvm/IR/IntrinsicsARM.td @@ -1,9 +1,8 @@ //===- IntrinsicsARM.td - Defines ARM intrinsics -----------*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -20,7 +19,7 @@ let TargetPrefix = "arm" in { // All intrinsics start with "llvm.arm.". // A space-consuming intrinsic primarily for testing ARMConstantIslands. The // first argument is the number of bytes this "instruction" takes up, the second // and return value are essentially chains, used to force ordering during ISel. -def int_arm_space : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], []>; +def int_arm_space : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [ImmArg<0>]>; // 16-bit multiplications def int_arm_smulbb : GCCBuiltin<"__builtin_arm_smulbb">, @@ -263,59 +262,59 @@ def int_arm_vcvtru : Intrinsic<[llvm_float_ty], [llvm_anyfloat_ty], // Coprocessor def int_arm_ldc : GCCBuiltin<"__builtin_arm_ldc">, - Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], []>; + Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], [ImmArg<0>, ImmArg<1>]>; def int_arm_ldcl : GCCBuiltin<"__builtin_arm_ldcl">, - Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], []>; + Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], [ImmArg<0>, ImmArg<1>]>; def int_arm_ldc2 : GCCBuiltin<"__builtin_arm_ldc2">, - Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], []>; + Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], [ImmArg<0>, ImmArg<1>]>; def int_arm_ldc2l : GCCBuiltin<"__builtin_arm_ldc2l">, - Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], []>; + Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], [ImmArg<0>, ImmArg<1>]>; def int_arm_stc : GCCBuiltin<"__builtin_arm_stc">, - Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], []>; + Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], [ImmArg<0>, ImmArg<1>]>; def int_arm_stcl : GCCBuiltin<"__builtin_arm_stcl">, - Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], []>; + Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], [ImmArg<0>, ImmArg<1>]>; def int_arm_stc2 : GCCBuiltin<"__builtin_arm_stc2">, - Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], []>; + Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], [ImmArg<0>, ImmArg<1>]>; def int_arm_stc2l : GCCBuiltin<"__builtin_arm_stc2l">, - Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], []>; + Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty], [ImmArg<0>, ImmArg<1>]>; // Move to coprocessor def int_arm_mcr : GCCBuiltin<"__builtin_arm_mcr">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, - llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>; + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<0>, ImmArg<1>, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; def int_arm_mcr2 : GCCBuiltin<"__builtin_arm_mcr2">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, - llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>; + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<0>, ImmArg<1>, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; // Move from coprocessor def int_arm_mrc : GCCBuiltin<"__builtin_arm_mrc">, MSBuiltin<"_MoveFromCoprocessor">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, - llvm_i32_ty, llvm_i32_ty], []>; + llvm_i32_ty, llvm_i32_ty], [ImmArg<0>, ImmArg<1>, ImmArg<2>, ImmArg<3>, ImmArg<4>]>; def int_arm_mrc2 : GCCBuiltin<"__builtin_arm_mrc2">, MSBuiltin<"_MoveFromCoprocessor2">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, - llvm_i32_ty, llvm_i32_ty], []>; + llvm_i32_ty, llvm_i32_ty], [ImmArg<0>, ImmArg<1>, ImmArg<2>, ImmArg<3>, ImmArg<4>]>; // Coprocessor data processing def int_arm_cdp : GCCBuiltin<"__builtin_arm_cdp">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, - llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>; + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<0>, ImmArg<1>, ImmArg<2>, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; def int_arm_cdp2 : GCCBuiltin<"__builtin_arm_cdp2">, Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, - llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], []>; + llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<0>, ImmArg<1>, ImmArg<2>, ImmArg<3>, ImmArg<4>, ImmArg<5>]>; // Move from two registers to coprocessor def int_arm_mcrr : Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, - llvm_i32_ty, llvm_i32_ty], []>; + llvm_i32_ty, llvm_i32_ty], [ImmArg<0>, ImmArg<1>, ImmArg<4>]>; def int_arm_mcrr2 : Intrinsic<[], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, - llvm_i32_ty, llvm_i32_ty], []>; + llvm_i32_ty, llvm_i32_ty], [ImmArg<0>, ImmArg<1>, ImmArg<4>]>; def int_arm_mrrc : Intrinsic<[llvm_i32_ty, llvm_i32_ty], [llvm_i32_ty, - llvm_i32_ty, llvm_i32_ty], []>; + llvm_i32_ty, llvm_i32_ty], [ImmArg<0>, ImmArg<1>, ImmArg<2>]>; def int_arm_mrrc2 : Intrinsic<[llvm_i32_ty, llvm_i32_ty], [llvm_i32_ty, - llvm_i32_ty, llvm_i32_ty], []>; + llvm_i32_ty, llvm_i32_ty], [ImmArg<0>, ImmArg<1>, ImmArg<2>]>; //===----------------------------------------------------------------------===// // CRC32 @@ -333,6 +332,18 @@ def int_arm_crc32w : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], def int_arm_crc32cw : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; +//===----------------------------------------------------------------------===// +// CMSE + +def int_arm_cmse_tt : GCCBuiltin<"__builtin_arm_cmse_TT">, + Intrinsic<[llvm_i32_ty], [llvm_ptr_ty], [IntrNoMem]>; +def int_arm_cmse_ttt : GCCBuiltin<"__builtin_arm_cmse_TTT">, + Intrinsic<[llvm_i32_ty], [llvm_ptr_ty], [IntrNoMem]>; +def int_arm_cmse_tta : GCCBuiltin<"__builtin_arm_cmse_TTA">, + Intrinsic<[llvm_i32_ty], [llvm_ptr_ty], [IntrNoMem]>; +def int_arm_cmse_ttat : GCCBuiltin<"__builtin_arm_cmse_TTAT">, + Intrinsic<[llvm_i32_ty], [llvm_ptr_ty], [IntrNoMem]>; + //===----------------------------------------------------------------------===// // HINT diff --git a/include/llvm/IR/IntrinsicsBPF.td b/include/llvm/IR/IntrinsicsBPF.td index 94eca8e40332..d7595a2a7700 100644 --- a/include/llvm/IR/IntrinsicsBPF.td +++ b/include/llvm/IR/IntrinsicsBPF.td @@ -1,9 +1,8 @@ //===- IntrinsicsBPF.td - Defines BPF intrinsics -----------*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/IntrinsicsHexagon.td b/include/llvm/IR/IntrinsicsHexagon.td index ecc69a679553..2abc1dc07ebd 100644 --- a/include/llvm/IR/IntrinsicsHexagon.td +++ b/include/llvm/IR/IntrinsicsHexagon.td @@ -1,8 +1,7 @@ //===- IntrinsicsHexagon.td - Defines Hexagon intrinsics ---*- tablegen -*-===// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -52,19 +51,19 @@ class Hexagon_mem_memmemsisi_Intrinsic : Hexagon_Intrinsic; + [IntrArgMemOnly, ImmArg<3>]>; class Hexagon_mem_memsisisi_Intrinsic : Hexagon_Intrinsic; + [IntrWriteMem, ImmArg<3>]>; class Hexagon_mem_memdisisi_Intrinsic : Hexagon_Intrinsic; + [IntrWriteMem, ImmArg<3>]>; // // BUILTIN_INFO_NONCONST(circ_ldd,PTR_ftype_PTRPTRSISI,4) @@ -554,16 +553,18 @@ class Hexagon_v32i32_v32i32v32i32_Intrinsic [IntrNoMem]>; // tag : V6_vaslw_acc -class Hexagon_v16i32_v16i32v16i32i32_Intrinsic +class Hexagon_v16i32_v16i32v16i32i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : V6_vaslw_acc -class Hexagon_v32i32_v32i32v32i32i32_Intrinsic +class Hexagon_v32i32_v32i32v32i32i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : V6_vmux class Hexagon_v16i32_v512i1v16i32v16i32_Intrinsic @@ -581,7 +582,7 @@ class Hexagon_v32i32_v1024i1v32i32v32i32_Intrinsic class Hexagon_i32_i32i32i32i32_Intrinsic : Hexagon_Intrinsic; + [IntrNoMem, ImmArg<2>, ImmArg<3>]>; // tag : V6_vandnqrt_acc class Hexagon_v16i32_v16i32v512i1i32_Intrinsic @@ -596,58 +597,62 @@ class Hexagon_v32i32_v32i32v1024i1i32_Intrinsic [IntrNoMem]>; // tag : V6_vrmpybusi -class Hexagon_v32i32_v32i32i32i32_Intrinsic +class Hexagon_v32i32_v32i32i32i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : V6_vrmpybusi -class Hexagon_v64i32_v64i32i32i32_Intrinsic +class Hexagon_v64i32_v64i32i32i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : V6_vsubb_dv -class Hexagon_v64i32_v64i32v64i32_Intrinsic +class Hexagon_v64i32_v64i32v64i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : M2_mpysu_up -class Hexagon_i32_i32i32_Intrinsic +class Hexagon_i32_i32i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : M2_mpyud_acc_ll_s0 -class Hexagon_i64_i64i32i32_Intrinsic +class Hexagon_i64_i64i32i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : S2_lsr_i_r_nac -class Hexagon_i32_i32i32i32_Intrinsic +class Hexagon_i32_i32i32i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : M2_cmpysc_s0 -class Hexagon_i64_i32i32_Intrinsic +class Hexagon_i64_i32i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : V6_lo -class Hexagon_v16i32_v32i32_Intrinsic +class Hexagon_v16i32_v32i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : V6_lo -class Hexagon_v32i32_v64i32_Intrinsic +class Hexagon_v32i32_v64i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : S2_shuffoh class Hexagon_i64_i64i64_Intrinsic @@ -698,10 +703,10 @@ class Hexagon_v32i32_v32i32i32_Intrinsic [IntrNoMem]>; // tag : A4_vcmphgti -class Hexagon_i32_i64i32_Intrinsic +class Hexagon_i32_i64i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : class Hexagon_v32i32_v16i32i32_Intrinsic @@ -710,10 +715,11 @@ class Hexagon_v32i32_v16i32i32_Intrinsic [IntrNoMem]>; // tag : S6_rol_i_p_or -class Hexagon_i64_i64i64i32_Intrinsic +class Hexagon_i64_i64i64i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : V6_vgtuh_and class Hexagon_v512i1_v512i1v16i32v16i32_Intrinsic @@ -728,16 +734,18 @@ class Hexagon_v1024i1_v1024i1v32i32v32i32_Intrinsic [IntrNoMem]>; // tag : A2_abssat -class Hexagon_i32_i32_Intrinsic +class Hexagon_i32_i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : A2_vcmpwgtu -class Hexagon_i32_i64i64_Intrinsic +class Hexagon_i32_i64i64_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : V6_vtmpybus_acc class Hexagon_v64i32_v64i32v64i32i32_Intrinsic @@ -764,16 +772,18 @@ class Hexagon_v1024i1_v1024i1v1024i1_Intrinsic [IntrNoMem]>; // tag : S2_asr_i_p_rnd_goodsyntax -class Hexagon_i64_i64i32_Intrinsic +class Hexagon_i64_i64i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : F2_conv_w2df -class Hexagon_double_i32_Intrinsic +class Hexagon_double_i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : V6_vunpackuh class Hexagon_v32i32_v16i32_Intrinsic @@ -866,16 +876,18 @@ class Hexagon_i32_v32i32i32_Intrinsic [IntrNoMem]>; // tag : V6_vlutvwhi -class Hexagon_v32i32_v16i32v16i32i32_Intrinsic +class Hexagon_v32i32_v16i32v16i32i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : V6_vlutvwhi -class Hexagon_v64i32_v32i32v32i32i32_Intrinsic +class Hexagon_v64i32_v32i32v32i32i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : V6_vgtuh class Hexagon_v512i1_v16i32v16i32_Intrinsic @@ -902,10 +914,11 @@ class Hexagon_double_i64_Intrinsic [IntrNoMem]>; // tag : S2_vzxthw -class Hexagon_i64_i32_Intrinsic +class Hexagon_i64_i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : V6_vtmpyhb class Hexagon_v64i32_v64i32i32_Intrinsic @@ -944,10 +957,11 @@ class Hexagon_v16i32_v16i32_Intrinsic [IntrNoMem]>; // tag : F2_conv_uw2sf -class Hexagon_float_i32_Intrinsic +class Hexagon_float_i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : V6_vswap class Hexagon_v32i32_v512i1v16i32v16i32_Intrinsic @@ -1022,16 +1036,17 @@ class Hexagon_v32i32_v32i32v32i32v1024i1_Intrinsic [IntrNoMem]>; // tag : V6_vlutvvb_oracc -class Hexagon_v16i32_v16i32v16i32v16i32i32_Intrinsic +class Hexagon_v16i32_v16i32v16i32v16i32i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : V6_vlutvvb_oracc -class Hexagon_v32i32_v32i32v32i32v32i32i32_Intrinsic +class Hexagon_v32i32_v32i32v32i32v32i32i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : V6_vrmpybub_rtt class Hexagon_v32i32_v16i32i64_Intrinsic @@ -1052,16 +1067,18 @@ class Hexagon_i64i32_i64i64i32_Intrinsic [IntrNoMem]>; // tag : V6_vrsadubi_acc -class Hexagon_v32i32_v32i32v32i32i32i32_Intrinsic +class Hexagon_v32i32_v32i32v32i32i32i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : V6_vrsadubi_acc -class Hexagon_v64i32_v64i32v64i32i32i32_Intrinsic +class Hexagon_v64i32_v64i32v64i32i32i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : F2_conv_df2sf class Hexagon_float_double_Intrinsic @@ -1166,10 +1183,11 @@ class Hexagon_v32i32_v32i32v32i32v32i32_Intrinsic [IntrNoMem]>; // tag : S2_insertp -class Hexagon_i64_i64i64i32i32_Intrinsic +class Hexagon_i64_i64i64i32i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : F2_sfinvsqrta class Hexagon_floati32_float_Intrinsic @@ -1190,16 +1208,18 @@ class Hexagon_v32i32v32i32_v32i32v32i32i32_Intrinsic [IntrNoMem]>; // tag : V6_vlutvwh_oracc -class Hexagon_v32i32_v32i32v16i32v16i32i32_Intrinsic +class Hexagon_v32i32_v32i32v16i32v16i32i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : V6_vlutvwh_oracc -class Hexagon_v64i32_v64i32v32i32v32i32i32_Intrinsic +class Hexagon_v64i32_v64i32v32i32v32i32i32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem], intr_properties)>; // tag : F2_dfcmpge class Hexagon_i32_doubledouble_Intrinsic @@ -1223,7 +1243,7 @@ class Hexagon_i32_float_Intrinsic class Hexagon_i32_floati32_Intrinsic : Hexagon_Intrinsic; + [IntrNoMem, Throws, ImmArg<1>]>; // tag : F2_conv_sf2ud_chop class Hexagon_i64_float_Intrinsic @@ -1292,10 +1312,11 @@ class Hexagon_float_floatfloatfloati32_Intrinsic [IntrNoMem, Throws]>; // tag : F2_dfclass -class Hexagon_i32_doublei32_Intrinsic +class Hexagon_i32_doublei32_Intrinsic intr_properties = []> : Hexagon_Intrinsic; + !listconcat([IntrNoMem, Throws], intr_properties)>; // tag : V6_vd0 class Hexagon_v16i32__Intrinsic @@ -1393,13 +1414,13 @@ def int_hexagon_A2_vabswsat : Hexagon_i64_i64_Intrinsic<"HEXAGON_A2_vabswsat">; def int_hexagon_S2_asr_i_r : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_asr_i_r">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_asr_i_r", [ImmArg<1>]>; def int_hexagon_S2_asr_i_p : -Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_asr_i_p">; +Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_asr_i_p", [ImmArg<1>]>; def int_hexagon_A4_combineri : -Hexagon_i64_i32i32_Intrinsic<"HEXAGON_A4_combineri">; +Hexagon_i64_i32i32_Intrinsic<"HEXAGON_A4_combineri", [ImmArg<1>]>; def int_hexagon_M2_mpy_nac_sat_hl_s1 : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M2_mpy_nac_sat_hl_s1">; @@ -1450,7 +1471,7 @@ def int_hexagon_A2_maxup : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_maxup">; def int_hexagon_A4_vcmphgti : -Hexagon_i32_i64i32_Intrinsic<"HEXAGON_A4_vcmphgti">; +Hexagon_i32_i64i32_Intrinsic<"HEXAGON_A4_vcmphgti", [ImmArg<1>]>; def int_hexagon_S2_interleave : Hexagon_i64_i64_Intrinsic<"HEXAGON_S2_interleave">; @@ -1471,10 +1492,10 @@ def int_hexagon_C2_cmpgtp : Hexagon_i32_i64i64_Intrinsic<"HEXAGON_C2_cmpgtp">; def int_hexagon_A4_cmphgtui : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_cmphgtui">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_cmphgtui", [ImmArg<1>]>; def int_hexagon_C2_cmpgti : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C2_cmpgti">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C2_cmpgti", [ImmArg<1>]>; def int_hexagon_M2_mpyi : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_mpyi">; @@ -1492,16 +1513,16 @@ def int_hexagon_M2_mpy_lh_s0 : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_mpy_lh_s0">; def int_hexagon_S2_lsr_i_r_xacc : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_lsr_i_r_xacc">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_lsr_i_r_xacc", [ImmArg<2>]>; def int_hexagon_S2_vrcnegh : Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_vrcnegh">; def int_hexagon_S2_extractup : -Hexagon_i64_i64i32i32_Intrinsic<"HEXAGON_S2_extractup">; +Hexagon_i64_i64i32i32_Intrinsic<"HEXAGON_S2_extractup", [ImmArg<1>, ImmArg<2>]>; def int_hexagon_S2_asr_i_p_rnd_goodsyntax : -Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_asr_i_p_rnd_goodsyntax">; +Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_asr_i_p_rnd_goodsyntax", [ImmArg<1>]>; def int_hexagon_S4_ntstbit_r : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S4_ntstbit_r">; @@ -1528,10 +1549,10 @@ def int_hexagon_S2_asr_r_r_and : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asr_r_r_and">; def int_hexagon_A4_rcmpneqi : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_rcmpneqi">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_rcmpneqi", [ImmArg<1>]>; def int_hexagon_S2_asl_i_r_nac : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asl_i_r_nac">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asl_i_r_nac", [ImmArg<2>]>; def int_hexagon_M2_subacc : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M2_subacc">; @@ -1546,10 +1567,10 @@ def int_hexagon_M2_mpy_acc_sat_lh_s1 : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M2_mpy_acc_sat_lh_s1">; def int_hexagon_S2_asr_i_vh : -Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_asr_i_vh">; +Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_asr_i_vh", [ImmArg<1>]>; def int_hexagon_S2_asr_i_vw : -Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_asr_i_vw">; +Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_asr_i_vw", [ImmArg<1>]>; def int_hexagon_A4_cmpbgtu : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_cmpbgtu">; @@ -1558,7 +1579,7 @@ def int_hexagon_A4_vcmpbeq_any : Hexagon_i32_i64i64_Intrinsic<"HEXAGON_A4_vcmpbeq_any">; def int_hexagon_A4_cmpbgti : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_cmpbgti">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_cmpbgti", [ImmArg<1>]>; def int_hexagon_M2_mpyd_lh_s1 : Hexagon_i64_i32i32_Intrinsic<"HEXAGON_M2_mpyd_lh_s1">; @@ -1567,7 +1588,7 @@ def int_hexagon_S2_asl_r_p_nac : Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_asl_r_p_nac">; def int_hexagon_S2_lsr_i_r_nac : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_lsr_i_r_nac">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_lsr_i_r_nac", [ImmArg<2>]>; def int_hexagon_A2_addsp : Hexagon_i64_i32i64_Intrinsic<"HEXAGON_A2_addsp">; @@ -1576,7 +1597,7 @@ def int_hexagon_S4_vxsubaddw : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_S4_vxsubaddw">; def int_hexagon_A4_vcmpheqi : -Hexagon_i32_i64i32_Intrinsic<"HEXAGON_A4_vcmpheqi">; +Hexagon_i32_i64i32_Intrinsic<"HEXAGON_A4_vcmpheqi", [ImmArg<1>]>; def int_hexagon_S4_vxsubaddh : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_S4_vxsubaddh">; @@ -1603,16 +1624,16 @@ def int_hexagon_A2_pxorf : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_A2_pxorf">; def int_hexagon_C2_cmpgei : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C2_cmpgei">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C2_cmpgei", [ImmArg<1>]>; def int_hexagon_A2_vsubub : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_vsubub">; def int_hexagon_S2_asl_i_p : -Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_asl_i_p">; +Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_asl_i_p", [ImmArg<1>]>; def int_hexagon_S2_asl_i_r : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_asl_i_r">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_asl_i_r", [ImmArg<1>]>; def int_hexagon_A4_vrminuw : Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_A4_vrminuw">; @@ -1642,10 +1663,10 @@ def int_hexagon_C2_bitsset : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C2_bitsset">; def int_hexagon_M2_mpysip : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_mpysip">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_mpysip", [ImmArg<1>]>; def int_hexagon_M2_mpysin : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_mpysin">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_mpysin", [ImmArg<1>]>; def int_hexagon_A4_boundscheck : Hexagon_i32_i32i64_Intrinsic<"HEXAGON_A4_boundscheck">; @@ -1684,10 +1705,10 @@ def int_hexagon_A2_vnavgw : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_vnavgw">; def int_hexagon_S2_asl_i_r_acc : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asl_i_r_acc">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asl_i_r_acc", [ImmArg<2>]>; def int_hexagon_S4_subi_lsr_ri : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_subi_lsr_ri">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_subi_lsr_ri", [ImmArg<0>, ImmArg<2>]>; def int_hexagon_S2_vzxthw : Hexagon_i64_i32_Intrinsic<"HEXAGON_S2_vzxthw">; @@ -1714,7 +1735,7 @@ def int_hexagon_S2_packhl : Hexagon_i64_i32i32_Intrinsic<"HEXAGON_S2_packhl">; def int_hexagon_A4_vcmpwgti : -Hexagon_i32_i64i32_Intrinsic<"HEXAGON_A4_vcmpwgti">; +Hexagon_i32_i64i32_Intrinsic<"HEXAGON_A4_vcmpwgti", [ImmArg<1>]>; def int_hexagon_A2_vavguwr : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_vavguwr">; @@ -1735,7 +1756,7 @@ def int_hexagon_F2_conv_d2df : Hexagon_double_i64_Intrinsic<"HEXAGON_F2_conv_d2df">; def int_hexagon_C2_cmpgtui : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C2_cmpgtui">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C2_cmpgtui", [ImmArg<1>]>; def int_hexagon_A2_vconj : Hexagon_i64_i64_Intrinsic<"HEXAGON_A2_vconj">; @@ -1765,7 +1786,7 @@ def int_hexagon_S2_togglebit_r : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_togglebit_r">; def int_hexagon_S2_togglebit_i : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_togglebit_i">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_togglebit_i", [ImmArg<1>]>; def int_hexagon_F2_conv_uw2sf : Hexagon_float_i32_Intrinsic<"HEXAGON_F2_conv_uw2sf">; @@ -1801,10 +1822,10 @@ def int_hexagon_S2_asl_r_r_nac : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asl_r_r_nac">; def int_hexagon_S2_asl_i_p_acc : -Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_asl_i_p_acc">; +Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_asl_i_p_acc", [ImmArg<2>]>; def int_hexagon_A4_vcmpwgtui : -Hexagon_i32_i64i32_Intrinsic<"HEXAGON_A4_vcmpwgtui">; +Hexagon_i32_i64i32_Intrinsic<"HEXAGON_A4_vcmpwgtui", [ImmArg<1>]>; def int_hexagon_M4_vrmpyoh_acc_s0 : Hexagon_i64_i64i64i64_Intrinsic<"HEXAGON_M4_vrmpyoh_acc_s0">; @@ -1831,7 +1852,7 @@ def int_hexagon_A2_vavgwcr : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_vavgwcr">; def int_hexagon_S2_asl_i_p_xacc : -Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_asl_i_p_xacc">; +Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_asl_i_p_xacc", [ImmArg<2>]>; def int_hexagon_A4_vrmaxw : Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_A4_vrmaxw">; @@ -1843,22 +1864,22 @@ def int_hexagon_M4_cmpyi_wh : Hexagon_i32_i64i32_Intrinsic<"HEXAGON_M4_cmpyi_wh">; def int_hexagon_A2_tfrsi : -Hexagon_i32_i32_Intrinsic<"HEXAGON_A2_tfrsi">; +Hexagon_i32_i32_Intrinsic<"HEXAGON_A2_tfrsi", [ImmArg<0>]>; def int_hexagon_S2_asr_i_r_acc : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asr_i_r_acc">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asr_i_r_acc", [ImmArg<2>]>; def int_hexagon_A2_svnavgh : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_svnavgh">; def int_hexagon_S2_lsr_i_r : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_lsr_i_r">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_lsr_i_r", [ImmArg<1>]>; def int_hexagon_M2_vmac2 : Hexagon_i64_i64i32i32_Intrinsic<"HEXAGON_M2_vmac2">; def int_hexagon_A4_vcmphgtui : -Hexagon_i32_i64i32_Intrinsic<"HEXAGON_A4_vcmphgtui">; +Hexagon_i32_i64i32_Intrinsic<"HEXAGON_A4_vcmphgtui", [ImmArg<1>]>; def int_hexagon_A2_svavgh : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_svavgh">; @@ -1870,7 +1891,7 @@ def int_hexagon_M4_vrmpyeh_acc_s1 : Hexagon_i64_i64i64i64_Intrinsic<"HEXAGON_M4_vrmpyeh_acc_s1">; def int_hexagon_S2_lsr_i_p : -Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_lsr_i_p">; +Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_lsr_i_p", [ImmArg<1>]>; def int_hexagon_A2_combine_hl : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_combine_hl">; @@ -1909,7 +1930,7 @@ def int_hexagon_M2_mmpyul_rs0 : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_M2_mmpyul_rs0">; def int_hexagon_S2_asr_i_r_rnd_goodsyntax : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_asr_i_r_rnd_goodsyntax">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_asr_i_r_rnd_goodsyntax", [ImmArg<1>]>; def int_hexagon_S2_lsr_r_p_nac : Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_lsr_r_p_nac">; @@ -1924,10 +1945,10 @@ def int_hexagon_M4_or_and : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M4_or_and">; def int_hexagon_M4_mpyrr_addi : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M4_mpyrr_addi">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M4_mpyrr_addi", [ImmArg<0>]>; def int_hexagon_S4_or_andi : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_or_andi">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_or_andi", [ImmArg<2>]>; def int_hexagon_M2_mpy_sat_hl_s0 : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_mpy_sat_hl_s0">; @@ -2032,7 +2053,7 @@ def int_hexagon_F2_sffms_lib : Hexagon_float_floatfloatfloat_Intrinsic<"HEXAGON_F2_sffms_lib">; def int_hexagon_C4_cmpneqi : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C4_cmpneqi">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C4_cmpneqi", [ImmArg<1>]>; def int_hexagon_M4_and_xor : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M4_and_xor">; @@ -2056,7 +2077,7 @@ def int_hexagon_A2_vrsadub_acc : Hexagon_i64_i64i64i64_Intrinsic<"HEXAGON_A2_vrsadub_acc">; def int_hexagon_C2_bitsclri : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C2_bitsclri">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C2_bitsclri", [ImmArg<1>]>; def int_hexagon_A2_subh_h16_sat_hh : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_subh_h16_sat_hh">; @@ -2158,10 +2179,10 @@ def int_hexagon_S2_parityp : Hexagon_i32_i64i64_Intrinsic<"HEXAGON_S2_parityp">; def int_hexagon_S2_lsr_i_p_and : -Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_lsr_i_p_and">; +Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_lsr_i_p_and", [ImmArg<2>]>; def int_hexagon_S2_asr_i_r_or : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asr_i_r_or">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asr_i_r_or", [ImmArg<2>]>; def int_hexagon_M2_mpyu_nac_ll_s0 : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M2_mpyu_nac_ll_s0">; @@ -2191,7 +2212,7 @@ def int_hexagon_M2_cnacsc_s0 : Hexagon_i64_i64i32i32_Intrinsic<"HEXAGON_M2_cnacsc_s0">; def int_hexagon_S4_subaddi : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_subaddi">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_subaddi", [ImmArg<1>]>; def int_hexagon_M2_mpyud_nac_hl_s1 : Hexagon_i64_i64i32i32_Intrinsic<"HEXAGON_M2_mpyud_nac_hl_s1">; @@ -2200,13 +2221,13 @@ def int_hexagon_M2_mpyud_nac_hl_s0 : Hexagon_i64_i64i32i32_Intrinsic<"HEXAGON_M2_mpyud_nac_hl_s0">; def int_hexagon_S5_vasrhrnd_goodsyntax : -Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S5_vasrhrnd_goodsyntax">; +Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S5_vasrhrnd_goodsyntax", [ImmArg<1>]>; def int_hexagon_S2_tstbit_r : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_tstbit_r">; def int_hexagon_S4_vrcrotate : -Hexagon_i64_i64i32i32_Intrinsic<"HEXAGON_S4_vrcrotate">; +Hexagon_i64_i64i32i32_Intrinsic<"HEXAGON_S4_vrcrotate", [ImmArg<2>]>; def int_hexagon_M2_mmachs_s1 : Hexagon_i64_i64i64i64_Intrinsic<"HEXAGON_M2_mmachs_s1">; @@ -2215,7 +2236,7 @@ def int_hexagon_M2_mmachs_s0 : Hexagon_i64_i64i64i64_Intrinsic<"HEXAGON_M2_mmachs_s0">; def int_hexagon_S2_tstbit_i : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_tstbit_i">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_tstbit_i", [ImmArg<1>]>; def int_hexagon_M2_mpy_up_s1 : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_mpy_up_s1">; @@ -2227,7 +2248,7 @@ def int_hexagon_M2_mmpyuh_rs0 : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_M2_mmpyuh_rs0">; def int_hexagon_S2_lsr_i_vw : -Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_lsr_i_vw">; +Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_lsr_i_vw", [ImmArg<1>]>; def int_hexagon_M2_mpy_rnd_ll_s0 : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_mpy_rnd_ll_s0">; @@ -2266,16 +2287,16 @@ def int_hexagon_A2_subh_l16_sat_hl : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_subh_l16_sat_hl">; def int_hexagon_C2_cmpeqi : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C2_cmpeqi">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C2_cmpeqi", [ImmArg<1>]>; def int_hexagon_S2_asl_i_r_and : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asl_i_r_and">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asl_i_r_and", [ImmArg<2>]>; def int_hexagon_S2_vcnegh : Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_vcnegh">; def int_hexagon_A4_vcmpweqi : -Hexagon_i32_i64i32_Intrinsic<"HEXAGON_A4_vcmpweqi">; +Hexagon_i32_i64i32_Intrinsic<"HEXAGON_A4_vcmpweqi", [ImmArg<1>]>; def int_hexagon_M2_vdmpyrs_s0 : Hexagon_i32_i64i64_Intrinsic<"HEXAGON_M2_vdmpyrs_s0">; @@ -2308,7 +2329,7 @@ def int_hexagon_S2_cl0p : Hexagon_i32_i64_Intrinsic<"HEXAGON_S2_cl0p">; def int_hexagon_S2_valignib : -Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_valignib">; +Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_valignib", [ImmArg<2>]>; def int_hexagon_F2_sffixupd : Hexagon_float_floatfloat_Intrinsic<"HEXAGON_F2_sffixupd">; @@ -2338,7 +2359,7 @@ def int_hexagon_M2_mmpyul_rs1 : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_M2_mmpyul_rs1">; def int_hexagon_S4_ntstbit_i : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S4_ntstbit_i">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S4_ntstbit_i", [ImmArg<1>]> ; def int_hexagon_F2_sffixupr : Hexagon_float_float_Intrinsic<"HEXAGON_F2_sffixupr">; @@ -2362,7 +2383,7 @@ def int_hexagon_M2_vmpy2s_s0pack : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_vmpy2s_s0pack">; def int_hexagon_S4_addaddi : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_addaddi">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_addaddi", [ImmArg<2>]>; def int_hexagon_M2_mpyd_acc_ll_s0 : Hexagon_i64_i64i32i32_Intrinsic<"HEXAGON_M2_mpyd_acc_ll_s0">; @@ -2371,13 +2392,13 @@ def int_hexagon_M2_mpy_acc_sat_hl_s1 : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M2_mpy_acc_sat_hl_s1">; def int_hexagon_A4_rcmpeqi : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_rcmpeqi">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_rcmpeqi", [ImmArg<1>]>; def int_hexagon_M4_xor_and : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M4_xor_and">; def int_hexagon_S2_asl_i_p_and : -Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_asl_i_p_and">; +Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_asl_i_p_and", [ImmArg<2>]>; def int_hexagon_M2_mmpyuh_rs1 : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_M2_mmpyuh_rs1">; @@ -2386,7 +2407,7 @@ def int_hexagon_S2_asr_r_r_or : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asr_r_r_or">; def int_hexagon_A4_round_ri : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_round_ri">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_round_ri", [ImmArg<1>]>; def int_hexagon_A2_max : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_max">; @@ -2395,10 +2416,10 @@ def int_hexagon_A4_round_rr : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_round_rr">; def int_hexagon_A4_combineii : -Hexagon_i64_i32i32_Intrinsic<"HEXAGON_A4_combineii">; +Hexagon_i64_i32i32_Intrinsic<"HEXAGON_A4_combineii", [ImmArg<0>, ImmArg<1>]>; def int_hexagon_A4_combineir : -Hexagon_i64_i32i32_Intrinsic<"HEXAGON_A4_combineir">; +Hexagon_i64_i32i32_Intrinsic<"HEXAGON_A4_combineir", [ImmArg<0>]>; def int_hexagon_C4_and_orn : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_C4_and_orn">; @@ -2413,7 +2434,7 @@ def int_hexagon_M4_cmpyr_whc : Hexagon_i32_i64i32_Intrinsic<"HEXAGON_M4_cmpyr_whc">; def int_hexagon_S2_lsr_i_r_acc : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_lsr_i_r_acc">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_lsr_i_r_acc", [ImmArg<2>]>; def int_hexagon_S2_vzxtbh : Hexagon_i64_i32_Intrinsic<"HEXAGON_S2_vzxtbh">; @@ -2440,7 +2461,7 @@ def int_hexagon_S2_asl_r_p_or : Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_asl_r_p_or">; def int_hexagon_S4_ori_asl_ri : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_ori_asl_ri">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_ori_asl_ri", [ImmArg<0>, ImmArg<2>]>; def int_hexagon_C4_nbitsset : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C4_nbitsset">; @@ -2476,10 +2497,10 @@ def int_hexagon_M2_mpyd_acc_hh_s1 : Hexagon_i64_i64i32i32_Intrinsic<"HEXAGON_M2_mpyd_acc_hh_s1">; def int_hexagon_F2_sfimm_p : -Hexagon_float_i32_Intrinsic<"HEXAGON_F2_sfimm_p">; +Hexagon_float_i32_Intrinsic<"HEXAGON_F2_sfimm_p", [ImmArg<0>]>; def int_hexagon_F2_sfimm_n : -Hexagon_float_i32_Intrinsic<"HEXAGON_F2_sfimm_n">; +Hexagon_float_i32_Intrinsic<"HEXAGON_F2_sfimm_n", [ImmArg<0>]>; def int_hexagon_M4_cmpyr_wh : Hexagon_i32_i64i32_Intrinsic<"HEXAGON_M4_cmpyr_wh">; @@ -2497,7 +2518,7 @@ def int_hexagon_A2_vavguh : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_vavguh">; def int_hexagon_A4_cmpbeqi : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_cmpbeqi">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_cmpbeqi", [ImmArg<1>]>; def int_hexagon_F2_sfcmpuo : Hexagon_i32_floatfloat_Intrinsic<"HEXAGON_F2_sfcmpuo">; @@ -2506,7 +2527,7 @@ def int_hexagon_A2_vavguw : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_vavguw">; def int_hexagon_S2_asr_i_p_nac : -Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_asr_i_p_nac">; +Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_asr_i_p_nac", [ImmArg<2>]>; def int_hexagon_S2_vsatwh_nopack : Hexagon_i64_i64_Intrinsic<"HEXAGON_S2_vsatwh_nopack">; @@ -2533,7 +2554,7 @@ def int_hexagon_A2_minp : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_minp">; def int_hexagon_S4_or_andix : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_or_andix">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_or_andix", [ImmArg<2>]>; def int_hexagon_M2_mpy_rnd_lh_s0 : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_mpy_rnd_lh_s0">; @@ -2584,19 +2605,19 @@ def int_hexagon_S2_lsl_r_r_or : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_lsl_r_r_or">; def int_hexagon_C4_cmplteui : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C4_cmplteui">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C4_cmplteui", [ImmArg<1>]>; def int_hexagon_S4_addi_lsr_ri : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_addi_lsr_ri">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_addi_lsr_ri", [ImmArg<0>, ImmArg<2>]>; def int_hexagon_A4_tfrcpp : Hexagon_i64_i64_Intrinsic<"HEXAGON_A4_tfrcpp">; def int_hexagon_S2_asr_i_svw_trun : -Hexagon_i32_i64i32_Intrinsic<"HEXAGON_S2_asr_i_svw_trun">; +Hexagon_i32_i64i32_Intrinsic<"HEXAGON_S2_asr_i_svw_trun", [ImmArg<1>]>; def int_hexagon_A4_cmphgti : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_cmphgti">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_cmphgti", [ImmArg<1>]>; def int_hexagon_A4_vrminh : Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_A4_vrminh">; @@ -2614,7 +2635,7 @@ def int_hexagon_A2_vnavghcr : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_vnavghcr">; def int_hexagon_S4_subi_asl_ri : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_subi_asl_ri">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_subi_asl_ri", [ImmArg<0>, ImmArg<2>]>; def int_hexagon_S2_lsl_r_vh : Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_lsl_r_vh">; @@ -2638,7 +2659,7 @@ def int_hexagon_C2_cmpltu : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C2_cmpltu">; def int_hexagon_S2_insertp : -Hexagon_i64_i64i64i32i32_Intrinsic<"HEXAGON_S2_insertp">; +Hexagon_i64_i64i64i32i32_Intrinsic<"HEXAGON_S2_insertp", [ImmArg<2>, ImmArg<3>]>; def int_hexagon_M2_mpyd_rnd_ll_s1 : Hexagon_i64_i32i32_Intrinsic<"HEXAGON_M2_mpyd_rnd_ll_s1">; @@ -2647,7 +2668,7 @@ def int_hexagon_M2_mpyd_rnd_ll_s0 : Hexagon_i64_i32i32_Intrinsic<"HEXAGON_M2_mpyd_rnd_ll_s0">; def int_hexagon_S2_lsr_i_p_nac : -Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_lsr_i_p_nac">; +Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_lsr_i_p_nac", [ImmArg<2>]>; def int_hexagon_S2_extractup_rp : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_S2_extractup_rp">; @@ -2749,7 +2770,7 @@ def int_hexagon_M2_dpmpyss_rnd_s0 : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_dpmpyss_rnd_s0">; def int_hexagon_C2_muxri : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_C2_muxri">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_C2_muxri", [ImmArg<1>]>; def int_hexagon_M2_vmac2es_s0 : Hexagon_i64_i64i64i64_Intrinsic<"HEXAGON_M2_vmac2es_s0">; @@ -2767,7 +2788,7 @@ def int_hexagon_M2_mpyu_lh_s0 : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_mpyu_lh_s0">; def int_hexagon_S2_asl_i_r_or : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asl_i_r_or">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asl_i_r_or", [ImmArg<2>]>; def int_hexagon_M2_mpyd_acc_hl_s0 : Hexagon_i64_i64i32i32_Intrinsic<"HEXAGON_M2_mpyd_acc_hl_s0">; @@ -2782,7 +2803,7 @@ def int_hexagon_A2_vaddw : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_vaddw">; def int_hexagon_S2_asr_i_r_and : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asr_i_r_and">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asr_i_r_and", [ImmArg<2>]>; def int_hexagon_A2_vaddh : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_vaddh">; @@ -2797,22 +2818,22 @@ def int_hexagon_C2_cmpeqp : Hexagon_i32_i64i64_Intrinsic<"HEXAGON_C2_cmpeqp">; def int_hexagon_M4_mpyri_addi : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M4_mpyri_addi">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M4_mpyri_addi", [ImmArg<0>, ImmArg<2>]>; def int_hexagon_A2_not : Hexagon_i32_i32_Intrinsic<"HEXAGON_A2_not">; def int_hexagon_S4_andi_lsr_ri : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_andi_lsr_ri">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_andi_lsr_ri", [ImmArg<0>, ImmArg<2>]>; def int_hexagon_M2_macsip : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M2_macsip">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M2_macsip", [ImmArg<2>]>; def int_hexagon_A2_tfrcrr : Hexagon_i32_i32_Intrinsic<"HEXAGON_A2_tfrcrr">; def int_hexagon_M2_macsin : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M2_macsin">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M2_macsin", [ImmArg<2>]>; def int_hexagon_C2_orn : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C2_orn">; @@ -2875,7 +2896,7 @@ def int_hexagon_F2_dfcmpge : Hexagon_i32_doubledouble_Intrinsic<"HEXAGON_F2_dfcmpge">; def int_hexagon_M2_accii : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M2_accii">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M2_accii", [ImmArg<2>]>; def int_hexagon_A5_vaddhubs : Hexagon_i32_i64i64_Intrinsic<"HEXAGON_A5_vaddhubs">; @@ -2893,10 +2914,10 @@ def int_hexagon_S2_vsxthw : Hexagon_i64_i32_Intrinsic<"HEXAGON_S2_vsxthw">; def int_hexagon_S4_andi_asl_ri : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_andi_asl_ri">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_andi_asl_ri", [ImmArg<0>, ImmArg<2>]>; def int_hexagon_S2_asl_i_p_nac : -Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_asl_i_p_nac">; +Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_asl_i_p_nac", [ImmArg<2>]>; def int_hexagon_S2_lsl_r_p_xor : Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_lsl_r_p_xor">; @@ -2929,7 +2950,7 @@ def int_hexagon_M4_xor_andn : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M4_xor_andn">; def int_hexagon_S2_addasl_rrri : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_addasl_rrri">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_addasl_rrri", [ImmArg<2>]>; def int_hexagon_M5_vdmpybsu : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_M5_vdmpybsu">; @@ -2941,7 +2962,7 @@ def int_hexagon_M2_mpyu_nac_hh_s1 : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M2_mpyu_nac_hh_s1">; def int_hexagon_A2_addi : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_addi">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_addi", [ImmArg<1>]>; def int_hexagon_A2_addp : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_addp">; @@ -2962,7 +2983,7 @@ def int_hexagon_S2_shuffeh : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_S2_shuffeh">; def int_hexagon_S2_lsr_i_r_and : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_lsr_i_r_and">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_lsr_i_r_and", [ImmArg<2>]>; def int_hexagon_M2_mpy_sat_rnd_hh_s1 : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_mpy_sat_rnd_hh_s1">; @@ -3064,13 +3085,13 @@ def int_hexagon_S5_popcountp : Hexagon_i32_i64_Intrinsic<"HEXAGON_S5_popcountp">; def int_hexagon_S4_extractp : -Hexagon_i64_i64i32i32_Intrinsic<"HEXAGON_S4_extractp">; +Hexagon_i64_i64i32i32_Intrinsic<"HEXAGON_S4_extractp", [ImmArg<1>, ImmArg<2>]>; def int_hexagon_S2_cl0 : Hexagon_i32_i32_Intrinsic<"HEXAGON_S2_cl0">; def int_hexagon_A4_vcmpbgti : -Hexagon_i32_i64i32_Intrinsic<"HEXAGON_A4_vcmpbgti">; +Hexagon_i32_i64i32_Intrinsic<"HEXAGON_A4_vcmpbgti", [ImmArg<1>]>; def int_hexagon_M2_mmacls_s1 : Hexagon_i64_i64i64i64_Intrinsic<"HEXAGON_M2_mmacls_s1">; @@ -3118,7 +3139,7 @@ def int_hexagon_A2_vmaxuh : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_vmaxuh">; def int_hexagon_A4_bitspliti : -Hexagon_i64_i32i32_Intrinsic<"HEXAGON_A4_bitspliti">; +Hexagon_i64_i32i32_Intrinsic<"HEXAGON_A4_bitspliti", [ImmArg<1>]>; def int_hexagon_A2_vmaxub : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_vmaxub">; @@ -3145,13 +3166,13 @@ def int_hexagon_S2_asr_r_r_nac : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asr_r_r_nac">; def int_hexagon_F2_dfimm_n : -Hexagon_double_i32_Intrinsic<"HEXAGON_F2_dfimm_n">; +Hexagon_double_i32_Intrinsic<"HEXAGON_F2_dfimm_n", [ImmArg<0>]>; def int_hexagon_A4_cmphgt : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_cmphgt">; def int_hexagon_F2_dfimm_p : -Hexagon_double_i32_Intrinsic<"HEXAGON_F2_dfimm_p">; +Hexagon_double_i32_Intrinsic<"HEXAGON_F2_dfimm_p", [ImmArg<0>]>; def int_hexagon_M2_mpyud_acc_lh_s1 : Hexagon_i64_i64i32i32_Intrinsic<"HEXAGON_M2_mpyud_acc_lh_s1">; @@ -3160,7 +3181,7 @@ def int_hexagon_M2_vcmpy_s1_sat_r : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_M2_vcmpy_s1_sat_r">; def int_hexagon_M4_mpyri_addr_u2 : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M4_mpyri_addr_u2">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M4_mpyri_addr_u2", [ImmArg<1>]>; def int_hexagon_M2_vcmpy_s1_sat_i : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_M2_vcmpy_s1_sat_i">; @@ -3172,10 +3193,10 @@ def int_hexagon_M5_vrmacbuu : Hexagon_i64_i64i64i64_Intrinsic<"HEXAGON_M5_vrmacbuu">; def int_hexagon_S5_asrhub_rnd_sat_goodsyntax : -Hexagon_i32_i64i32_Intrinsic<"HEXAGON_S5_asrhub_rnd_sat_goodsyntax">; +Hexagon_i32_i64i32_Intrinsic<"HEXAGON_S5_asrhub_rnd_sat_goodsyntax", [ImmArg<1>]>; def int_hexagon_S2_vspliceib : -Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_vspliceib">; +Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_vspliceib", [ImmArg<2>]>; def int_hexagon_M2_dpmpyss_acc_s0 : Hexagon_i64_i64i32i32_Intrinsic<"HEXAGON_M2_dpmpyss_acc_s0">; @@ -3193,25 +3214,25 @@ def int_hexagon_A2_maxp : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_maxp">; def int_hexagon_A2_andir : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_andir">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_andir", [ImmArg<1>]>; def int_hexagon_F2_sfrecipa : Hexagon_floati32_floatfloat_Intrinsic<"HEXAGON_F2_sfrecipa">; def int_hexagon_A2_combineii : -Hexagon_i64_i32i32_Intrinsic<"HEXAGON_A2_combineii">; +Hexagon_i64_i32i32_Intrinsic<"HEXAGON_A2_combineii", [ImmArg<0>, ImmArg<1>]>; def int_hexagon_A4_orn : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_orn">; def int_hexagon_A4_cmpbgtui : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_cmpbgtui">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_cmpbgtui", [ImmArg<1>]>; def int_hexagon_S2_lsr_r_r_or : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_lsr_r_r_or">; def int_hexagon_A4_vcmpbeqi : -Hexagon_i32_i64i32_Intrinsic<"HEXAGON_A4_vcmpbeqi">; +Hexagon_i32_i64i32_Intrinsic<"HEXAGON_A4_vcmpbeqi", [ImmArg<1>]>; def int_hexagon_S2_lsl_r_r : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_lsl_r_r">; @@ -3247,19 +3268,19 @@ def int_hexagon_M2_vrcmpys_s1 : Hexagon_i64_i64i32_Intrinsic<"HEXAGON_M2_vrcmpys_s1">; def int_hexagon_S4_or_ori : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_or_ori">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_or_ori", [ImmArg<2>]>; def int_hexagon_C4_fastcorner9_not : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C4_fastcorner9_not">; def int_hexagon_A2_tfrih : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_tfrih">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_tfrih", [ImmArg<1>]>; def int_hexagon_A2_tfril : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_tfril">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_tfril", [ImmArg<1>]>; def int_hexagon_M4_mpyri_addr : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M4_mpyri_addr">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M4_mpyri_addr", [ImmArg<2>]>; def int_hexagon_S2_vtrunehb : Hexagon_i32_i64_Intrinsic<"HEXAGON_S2_vtrunehb">; @@ -3274,16 +3295,16 @@ def int_hexagon_F2_sfsub : Hexagon_float_floatfloat_Intrinsic<"HEXAGON_F2_sfsub">; def int_hexagon_C2_muxii : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_C2_muxii">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_C2_muxii", [ImmArg<1>, ImmArg<2>]>; def int_hexagon_C2_muxir : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_C2_muxir">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_C2_muxir", [ImmArg<2>]>; def int_hexagon_A2_swiz : Hexagon_i32_i32_Intrinsic<"HEXAGON_A2_swiz">; def int_hexagon_S2_asr_i_p_and : -Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_asr_i_p_and">; +Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_asr_i_p_and", [ImmArg<2>]>; def int_hexagon_M2_cmpyrsc_s0 : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_cmpyrsc_s0">; @@ -3313,7 +3334,7 @@ def int_hexagon_M2_mpy_nac_sat_ll_s0 : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M2_mpy_nac_sat_ll_s0">; def int_hexagon_S4_extract : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_extract">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_extract", [ImmArg<1>, ImmArg<2>]>; def int_hexagon_A2_vcmpweq : Hexagon_i32_i64i64_Intrinsic<"HEXAGON_A2_vcmpweq">; @@ -3322,10 +3343,10 @@ def int_hexagon_M2_acci : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M2_acci">; def int_hexagon_S2_lsr_i_p_acc : -Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_lsr_i_p_acc">; +Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_lsr_i_p_acc", [ImmArg<2>]>; def int_hexagon_S2_lsr_i_p_or : -Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_lsr_i_p_or">; +Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_lsr_i_p_or", [ImmArg<2>]>; def int_hexagon_F2_conv_ud2sf : Hexagon_float_i64_Intrinsic<"HEXAGON_F2_conv_ud2sf">; @@ -3334,10 +3355,10 @@ def int_hexagon_A2_tfr : Hexagon_i32_i32_Intrinsic<"HEXAGON_A2_tfr">; def int_hexagon_S2_asr_i_p_or : -Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_asr_i_p_or">; +Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_asr_i_p_or", [ImmArg<2>]>; def int_hexagon_A2_subri : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_subri">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_subri", [ImmArg<0>]>; def int_hexagon_A4_vrmaxuw : Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_A4_vrmaxuw">; @@ -3349,7 +3370,7 @@ def int_hexagon_A4_vrmaxuh : Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_A4_vrmaxuh">; def int_hexagon_S2_asl_i_vw : -Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_asl_i_vw">; +Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_asl_i_vw", [ImmArg<1>]>; def int_hexagon_A2_vavgw : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_vavgw">; @@ -3361,13 +3382,13 @@ def int_hexagon_A2_vavgh : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_vavgh">; def int_hexagon_S2_clrbit_i : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_clrbit_i">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_clrbit_i", [ImmArg<1>]>; def int_hexagon_S2_asl_i_vh : -Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_asl_i_vh">; +Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_asl_i_vh", [ImmArg<1>]>; def int_hexagon_S2_lsr_i_r_or : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_lsr_i_r_or">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_lsr_i_r_or", [ImmArg<2>]>; def int_hexagon_S2_lsl_r_r_nac : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_lsl_r_r_nac">; @@ -3385,7 +3406,7 @@ def int_hexagon_M2_mmpyl_s1 : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_M2_mmpyl_s1">; def int_hexagon_M2_naccii : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M2_naccii">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M2_naccii", [ImmArg<2>]>; def int_hexagon_S2_vrndpackwhs : Hexagon_i32_i64_Intrinsic<"HEXAGON_S2_vrndpackwhs">; @@ -3406,7 +3427,7 @@ def int_hexagon_M4_mac_up_s1_sat : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M4_mac_up_s1_sat">; def int_hexagon_S4_vrcrotate_acc : -Hexagon_i64_i64i64i32i32_Intrinsic<"HEXAGON_S4_vrcrotate_acc">; +Hexagon_i64_i64i64i32i32_Intrinsic<"HEXAGON_S4_vrcrotate_acc", [ImmArg<3>]>; def int_hexagon_F2_conv_uw2df : Hexagon_double_i32_Intrinsic<"HEXAGON_F2_conv_uw2df">; @@ -3418,7 +3439,7 @@ def int_hexagon_S2_asr_r_r_acc : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asr_r_r_acc">; def int_hexagon_A2_orir : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_orir">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_orir", [ImmArg<1>]>; def int_hexagon_A2_andp : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_andp">; @@ -3430,7 +3451,7 @@ def int_hexagon_A2_min : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_min">; def int_hexagon_M2_mpysmi : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_mpysmi">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_mpysmi", [ImmArg<1>]>; def int_hexagon_M2_vcmpy_s0_sat_r : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_M2_vcmpy_s0_sat_r">; @@ -3466,10 +3487,10 @@ def int_hexagon_F2_conv_df2w : Hexagon_i32_double_Intrinsic<"HEXAGON_F2_conv_df2w">; def int_hexagon_S5_asrhub_sat : -Hexagon_i32_i64i32_Intrinsic<"HEXAGON_S5_asrhub_sat">; +Hexagon_i32_i64i32_Intrinsic<"HEXAGON_S5_asrhub_sat", [ImmArg<1>]>; def int_hexagon_S2_asl_i_r_xacc : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asl_i_r_xacc">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asl_i_r_xacc", [ImmArg<2>]>; def int_hexagon_F2_conv_df2d : Hexagon_i64_double_Intrinsic<"HEXAGON_F2_conv_df2d">; @@ -3505,7 +3526,7 @@ def int_hexagon_F2_sffma_sc : Hexagon_float_floatfloatfloati32_Intrinsic<"HEXAGON_F2_sffma_sc">; def int_hexagon_F2_dfclass : -Hexagon_i32_doublei32_Intrinsic<"HEXAGON_F2_dfclass">; +Hexagon_i32_doublei32_Intrinsic<"HEXAGON_F2_dfclass", [ImmArg<1>]>; def int_hexagon_F2_conv_df2ud : Hexagon_i64_double_Intrinsic<"HEXAGON_F2_conv_df2ud">; @@ -3520,7 +3541,7 @@ def int_hexagon_M2_cmpyrs_s1 : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_cmpyrs_s1">; def int_hexagon_C4_cmpltei : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C4_cmpltei">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C4_cmpltei", [ImmArg<1>]>; def int_hexagon_C4_cmplteu : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C4_cmplteu">; @@ -3532,7 +3553,7 @@ def int_hexagon_A2_subh_l16_ll : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_subh_l16_ll">; def int_hexagon_S2_asr_i_r_rnd : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_asr_i_r_rnd">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_asr_i_r_rnd", [ImmArg<1>]>; def int_hexagon_M2_vrmpy_s0 : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_M2_vrmpy_s0">; @@ -3577,7 +3598,7 @@ def int_hexagon_M2_vrcmpyi_s0c : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_M2_vrcmpyi_s0c">; def int_hexagon_S2_asr_i_p_rnd : -Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_asr_i_p_rnd">; +Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_asr_i_p_rnd", [ImmArg<1>]>; def int_hexagon_A2_addpsat : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_addpsat">; @@ -3586,7 +3607,7 @@ def int_hexagon_A2_svaddhs : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_svaddhs">; def int_hexagon_S4_ori_lsr_ri : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_ori_lsr_ri">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_ori_lsr_ri", [ImmArg<0>, ImmArg<2>]>; def int_hexagon_M2_mpy_sat_rnd_ll_s1 : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_M2_mpy_sat_rnd_ll_s1">; @@ -3619,7 +3640,7 @@ def int_hexagon_S2_asl_r_r_or : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asl_r_r_or">; def int_hexagon_S4_lsli : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S4_lsli">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S4_lsli", [ImmArg<0>]>; def int_hexagon_S2_lsl_r_vw : Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_lsl_r_vw">; @@ -3664,7 +3685,7 @@ def int_hexagon_A2_negp : Hexagon_i64_i64_Intrinsic<"HEXAGON_A2_negp">; def int_hexagon_S2_asl_i_r_sat : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_asl_i_r_sat">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_asl_i_r_sat", [ImmArg<1>]>; def int_hexagon_A2_addh_l16_sat_hl : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_addh_l16_sat_hl">; @@ -3682,10 +3703,10 @@ def int_hexagon_C2_cmpgtup : Hexagon_i32_i64i64_Intrinsic<"HEXAGON_C2_cmpgtup">; def int_hexagon_A4_cround_ri : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_cround_ri">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_cround_ri", [ImmArg<1>]>; def int_hexagon_S4_clbpaddi : -Hexagon_i32_i64i32_Intrinsic<"HEXAGON_S4_clbpaddi">; +Hexagon_i32_i64i32_Intrinsic<"HEXAGON_S4_clbpaddi", [ImmArg<1>]>; def int_hexagon_A4_cround_rr : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_cround_rr">; @@ -3715,13 +3736,13 @@ def int_hexagon_A2_vminub : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A2_vminub">; def int_hexagon_S2_extractu : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_extractu">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_extractu", [ImmArg<1>, ImmArg<2>]>; def int_hexagon_A2_svsubh : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A2_svsubh">; def int_hexagon_S4_clbaddi : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S4_clbaddi">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S4_clbaddi", [ImmArg<1>]>; def int_hexagon_F2_sffms : Hexagon_float_floatfloatfloat_Intrinsic<"HEXAGON_F2_sffms">; @@ -3754,7 +3775,7 @@ def int_hexagon_M2_mpy_acc_hh_s0 : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M2_mpy_acc_hh_s0">; def int_hexagon_S4_addi_asl_ri : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_addi_asl_ri">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S4_addi_asl_ri", [ImmArg<0>, ImmArg<2>]>; def int_hexagon_M2_mpyd_nac_hh_s1 : Hexagon_i64_i64i32i32_Intrinsic<"HEXAGON_M2_mpyd_nac_hh_s1">; @@ -3763,10 +3784,10 @@ def int_hexagon_M2_mpyd_nac_hh_s0 : Hexagon_i64_i64i32i32_Intrinsic<"HEXAGON_M2_mpyd_nac_hh_s0">; def int_hexagon_S2_asr_i_r_nac : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asr_i_r_nac">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S2_asr_i_r_nac", [ImmArg<2>]>; def int_hexagon_A4_cmpheqi : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_cmpheqi">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_cmpheqi", [ImmArg<1>]>; def int_hexagon_S2_lsr_r_p_xor : Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_lsr_r_p_xor">; @@ -3781,7 +3802,7 @@ def int_hexagon_F2_conv_sf2ud_chop : Hexagon_i64_float_Intrinsic<"HEXAGON_F2_conv_sf2ud_chop">; def int_hexagon_C2_cmpgeui : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C2_cmpgeui">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C2_cmpgeui", [ImmArg<1>]>; def int_hexagon_M2_mpy_acc_sat_hh_s0 : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M2_mpy_acc_sat_hh_s0">; @@ -3808,7 +3829,7 @@ def int_hexagon_M2_mpyud_nac_lh_s0 : Hexagon_i64_i64i32i32_Intrinsic<"HEXAGON_M2_mpyud_nac_lh_s0">; def int_hexagon_A4_round_ri_sat : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_round_ri_sat">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_round_ri_sat", [ImmArg<1>]>; def int_hexagon_M2_mpy_nac_hl_s0 : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M2_mpy_nac_hl_s0">; @@ -3829,10 +3850,10 @@ def int_hexagon_M2_cmaci_s0 : Hexagon_i64_i64i32i32_Intrinsic<"HEXAGON_M2_cmaci_s0">; def int_hexagon_S2_setbit_i : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_setbit_i">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_setbit_i", [ImmArg<1>]>; def int_hexagon_S2_asl_i_p_or : -Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_asl_i_p_or">; +Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_asl_i_p_or", [ImmArg<2>]>; def int_hexagon_A4_andn : Hexagon_i32_i32i32_Intrinsic<"HEXAGON_A4_andn">; @@ -3856,13 +3877,13 @@ def int_hexagon_M2_xor_xacc : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M2_xor_xacc">; def int_hexagon_A4_vcmpbgtui : -Hexagon_i32_i64i32_Intrinsic<"HEXAGON_A4_vcmpbgtui">; +Hexagon_i32_i64i32_Intrinsic<"HEXAGON_A4_vcmpbgtui", [ImmArg<1>]>; def int_hexagon_A4_ornp : Hexagon_i64_i64i64_Intrinsic<"HEXAGON_A4_ornp">; def int_hexagon_A2_tfrpi : -Hexagon_i64_i32_Intrinsic<"HEXAGON_A2_tfrpi">; +Hexagon_i64_i32_Intrinsic<"HEXAGON_A2_tfrpi", [ImmArg<0>]>; def int_hexagon_C4_and_or : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_C4_and_or">; @@ -3886,16 +3907,16 @@ def int_hexagon_M2_vmpy2su_s0 : Hexagon_i64_i32i32_Intrinsic<"HEXAGON_M2_vmpy2su_s0">; def int_hexagon_S2_asr_i_p_acc : -Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_asr_i_p_acc">; +Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_asr_i_p_acc", [ImmArg<2>]>; def int_hexagon_C4_nbitsclri : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C4_nbitsclri">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_C4_nbitsclri", [ImmArg<1>]>; def int_hexagon_S2_lsr_i_vh : -Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_lsr_i_vh">; +Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S2_lsr_i_vh", [ImmArg<1>]>; def int_hexagon_S2_lsr_i_p_xacc : -Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_lsr_i_p_xacc">; +Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S2_lsr_i_p_xacc", [ImmArg<2>]>; // V55 Scalar Instructions. @@ -3905,40 +3926,40 @@ Hexagon_i64i32_i64i64i64_Intrinsic<"HEXAGON_A5_ACS">; // V60 Scalar Instructions. def int_hexagon_S6_rol_i_p_and : -Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S6_rol_i_p_and">; +Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S6_rol_i_p_and", [ImmArg<2>]>; def int_hexagon_S6_rol_i_r_xacc : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S6_rol_i_r_xacc">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S6_rol_i_r_xacc", [ImmArg<2>]>; def int_hexagon_S6_rol_i_r_and : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S6_rol_i_r_and">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S6_rol_i_r_and", [ImmArg<2>]>; def int_hexagon_S6_rol_i_r_acc : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S6_rol_i_r_acc">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S6_rol_i_r_acc", [ImmArg<2>]>; def int_hexagon_S6_rol_i_p_xacc : -Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S6_rol_i_p_xacc">; +Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S6_rol_i_p_xacc", [ImmArg<2>]>; def int_hexagon_S6_rol_i_p : -Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S6_rol_i_p">; +Hexagon_i64_i64i32_Intrinsic<"HEXAGON_S6_rol_i_p", [ImmArg<1>]>; def int_hexagon_S6_rol_i_p_nac : -Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S6_rol_i_p_nac">; +Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S6_rol_i_p_nac", [ImmArg<2>]>; def int_hexagon_S6_rol_i_p_acc : -Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S6_rol_i_p_acc">; +Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S6_rol_i_p_acc", [ImmArg<2>]>; def int_hexagon_S6_rol_i_r_or : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S6_rol_i_r_or">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S6_rol_i_r_or", [ImmArg<2>]>; def int_hexagon_S6_rol_i_r : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S6_rol_i_r">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S6_rol_i_r", [ImmArg<1>]>; def int_hexagon_S6_rol_i_r_nac : -Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S6_rol_i_r_nac">; +Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_S6_rol_i_r_nac", [ImmArg<2>]>; def int_hexagon_S6_rol_i_p_or : -Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S6_rol_i_p_or">; +Hexagon_i64_i64i64i32_Intrinsic<"HEXAGON_S6_rol_i_p_or", [ImmArg<2>]>; // V62 Scalar Instructions. @@ -3980,7 +4001,7 @@ def int_hexagon_M2_mnaci : Hexagon_i32_i32i32i32_Intrinsic<"HEXAGON_M2_mnaci">; def int_hexagon_S2_mask : -Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_mask">; +Hexagon_i32_i32i32_Intrinsic<"HEXAGON_S2_mask", [ImmArg<0>, ImmArg<1>]>; // V60 HVX Instructions. @@ -4021,10 +4042,10 @@ def int_hexagon_V6_vaddh_dv_128B : Hexagon_v64i32_v64i32v64i32_Intrinsic<"HEXAGON_V6_vaddh_dv_128B">; def int_hexagon_V6_vrmpybusi : -Hexagon_v32i32_v32i32i32i32_Intrinsic<"HEXAGON_V6_vrmpybusi">; +Hexagon_v32i32_v32i32i32i32_Intrinsic<"HEXAGON_V6_vrmpybusi", [ImmArg<2>]>; def int_hexagon_V6_vrmpybusi_128B : -Hexagon_v64i32_v64i32i32i32_Intrinsic<"HEXAGON_V6_vrmpybusi_128B">; +Hexagon_v64i32_v64i32i32i32_Intrinsic<"HEXAGON_V6_vrmpybusi_128B", [ImmArg<2>]>; def int_hexagon_V6_vshufoh : Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vshufoh">; @@ -4045,10 +4066,10 @@ def int_hexagon_V6_vdmpyhsuisat_128B : Hexagon_v32i32_v64i32i32_Intrinsic<"HEXAGON_V6_vdmpyhsuisat_128B">; def int_hexagon_V6_vrsadubi_acc : -Hexagon_v32i32_v32i32v32i32i32i32_Intrinsic<"HEXAGON_V6_vrsadubi_acc">; +Hexagon_v32i32_v32i32v32i32i32i32_Intrinsic<"HEXAGON_V6_vrsadubi_acc", [ImmArg<3>]>; def int_hexagon_V6_vrsadubi_acc_128B : -Hexagon_v64i32_v64i32v64i32i32i32_Intrinsic<"HEXAGON_V6_vrsadubi_acc_128B">; +Hexagon_v64i32_v64i32v64i32i32i32_Intrinsic<"HEXAGON_V6_vrsadubi_acc_128B", [ImmArg<3>]>; def int_hexagon_V6_vnavgw : Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vnavgw">; @@ -4915,10 +4936,10 @@ def int_hexagon_V6_vsubhsat_128B : Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vsubhsat_128B">; def int_hexagon_V6_vrmpyubi_acc : -Hexagon_v32i32_v32i32v32i32i32i32_Intrinsic<"HEXAGON_V6_vrmpyubi_acc">; +Hexagon_v32i32_v32i32v32i32i32i32_Intrinsic<"HEXAGON_V6_vrmpyubi_acc", [ImmArg<3>]>; def int_hexagon_V6_vrmpyubi_acc_128B : -Hexagon_v64i32_v64i32v64i32i32i32_Intrinsic<"HEXAGON_V6_vrmpyubi_acc_128B">; +Hexagon_v64i32_v64i32v64i32i32i32_Intrinsic<"HEXAGON_V6_vrmpyubi_acc_128B", [ImmArg<3>]>; def int_hexagon_V6_vabsw : Hexagon_v16i32_v16i32_Intrinsic<"HEXAGON_V6_vabsw">; @@ -5095,10 +5116,10 @@ def int_hexagon_V6_vmpybv_acc_128B : Hexagon_v64i32_v64i32v32i32v32i32_Intrinsic<"HEXAGON_V6_vmpybv_acc_128B">; def int_hexagon_V6_vrsadubi : -Hexagon_v32i32_v32i32i32i32_Intrinsic<"HEXAGON_V6_vrsadubi">; +Hexagon_v32i32_v32i32i32i32_Intrinsic<"HEXAGON_V6_vrsadubi", [ImmArg<2>]>; def int_hexagon_V6_vrsadubi_128B : -Hexagon_v64i32_v64i32i32i32_Intrinsic<"HEXAGON_V6_vrsadubi_128B">; +Hexagon_v64i32_v64i32i32i32_Intrinsic<"HEXAGON_V6_vrsadubi_128B", [ImmArg<2>]>; def int_hexagon_V6_vdmpyhb_dv_acc : Hexagon_v32i32_v32i32v32i32i32_Intrinsic<"HEXAGON_V6_vdmpyhb_dv_acc">; @@ -5377,10 +5398,10 @@ def int_hexagon_V6_vaddbnq_128B : Hexagon_v32i32_v1024i1v32i32v32i32_Intrinsic<"HEXAGON_V6_vaddbnq_128B">; def int_hexagon_V6_vlalignbi : -Hexagon_v16i32_v16i32v16i32i32_Intrinsic<"HEXAGON_V6_vlalignbi">; +Hexagon_v16i32_v16i32v16i32i32_Intrinsic<"HEXAGON_V6_vlalignbi", [ImmArg<2>]>; def int_hexagon_V6_vlalignbi_128B : -Hexagon_v32i32_v32i32v32i32i32_Intrinsic<"HEXAGON_V6_vlalignbi_128B">; +Hexagon_v32i32_v32i32v32i32i32_Intrinsic<"HEXAGON_V6_vlalignbi_128B", [ImmArg<2>]>; def int_hexagon_V6_vsatwh : Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vsatwh">; @@ -5443,10 +5464,10 @@ def int_hexagon_V6_veqh_and_128B : Hexagon_v1024i1_v1024i1v32i32v32i32_Intrinsic<"HEXAGON_V6_veqh_and_128B">; def int_hexagon_V6_valignbi : -Hexagon_v16i32_v16i32v16i32i32_Intrinsic<"HEXAGON_V6_valignbi">; +Hexagon_v16i32_v16i32v16i32i32_Intrinsic<"HEXAGON_V6_valignbi", [ImmArg<2>]>; def int_hexagon_V6_valignbi_128B : -Hexagon_v32i32_v32i32v32i32i32_Intrinsic<"HEXAGON_V6_valignbi_128B">; +Hexagon_v32i32_v32i32v32i32i32_Intrinsic<"HEXAGON_V6_valignbi_128B", [ImmArg<2>]>; def int_hexagon_V6_vaddwsat : Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vaddwsat">; @@ -5689,10 +5710,10 @@ def int_hexagon_V6_vsubh_128B : Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vsubh_128B">; def int_hexagon_V6_vrmpyubi : -Hexagon_v32i32_v32i32i32i32_Intrinsic<"HEXAGON_V6_vrmpyubi">; +Hexagon_v32i32_v32i32i32i32_Intrinsic<"HEXAGON_V6_vrmpyubi", [ImmArg<2>]>; def int_hexagon_V6_vrmpyubi_128B : -Hexagon_v64i32_v64i32i32i32_Intrinsic<"HEXAGON_V6_vrmpyubi_128B">; +Hexagon_v64i32_v64i32i32i32_Intrinsic<"HEXAGON_V6_vrmpyubi_128B", [ImmArg<2>]>; def int_hexagon_V6_vminw : Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vminw">; @@ -5755,10 +5776,10 @@ def int_hexagon_V6_vsubuhw_128B : Hexagon_v64i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vsubuhw_128B">; def int_hexagon_V6_vrmpybusi_acc : -Hexagon_v32i32_v32i32v32i32i32i32_Intrinsic<"HEXAGON_V6_vrmpybusi_acc">; +Hexagon_v32i32_v32i32v32i32i32i32_Intrinsic<"HEXAGON_V6_vrmpybusi_acc", [ImmArg<3>]>; def int_hexagon_V6_vrmpybusi_acc_128B : -Hexagon_v64i32_v64i32v64i32i32i32_Intrinsic<"HEXAGON_V6_vrmpybusi_acc_128B">; +Hexagon_v64i32_v64i32v64i32i32i32_Intrinsic<"HEXAGON_V6_vrmpybusi_acc_128B", [ImmArg<3>]>; def int_hexagon_V6_vasrw : Hexagon_v16i32_v16i32i32_Intrinsic<"HEXAGON_V6_vasrw">; @@ -5883,10 +5904,10 @@ def int_hexagon_V6_vlsrb_128B : Hexagon_v32i32_v32i32i32_Intrinsic<"HEXAGON_V6_vlsrb_128B">; def int_hexagon_V6_vlutvwhi : -Hexagon_v32i32_v16i32v16i32i32_Intrinsic<"HEXAGON_V6_vlutvwhi">; +Hexagon_v32i32_v16i32v16i32i32_Intrinsic<"HEXAGON_V6_vlutvwhi", [ImmArg<2>]>; def int_hexagon_V6_vlutvwhi_128B : -Hexagon_v64i32_v32i32v32i32i32_Intrinsic<"HEXAGON_V6_vlutvwhi_128B">; +Hexagon_v64i32_v32i32v32i32i32_Intrinsic<"HEXAGON_V6_vlutvwhi_128B", [ImmArg<2>]>; def int_hexagon_V6_vaddububb_sat : Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vaddububb_sat">; @@ -5907,10 +5928,10 @@ def int_hexagon_V6_ldtp0_128B : Hexagon_v32i32_i32i32_Intrinsic<"HEXAGON_V6_ldtp0_128B">; def int_hexagon_V6_vlutvvb_oracci : -Hexagon_v16i32_v16i32v16i32v16i32i32_Intrinsic<"HEXAGON_V6_vlutvvb_oracci">; +Hexagon_v16i32_v16i32v16i32v16i32i32_Intrinsic<"HEXAGON_V6_vlutvvb_oracci", [ImmArg<3>]>; def int_hexagon_V6_vlutvvb_oracci_128B : -Hexagon_v32i32_v32i32v32i32v32i32i32_Intrinsic<"HEXAGON_V6_vlutvvb_oracci_128B">; +Hexagon_v32i32_v32i32v32i32v32i32i32_Intrinsic<"HEXAGON_V6_vlutvvb_oracci_128B", [ImmArg<3>]>; def int_hexagon_V6_vsubuwsat_dv : Hexagon_v32i32_v32i32v32i32_Intrinsic<"HEXAGON_V6_vsubuwsat_dv">; @@ -6045,10 +6066,10 @@ def int_hexagon_V6_vasrwuhrndsat_128B : Hexagon_v32i32_v32i32v32i32i32_Intrinsic<"HEXAGON_V6_vasrwuhrndsat_128B">; def int_hexagon_V6_vlutvvbi : -Hexagon_v16i32_v16i32v16i32i32_Intrinsic<"HEXAGON_V6_vlutvvbi">; +Hexagon_v16i32_v16i32v16i32i32_Intrinsic<"HEXAGON_V6_vlutvvbi", [ImmArg<2>]>; def int_hexagon_V6_vlutvvbi_128B : -Hexagon_v32i32_v32i32v32i32i32_Intrinsic<"HEXAGON_V6_vlutvvbi_128B">; +Hexagon_v32i32_v32i32v32i32i32_Intrinsic<"HEXAGON_V6_vlutvvbi_128B", [ImmArg<2>]>; def int_hexagon_V6_vsubuwsat : Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vsubuwsat">; @@ -6141,10 +6162,10 @@ def int_hexagon_V6_ldcnp0_128B : Hexagon_v32i32_i32i32_Intrinsic<"HEXAGON_V6_ldcnp0_128B">; def int_hexagon_V6_vlutvwh_oracci : -Hexagon_v32i32_v32i32v16i32v16i32i32_Intrinsic<"HEXAGON_V6_vlutvwh_oracci">; +Hexagon_v32i32_v32i32v16i32v16i32i32_Intrinsic<"HEXAGON_V6_vlutvwh_oracci", [ImmArg<3>]>; def int_hexagon_V6_vlutvwh_oracci_128B : -Hexagon_v64i32_v64i32v32i32v32i32i32_Intrinsic<"HEXAGON_V6_vlutvwh_oracci_128B">; +Hexagon_v64i32_v64i32v32i32v32i32i32_Intrinsic<"HEXAGON_V6_vlutvwh_oracci_128B", [ImmArg<3>]>; def int_hexagon_V6_vsubbsat : Hexagon_v16i32_v16i32v16i32_Intrinsic<"HEXAGON_V6_vsubbsat">; diff --git a/include/llvm/IR/IntrinsicsMips.td b/include/llvm/IR/IntrinsicsMips.td index 421a79be4ebc..6393a9ca35d5 100644 --- a/include/llvm/IR/IntrinsicsMips.td +++ b/include/llvm/IR/IntrinsicsMips.td @@ -1,9 +1,8 @@ //===- IntrinsicsMips.td - Defines Mips intrinsics ---------*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -235,9 +234,9 @@ def int_mips_extpdp: GCCBuiltin<"__builtin_mips_extpdp">, // Misc def int_mips_wrdsp: GCCBuiltin<"__builtin_mips_wrdsp">, - Intrinsic<[], [llvm_i32_ty, llvm_i32_ty], []>; + Intrinsic<[], [llvm_i32_ty, llvm_i32_ty], [ImmArg<1>]>; def int_mips_rddsp: GCCBuiltin<"__builtin_mips_rddsp">, - Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrReadMem]>; + Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrReadMem, ImmArg<0>]>; def int_mips_insv: GCCBuiltin<"__builtin_mips_insv">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrReadMem]>; @@ -303,10 +302,10 @@ def int_mips_adduh_r_qb: GCCBuiltin<"__builtin_mips_adduh_r_qb">, def int_mips_append: GCCBuiltin<"__builtin_mips_append">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_mips_balign: GCCBuiltin<"__builtin_mips_balign">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_mips_cmpgdu_eq_qb: GCCBuiltin<"__builtin_mips_cmpgdu_eq_qb">, Intrinsic<[llvm_i32_ty], [llvm_v4i8_ty, llvm_v4i8_ty], [Commutative]>; @@ -356,14 +355,14 @@ def int_mips_precr_qb_ph: GCCBuiltin<"__builtin_mips_precr_qb_ph">, Intrinsic<[llvm_v4i8_ty], [llvm_v2i16_ty, llvm_v2i16_ty], []>; def int_mips_precr_sra_ph_w: GCCBuiltin<"__builtin_mips_precr_sra_ph_w">, Intrinsic<[llvm_v2i16_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_mips_precr_sra_r_ph_w: GCCBuiltin<"__builtin_mips_precr_sra_r_ph_w">, Intrinsic<[llvm_v2i16_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_mips_prepend: GCCBuiltin<"__builtin_mips_prepend">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_mips_shra_qb: GCCBuiltin<"__builtin_mips_shra_qb">, Intrinsic<[llvm_v4i8_ty], [llvm_v4i8_ty, llvm_i32_ty], [IntrNoMem]>; @@ -464,22 +463,22 @@ def int_mips_addv_d : GCCBuiltin<"__builtin_msa_addv_d">, def int_mips_addvi_b : GCCBuiltin<"__builtin_msa_addvi_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], - [Commutative, IntrNoMem]>; + [Commutative, IntrNoMem, ImmArg<1>]>; def int_mips_addvi_h : GCCBuiltin<"__builtin_msa_addvi_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], - [Commutative, IntrNoMem]>; + [Commutative, IntrNoMem, ImmArg<1>]>; def int_mips_addvi_w : GCCBuiltin<"__builtin_msa_addvi_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], - [Commutative, IntrNoMem]>; + [Commutative, IntrNoMem, ImmArg<1>]>; def int_mips_addvi_d : GCCBuiltin<"__builtin_msa_addvi_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], - [Commutative, IntrNoMem]>; + [Commutative, IntrNoMem, ImmArg<1>]>; def int_mips_and_v : GCCBuiltin<"__builtin_msa_and_v">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; def int_mips_andi_b : GCCBuiltin<"__builtin_msa_andi_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_asub_s_b : GCCBuiltin<"__builtin_msa_asub_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; @@ -561,13 +560,13 @@ def int_mips_bclr_d : GCCBuiltin<"__builtin_msa_bclr_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; def int_mips_bclri_b : GCCBuiltin<"__builtin_msa_bclri_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_bclri_h : GCCBuiltin<"__builtin_msa_bclri_h">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_bclri_w : GCCBuiltin<"__builtin_msa_bclri_w">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_bclri_d : GCCBuiltin<"__builtin_msa_bclri_d">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_binsl_b : GCCBuiltin<"__builtin_msa_binsl_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty], @@ -584,16 +583,16 @@ def int_mips_binsl_d : GCCBuiltin<"__builtin_msa_binsl_d">, def int_mips_binsli_b : GCCBuiltin<"__builtin_msa_binsli_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_mips_binsli_h : GCCBuiltin<"__builtin_msa_binsli_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_mips_binsli_w : GCCBuiltin<"__builtin_msa_binsli_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_mips_binsli_d : GCCBuiltin<"__builtin_msa_binsli_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_mips_binsr_b : GCCBuiltin<"__builtin_msa_binsr_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty], @@ -610,16 +609,16 @@ def int_mips_binsr_d : GCCBuiltin<"__builtin_msa_binsr_d">, def int_mips_binsri_b : GCCBuiltin<"__builtin_msa_binsri_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_mips_binsri_h : GCCBuiltin<"__builtin_msa_binsri_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_mips_binsri_w : GCCBuiltin<"__builtin_msa_binsri_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_mips_binsri_d : GCCBuiltin<"__builtin_msa_binsri_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_mips_bmnz_v : GCCBuiltin<"__builtin_msa_bmnz_v">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty], @@ -627,7 +626,7 @@ def int_mips_bmnz_v : GCCBuiltin<"__builtin_msa_bmnz_v">, def int_mips_bmnzi_b : GCCBuiltin<"__builtin_msa_bmnzi_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_mips_bmz_v : GCCBuiltin<"__builtin_msa_bmz_v">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty], @@ -635,7 +634,7 @@ def int_mips_bmz_v : GCCBuiltin<"__builtin_msa_bmz_v">, def int_mips_bmzi_b : GCCBuiltin<"__builtin_msa_bmzi_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_mips_bneg_b : GCCBuiltin<"__builtin_msa_bneg_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; @@ -647,13 +646,13 @@ def int_mips_bneg_d : GCCBuiltin<"__builtin_msa_bneg_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; def int_mips_bnegi_b : GCCBuiltin<"__builtin_msa_bnegi_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_bnegi_h : GCCBuiltin<"__builtin_msa_bnegi_h">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_bnegi_w : GCCBuiltin<"__builtin_msa_bnegi_w">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_bnegi_d : GCCBuiltin<"__builtin_msa_bnegi_d">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_bnz_b : GCCBuiltin<"__builtin_msa_bnz_b">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty], [IntrNoMem]>; @@ -673,7 +672,7 @@ def int_mips_bsel_v : GCCBuiltin<"__builtin_msa_bsel_v">, def int_mips_bseli_b : GCCBuiltin<"__builtin_msa_bseli_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_mips_bset_b : GCCBuiltin<"__builtin_msa_bset_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; @@ -685,13 +684,13 @@ def int_mips_bset_d : GCCBuiltin<"__builtin_msa_bset_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; def int_mips_bseti_b : GCCBuiltin<"__builtin_msa_bseti_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_bseti_h : GCCBuiltin<"__builtin_msa_bseti_h">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_bseti_w : GCCBuiltin<"__builtin_msa_bseti_w">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_bseti_d : GCCBuiltin<"__builtin_msa_bseti_d">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_bz_b : GCCBuiltin<"__builtin_msa_bz_b">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty], [IntrNoMem]>; @@ -715,16 +714,16 @@ def int_mips_ceq_d : GCCBuiltin<"__builtin_msa_ceq_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; def int_mips_ceqi_b : GCCBuiltin<"__builtin_msa_ceqi_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_ceqi_h : GCCBuiltin<"__builtin_msa_ceqi_h">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_ceqi_w : GCCBuiltin<"__builtin_msa_ceqi_w">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_ceqi_d : GCCBuiltin<"__builtin_msa_ceqi_d">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_cfcmsa : GCCBuiltin<"__builtin_msa_cfcmsa">, - Intrinsic<[llvm_i32_ty], [llvm_i32_ty], []>; + Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [ImmArg<0>]>; def int_mips_cle_s_b : GCCBuiltin<"__builtin_msa_cle_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; @@ -745,22 +744,22 @@ def int_mips_cle_u_d : GCCBuiltin<"__builtin_msa_cle_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; def int_mips_clei_s_b : GCCBuiltin<"__builtin_msa_clei_s_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_clei_s_h : GCCBuiltin<"__builtin_msa_clei_s_h">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_clei_s_w : GCCBuiltin<"__builtin_msa_clei_s_w">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_clei_s_d : GCCBuiltin<"__builtin_msa_clei_s_d">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_clei_u_b : GCCBuiltin<"__builtin_msa_clei_u_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_clei_u_h : GCCBuiltin<"__builtin_msa_clei_u_h">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_clei_u_w : GCCBuiltin<"__builtin_msa_clei_u_w">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_clei_u_d : GCCBuiltin<"__builtin_msa_clei_u_d">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_clt_s_b : GCCBuiltin<"__builtin_msa_clt_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; @@ -781,22 +780,22 @@ def int_mips_clt_u_d : GCCBuiltin<"__builtin_msa_clt_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; def int_mips_clti_s_b : GCCBuiltin<"__builtin_msa_clti_s_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_clti_s_h : GCCBuiltin<"__builtin_msa_clti_s_h">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_clti_s_w : GCCBuiltin<"__builtin_msa_clti_s_w">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_clti_s_d : GCCBuiltin<"__builtin_msa_clti_s_d">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_clti_u_b : GCCBuiltin<"__builtin_msa_clti_u_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_clti_u_h : GCCBuiltin<"__builtin_msa_clti_u_h">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_clti_u_w : GCCBuiltin<"__builtin_msa_clti_u_w">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_clti_u_d : GCCBuiltin<"__builtin_msa_clti_u_d">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_copy_s_b : GCCBuiltin<"__builtin_msa_copy_s_b">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; @@ -817,7 +816,7 @@ def int_mips_copy_u_d : GCCBuiltin<"__builtin_msa_copy_u_d">, Intrinsic<[llvm_i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; def int_mips_ctcmsa : GCCBuiltin<"__builtin_msa_ctcmsa">, - Intrinsic<[], [llvm_i32_ty, llvm_i32_ty], []>; + Intrinsic<[], [llvm_i32_ty, llvm_i32_ty], [ImmArg<0>]>; def int_mips_div_s_b : GCCBuiltin<"__builtin_msa_div_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; @@ -1245,41 +1244,41 @@ def int_mips_insert_d : GCCBuiltin<"__builtin_msa_insert_d">, def int_mips_insve_b : GCCBuiltin<"__builtin_msa_insve_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; def int_mips_insve_h : GCCBuiltin<"__builtin_msa_insve_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty, llvm_v8i16_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; def int_mips_insve_w : GCCBuiltin<"__builtin_msa_insve_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; def int_mips_insve_d : GCCBuiltin<"__builtin_msa_insve_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty, llvm_v2i64_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; def int_mips_ld_b : GCCBuiltin<"__builtin_msa_ld_b">, Intrinsic<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, IntrArgMemOnly, ImmArg<1>]>; def int_mips_ld_h : GCCBuiltin<"__builtin_msa_ld_h">, Intrinsic<[llvm_v8i16_ty], [llvm_ptr_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, IntrArgMemOnly, ImmArg<1>]>; def int_mips_ld_w : GCCBuiltin<"__builtin_msa_ld_w">, Intrinsic<[llvm_v4i32_ty], [llvm_ptr_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, IntrArgMemOnly, ImmArg<1>]>; def int_mips_ld_d : GCCBuiltin<"__builtin_msa_ld_d">, Intrinsic<[llvm_v2i64_ty], [llvm_ptr_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, IntrArgMemOnly, ImmArg<1>]>; def int_mips_ldi_b : GCCBuiltin<"__builtin_msa_ldi_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_i32_ty], [IntrNoMem, ImmArg<0>]>; def int_mips_ldi_h : GCCBuiltin<"__builtin_msa_ldi_h">, - Intrinsic<[llvm_v8i16_ty], [llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_i32_ty], [IntrNoMem, ImmArg<0>]>; def int_mips_ldi_w : GCCBuiltin<"__builtin_msa_ldi_w">, - Intrinsic<[llvm_v4i32_ty], [llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_i32_ty], [IntrNoMem, ImmArg<0>]>; def int_mips_ldi_d : GCCBuiltin<"__builtin_msa_ldi_d">, - Intrinsic<[llvm_v2i64_ty], [llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_i32_ty], [IntrNoMem, ImmArg<0>]>; // This instruction is part of the MSA spec but it does not share the // __builtin_msa prefix because it operates on the GPR registers. @@ -1342,22 +1341,22 @@ def int_mips_max_u_d : GCCBuiltin<"__builtin_msa_max_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; def int_mips_maxi_s_b : GCCBuiltin<"__builtin_msa_maxi_s_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_maxi_s_h : GCCBuiltin<"__builtin_msa_maxi_s_h">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_maxi_s_w : GCCBuiltin<"__builtin_msa_maxi_s_w">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_maxi_s_d : GCCBuiltin<"__builtin_msa_maxi_s_d">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_maxi_u_b : GCCBuiltin<"__builtin_msa_maxi_u_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_maxi_u_h : GCCBuiltin<"__builtin_msa_maxi_u_h">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_maxi_u_w : GCCBuiltin<"__builtin_msa_maxi_u_w">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_maxi_u_d : GCCBuiltin<"__builtin_msa_maxi_u_d">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_min_a_b : GCCBuiltin<"__builtin_msa_min_a_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; @@ -1387,22 +1386,22 @@ def int_mips_min_u_d : GCCBuiltin<"__builtin_msa_min_u_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; def int_mips_mini_s_b : GCCBuiltin<"__builtin_msa_mini_s_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_mini_s_h : GCCBuiltin<"__builtin_msa_mini_s_h">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_mini_s_w : GCCBuiltin<"__builtin_msa_mini_s_w">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_mini_s_d : GCCBuiltin<"__builtin_msa_mini_s_d">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_mini_u_b : GCCBuiltin<"__builtin_msa_mini_u_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_mini_u_h : GCCBuiltin<"__builtin_msa_mini_u_h">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_mini_u_w : GCCBuiltin<"__builtin_msa_mini_u_w">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_mini_u_d : GCCBuiltin<"__builtin_msa_mini_u_d">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_mod_s_b : GCCBuiltin<"__builtin_msa_mod_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; @@ -1493,13 +1492,13 @@ def int_mips_nor_v : GCCBuiltin<"__builtin_msa_nor_v">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; def int_mips_nori_b : GCCBuiltin<"__builtin_msa_nori_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_or_v : GCCBuiltin<"__builtin_msa_or_v">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; def int_mips_ori_b : GCCBuiltin<"__builtin_msa_ori_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_pckev_b : GCCBuiltin<"__builtin_msa_pckev_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; @@ -1529,29 +1528,29 @@ def int_mips_pcnt_d : GCCBuiltin<"__builtin_msa_pcnt_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty], [IntrNoMem]>; def int_mips_sat_s_b : GCCBuiltin<"__builtin_msa_sat_s_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_sat_s_h : GCCBuiltin<"__builtin_msa_sat_s_h">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_sat_s_w : GCCBuiltin<"__builtin_msa_sat_s_w">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_sat_s_d : GCCBuiltin<"__builtin_msa_sat_s_d">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_sat_u_b : GCCBuiltin<"__builtin_msa_sat_u_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_sat_u_h : GCCBuiltin<"__builtin_msa_sat_u_h">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_sat_u_w : GCCBuiltin<"__builtin_msa_sat_u_w">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_sat_u_d : GCCBuiltin<"__builtin_msa_sat_u_d">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_shf_b : GCCBuiltin<"__builtin_msa_shf_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_shf_h : GCCBuiltin<"__builtin_msa_shf_h">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_shf_w : GCCBuiltin<"__builtin_msa_shf_w">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_sld_b : GCCBuiltin<"__builtin_msa_sld_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; @@ -1564,16 +1563,16 @@ def int_mips_sld_d : GCCBuiltin<"__builtin_msa_sld_d">, def int_mips_sldi_b : GCCBuiltin<"__builtin_msa_sldi_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_mips_sldi_h : GCCBuiltin<"__builtin_msa_sldi_h">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_mips_sldi_w : GCCBuiltin<"__builtin_msa_sldi_w">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_mips_sldi_d : GCCBuiltin<"__builtin_msa_sldi_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_mips_sll_b : GCCBuiltin<"__builtin_msa_sll_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; @@ -1585,13 +1584,13 @@ def int_mips_sll_d : GCCBuiltin<"__builtin_msa_sll_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; def int_mips_slli_b : GCCBuiltin<"__builtin_msa_slli_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_slli_h : GCCBuiltin<"__builtin_msa_slli_h">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_slli_w : GCCBuiltin<"__builtin_msa_slli_w">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_slli_d : GCCBuiltin<"__builtin_msa_slli_d">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_splat_b : GCCBuiltin<"__builtin_msa_splat_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; @@ -1603,13 +1602,13 @@ def int_mips_splat_d : GCCBuiltin<"__builtin_msa_splat_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; def int_mips_splati_b : GCCBuiltin<"__builtin_msa_splati_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_splati_h : GCCBuiltin<"__builtin_msa_splati_h">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_splati_w : GCCBuiltin<"__builtin_msa_splati_w">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_splati_d : GCCBuiltin<"__builtin_msa_splati_d">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_sra_b : GCCBuiltin<"__builtin_msa_sra_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; @@ -1621,13 +1620,13 @@ def int_mips_sra_d : GCCBuiltin<"__builtin_msa_sra_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; def int_mips_srai_b : GCCBuiltin<"__builtin_msa_srai_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_srai_h : GCCBuiltin<"__builtin_msa_srai_h">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_srai_w : GCCBuiltin<"__builtin_msa_srai_w">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_srai_d : GCCBuiltin<"__builtin_msa_srai_d">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_srar_b : GCCBuiltin<"__builtin_msa_srar_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; @@ -1639,13 +1638,13 @@ def int_mips_srar_d : GCCBuiltin<"__builtin_msa_srar_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; def int_mips_srari_b : GCCBuiltin<"__builtin_msa_srari_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_srari_h : GCCBuiltin<"__builtin_msa_srari_h">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_srari_w : GCCBuiltin<"__builtin_msa_srari_w">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_srari_d : GCCBuiltin<"__builtin_msa_srari_d">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_srl_b : GCCBuiltin<"__builtin_msa_srl_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; @@ -1657,13 +1656,13 @@ def int_mips_srl_d : GCCBuiltin<"__builtin_msa_srl_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; def int_mips_srli_b : GCCBuiltin<"__builtin_msa_srli_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_srli_h : GCCBuiltin<"__builtin_msa_srli_h">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_srli_w : GCCBuiltin<"__builtin_msa_srli_w">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_srli_d : GCCBuiltin<"__builtin_msa_srli_d">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_srlr_b : GCCBuiltin<"__builtin_msa_srlr_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; @@ -1675,26 +1674,26 @@ def int_mips_srlr_d : GCCBuiltin<"__builtin_msa_srlr_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; def int_mips_srlri_b : GCCBuiltin<"__builtin_msa_srlri_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_srlri_h : GCCBuiltin<"__builtin_msa_srlri_h">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_srlri_w : GCCBuiltin<"__builtin_msa_srlri_w">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_srlri_d : GCCBuiltin<"__builtin_msa_srlri_d">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_st_b : GCCBuiltin<"__builtin_msa_st_b">, Intrinsic<[], [llvm_v16i8_ty, llvm_ptr_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [IntrArgMemOnly, ImmArg<2>]>; def int_mips_st_h : GCCBuiltin<"__builtin_msa_st_h">, Intrinsic<[], [llvm_v8i16_ty, llvm_ptr_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [IntrArgMemOnly, ImmArg<2>]>; def int_mips_st_w : GCCBuiltin<"__builtin_msa_st_w">, Intrinsic<[], [llvm_v4i32_ty, llvm_ptr_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [IntrArgMemOnly, ImmArg<2>]>; def int_mips_st_d : GCCBuiltin<"__builtin_msa_st_d">, Intrinsic<[], [llvm_v2i64_ty, llvm_ptr_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [IntrArgMemOnly, ImmArg<2>]>; def int_mips_subs_s_b : GCCBuiltin<"__builtin_msa_subs_s_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; @@ -1742,13 +1741,13 @@ def int_mips_subv_d : GCCBuiltin<"__builtin_msa_subv_d">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; def int_mips_subvi_b : GCCBuiltin<"__builtin_msa_subvi_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_subvi_h : GCCBuiltin<"__builtin_msa_subvi_h">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_subvi_w : GCCBuiltin<"__builtin_msa_subvi_w">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_subvi_d : GCCBuiltin<"__builtin_msa_subvi_d">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_mips_vshf_b : GCCBuiltin<"__builtin_msa_vshf_b">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty], @@ -1767,5 +1766,5 @@ def int_mips_xor_v : GCCBuiltin<"__builtin_msa_xor_v">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; def int_mips_xori_b : GCCBuiltin<"__builtin_msa_xori_b">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; } diff --git a/include/llvm/IR/IntrinsicsNVVM.td b/include/llvm/IR/IntrinsicsNVVM.td index 7f694f68969e..dba7dd76c4ff 100644 --- a/include/llvm/IR/IntrinsicsNVVM.td +++ b/include/llvm/IR/IntrinsicsNVVM.td @@ -1,9 +1,8 @@ //===- IntrinsicsNVVM.td - Defines NVVM intrinsics ---------*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -38,6 +37,245 @@ def llvm_anyi64ptr_ty : LLVMAnyPointerType; // (space)i64* // MISC // +// Helper class for construction of n-element list [t,t,...,t] +class RepLLVMType { + list ret = !if(N, !listconcat(RepLLVMType.ret, [T]), []); +} + +// Helper class that represents a 'fragment' of an NVPTX *MMA instruction. +// Geom: mnk. E.g. m8n32k16 +// Frag: [abcd] +// PtxEltType: PTX type for the element. +class WMMA_REGS { + string geom = Geom; + string frag = Frag; + string ptx_elt_type = PtxEltType; + string gft = Geom#":"#Frag#":"#ptx_elt_type; + string ft = frag#":"#ptx_elt_type; + list regs = !cond( + // fp16 -> fp16/fp32 @ m16n16k16/m8n32k16/m32n8k16 + // All currently supported geometries use the same fragment format, + // so we only need to consider {fragment, type}. + !eq(ft,"a:f16") : RepLLVMType<8, llvm_v2f16_ty>.ret, + !eq(ft,"b:f16") : RepLLVMType<8, llvm_v2f16_ty>.ret, + !eq(ft,"c:f16") : RepLLVMType<4, llvm_v2f16_ty>.ret, + !eq(ft,"d:f16") : RepLLVMType<4, llvm_v2f16_ty>.ret, + !eq(ft,"c:f32") : RepLLVMType<8, llvm_float_ty>.ret, + !eq(ft,"d:f32") : RepLLVMType<8, llvm_float_ty>.ret, + + // u8/s8 -> s32 @ m16n16k16/m8n32k16/m32n8k16 + !eq(gft,"m16n16k16:a:u8") : RepLLVMType<2, llvm_i32_ty>.ret, + !eq(gft,"m16n16k16:a:s8") : RepLLVMType<2, llvm_i32_ty>.ret, + !eq(gft,"m16n16k16:b:u8") : RepLLVMType<2, llvm_i32_ty>.ret, + !eq(gft,"m16n16k16:b:s8") : RepLLVMType<2, llvm_i32_ty>.ret, + !eq(gft,"m16n16k16:c:s32") : RepLLVMType<8, llvm_i32_ty>.ret, + !eq(gft,"m16n16k16:d:s32") : RepLLVMType<8, llvm_i32_ty>.ret, + + !eq(gft,"m8n32k16:a:u8") : [llvm_i32_ty], + !eq(gft,"m8n32k16:a:s8") : [llvm_i32_ty], + !eq(gft,"m8n32k16:b:u8") : RepLLVMType<4, llvm_i32_ty>.ret, + !eq(gft,"m8n32k16:b:s8") : RepLLVMType<4, llvm_i32_ty>.ret, + !eq(gft,"m8n32k16:c:s32") : RepLLVMType<8, llvm_i32_ty>.ret, + !eq(gft,"m8n32k16:d:s32") : RepLLVMType<8, llvm_i32_ty>.ret, + + !eq(gft,"m32n8k16:a:u8") : RepLLVMType<4, llvm_i32_ty>.ret, + !eq(gft,"m32n8k16:a:s8") : RepLLVMType<4, llvm_i32_ty>.ret, + !eq(gft,"m32n8k16:b:u8") : [llvm_i32_ty], + !eq(gft,"m32n8k16:b:s8") : [llvm_i32_ty], + !eq(gft,"m32n8k16:c:s32") : RepLLVMType<8, llvm_i32_ty>.ret, + !eq(gft,"m32n8k16:d:s32") : RepLLVMType<8, llvm_i32_ty>.ret, + + // u4/s4/b1 -> s32 @ m8n8k32 (u4/s4), m8n8k128(b1) + !eq(gft,"m8n8k128:a:b1") : [llvm_i32_ty], + !eq(gft,"m8n8k32:a:u4") : [llvm_i32_ty], + !eq(gft,"m8n8k32:a:s4") : [llvm_i32_ty], + !eq(gft,"m8n8k128:b:b1") : [llvm_i32_ty], + !eq(gft,"m8n8k32:b:u4") : [llvm_i32_ty], + !eq(gft,"m8n8k32:b:s4") : [llvm_i32_ty], + !eq(gft,"m8n8k128:c:s32") : RepLLVMType<2, llvm_i32_ty>.ret, + !eq(gft,"m8n8k128:d:s32") : RepLLVMType<2, llvm_i32_ty>.ret, + !eq(gft,"m8n8k32:c:s32") : RepLLVMType<2, llvm_i32_ty>.ret, + !eq(gft,"m8n8k32:d:s32") : RepLLVMType<2, llvm_i32_ty>.ret, + ); +} + +class WMMA_NAME_LDST { + string intr = "llvm.nvvm.wmma." + # Frag.geom + # "." # Op + # "." # Frag.frag + # "." # Layout + # !if(WithStride, ".stride", "") + # "." # Frag.ptx_elt_type + ; + // TODO(tra): record name should ideally use the same field order as the intrinsic. + // E.g. string record = !subst("llvm", "int", + // !subst(".", "_", llvm)); + string record = "int_nvvm_wmma_" + # Frag.geom + # "_" # Op + # "_" # Frag.frag + # "_" # Frag.ptx_elt_type + # "_" # Layout + # !if(WithStride, "_stride", ""); +} + +class MMA_SIGNATURE { + list id_frags = !cond( + // int and sub-int ops are identified by input type. + !eq(A.ptx_elt_type, "s8") : [A], + !eq(A.ptx_elt_type, "u8") : [A], + !eq(A.ptx_elt_type, "s4") : [A], + !eq(A.ptx_elt_type, "u4") : [A], + !eq(A.ptx_elt_type, "b1") : [A], + // the rest are FP ops identified by accumulator & result type. + 1: [D, C] + ); + string ret = !foldl("", id_frags, a, b, !strconcat(a, ".", b.ptx_elt_type)); +} + +class WMMA_NAME_MMA { + string signature = MMA_SIGNATURE.ret; + string llvm = "llvm.nvvm.wmma." + # A.geom + # ".mma" + # "." # ALayout + # "." # BLayout + # signature + # !if(Satfinite, ".satfinite", ""); + + string record = !subst(".", "_", + !subst("llvm.", "int_", llvm)); +} + +// Generates list of 4-tuples of WMMA_REGS representing a valid MMA op. +// Geom: list of supported geometries. +// TypeN: PTX type of the corresponding fragment's element. +// TypeB and TypeD may be empty if it must match that of TypeA or TypeC. +class MMA_OPS Geom, list TypeA, list TypeB, + list TypeC, list TypeD> { + list> ret = + !foldl([]>, Geom, t1, geom, !listconcat(t1, + !foldl([]>, TypeA, t2, type_a, !listconcat(t2, + !foldl([]>, !if(!size(TypeB), TypeB, [type_a]), t3, type_b, !listconcat(t3, + !foldl([]>, TypeC, t4, type_c, !listconcat(t4, + !foldl([]>, !if(!size(TypeC), TypeC, [type_c]), t5, type_d, !listconcat(t5, + [[WMMA_REGS, + WMMA_REGS, + WMMA_REGS, + WMMA_REGS]])))))))))); + // Debugging aid for readable representation of the list above. + list> ops = !foreach(x, ret, [x[0].gft, x[1].gft, x[2].gft, x[3].gft]); +} + +class MMA_LDST_OPS Geom, list Frags, list Types> { + list ret = + !foldl([], Geom, t1, geom, !listconcat(t1, + !foldl([], Frags, t2, frag, !listconcat(t2, + !foldl([], Types, t3, type, !listconcat(t3, + [WMMA_REGS])))))); + // Debugging aid for readable representation of the list above. + list ops = !foreach(x, ret, x.gft); +} + + + +// Creates list of valid combinations of fragments. This is the master list that +// drives generation of corresponding intrinsics and instructions. +class NVVM_MMA_OPS { + list> fp_mma_ops = MMA_OPS< + ["m16n16k16", "m32n8k16", "m8n32k16"], + ["f16"], [], ["f16", "f32"], ["f16", "f32"]>.ret; + list> int_mma_ops = MMA_OPS< + ["m16n16k16", "m32n8k16", "m8n32k16"], + ["s8", "u8"], [], ["s32"], []>.ret; + list> subint_mma_ops = MMA_OPS< + ["m8n8k32"], + ["s4", "u4"], [], ["s32"], []>.ret; + list> bit_mma_ops = MMA_OPS< + ["m8n8k128"], + ["b1"], [], ["s32"], []>.ret; + list> all_mma_ops = !listconcat(fp_mma_ops, int_mma_ops, + subint_mma_ops, bit_mma_ops); + + list ldst_ab_ops = MMA_LDST_OPS< + ["m16n16k16", "m32n8k16", "m8n32k16"], + ["a", "b"], ["f16", "u8", "s8"]>.ret; + list ldst_cd_ops = MMA_LDST_OPS< + ["m16n16k16", "m32n8k16", "m8n32k16"], + ["c", "d"], ["f16", "f32", "s32"]>.ret; + list ldst_subint_ab_ops = MMA_LDST_OPS< + ["m8n8k32"], ["a", "b"], ["s4","u4"]>.ret; + list ldst_bit_ab_ops = MMA_LDST_OPS< + ["m8n8k128"], ["a", "b"], ["b1"]>.ret; + list ldst_subint_cd_ops = MMA_LDST_OPS< + ["m8n8k32", "m8n8k128"], ["c", "d"], ["s32"]>.ret; + list all_ldst_ops = !listconcat(ldst_ab_ops, ldst_cd_ops, + ldst_subint_ab_ops, + ldst_bit_ab_ops, + ldst_subint_cd_ops); + // Separate A/B/C fragments (loads) from D (stores). + list all_ld_ops = !foldl([], all_ldst_ops, a, b, + !listconcat(a, !if(!eq(b.frag,"d"), [],[b]))); + list all_st_ops = !foldl([], all_ldst_ops, a, b, + !listconcat(a, !if(!eq(b.frag,"d"), [b],[]))); +} + +def NVVM_MMA_OPS : NVVM_MMA_OPS; + +// Returns [1] if this combination of layout/satf is supported, [] otherwise. +// MMA ops must provide all parameters. Loads and stores -- only frags and layout_a. +// The class is used to prevent generation of records for the unsupported variants. +// E.g. +// foreach _ = NVVM_MMA_SUPPORTED<...>.ret in = +// def : FOO<>; // The record will only be defined for supported ops. +// +class NVVM_MMA_SUPPORTED frags, string layout_a, string layout_b="-", int satf=-1> { + // MMA ops check both layouts. + string mma = frags[0].ptx_elt_type + # ":" # layout_a + # ":" # layout_b; + // Load ops only need type/fragment/layout. + string ld = frags[0].ptx_elt_type + # ":" # frags[0].frag + # ":" # layout_a + ; + string ldf = frags[0].ptx_elt_type + # ":" # frags[0].frag + ; + string t = frags[0].ptx_elt_type; + list ret = !cond( + // Sub-int MMA only supports fixed A/B layout. + // b1 does not support .satf. + !eq(mma#":"#satf, "b1:row:col:0") : [1], + !eq(mma, "s4:row:col") : [1], + !eq(mma, "u4:row:col") : [1], + !eq(mma, "s4:row:col") : [1], + !eq(mma, "u4:row:col") : [1], + // Sub-int load/stores have fixed layout for A and B. + !and(!eq(layout_b, "-"), // It's a Load or Store op + !or(!eq(ld, "b1:a:row"), + !eq(ld, "b1:b:col"), + !eq(ldf, "b1:c"), + !eq(ldf, "b1:d"), + !eq(ld, "s4:a:row"), + !eq(ld, "s4:b:col"), + !eq(ldf, "s4:c"), + !eq(ldf, "s4:d"), + !eq(ld, "u4:a:row"), + !eq(ld, "u4:b:col"), + !eq(ldf, "u4:c"), + !eq(ldf, "u4:d"))) : [1], + // All other sub-int ops are not supported. + !eq(t, "b1") : [], + !eq(t, "s4") : [], + !eq(t, "u4") : [], + // All other (non sub-int) are OK. + 1: [1] + ); +} + let TargetPrefix = "nvvm" in { def int_nvvm_prmt : GCCBuiltin<"__nvvm_prmt">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], @@ -689,14 +927,6 @@ let TargetPrefix = "nvvm" in { [IntrNoMem]>; // Atomics not available as llvm intrinsics. - def int_nvvm_atomic_load_add_f32 : Intrinsic<[llvm_float_ty], - [LLVMAnyPointerType, llvm_float_ty], - [IntrArgMemOnly, NoCapture<0>]>; - // Atomic add of f64 requires sm_60. - def int_nvvm_atomic_load_add_f64 : Intrinsic<[llvm_double_ty], - [LLVMAnyPointerType, llvm_double_ty], - [IntrArgMemOnly, NoCapture<0>]>; - def int_nvvm_atomic_load_inc_32 : Intrinsic<[llvm_i32_ty], [LLVMAnyPointerType, llvm_i32_ty], [IntrArgMemOnly, NoCapture<0>]>; @@ -3674,11 +3904,19 @@ multiclass PTXReadSRegIntrinsic_v4i32 { class PTXReadSRegIntrinsic_r32 : Intrinsic<[llvm_i32_ty], [], [IntrNoMem]>, GCCBuiltin<"__nvvm_read_ptx_sreg_" # name>; - class PTXReadSRegIntrinsic_r64 : Intrinsic<[llvm_i64_ty], [], [IntrNoMem]>, GCCBuiltin<"__nvvm_read_ptx_sreg_" # name>; +// Intrinsics to read registers with non-constant values. E.g. the values that +// do change over the kernel lifetime. Such reads should not be CSE'd. +class PTXReadNCSRegIntrinsic_r32 + : Intrinsic<[llvm_i32_ty], [], [IntrInaccessibleMemOnly]>, + GCCBuiltin<"__nvvm_read_ptx_sreg_" # name>; +class PTXReadNCSRegIntrinsic_r64 + : Intrinsic<[llvm_i64_ty], [], [IntrInaccessibleMemOnly]>, + GCCBuiltin<"__nvvm_read_ptx_sreg_" # name>; + defm int_nvvm_read_ptx_sreg_tid : PTXReadSRegIntrinsic_v4i32<"tid">; defm int_nvvm_read_ptx_sreg_ntid : PTXReadSRegIntrinsic_v4i32<"ntid">; @@ -3704,13 +3942,13 @@ def int_nvvm_read_ptx_sreg_lanemask_ge : def int_nvvm_read_ptx_sreg_lanemask_gt : PTXReadSRegIntrinsic_r32<"lanemask_gt">; -def int_nvvm_read_ptx_sreg_clock : PTXReadSRegIntrinsic_r32<"clock">; -def int_nvvm_read_ptx_sreg_clock64 : PTXReadSRegIntrinsic_r64<"clock64">; +def int_nvvm_read_ptx_sreg_clock : PTXReadNCSRegIntrinsic_r32<"clock">; +def int_nvvm_read_ptx_sreg_clock64 : PTXReadNCSRegIntrinsic_r64<"clock64">; -def int_nvvm_read_ptx_sreg_pm0 : PTXReadSRegIntrinsic_r32<"pm0">; -def int_nvvm_read_ptx_sreg_pm1 : PTXReadSRegIntrinsic_r32<"pm1">; -def int_nvvm_read_ptx_sreg_pm2 : PTXReadSRegIntrinsic_r32<"pm2">; -def int_nvvm_read_ptx_sreg_pm3 : PTXReadSRegIntrinsic_r32<"pm3">; +def int_nvvm_read_ptx_sreg_pm0 : PTXReadNCSRegIntrinsic_r32<"pm0">; +def int_nvvm_read_ptx_sreg_pm1 : PTXReadNCSRegIntrinsic_r32<"pm1">; +def int_nvvm_read_ptx_sreg_pm2 : PTXReadNCSRegIntrinsic_r32<"pm2">; +def int_nvvm_read_ptx_sreg_pm3 : PTXReadNCSRegIntrinsic_r32<"pm3">; def int_nvvm_read_ptx_sreg_warpsize : PTXReadSRegIntrinsic_r32<"warpsize">; @@ -3882,166 +4120,59 @@ def int_nvvm_match_all_sync_i64p : // // WMMA instructions // - // WMMA.LOAD -class NVVM_WMMA_LD_GALSTS - : Intrinsic + : Intrinsic, NoCapture<0>], - "llvm.nvvm.wmma." - # Geometry - # ".load" - # "." # Abc - # "." # Layout - # !if(WithStride, ".stride", "") - # "." # Type>; - -multiclass NVVM_WMMA_LD_GALT { - def _stride: NVVM_WMMA_LD_GALSTS; - def NAME : NVVM_WMMA_LD_GALSTS; -} - -multiclass NVVM_WMMA_LD_GAT { - defm _row: NVVM_WMMA_LD_GALT; - defm _col: NVVM_WMMA_LD_GALT; -} - -multiclass NVVM_WMMA_LD_G { - defm _a_f16: NVVM_WMMA_LD_GAT; - defm _b_f16: NVVM_WMMA_LD_GAT; - defm _c_f16: NVVM_WMMA_LD_GAT; - defm _c_f32: NVVM_WMMA_LD_GAT; -} - -multiclass NVVM_WMMA_LD { - defm _m32n8k16_load: NVVM_WMMA_LD_G<"m32n8k16">; - defm _m16n16k16_load: NVVM_WMMA_LD_G<"m16n16k16">; - defm _m8n32k16_load: NVVM_WMMA_LD_G<"m8n32k16">; -} - -defm int_nvvm_wmma: NVVM_WMMA_LD; + WMMA_NAME_LDST<"load", Frag, Layout, WithStride>.intr>; // WMMA.STORE.D -class NVVM_WMMA_STD_GLSTSEmpty=[]> +class NVVM_WMMA_ST : Intrinsic<[], !listconcat( [llvm_anyptr_ty], - !if(!eq(Type,"f16"), - [regty, regty, regty, regty], - [regty, regty, regty, regty, - regty, regty, regty, regty]), - !if(WithStride, [llvm_i32_ty], Empty)), + Frag.regs, + !if(WithStride, [llvm_i32_ty], [])), [IntrWriteMem, IntrArgMemOnly, WriteOnly<0>, NoCapture<0>], - "llvm.nvvm.wmma." - # Geometry - # ".store.d" - # "." # Layout - # !if(WithStride, ".stride", "") - # "." # Type>; - -multiclass NVVM_WMMA_STD_GLT { - def _stride: NVVM_WMMA_STD_GLSTS; - def NAME: NVVM_WMMA_STD_GLSTS; -} - -multiclass NVVM_WMMA_STD_GT { - defm _row: NVVM_WMMA_STD_GLT; - defm _col: NVVM_WMMA_STD_GLT; -} -multiclass NVVM_WMMA_STD_G { - defm _d_f16: NVVM_WMMA_STD_GT; - defm _d_f32: NVVM_WMMA_STD_GT; -} - -multiclass NVVM_WMMA_STD { - defm _m32n8k16_store: NVVM_WMMA_STD_G<"m32n8k16">; - defm _m16n16k16_store: NVVM_WMMA_STD_G<"m16n16k16">; - defm _m8n32k16_store: NVVM_WMMA_STD_G<"m8n32k16">; + WMMA_NAME_LDST<"store", Frag, Layout, WithStride>.intr>; + +// Create all load/store variants +foreach layout = ["row", "col"] in { + foreach stride = [0, 1] in { + foreach frag = NVVM_MMA_OPS.all_ld_ops in + foreach _ = NVVM_MMA_SUPPORTED<[frag], layout>.ret in + def WMMA_NAME_LDST<"load", frag, layout, stride>.record + : NVVM_WMMA_LD; + foreach frag = NVVM_MMA_OPS.all_st_ops in + foreach _ = NVVM_MMA_SUPPORTED<[frag], layout>.ret in + def WMMA_NAME_LDST<"store", frag, layout, stride>.record + : NVVM_WMMA_ST; + } } -defm int_nvvm_wmma: NVVM_WMMA_STD; - // WMMA.MMA -class NVVM_WMMA_MMA_GABDCS - : Intrinsic + : Intrinsic { -} - -multiclass NVVM_WMMA_MMA_GABDC { - def NAME : NVVM_WMMA_MMA_GABDCS; - def _satfinite: NVVM_WMMA_MMA_GABDCS; -} - -multiclass NVVM_WMMA_MMA_GABD { - defm _f16: NVVM_WMMA_MMA_GABDC; - defm _f32: NVVM_WMMA_MMA_GABDC; -} - -multiclass NVVM_WMMA_MMA_GAB { - defm _f16: NVVM_WMMA_MMA_GABD; - defm _f32: NVVM_WMMA_MMA_GABD; -} - -multiclass NVVM_WMMA_MMA_GA { - defm _col: NVVM_WMMA_MMA_GAB; - defm _row: NVVM_WMMA_MMA_GAB; -} - -multiclass NVVM_WMMA_MMA_G { - defm _col: NVVM_WMMA_MMA_GA; - defm _row: NVVM_WMMA_MMA_GA; -} - -multiclass NVVM_WMMA_MMA { - defm _m32n8k16_mma : NVVM_WMMA_MMA_G<"m32n8k16">; - defm _m16n16k16_mma : NVVM_WMMA_MMA_G<"m16n16k16">; - defm _m8n32k16_mma : NVVM_WMMA_MMA_G<"m8n32k16">; -} - -defm int_nvvm_wmma : NVVM_WMMA_MMA; + WMMA_NAME_MMA.llvm>; + +foreach layout_a = ["row", "col"] in { + foreach layout_b = ["row", "col"] in { + foreach satf = [0, 1] in { + foreach op = NVVM_MMA_OPS.all_mma_ops in { + foreach _ = NVVM_MMA_SUPPORTED.ret in { + def WMMA_NAME_MMA.record + : NVVM_WMMA_MMA; + } + } + } // satf + } // layout_b +} // layout_a } // let TargetPrefix = "nvvm" diff --git a/include/llvm/IR/IntrinsicsPowerPC.td b/include/llvm/IR/IntrinsicsPowerPC.td index 62b2e8f77e7d..f87317445753 100644 --- a/include/llvm/IR/IntrinsicsPowerPC.td +++ b/include/llvm/IR/IntrinsicsPowerPC.td @@ -1,9 +1,8 @@ //===- IntrinsicsPowerPC.td - Defines PowerPC intrinsics ---*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,7 +18,8 @@ let TargetPrefix = "ppc" in { // All intrinsics start with "llvm.ppc.". // dcba/dcbf/dcbi/dcbst/dcbt/dcbz/dcbzl(PPC970) instructions. def int_ppc_dcba : Intrinsic<[], [llvm_ptr_ty], []>; - def int_ppc_dcbf : Intrinsic<[], [llvm_ptr_ty], []>; + def int_ppc_dcbf : GCCBuiltin<"__builtin_dcbf">, + Intrinsic<[], [llvm_ptr_ty], []>; def int_ppc_dcbi : Intrinsic<[], [llvm_ptr_ty], []>; def int_ppc_dcbst : Intrinsic<[], [llvm_ptr_ty], []>; def int_ppc_dcbt : Intrinsic<[], [llvm_ptr_ty], @@ -610,16 +610,16 @@ let TargetPrefix = "ppc" in { // All PPC intrinsics start with "llvm.ppc.". // FP <-> integer conversion. def int_ppc_altivec_vcfsx : GCCBuiltin<"__builtin_altivec_vcfsx">, Intrinsic<[llvm_v4f32_ty], [llvm_v4i32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; def int_ppc_altivec_vcfux : GCCBuiltin<"__builtin_altivec_vcfux">, Intrinsic<[llvm_v4f32_ty], [llvm_v4i32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; def int_ppc_altivec_vctsxs : GCCBuiltin<"__builtin_altivec_vctsxs">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; def int_ppc_altivec_vctuxs : GCCBuiltin<"__builtin_altivec_vctuxs">, Intrinsic<[llvm_v4i32_ty], [llvm_v4f32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; def int_ppc_altivec_vrfim : GCCBuiltin<"__builtin_altivec_vrfim">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty], [IntrNoMem]>; @@ -716,11 +716,11 @@ let TargetPrefix = "ppc" in { // All PPC intrinsics start with "llvm.ppc.". def int_ppc_altivec_crypto_vshasigmad : GCCBuiltin<"__builtin_altivec_crypto_vshasigmad">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, - llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>, ImmArg<2>]>; def int_ppc_altivec_crypto_vshasigmaw : GCCBuiltin<"__builtin_altivec_crypto_vshasigmaw">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, - llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>, ImmArg<2>]>; } def int_ppc_altivec_crypto_vcipher : PowerPC_Vec_DDD_Intrinsic<"crypto_vcipher">; @@ -915,10 +915,10 @@ def int_ppc_vsx_xvxsigsp : [llvm_v4f32_ty], [IntrNoMem]>; def int_ppc_vsx_xvtstdcdp : PowerPC_VSX_Intrinsic<"xvtstdcdp", [llvm_v2i64_ty], - [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem]>; + [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_ppc_vsx_xvtstdcsp : PowerPC_VSX_Intrinsic<"xvtstdcsp", [llvm_v4i32_ty], - [llvm_v4f32_ty,llvm_i32_ty], [IntrNoMem]>; + [llvm_v4f32_ty,llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_ppc_vsx_xvcvhpsp : PowerPC_VSX_Intrinsic<"xvcvhpsp", [llvm_v4f32_ty], [llvm_v8i16_ty],[IntrNoMem]>; @@ -1113,9 +1113,9 @@ let TargetPrefix = "ppc" in { // All intrinsics start with "llvm.ppc.". let TargetPrefix = "ppc" in { // All intrinsics start with "llvm.ppc.". def int_ppc_tbegin : GCCBuiltin<"__builtin_tbegin">, - Intrinsic<[llvm_i32_ty], [llvm_i32_ty], []>; + Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [ImmArg<0>]>; def int_ppc_tend : GCCBuiltin<"__builtin_tend">, - Intrinsic<[llvm_i32_ty], [llvm_i32_ty], []>; + Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [ImmArg<0>]>; def int_ppc_tabort : GCCBuiltin<"__builtin_tabort">, Intrinsic<[llvm_i32_ty], [llvm_i32_ty], []>; @@ -1167,4 +1167,9 @@ def int_ppc_ttest : GCCBuiltin<"__builtin_ttest">, Intrinsic<[llvm_i64_ty], [], []>; def int_ppc_cfence : Intrinsic<[], [llvm_anyint_ty], []>; + +// PowerPC set FPSCR Intrinsic Definitions. +def int_ppc_setrnd : GCCBuiltin<"__builtin_setrnd">, + Intrinsic<[llvm_double_ty], [llvm_i32_ty], []>; + } diff --git a/include/llvm/IR/IntrinsicsRISCV.td b/include/llvm/IR/IntrinsicsRISCV.td index 0ac7348b56db..60393189b830 100644 --- a/include/llvm/IR/IntrinsicsRISCV.td +++ b/include/llvm/IR/IntrinsicsRISCV.td @@ -1,9 +1,8 @@ //===- IntrinsicsRISCV.td - Defines RISCV intrinsics -------*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,13 +18,13 @@ let TargetPrefix = "riscv" in { class MaskedAtomicRMW32Intrinsic : Intrinsic<[llvm_i32_ty], [llvm_anyptr_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrArgMemOnly, NoCapture<0>]>; + [IntrArgMemOnly, NoCapture<0>, ImmArg<3>]>; class MaskedAtomicRMW32WithSextIntrinsic : Intrinsic<[llvm_i32_ty], [llvm_anyptr_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrArgMemOnly, NoCapture<0>]>; + [IntrArgMemOnly, NoCapture<0>, ImmArg<4>]>; def int_riscv_masked_atomicrmw_xchg_i32 : MaskedAtomicRMW32Intrinsic; def int_riscv_masked_atomicrmw_add_i32 : MaskedAtomicRMW32Intrinsic; @@ -39,6 +38,31 @@ def int_riscv_masked_atomicrmw_umin_i32 : MaskedAtomicRMW32Intrinsic; def int_riscv_masked_cmpxchg_i32 : Intrinsic<[llvm_i32_ty], [llvm_anyptr_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrArgMemOnly, NoCapture<0>]>; + [IntrArgMemOnly, NoCapture<0>, ImmArg<4>]>; + +class MaskedAtomicRMW64Intrinsic + : Intrinsic<[llvm_i64_ty], + [llvm_anyptr_ty, llvm_i64_ty, llvm_i64_ty, llvm_i64_ty], + [IntrArgMemOnly, NoCapture<0>, ImmArg<3>]>; + +class MaskedAtomicRMW64WithSextIntrinsic + : Intrinsic<[llvm_i64_ty], + [llvm_anyptr_ty, llvm_i64_ty, llvm_i64_ty, llvm_i64_ty, + llvm_i64_ty], + [IntrArgMemOnly, NoCapture<0>, ImmArg<4>]>; + +def int_riscv_masked_atomicrmw_xchg_i64 : MaskedAtomicRMW64Intrinsic; +def int_riscv_masked_atomicrmw_add_i64 : MaskedAtomicRMW64Intrinsic; +def int_riscv_masked_atomicrmw_sub_i64 : MaskedAtomicRMW64Intrinsic; +def int_riscv_masked_atomicrmw_nand_i64 : MaskedAtomicRMW64Intrinsic; +def int_riscv_masked_atomicrmw_max_i64 : MaskedAtomicRMW64WithSextIntrinsic; +def int_riscv_masked_atomicrmw_min_i64 : MaskedAtomicRMW64WithSextIntrinsic; +def int_riscv_masked_atomicrmw_umax_i64 : MaskedAtomicRMW64Intrinsic; +def int_riscv_masked_atomicrmw_umin_i64 : MaskedAtomicRMW64Intrinsic; + +def int_riscv_masked_cmpxchg_i64 + : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty, llvm_i64_ty, llvm_i64_ty, + llvm_i64_ty, llvm_i64_ty], + [IntrArgMemOnly, NoCapture<0>, ImmArg<4>]>; } // TargetPrefix = "riscv" diff --git a/include/llvm/IR/IntrinsicsSystemZ.td b/include/llvm/IR/IntrinsicsSystemZ.td index caa2ec209a31..40d6ba17eaf1 100644 --- a/include/llvm/IR/IntrinsicsSystemZ.td +++ b/include/llvm/IR/IntrinsicsSystemZ.td @@ -1,9 +1,8 @@ //===- IntrinsicsSystemZ.td - Defines SystemZ intrinsics ---*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -39,7 +38,8 @@ class SystemZBinaryConvCC : Intrinsic<[result, llvm_i32_ty], [arg, arg], [IntrNoMem]>; class SystemZBinaryConvIntCC - : Intrinsic<[result, llvm_i32_ty], [arg, llvm_i32_ty], [IntrNoMem]>; + : Intrinsic<[result, llvm_i32_ty], [arg, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; class SystemZBinaryCC : SystemZBinaryConvCC; @@ -48,23 +48,28 @@ class SystemZTernaryConv : GCCBuiltin<"__builtin_s390_" ## name>, Intrinsic<[result], [arg, arg, result], [IntrNoMem]>; +class SystemZTernaryConvCC + : Intrinsic<[result, llvm_i32_ty], [arg, arg, result], [IntrNoMem]>; + class SystemZTernary : SystemZTernaryConv; class SystemZTernaryInt : GCCBuiltin<"__builtin_s390_" ## name>, - Intrinsic<[type], [type, type, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[type], [type, type, llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; class SystemZTernaryIntCC - : Intrinsic<[type, llvm_i32_ty], [type, type, llvm_i32_ty], [IntrNoMem]>; + : Intrinsic<[type, llvm_i32_ty], [type, type, llvm_i32_ty], + [IntrNoMem, ImmArg<2>]>; class SystemZQuaternaryInt : GCCBuiltin<"__builtin_s390_" ## name>, - Intrinsic<[type], [type, type, type, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[type], [type, type, type, llvm_i32_ty], + [IntrNoMem, ImmArg<3>]>; class SystemZQuaternaryIntCC : Intrinsic<[type, llvm_i32_ty], [type, type, type, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; multiclass SystemZUnaryExtBHF { def b : SystemZUnaryConv; @@ -180,7 +185,8 @@ multiclass SystemZQuaternaryIntBHF { def f : SystemZQuaternaryInt; } -multiclass SystemZQuaternaryIntBHFG : SystemZQuaternaryIntBHF { +multiclass SystemZQuaternaryIntBHFG : + SystemZQuaternaryIntBHF { def g : SystemZQuaternaryInt; } @@ -232,11 +238,11 @@ let TargetPrefix = "s390" in { let TargetPrefix = "s390" in { def int_s390_lcbb : GCCBuiltin<"__builtin_s390_lcbb">, Intrinsic<[llvm_i32_ty], [llvm_ptr_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; def int_s390_vlbb : GCCBuiltin<"__builtin_s390_vlbb">, Intrinsic<[llvm_v16i8_ty], [llvm_ptr_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, IntrArgMemOnly, ImmArg<1>]>; def int_s390_vll : GCCBuiltin<"__builtin_s390_vll">, Intrinsic<[llvm_v16i8_ty], [llvm_i32_ty, llvm_ptr_ty], @@ -245,7 +251,7 @@ let TargetPrefix = "s390" in { def int_s390_vpdi : GCCBuiltin<"__builtin_s390_vpdi">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_s390_vperm : GCCBuiltin<"__builtin_s390_vperm">, Intrinsic<[llvm_v16i8_ty], @@ -311,7 +317,7 @@ let TargetPrefix = "s390" in { def int_s390_vsldb : GCCBuiltin<"__builtin_s390_vsldb">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; defm int_s390_vscbi : SystemZBinaryBHFG<"vscbi">; @@ -370,7 +376,7 @@ let TargetPrefix = "s390" in { def int_s390_vfidb : Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>, ImmArg<2>]>; // Instructions from the Vector Enhancements Facility 1 def int_s390_vbperm : SystemZBinaryConv<"vbperm", llvm_v2i64_ty, @@ -379,20 +385,20 @@ let TargetPrefix = "s390" in { def int_s390_vmslg : GCCBuiltin<"__builtin_s390_vmslg">, Intrinsic<[llvm_v16i8_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v16i8_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<3>]>; def int_s390_vfmaxdb : Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_s390_vfmindb : Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_s390_vfmaxsb : Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_s390_vfminsb : Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_s390_vfcesbs : SystemZBinaryConvCC; def int_s390_vfchsbs : SystemZBinaryConvCC; @@ -402,7 +408,7 @@ let TargetPrefix = "s390" in { def int_s390_vfisb : Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>, ImmArg<2>]>; // Instructions from the Vector Packed Decimal Facility def int_s390_vlrl : GCCBuiltin<"__builtin_s390_vlrl">, @@ -412,6 +418,24 @@ let TargetPrefix = "s390" in { def int_s390_vstrl : GCCBuiltin<"__builtin_s390_vstrl">, Intrinsic<[], [llvm_v16i8_ty, llvm_i32_ty, llvm_ptr_ty], [IntrArgMemOnly, IntrWriteMem]>; + + // Instructions from the Vector Enhancements Facility 2 + def int_s390_vsld : GCCBuiltin<"__builtin_s390_vsld">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<2>]>; + + def int_s390_vsrd : GCCBuiltin<"__builtin_s390_vsrd">, + Intrinsic<[llvm_v16i8_ty], + [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<2>]>; + + def int_s390_vstrsb : SystemZTernaryConvCC; + def int_s390_vstrsh : SystemZTernaryConvCC; + def int_s390_vstrsf : SystemZTernaryConvCC; + def int_s390_vstrszb : SystemZTernaryConvCC; + def int_s390_vstrszh : SystemZTernaryConvCC; + def int_s390_vstrszf : SystemZTernaryConvCC; } //===----------------------------------------------------------------------===// diff --git a/include/llvm/IR/IntrinsicsWebAssembly.td b/include/llvm/IR/IntrinsicsWebAssembly.td index b015650906e0..1b892727547d 100644 --- a/include/llvm/IR/IntrinsicsWebAssembly.td +++ b/include/llvm/IR/IntrinsicsWebAssembly.td @@ -1,9 +1,8 @@ //===- IntrinsicsWebAssembly.td - Defines wasm intrinsics --*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -41,8 +40,8 @@ def int_wasm_trunc_saturate_unsigned : Intrinsic<[llvm_anyint_ty], // throw / rethrow def int_wasm_throw : Intrinsic<[], [llvm_i32_ty, llvm_ptr_ty], - [Throws, IntrNoReturn]>; -def int_wasm_rethrow : Intrinsic<[], [], [Throws, IntrNoReturn]>; + [Throws, IntrNoReturn, ImmArg<0>]>; +def int_wasm_rethrow_in_catch : Intrinsic<[], [], [Throws, IntrNoReturn]>; // Since wasm does not use landingpad instructions, these instructions return // exception pointer and selector values until we lower them in WasmEHPrepare. @@ -50,17 +49,16 @@ def int_wasm_get_exception : Intrinsic<[llvm_ptr_ty], [llvm_token_ty], [IntrHasSideEffects]>; def int_wasm_get_ehselector : Intrinsic<[llvm_i32_ty], [llvm_token_ty], [IntrHasSideEffects]>; - -// wasm.catch returns the pointer to the exception object caught by wasm 'catch' -// instruction. -def int_wasm_catch : Intrinsic<[llvm_ptr_ty], [llvm_i32_ty], - [IntrHasSideEffects]>; +// This is the same as llvm.wasm.get.exception except that it does not take a +// token operand. This is only for instruction selection purpose. +def int_wasm_extract_exception : Intrinsic<[llvm_ptr_ty], [], + [IntrHasSideEffects]>; // WebAssembly EH must maintain the landingpads in the order assigned to them // by WasmEHPrepare pass to generate landingpad table in EHStreamer. This is // used in order to give them the indices in WasmEHPrepare. def int_wasm_landingpad_index: Intrinsic<[], [llvm_token_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; // Returns LSDA address of the current function. def int_wasm_lsda : Intrinsic<[llvm_ptr_ty], [], [IntrNoMem]>; @@ -112,4 +110,27 @@ def int_wasm_alltrue : [llvm_anyvector_ty], [IntrNoMem, IntrSpeculatable]>; +//===----------------------------------------------------------------------===// +// Bulk memory intrinsics +//===----------------------------------------------------------------------===// + +def int_wasm_memory_init : + Intrinsic<[], + [llvm_i32_ty, llvm_i32_ty, llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], + [IntrWriteMem, IntrInaccessibleMemOrArgMemOnly, WriteOnly<2>, + IntrHasSideEffects, ImmArg<0>, ImmArg<1>]>; +def int_wasm_data_drop : + Intrinsic<[], + [llvm_i32_ty], + [IntrNoDuplicate, IntrHasSideEffects, ImmArg<0>]>; + +//===----------------------------------------------------------------------===// +// Thread-local storage intrinsics +//===----------------------------------------------------------------------===// + +def int_wasm_tls_size : + Intrinsic<[llvm_anyint_ty], + [], + [IntrNoMem, IntrSpeculatable]>; + } // TargetPrefix = "wasm" diff --git a/include/llvm/IR/IntrinsicsX86.td b/include/llvm/IR/IntrinsicsX86.td index 8d8cc8e97678..236d312d7d78 100644 --- a/include/llvm/IR/IntrinsicsX86.td +++ b/include/llvm/IR/IntrinsicsX86.td @@ -1,9 +1,8 @@ //===- IntrinsicsX86.td - Defines X86 intrinsics -----------*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,7 +13,7 @@ //===----------------------------------------------------------------------===// // Interrupt traps let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_int : Intrinsic<[], [llvm_i8_ty]>; + def int_x86_int : Intrinsic<[], [llvm_i8_ty], [ImmArg<0>]>; } //===----------------------------------------------------------------------===// @@ -204,12 +203,12 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_sse_cmp_ss : GCCBuiltin<"__builtin_ia32_cmpss">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, - llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<2>]>; // NOTE: This comparison intrinsic is not used by clang as long as the // distinction in signaling behaviour is not implemented. def int_x86_sse_cmp_ps : Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, - llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_sse_comieq_ss : GCCBuiltin<"__builtin_ia32_comieq">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_v4f32_ty], [IntrNoMem]>; @@ -278,9 +277,17 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Control register. let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_sse_stmxcsr : - Intrinsic<[], [llvm_ptr_ty], []>; + Intrinsic<[], [llvm_ptr_ty], + [IntrWriteMem, IntrArgMemOnly, + // This prevents reordering with ldmxcsr + IntrHasSideEffects]>; def int_x86_sse_ldmxcsr : - Intrinsic<[], [llvm_ptr_ty], []>; + Intrinsic<[], [llvm_ptr_ty], + [IntrReadMem, IntrArgMemOnly, IntrHasSideEffects, + // FIXME: LDMXCSR does not actualy write to memory, + // but Fast and DAG Isel both use writing to memory + // as a proxy for having side effects. + IntrWriteMem]>; } // Misc. @@ -312,12 +319,12 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_sse2_cmp_sd : GCCBuiltin<"__builtin_ia32_cmpsd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, - llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem, ImmArg<2>]>; // NOTE: This comparison intrinsic is not used by clang as long as the // distinction in signaling behaviour is not implemented. def int_x86_sse2_cmp_pd : Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, - llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_sse2_comieq_sd : GCCBuiltin<"__builtin_ia32_comisdeq">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_v2f64_ty], [IntrNoMem]>; @@ -367,6 +374,12 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_sse2_pmadd_wd : GCCBuiltin<"__builtin_ia32_pmaddwd128">, Intrinsic<[llvm_v4i32_ty], [llvm_v8i16_ty, llvm_v8i16_ty], [IntrNoMem, Commutative]>; + def int_x86_sse2_pavg_b : GCCBuiltin<"__builtin_ia32_pavgb128">, + Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, + llvm_v16i8_ty], [IntrNoMem, Commutative]>; + def int_x86_sse2_pavg_w : GCCBuiltin<"__builtin_ia32_pavgw128">, + Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, + llvm_v8i16_ty], [IntrNoMem, Commutative]>; def int_x86_sse2_psad_bw : GCCBuiltin<"__builtin_ia32_psadbw128">, Intrinsic<[llvm_v2i64_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem, Commutative]>; @@ -399,6 +412,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; + // Oddly these don't require an immediate due to a gcc compatibility issue. def int_x86_sse2_pslli_w : GCCBuiltin<"__builtin_ia32_psllwi128">, Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_i32_ty], [IntrNoMem]>; @@ -604,7 +618,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_v16i8_ty], [IntrNoMem]>; def int_x86_sse_pshuf_w : GCCBuiltin<"__builtin_ia32_pshufw">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; } // Sign ops @@ -650,16 +664,16 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_sse41_round_ss : GCCBuiltin<"__builtin_ia32_roundss">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_sse41_round_ps : GCCBuiltin<"__builtin_ia32_roundps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_x86_sse41_round_sd : GCCBuiltin<"__builtin_ia32_roundsd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_sse41_round_pd : GCCBuiltin<"__builtin_ia32_roundpd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; } // Vector min element @@ -722,20 +736,20 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_aesni_aeskeygenassist : GCCBuiltin<"__builtin_ia32_aeskeygenassist128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; } // PCLMUL instructions let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_pclmulqdq : GCCBuiltin<"__builtin_ia32_pclmulqdq128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_x86_pclmulqdq_256 : GCCBuiltin<"__builtin_ia32_pclmulqdq256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_x86_pclmulqdq_512 : GCCBuiltin<"__builtin_ia32_pclmulqdq512">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; } // Vector pack @@ -749,7 +763,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_sse41_insertps : GCCBuiltin<"__builtin_ia32_insertps128">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; } // Vector blend @@ -769,17 +783,17 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_sse41_dppd : GCCBuiltin<"__builtin_ia32_dppd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], - [IntrNoMem, Commutative]>; + [IntrNoMem, Commutative, ImmArg<2>]>; def int_x86_sse41_dpps : GCCBuiltin<"__builtin_ia32_dpps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], - [IntrNoMem, Commutative]>; + [IntrNoMem, Commutative, ImmArg<2>]>; } // Vector sum of absolute differences let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_sse41_mpsadbw : GCCBuiltin<"__builtin_ia32_mpsadbw128">, Intrinsic<[llvm_v8i16_ty], [llvm_v16i8_ty, llvm_v16i8_ty,llvm_i8_ty], - [IntrNoMem, Commutative]>; + [IntrNoMem, Commutative, ImmArg<2>]>; } // Test instruction with bitwise comparison. @@ -820,66 +834,66 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_sse42_pcmpistrm128 : GCCBuiltin<"__builtin_ia32_pcmpistrm128">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_x86_sse42_pcmpistri128 : GCCBuiltin<"__builtin_ia32_pcmpistri128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_x86_sse42_pcmpistria128 : GCCBuiltin<"__builtin_ia32_pcmpistria128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_x86_sse42_pcmpistric128 : GCCBuiltin<"__builtin_ia32_pcmpistric128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_x86_sse42_pcmpistrio128 : GCCBuiltin<"__builtin_ia32_pcmpistrio128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_x86_sse42_pcmpistris128 : GCCBuiltin<"__builtin_ia32_pcmpistris128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_x86_sse42_pcmpistriz128 : GCCBuiltin<"__builtin_ia32_pcmpistriz128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_x86_sse42_pcmpestrm128 : GCCBuiltin<"__builtin_ia32_pcmpestrm128">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty, llvm_i32_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<4>]>; def int_x86_sse42_pcmpestri128 : GCCBuiltin<"__builtin_ia32_pcmpestri128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty, llvm_i32_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<4>]>; def int_x86_sse42_pcmpestria128 : GCCBuiltin<"__builtin_ia32_pcmpestria128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty, llvm_i32_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<4>]>; def int_x86_sse42_pcmpestric128 : GCCBuiltin<"__builtin_ia32_pcmpestric128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty, llvm_i32_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<4>]>; def int_x86_sse42_pcmpestrio128 : GCCBuiltin<"__builtin_ia32_pcmpestrio128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty, llvm_i32_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<4>]>; def int_x86_sse42_pcmpestris128 : GCCBuiltin<"__builtin_ia32_pcmpestris128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty, llvm_i32_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<4>]>; def int_x86_sse42_pcmpestriz128 : GCCBuiltin<"__builtin_ia32_pcmpestriz128">, Intrinsic<[llvm_i32_ty], [llvm_v16i8_ty, llvm_i32_ty, llvm_v16i8_ty, llvm_i32_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<4>]>; } //===----------------------------------------------------------------------===// @@ -888,13 +902,14 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_sse4a_extrqi : GCCBuiltin<"__builtin_ia32_extrqi">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i8_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>, ImmArg<2>]>; def int_x86_sse4a_extrq : GCCBuiltin<"__builtin_ia32_extrq">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v16i8_ty], [IntrNoMem]>; def int_x86_sse4a_insertqi : GCCBuiltin<"__builtin_ia32_insertqi">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, - llvm_i8_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_i8_ty, llvm_i8_ty], + [IntrNoMem, ImmArg<2>, ImmArg<3>]>; def int_x86_sse4a_insertq : GCCBuiltin<"__builtin_ia32_insertq">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; } @@ -931,10 +946,10 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx_round_pd_256 : GCCBuiltin<"__builtin_ia32_roundpd256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; def int_x86_avx_round_ps_256 : GCCBuiltin<"__builtin_ia32_roundps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<1>]>; } // Horizontal ops @@ -1086,33 +1101,33 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". GCCBuiltin<"__builtin_ia32_vgf2p8affineinvqb_v16qi">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_x86_vgf2p8affineinvqb_256 : GCCBuiltin<"__builtin_ia32_vgf2p8affineinvqb_v32qi">, Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_x86_vgf2p8affineinvqb_512 : GCCBuiltin<"__builtin_ia32_vgf2p8affineinvqb_v64qi">, Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_x86_vgf2p8affineqb_128 : GCCBuiltin<"__builtin_ia32_vgf2p8affineqb_v16qi">, Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_x86_vgf2p8affineqb_256 : GCCBuiltin<"__builtin_ia32_vgf2p8affineqb_v32qi">, Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_x86_vgf2p8affineqb_512 : GCCBuiltin<"__builtin_ia32_vgf2p8affineqb_v64qi">, Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_x86_vgf2p8mulb_128 : GCCBuiltin<"__builtin_ia32_vgf2p8mulb_v16qi">, @@ -1145,17 +1160,18 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx_dp_ps_256 : GCCBuiltin<"__builtin_ia32_dpps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, - llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem, Commutative]>; + llvm_v8f32_ty, llvm_i8_ty], + [IntrNoMem, Commutative, ImmArg<2>]>; } // Vector compare let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx_cmp_pd_256 : Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, - llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_avx_cmp_ps_256 : Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, - llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem, ImmArg<2>]>; } // Vector convert @@ -1222,30 +1238,30 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_fpclass_pd_128 : Intrinsic<[llvm_v2i1_ty], [llvm_v2f64_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_fpclass_pd_256 : Intrinsic<[llvm_v4i1_ty], [llvm_v4f64_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_fpclass_pd_512 : Intrinsic<[llvm_v8i1_ty], [llvm_v8f64_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_fpclass_ps_128 : Intrinsic<[llvm_v4i1_ty], [llvm_v4f32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_fpclass_ps_256 : Intrinsic<[llvm_v8i1_ty], [llvm_v8f32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_fpclass_ps_512 : Intrinsic<[llvm_v16i1_ty], [llvm_v16f32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_mask_fpclass_sd : GCCBuiltin<"__builtin_ia32_fpclasssd_mask">, Intrinsic<[llvm_i8_ty], [llvm_v2f64_ty, llvm_i32_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_mask_fpclass_ss : GCCBuiltin<"__builtin_ia32_fpclassss_mask">, Intrinsic<[llvm_i8_ty], [llvm_v4f32_ty, llvm_i32_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; } // Vector extract sign mask @@ -1328,6 +1344,12 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx2_pmadd_wd : GCCBuiltin<"__builtin_ia32_pmaddwd256">, Intrinsic<[llvm_v8i32_ty], [llvm_v16i16_ty, llvm_v16i16_ty], [IntrNoMem, Commutative]>; + def int_x86_avx2_pavg_b : GCCBuiltin<"__builtin_ia32_pavgb256">, + Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, + llvm_v32i8_ty], [IntrNoMem, Commutative]>; + def int_x86_avx2_pavg_w : GCCBuiltin<"__builtin_ia32_pavgw256">, + Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, + llvm_v16i16_ty], [IntrNoMem, Commutative]>; def int_x86_avx2_psad_bw : GCCBuiltin<"__builtin_ia32_psadbw256">, Intrinsic<[llvm_v4i64_ty], [llvm_v32i8_ty, llvm_v32i8_ty], [IntrNoMem, Commutative]>; @@ -1360,6 +1382,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v4i32_ty], [IntrNoMem]>; + // Oddly these don't require an immediate due to a gcc compatibility issue. def int_x86_avx2_pslli_w : GCCBuiltin<"__builtin_ia32_psllwi256">, Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_i32_ty], [IntrNoMem]>; @@ -1392,6 +1415,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v2i64_ty], [IntrNoMem]>; + // Oddly these don't require an immediate due to a gcc compatibility issue. def int_x86_avx512_psrai_q_128 : GCCBuiltin<"__builtin_ia32_psraqi128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_i32_ty], [IntrNoMem]>; @@ -1427,6 +1451,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v2i64_ty], [IntrNoMem]>; + // Oddly these don't require an immediate due to a gcc compatibility issue. def int_x86_avx512_pslli_w_512 : GCCBuiltin<"__builtin_ia32_psllwi512">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_i32_ty], [IntrNoMem]>; @@ -1677,71 +1702,73 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Gather ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". + // NOTE: These can't be ArgMemOnly because you can put the address completely + // in the index register. def int_x86_avx2_gather_d_pd : GCCBuiltin<"__builtin_ia32_gatherd_pd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v2f64_ty, llvm_i8_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx2_gather_d_pd_256 : GCCBuiltin<"__builtin_ia32_gatherd_pd256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4f64_ty, llvm_i8_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx2_gather_q_pd : GCCBuiltin<"__builtin_ia32_gatherq_pd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v2f64_ty, llvm_i8_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx2_gather_q_pd_256 : GCCBuiltin<"__builtin_ia32_gatherq_pd256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4f64_ty, llvm_i8_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx2_gather_d_ps : GCCBuiltin<"__builtin_ia32_gatherd_ps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4f32_ty, llvm_i8_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx2_gather_d_ps_256 : GCCBuiltin<"__builtin_ia32_gatherd_ps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_v8f32_ty, llvm_i8_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx2_gather_q_ps : GCCBuiltin<"__builtin_ia32_gatherq_ps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v4f32_ty, llvm_i8_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx2_gather_q_ps_256 : GCCBuiltin<"__builtin_ia32_gatherq_ps256">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4f32_ty, llvm_i8_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx2_gather_d_q : GCCBuiltin<"__builtin_ia32_gatherd_q">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v2i64_ty, llvm_i8_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx2_gather_d_q_256 : GCCBuiltin<"__builtin_ia32_gatherd_q256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4i64_ty, llvm_i8_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx2_gather_q_q : GCCBuiltin<"__builtin_ia32_gatherq_q">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx2_gather_q_q_256 : GCCBuiltin<"__builtin_ia32_gatherq_q256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx2_gather_d_d : GCCBuiltin<"__builtin_ia32_gatherd_d">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx2_gather_d_d_256 : GCCBuiltin<"__builtin_ia32_gatherd_d256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx2_gather_q_d : GCCBuiltin<"__builtin_ia32_gatherq_d">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v4i32_ty, llvm_i8_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx2_gather_q_d_256 : GCCBuiltin<"__builtin_ia32_gatherq_d256">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4i32_ty, llvm_i8_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; } // Misc. @@ -1753,7 +1780,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_v32i8_ty], [IntrNoMem]>; def int_x86_avx2_mpsadbw : GCCBuiltin<"__builtin_ia32_mpsadbw256">, Intrinsic<[llvm_v16i16_ty], [llvm_v32i8_ty, llvm_v32i8_ty, - llvm_i8_ty], [IntrNoMem, Commutative]>; + llvm_i8_ty], [IntrNoMem, Commutative, ImmArg<2>]>; } //===----------------------------------------------------------------------===// @@ -1763,32 +1790,32 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_vfmadd_pd_512 : Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_vfmadd_ps_512 : Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; // TODO: Can we use 2 vfmadds+shufflevector? def int_x86_avx512_vfmaddsub_pd_512 : Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8f64_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_vfmaddsub_ps_512 : Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16f32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_vfmadd_f64 : Intrinsic<[llvm_double_ty], [llvm_double_ty, llvm_double_ty, llvm_double_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_vfmadd_f32 : Intrinsic<[llvm_float_ty], [llvm_float_ty, llvm_float_ty, llvm_float_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_vpmadd52h_uq_128 : GCCBuiltin<"__builtin_ia32_vpmadd52huq128">, @@ -1878,23 +1905,23 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_xop_vpermil2pd : GCCBuiltin<"__builtin_ia32_vpermil2pd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2i64_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_xop_vpermil2pd_256 : GCCBuiltin<"__builtin_ia32_vpermil2pd256">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4i64_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_xop_vpermil2ps : GCCBuiltin<"__builtin_ia32_vpermil2ps">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4i32_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_xop_vpermil2ps_256 : GCCBuiltin<"__builtin_ia32_vpermil2ps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8i32_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_xop_vfrcz_pd : GCCBuiltin<"__builtin_ia32_vfrczpd">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty], [IntrNoMem]>; @@ -1909,31 +1936,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_xop_vfrcz_ps_256 : GCCBuiltin<"__builtin_ia32_vfrczps256">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty], [IntrNoMem]>; - def int_x86_xop_vpcomb : GCCBuiltin<"__builtin_ia32_vpcomb">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_xop_vpcomw : GCCBuiltin<"__builtin_ia32_vpcomw">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_xop_vpcomd : GCCBuiltin<"__builtin_ia32_vpcomd">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_xop_vpcomq : GCCBuiltin<"__builtin_ia32_vpcomq">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_xop_vpcomub : GCCBuiltin<"__builtin_ia32_vpcomub">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_xop_vpcomuw : GCCBuiltin<"__builtin_ia32_vpcomuw">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_xop_vpcomud : GCCBuiltin<"__builtin_ia32_vpcomud">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_xop_vpcomuq : GCCBuiltin<"__builtin_ia32_vpcomuq">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_xop_vphaddbd : GCCBuiltin<"__builtin_ia32_vphaddbd">, Intrinsic<[llvm_v4i32_ty], [llvm_v16i8_ty], [IntrNoMem]>; @@ -2261,6 +2263,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_x86mmx_ty], [IntrNoMem]>; + // Oddly these don't require an immediate due to a gcc compatibility issue. def int_x86_mmx_pslli_w : GCCBuiltin<"__builtin_ia32_psllwi">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, llvm_i32_ty], [IntrNoMem]>; @@ -2398,15 +2401,15 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_mmx_palignr_b : GCCBuiltin<"__builtin_ia32_palignr">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, - llvm_x86mmx_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_x86mmx_ty, llvm_i8_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_mmx_pextr_w : GCCBuiltin<"__builtin_ia32_vec_ext_v4hi">, Intrinsic<[llvm_i32_ty], [llvm_x86mmx_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; def int_x86_mmx_pinsr_w : GCCBuiltin<"__builtin_ia32_vec_set_v4hi">, Intrinsic<[llvm_x86mmx_ty], [llvm_x86mmx_ty, - llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; } //===----------------------------------------------------------------------===// @@ -2527,13 +2530,14 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". Intrinsic<[llvm_v8f32_ty], [llvm_v8i16_ty], [IntrNoMem]>; def int_x86_vcvtps2ph_128 : GCCBuiltin<"__builtin_ia32_vcvtps2ph">, Intrinsic<[llvm_v8i16_ty], [llvm_v4f32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; def int_x86_vcvtps2ph_256 : GCCBuiltin<"__builtin_ia32_vcvtps2ph256">, Intrinsic<[llvm_v8i16_ty], [llvm_v8f32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_mask_vcvtph2ps_512 : GCCBuiltin<"__builtin_ia32_vcvtph2ps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16i16_ty, llvm_v16f32_ty, - llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i16_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_vcvtph2ps_256 : GCCBuiltin<"__builtin_ia32_vcvtph2ps256_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8i16_ty, llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; @@ -2542,13 +2546,16 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_vcvtps2ph_512 : GCCBuiltin<"__builtin_ia32_vcvtps2ph512_mask">, Intrinsic<[llvm_v16i16_ty], [llvm_v16f32_ty, llvm_i32_ty, - llvm_v16i16_ty, llvm_i16_ty], [IntrNoMem]>; + llvm_v16i16_ty, llvm_i16_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_mask_vcvtps2ph_256 : GCCBuiltin<"__builtin_ia32_vcvtps2ph256_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v8f32_ty, llvm_i32_ty, - llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_mask_vcvtps2ph_128 : GCCBuiltin<"__builtin_ia32_vcvtps2ph_mask">, Intrinsic<[llvm_v8i16_ty], [llvm_v4f32_ty, llvm_i32_ty, - llvm_v8i16_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_v8i16_ty, llvm_i8_ty], + [IntrNoMem, ImmArg<1>]>; } //===----------------------------------------------------------------------===// @@ -2556,9 +2563,11 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_tbm_bextri_u32 : GCCBuiltin<"__builtin_ia32_bextri_u32">, - Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_tbm_bextri_u64 : GCCBuiltin<"__builtin_ia32_bextri_u64">, - Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; + Intrinsic<[llvm_i64_ty], [llvm_i64_ty, llvm_i64_ty], + [IntrNoMem, ImmArg<1>]>; } //===----------------------------------------------------------------------===// @@ -2604,7 +2613,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_xend : GCCBuiltin<"__builtin_ia32_xend">, Intrinsic<[], [], []>; def int_x86_xabort : GCCBuiltin<"__builtin_ia32_xabort">, - Intrinsic<[], [llvm_i8_ty], []>; + Intrinsic<[], [llvm_i8_ty], [ImmArg<0>]>; def int_x86_xtest : GCCBuiltin<"__builtin_ia32_xtest">, Intrinsic<[llvm_i32_ty], [], []>; } @@ -2645,55 +2654,71 @@ let TargetPrefix = "x86" in { // Conversion ops let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_cvttss2si : GCCBuiltin<"__builtin_ia32_vcvttss2si32">, - Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_cvttss2si64 : GCCBuiltin<"__builtin_ia32_vcvttss2si64">, - Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_cvttss2usi : GCCBuiltin<"__builtin_ia32_vcvttss2usi32">, - Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_cvttss2usi64 : GCCBuiltin<"__builtin_ia32_vcvttss2usi64">, - Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_cvtusi2ss : GCCBuiltin<"__builtin_ia32_cvtusi2ss32">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, - llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_cvtusi642ss : GCCBuiltin<"__builtin_ia32_cvtusi2ss64">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, - llvm_i64_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_cvttsd2si : GCCBuiltin<"__builtin_ia32_vcvttsd2si32">, - Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_cvttsd2si64 : GCCBuiltin<"__builtin_ia32_vcvttsd2si64">, - Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_cvttsd2usi : GCCBuiltin<"__builtin_ia32_vcvttsd2usi32">, - Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_cvttsd2usi64 : GCCBuiltin<"__builtin_ia32_vcvttsd2usi64">, - Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_cvtusi642sd : GCCBuiltin<"__builtin_ia32_cvtusi2sd64">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, - llvm_i64_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_vcvtss2usi32 : GCCBuiltin<"__builtin_ia32_vcvtss2usi32">, - Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_vcvtss2usi64 : GCCBuiltin<"__builtin_ia32_vcvtss2usi64">, - Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_vcvtss2si32 : GCCBuiltin<"__builtin_ia32_vcvtss2si32">, - Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_vcvtss2si64 : GCCBuiltin<"__builtin_ia32_vcvtss2si64">, - Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_i64_ty], [llvm_v4f32_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_vcvtsd2usi32 : GCCBuiltin<"__builtin_ia32_vcvtsd2usi32">, - Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_vcvtsd2usi64 : GCCBuiltin<"__builtin_ia32_vcvtsd2usi64">, - Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_vcvtsd2si32 : GCCBuiltin<"__builtin_ia32_vcvtsd2si32">, - Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_vcvtsd2si64 : GCCBuiltin<"__builtin_ia32_vcvtsd2si64">, - Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_i64_ty], [llvm_v2f64_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_cvtsi2ss32 : GCCBuiltin<"__builtin_ia32_cvtsi2ss32">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, - llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_cvtsi2ss64 : GCCBuiltin<"__builtin_ia32_cvtsi2ss64">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, - llvm_i64_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_cvtsi2sd64 : GCCBuiltin<"__builtin_ia32_cvtsi2sd64">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, - llvm_i64_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i64_ty, llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; } // Pack ops. @@ -2714,11 +2739,13 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". // Vector convert let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". - def int_x86_avx512_mask_cvtdq2ps_512 : - GCCBuiltin<"__builtin_ia32_cvtdq2ps512_mask">, - Intrinsic<[llvm_v16f32_ty], - [llvm_v16i32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], - [IntrNoMem]>; + def int_x86_avx512_sitofp_round : + Intrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; + + def int_x86_avx512_uitofp_round : + Intrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_mask_cvtpd2dq_128 : GCCBuiltin<"__builtin_ia32_cvtpd2dq128_mask">, @@ -2730,25 +2757,25 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". GCCBuiltin<"__builtin_ia32_cvtpd2dq512_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8f64_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_cvtpd2ps_512 : GCCBuiltin<"__builtin_ia32_cvtpd2ps512_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f64_ty, llvm_v8f32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_cvtsd2ss_round : GCCBuiltin<"__builtin_ia32_cvtsd2ss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v2f64_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_mask_cvtss2sd_round : GCCBuiltin<"__builtin_ia32_cvtss2sd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v4f32_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_mask_cvtpd2ps : GCCBuiltin<"__builtin_ia32_cvtpd2ps_mask">, @@ -2772,7 +2799,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". GCCBuiltin<"__builtin_ia32_cvtpd2qq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f64_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_cvtpd2udq_128 : GCCBuiltin<"__builtin_ia32_cvtpd2udq128_mask">, @@ -2790,7 +2817,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". GCCBuiltin<"__builtin_ia32_cvtpd2udq512_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8f64_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_cvtpd2uqq_128 : GCCBuiltin<"__builtin_ia32_cvtpd2uqq128_mask">, @@ -2808,7 +2835,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". GCCBuiltin<"__builtin_ia32_cvtpd2uqq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f64_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_cvtps2dq_128 : GCCBuiltin<"__builtin_ia32_cvtps2dq128_mask">, @@ -2826,13 +2853,13 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". GCCBuiltin<"__builtin_ia32_cvtps2dq512_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_v16f32_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_cvtps2pd_512 : GCCBuiltin<"__builtin_ia32_cvtps2pd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f32_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_cvtps2qq_128 : GCCBuiltin<"__builtin_ia32_cvtps2qq128_mask">, @@ -2850,7 +2877,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". GCCBuiltin<"__builtin_ia32_cvtps2qq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f32_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_cvtps2udq_128 : GCCBuiltin<"__builtin_ia32_cvtps2udq128_mask">, @@ -2868,7 +2895,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". GCCBuiltin<"__builtin_ia32_cvtps2udq512_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_v16f32_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_cvtps2uqq_128 : GCCBuiltin<"__builtin_ia32_cvtps2uqq128_mask">, @@ -2886,13 +2913,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". GCCBuiltin<"__builtin_ia32_cvtps2uqq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f32_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_cvtqq2pd_512 : - GCCBuiltin<"__builtin_ia32_cvtqq2pd512_mask">, - Intrinsic<[llvm_v8f64_ty], - [llvm_v8i64_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_cvtqq2ps_128 : GCCBuiltin<"__builtin_ia32_cvtqq2ps128_mask">, @@ -2900,18 +2921,6 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". [llvm_v2i64_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtqq2ps_256 : - GCCBuiltin<"__builtin_ia32_cvtqq2ps256_mask">, - Intrinsic<[llvm_v4f32_ty], - [llvm_v4i64_ty, llvm_v4f32_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_cvtqq2ps_512 : - GCCBuiltin<"__builtin_ia32_cvtqq2ps512_mask">, - Intrinsic<[llvm_v8f32_ty], - [llvm_v8i64_ty, llvm_v8f32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_cvttpd2dq_128 : GCCBuiltin<"__builtin_ia32_cvttpd2dq128_mask">, Intrinsic<[llvm_v4i32_ty], @@ -2922,7 +2931,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". GCCBuiltin<"__builtin_ia32_cvttpd2dq512_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8f64_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_cvttpd2qq_128 : GCCBuiltin<"__builtin_ia32_cvttpd2qq128_mask">, @@ -2940,7 +2949,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". GCCBuiltin<"__builtin_ia32_cvttpd2qq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f64_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_cvttpd2udq_128 : GCCBuiltin<"__builtin_ia32_cvttpd2udq128_mask">, @@ -2958,7 +2967,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". GCCBuiltin<"__builtin_ia32_cvttpd2udq512_mask">, Intrinsic<[llvm_v8i32_ty], [llvm_v8f64_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_cvttpd2uqq_128 : GCCBuiltin<"__builtin_ia32_cvttpd2uqq128_mask">, @@ -2976,13 +2985,13 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". GCCBuiltin<"__builtin_ia32_cvttpd2uqq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f64_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_cvttps2dq_512 : GCCBuiltin<"__builtin_ia32_cvttps2dq512_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_v16f32_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_cvttps2qq_128 : GCCBuiltin<"__builtin_ia32_cvttps2qq128_mask">, @@ -3000,7 +3009,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". GCCBuiltin<"__builtin_ia32_cvttps2qq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f32_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_cvttps2udq_128 : GCCBuiltin<"__builtin_ia32_cvttps2udq128_mask">, @@ -3018,7 +3027,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". GCCBuiltin<"__builtin_ia32_cvttps2udq512_mask">, Intrinsic<[llvm_v16i32_ty], [llvm_v16f32_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_cvttps2uqq_128 : GCCBuiltin<"__builtin_ia32_cvttps2uqq128_mask">, @@ -3036,19 +3045,7 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". GCCBuiltin<"__builtin_ia32_cvttps2uqq512_mask">, Intrinsic<[llvm_v8i64_ty], [llvm_v8f32_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_cvtudq2ps_512 : - GCCBuiltin<"__builtin_ia32_cvtudq2ps512_mask">, - Intrinsic<[llvm_v16f32_ty], - [llvm_v16i32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_cvtuqq2pd_512 : - GCCBuiltin<"__builtin_ia32_cvtuqq2pd512_mask">, - Intrinsic<[llvm_v8f64_ty], - [llvm_v8i64_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_cvtuqq2ps_128 : GCCBuiltin<"__builtin_ia32_cvtuqq2ps128_mask">, @@ -3056,72 +3053,78 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". [llvm_v2i64_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_cvtuqq2ps_256 : - GCCBuiltin<"__builtin_ia32_cvtuqq2ps256_mask">, - Intrinsic<[llvm_v4f32_ty], - [llvm_v4i64_ty, llvm_v4f32_ty, llvm_i8_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_cvtuqq2ps_512 : - GCCBuiltin<"__builtin_ia32_cvtuqq2ps512_mask">, - Intrinsic<[llvm_v8f32_ty], - [llvm_v8i64_ty, llvm_v8f32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_rndscale_pd_128 : GCCBuiltin<"__builtin_ia32_rndscalepd_128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_i32_ty, - llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_v2f64_ty, llvm_i8_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_mask_rndscale_pd_256 : GCCBuiltin<"__builtin_ia32_rndscalepd_256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_i32_ty, - llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_v4f64_ty, llvm_i8_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_mask_rndscale_pd_512 : GCCBuiltin<"__builtin_ia32_rndscalepd_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_i32_ty, llvm_v8f64_ty, - llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i8_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>, ImmArg<4>]>; def int_x86_avx512_mask_rndscale_ps_128 : GCCBuiltin<"__builtin_ia32_rndscaleps_128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i32_ty, - llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_v4f32_ty, llvm_i8_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_mask_rndscale_ps_256 : GCCBuiltin<"__builtin_ia32_rndscaleps_256_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_i32_ty, - llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_v8f32_ty, llvm_i8_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_mask_rndscale_ps_512 : GCCBuiltin<"__builtin_ia32_rndscaleps_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_i32_ty, llvm_v16f32_ty, - llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i16_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>, ImmArg<4>]>; def int_x86_avx512_mask_reduce_pd_128 : GCCBuiltin<"__builtin_ia32_reducepd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_i32_ty, - llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_v2f64_ty, llvm_i8_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_mask_reduce_pd_256 : GCCBuiltin<"__builtin_ia32_reducepd256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_i32_ty, - llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_v4f64_ty, llvm_i8_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_mask_reduce_pd_512 : GCCBuiltin<"__builtin_ia32_reducepd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_i32_ty, llvm_v8f64_ty, - llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i8_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>, ImmArg<4>]>; def int_x86_avx512_mask_reduce_ps_128 : GCCBuiltin<"__builtin_ia32_reduceps128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i32_ty, - llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_v4f32_ty, llvm_i8_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_mask_reduce_ps_256 : GCCBuiltin<"__builtin_ia32_reduceps256_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_i32_ty, - llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_v8f32_ty, llvm_i8_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_mask_reduce_ps_512 : GCCBuiltin<"__builtin_ia32_reduceps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_i32_ty, llvm_v16f32_ty, - llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i16_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>, ImmArg<4>]>; def int_x86_avx512_mask_range_pd_128 : GCCBuiltin<"__builtin_ia32_rangepd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i32_ty, - llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_v2f64_ty, llvm_i8_ty], + [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_mask_range_pd_256 : GCCBuiltin<"__builtin_ia32_rangepd256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_i32_ty, - llvm_v4f64_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_v4f64_ty, llvm_i8_ty], + [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_mask_range_pd_512 : GCCBuiltin<"__builtin_ia32_rangepd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i32_ty, - llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<2>, ImmArg<5>]>; def int_x86_avx512_mask_range_ps_128 : GCCBuiltin<"__builtin_ia32_rangeps128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i32_ty, - llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_v4f32_ty, llvm_i8_ty], + [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_mask_range_ps_256 : GCCBuiltin<"__builtin_ia32_rangeps256_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_i32_ty, - llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; + llvm_v8f32_ty, llvm_i8_ty], + [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_mask_range_ps_512 : GCCBuiltin<"__builtin_ia32_rangeps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i32_ty, - llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<2>, ImmArg<5>]>; } // Vector load with broadcast @@ -3151,109 +3154,111 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_add_ps_512 : GCCBuiltin<"__builtin_ia32_addps512">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_add_pd_512 : GCCBuiltin<"__builtin_ia32_addpd512">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_sub_ps_512 : GCCBuiltin<"__builtin_ia32_subps512">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_sub_pd_512 : GCCBuiltin<"__builtin_ia32_subpd512">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_mul_ps_512 : GCCBuiltin<"__builtin_ia32_mulps512">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_mul_pd_512 : GCCBuiltin<"__builtin_ia32_mulpd512">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_div_ps_512 : GCCBuiltin<"__builtin_ia32_divps512">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_div_pd_512 : GCCBuiltin<"__builtin_ia32_divpd512">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_max_ps_512 : GCCBuiltin<"__builtin_ia32_maxps512">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_max_pd_512 : GCCBuiltin<"__builtin_ia32_maxpd512">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_min_ps_512 : GCCBuiltin<"__builtin_ia32_minps512">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_min_pd_512 : GCCBuiltin<"__builtin_ia32_minpd512">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_mask_add_ss_round : GCCBuiltin<"__builtin_ia32_addss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, - llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_mask_div_ss_round : GCCBuiltin<"__builtin_ia32_divss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, - llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_mask_mul_ss_round : GCCBuiltin<"__builtin_ia32_mulss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, - llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_mask_sub_ss_round : GCCBuiltin<"__builtin_ia32_subss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, - llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_mask_max_ss_round : GCCBuiltin<"__builtin_ia32_maxss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, - llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_mask_min_ss_round : GCCBuiltin<"__builtin_ia32_minss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, - llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_mask_add_sd_round : GCCBuiltin<"__builtin_ia32_addsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, - llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_mask_div_sd_round : GCCBuiltin<"__builtin_ia32_divsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, - llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_mask_mul_sd_round : GCCBuiltin<"__builtin_ia32_mulsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, - llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_mask_sub_sd_round : GCCBuiltin<"__builtin_ia32_subsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, - llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_mask_max_sd_round : GCCBuiltin<"__builtin_ia32_maxsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, - llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_mask_min_sd_round : GCCBuiltin<"__builtin_ia32_minsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, - llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_mask_rndscale_ss : GCCBuiltin<"__builtin_ia32_rndscaless_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<4>, ImmArg<5>]>; def int_x86_avx512_mask_rndscale_sd : GCCBuiltin<"__builtin_ia32_rndscalesd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<4>, ImmArg<5>]>; def int_x86_avx512_mask_range_ss : GCCBuiltin<"__builtin_ia32_rangess128_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<4>, ImmArg<5>]>; def int_x86_avx512_mask_range_sd : GCCBuiltin<"__builtin_ia32_rangesd128_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<4>, ImmArg<5>]>; def int_x86_avx512_mask_reduce_ss : GCCBuiltin<"__builtin_ia32_reducess_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<4>, ImmArg<5>]>; def int_x86_avx512_mask_reduce_sd : GCCBuiltin<"__builtin_ia32_reducesd_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<4>, ImmArg<5>]>; def int_x86_avx512_mask_scalef_sd : GCCBuiltin<"__builtin_ia32_scalefsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, - llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_mask_scalef_ss : GCCBuiltin<"__builtin_ia32_scalefss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, - llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_mask_scalef_pd_128 : GCCBuiltin<"__builtin_ia32_scalefpd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; @@ -3262,7 +3267,8 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_v4f64_ty, llvm_i8_ty],[IntrNoMem]>; def int_x86_avx512_mask_scalef_pd_512 : GCCBuiltin<"__builtin_ia32_scalefpd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, - llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_mask_scalef_ps_128 : GCCBuiltin<"__builtin_ia32_scalefps128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; @@ -3271,99 +3277,104 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_v8f32_ty, llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_scalef_ps_512 : GCCBuiltin<"__builtin_ia32_scalefps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, - llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_mask_sqrt_ss : Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, - llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i8_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_mask_sqrt_sd : Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, - llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i8_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_sqrt_pd_512 : - Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_sqrt_ps_512 : - Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_i32_ty], [IntrNoMem]>; + Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_mask_fixupimm_pd_128 : GCCBuiltin<"__builtin_ia32_fixupimmpd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_maskz_fixupimm_pd_128 : GCCBuiltin<"__builtin_ia32_fixupimmpd128_maskz">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_fixupimm_pd_256 : GCCBuiltin<"__builtin_ia32_fixupimmpd256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4i64_ty, llvm_i32_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_maskz_fixupimm_pd_256 : GCCBuiltin<"__builtin_ia32_fixupimmpd256_maskz">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, llvm_v4i64_ty, llvm_i32_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_fixupimm_pd_512 : GCCBuiltin<"__builtin_ia32_fixupimmpd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8i64_ty, llvm_i32_ty, llvm_i8_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<3>, ImmArg<5>]>; def int_x86_avx512_maskz_fixupimm_pd_512 : GCCBuiltin<"__builtin_ia32_fixupimmpd512_maskz">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_v8i64_ty, llvm_i32_ty, llvm_i8_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<3>, ImmArg<5>]>; def int_x86_avx512_mask_fixupimm_ps_128 : GCCBuiltin<"__builtin_ia32_fixupimmps128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_maskz_fixupimm_ps_128 : GCCBuiltin<"__builtin_ia32_fixupimmps128_maskz">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_fixupimm_ps_256 : GCCBuiltin<"__builtin_ia32_fixupimmps256_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8i32_ty, llvm_i32_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_maskz_fixupimm_ps_256 : GCCBuiltin<"__builtin_ia32_fixupimmps256_maskz">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, llvm_v8i32_ty, llvm_i32_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_fixupimm_ps_512 : GCCBuiltin<"__builtin_ia32_fixupimmps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16i32_ty, llvm_i32_ty, - llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<3>, ImmArg<5>]>; def int_x86_avx512_maskz_fixupimm_ps_512 : GCCBuiltin<"__builtin_ia32_fixupimmps512_maskz">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_v16i32_ty, llvm_i32_ty, - llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<3>, ImmArg<5>]>; def int_x86_avx512_mask_fixupimm_sd : GCCBuiltin<"__builtin_ia32_fixupimmsd_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_i8_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<3>, ImmArg<5>]>; def int_x86_avx512_maskz_fixupimm_sd : GCCBuiltin<"__builtin_ia32_fixupimmsd_maskz">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2i64_ty, llvm_i32_ty, llvm_i8_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<3>, ImmArg<5>]>; def int_x86_avx512_mask_fixupimm_ss : GCCBuiltin<"__builtin_ia32_fixupimmss_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_i8_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<3>, ImmArg<5>]>; def int_x86_avx512_maskz_fixupimm_ss : GCCBuiltin<"__builtin_ia32_fixupimmss_maskz">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4i32_ty, llvm_i32_ty, llvm_i8_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<3>, ImmArg<5>]>; def int_x86_avx512_mask_getexp_pd_128 : GCCBuiltin<"__builtin_ia32_getexppd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty], [IntrNoMem]>; @@ -3372,7 +3383,8 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_getexp_pd_512 : GCCBuiltin<"__builtin_ia32_getexppd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, - llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i8_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_getexp_ps_128 : GCCBuiltin<"__builtin_ia32_getexpps128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty], [IntrNoMem]>; @@ -3381,62 +3393,65 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". llvm_i8_ty], [IntrNoMem]>; def int_x86_avx512_mask_getexp_ps_512 : GCCBuiltin<"__builtin_ia32_getexpps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, - llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i16_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_mask_getexp_ss : GCCBuiltin<"__builtin_ia32_getexpss128_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, - llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i8_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_mask_getexp_sd : GCCBuiltin<"__builtin_ia32_getexpsd128_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, - llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i8_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_mask_getmant_pd_128 : GCCBuiltin<"__builtin_ia32_getmantpd128_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty,llvm_i32_ty, llvm_v2f64_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_mask_getmant_pd_256 : GCCBuiltin<"__builtin_ia32_getmantpd256_mask">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty,llvm_i32_ty, llvm_v4f64_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_mask_getmant_pd_512 : GCCBuiltin<"__builtin_ia32_getmantpd512_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty,llvm_i32_ty, llvm_v8f64_ty, llvm_i8_ty,llvm_i32_ty ], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>, ImmArg<4>]>; def int_x86_avx512_mask_getmant_ps_128 : GCCBuiltin<"__builtin_ia32_getmantps128_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_i32_ty, llvm_v4f32_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_mask_getmant_ps_256 : GCCBuiltin<"__builtin_ia32_getmantps256_mask">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_i32_ty, llvm_v8f32_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>]>; def int_x86_avx512_mask_getmant_ps_512 : GCCBuiltin<"__builtin_ia32_getmantps512_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty,llvm_i32_ty, llvm_v16f32_ty,llvm_i16_ty,llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<1>, ImmArg<4>]>; def int_x86_avx512_mask_getmant_ss : GCCBuiltin<"__builtin_ia32_getmantss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_i32_ty, llvm_v4f32_ty, - llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<2>, ImmArg<5>]>; def int_x86_avx512_mask_getmant_sd : GCCBuiltin<"__builtin_ia32_getmantsd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_i32_ty, llvm_v2f64_ty, - llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<2>, ImmArg<5>]>; def int_x86_avx512_rsqrt14_ss : GCCBuiltin<"__builtin_ia32_rsqrt14ss_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, @@ -3491,41 +3506,41 @@ let TargetPrefix = "x86" in { // All intrinsics start with "llvm.x86.". def int_x86_avx512_rcp28_ps : GCCBuiltin<"__builtin_ia32_rcp28ps_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, - llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_rcp28_pd : GCCBuiltin<"__builtin_ia32_rcp28pd_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, - llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_exp2_ps : GCCBuiltin<"__builtin_ia32_exp2ps_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, - llvm_i16_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i16_ty, llvm_i32_ty], [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_exp2_pd : GCCBuiltin<"__builtin_ia32_exp2pd_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, - llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i8_ty, llvm_i32_ty], [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_rcp28_ss : GCCBuiltin<"__builtin_ia32_rcp28ss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_rcp28_sd : GCCBuiltin<"__builtin_ia32_rcp28sd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_rsqrt28_ps : GCCBuiltin<"__builtin_ia32_rsqrt28ps_mask">, Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, llvm_i16_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_rsqrt28_pd : GCCBuiltin<"__builtin_ia32_rsqrt28pd_mask">, Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_rsqrt28_ss : GCCBuiltin<"__builtin_ia32_rsqrt28ss_round_mask">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, llvm_v4f32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_rsqrt28_sd : GCCBuiltin<"__builtin_ia32_rsqrt28sd_round_mask">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, llvm_v2f64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<4>]>; def int_x86_avx512_psad_bw_512 : GCCBuiltin<"__builtin_ia32_psadbw512">, Intrinsic<[llvm_v8i64_ty], [llvm_v64i8_ty, llvm_v64i8_ty], [IntrNoMem, Commutative]>; @@ -3538,6 +3553,12 @@ let TargetPrefix = "x86" in { def int_x86_avx512_pmulh_w_512 : GCCBuiltin<"__builtin_ia32_pmulhw512">, Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty], [IntrNoMem, Commutative]>; + def int_x86_avx512_pavg_b_512 : GCCBuiltin<"__builtin_ia32_pavgb512">, + Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty], + [IntrNoMem]>; + def int_x86_avx512_pavg_w_512 : GCCBuiltin<"__builtin_ia32_pavgw512">, + Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty], + [IntrNoMem]>; def int_x86_avx512_pmaddw_d_512 : GCCBuiltin<"__builtin_ia32_pmaddwd512">, Intrinsic<[llvm_v16i32_ty], [llvm_v32i16_ty, llvm_v32i16_ty], [IntrNoMem, Commutative]>; @@ -3548,582 +3569,553 @@ let TargetPrefix = "x86" in { def int_x86_avx512_dbpsadbw_128 : GCCBuiltin<"__builtin_ia32_dbpsadbw128">, Intrinsic<[llvm_v8i16_ty], - [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], [IntrNoMem]>; + [llvm_v16i8_ty, llvm_v16i8_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_dbpsadbw_256 : GCCBuiltin<"__builtin_ia32_dbpsadbw256">, Intrinsic<[llvm_v16i16_ty], - [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], [IntrNoMem]>; + [llvm_v32i8_ty, llvm_v32i8_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_dbpsadbw_512 : GCCBuiltin<"__builtin_ia32_dbpsadbw512">, Intrinsic<[llvm_v32i16_ty], - [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i32_ty], [IntrNoMem]>; + [llvm_v64i8_ty, llvm_v64i8_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<2>]>; } // Gather and Scatter ops let TargetPrefix = "x86" in { // NOTE: These are deprecated in favor of the versions that take a vXi1 mask. - def int_x86_avx512_gather_dpd_512 : GCCBuiltin<"__builtin_ia32_gathersiv8df">, + // NOTE: These can't be ArgMemOnly because you can put the address completely + // in the index register. + def int_x86_avx512_gather_dpd_512 : Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx512_gather_dps_512 : GCCBuiltin<"__builtin_ia32_gathersiv16sf">, + [IntrReadMem, ImmArg<4>]>; + def int_x86_avx512_gather_dps_512 : Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx512_gather_qpd_512 : GCCBuiltin<"__builtin_ia32_gatherdiv8df">, + [IntrReadMem, ImmArg<4>]>; + def int_x86_avx512_gather_qpd_512 : Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx512_gather_qps_512 : GCCBuiltin<"__builtin_ia32_gatherdiv16sf">, + [IntrReadMem, ImmArg<4>]>; + def int_x86_avx512_gather_qps_512 : Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; - def int_x86_avx512_gather_dpq_512 : GCCBuiltin<"__builtin_ia32_gathersiv8di">, + def int_x86_avx512_gather_dpq_512 : Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx512_gather_dpi_512 : GCCBuiltin<"__builtin_ia32_gathersiv16si">, + [IntrReadMem, ImmArg<4>]>; + def int_x86_avx512_gather_dpi_512 : Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_ptr_ty, llvm_v16i32_ty, llvm_i16_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx512_gather_qpq_512 : GCCBuiltin<"__builtin_ia32_gatherdiv8di">, + [IntrReadMem, ImmArg<4>]>; + def int_x86_avx512_gather_qpq_512 : Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; - def int_x86_avx512_gather_qpi_512 : GCCBuiltin<"__builtin_ia32_gatherdiv16si">, + [IntrReadMem, ImmArg<4>]>; + def int_x86_avx512_gather_qpi_512 : Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_ptr_ty, llvm_v8i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_gather3div2_df : - GCCBuiltin<"__builtin_ia32_gather3div2df">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_gather3div2_di : - GCCBuiltin<"__builtin_ia32_gather3div2di">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_gather3div4_df : - GCCBuiltin<"__builtin_ia32_gather3div4df">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_gather3div4_di : - GCCBuiltin<"__builtin_ia32_gather3div4di">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_gather3div4_sf : - GCCBuiltin<"__builtin_ia32_gather3div4sf">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_gather3div4_si : - GCCBuiltin<"__builtin_ia32_gather3div4si">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_gather3div8_sf : - GCCBuiltin<"__builtin_ia32_gather3div8sf">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_gather3div8_si : - GCCBuiltin<"__builtin_ia32_gather3div8si">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_gather3siv2_df : - GCCBuiltin<"__builtin_ia32_gather3siv2df">, Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_gather3siv2_di : - GCCBuiltin<"__builtin_ia32_gather3siv2di">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_gather3siv4_df : - GCCBuiltin<"__builtin_ia32_gather3siv4df">, Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_gather3siv4_di : - GCCBuiltin<"__builtin_ia32_gather3siv4di">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_gather3siv4_sf : - GCCBuiltin<"__builtin_ia32_gather3siv4sf">, Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_gather3siv4_si : - GCCBuiltin<"__builtin_ia32_gather3siv4si">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_gather3siv8_sf : - GCCBuiltin<"__builtin_ia32_gather3siv8sf">, Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_gather3siv8_si : - GCCBuiltin<"__builtin_ia32_gather3siv8si">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_i8_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; // scatter // NOTE: These are deprecated in favor of the versions that take a vXi1 mask. - def int_x86_avx512_scatter_dpd_512 : GCCBuiltin<"__builtin_ia32_scattersiv8df">, + // NOTE: These can't be ArgMemOnly because you can put the address completely + // in the index register. + def int_x86_avx512_scatter_dpd_512 : Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v8i32_ty, llvm_v8f64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; - def int_x86_avx512_scatter_dps_512 : GCCBuiltin<"__builtin_ia32_scattersiv16sf">, + [ImmArg<4>]>; + def int_x86_avx512_scatter_dps_512 : Intrinsic<[], [llvm_ptr_ty, llvm_i16_ty, llvm_v16i32_ty, llvm_v16f32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; - def int_x86_avx512_scatter_qpd_512 : GCCBuiltin<"__builtin_ia32_scatterdiv8df">, + [ImmArg<4>]>; + def int_x86_avx512_scatter_qpd_512 : Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v8i64_ty, llvm_v8f64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; - def int_x86_avx512_scatter_qps_512 : GCCBuiltin<"__builtin_ia32_scatterdiv16sf">, + [ImmArg<4>]>; + def int_x86_avx512_scatter_qps_512 : Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v8i64_ty, llvm_v8f32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; - def int_x86_avx512_scatter_dpq_512 : GCCBuiltin<"__builtin_ia32_scattersiv8di">, + def int_x86_avx512_scatter_dpq_512 : Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v8i32_ty, llvm_v8i64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; - def int_x86_avx512_scatter_dpi_512 : GCCBuiltin<"__builtin_ia32_scattersiv16si">, + [ImmArg<4>]>; + def int_x86_avx512_scatter_dpi_512 : Intrinsic<[], [llvm_ptr_ty, llvm_i16_ty, llvm_v16i32_ty, llvm_v16i32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; - def int_x86_avx512_scatter_qpq_512 : GCCBuiltin<"__builtin_ia32_scatterdiv8di">, + [ImmArg<4>]>; + def int_x86_avx512_scatter_qpq_512 : Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty,llvm_v8i64_ty, llvm_v8i64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; - def int_x86_avx512_scatter_qpi_512 : GCCBuiltin<"__builtin_ia32_scatterdiv16si">, + [ImmArg<4>]>; + def int_x86_avx512_scatter_qpi_512 : Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v8i64_ty, llvm_v8i32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_scatterdiv2_df : - GCCBuiltin<"__builtin_ia32_scatterdiv2df">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v2i64_ty, llvm_v2f64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_scatterdiv2_di : - GCCBuiltin<"__builtin_ia32_scatterdiv2di">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_scatterdiv4_df : - GCCBuiltin<"__builtin_ia32_scatterdiv4df">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v4i64_ty, llvm_v4f64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_scatterdiv4_di : - GCCBuiltin<"__builtin_ia32_scatterdiv4di">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_scatterdiv4_sf : - GCCBuiltin<"__builtin_ia32_scatterdiv4sf">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v2i64_ty, llvm_v4f32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_scatterdiv4_si : - GCCBuiltin<"__builtin_ia32_scatterdiv4si">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v2i64_ty, llvm_v4i32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_scatterdiv8_sf : - GCCBuiltin<"__builtin_ia32_scatterdiv8sf">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v4i64_ty, llvm_v4f32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_scatterdiv8_si : - GCCBuiltin<"__builtin_ia32_scatterdiv8si">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v4i64_ty, llvm_v4i32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_scattersiv2_df : - GCCBuiltin<"__builtin_ia32_scattersiv2df">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v2f64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_scattersiv2_di : - GCCBuiltin<"__builtin_ia32_scattersiv2di">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v2i64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_scattersiv4_df : - GCCBuiltin<"__builtin_ia32_scattersiv4df">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v4f64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_scattersiv4_di : - GCCBuiltin<"__builtin_ia32_scattersiv4di">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v4i64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_scattersiv4_sf : - GCCBuiltin<"__builtin_ia32_scattersiv4sf">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v4f32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_scattersiv4_si : - GCCBuiltin<"__builtin_ia32_scattersiv4si">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_scattersiv8_sf : - GCCBuiltin<"__builtin_ia32_scattersiv8sf">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v8i32_ty, llvm_v8f32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_scattersiv8_si : - GCCBuiltin<"__builtin_ia32_scattersiv8si">, Intrinsic<[], [llvm_ptr_ty, llvm_i8_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; // gather prefetch + // NOTE: These can't be ArgMemOnly because you can put the address completely + // in the index register. def int_x86_avx512_gatherpf_dpd_512 : GCCBuiltin<"__builtin_ia32_gatherpfdpd">, Intrinsic<[], [llvm_i8_ty, llvm_v8i32_ty, llvm_ptr_ty, - llvm_i32_ty, llvm_i32_ty], [IntrArgMemOnly]>; + llvm_i32_ty, llvm_i32_ty], [ImmArg<3>, ImmArg<4>]>; def int_x86_avx512_gatherpf_dps_512 : GCCBuiltin<"__builtin_ia32_gatherpfdps">, Intrinsic<[], [llvm_i16_ty, llvm_v16i32_ty, llvm_ptr_ty, - llvm_i32_ty, llvm_i32_ty], [IntrArgMemOnly]>; + llvm_i32_ty, llvm_i32_ty], [ImmArg<3>, ImmArg<4>]>; def int_x86_avx512_gatherpf_qpd_512 : GCCBuiltin<"__builtin_ia32_gatherpfqpd">, Intrinsic<[], [llvm_i8_ty, llvm_v8i64_ty, llvm_ptr_ty, - llvm_i32_ty, llvm_i32_ty], [IntrArgMemOnly]>; + llvm_i32_ty, llvm_i32_ty], [ImmArg<3>, ImmArg<4>]>; def int_x86_avx512_gatherpf_qps_512 : GCCBuiltin<"__builtin_ia32_gatherpfqps">, Intrinsic<[], [llvm_i8_ty, llvm_v8i64_ty, llvm_ptr_ty, - llvm_i32_ty, llvm_i32_ty], [IntrArgMemOnly]>; + llvm_i32_ty, llvm_i32_ty], [ImmArg<3>, ImmArg<4>]>; // scatter prefetch + // NOTE: These can't be ArgMemOnly because you can put the address completely + // in the index register. def int_x86_avx512_scatterpf_dpd_512 : GCCBuiltin<"__builtin_ia32_scatterpfdpd">, Intrinsic<[], [llvm_i8_ty, llvm_v8i32_ty, llvm_ptr_ty, - llvm_i32_ty, llvm_i32_ty], [IntrArgMemOnly]>; + llvm_i32_ty, llvm_i32_ty], [ImmArg<3>, ImmArg<4>]>; def int_x86_avx512_scatterpf_dps_512 : GCCBuiltin<"__builtin_ia32_scatterpfdps">, Intrinsic<[], [llvm_i16_ty, llvm_v16i32_ty, llvm_ptr_ty, - llvm_i32_ty, llvm_i32_ty], [IntrArgMemOnly]>; + llvm_i32_ty, llvm_i32_ty], [ImmArg<3>, ImmArg<4>]>; def int_x86_avx512_scatterpf_qpd_512 : GCCBuiltin<"__builtin_ia32_scatterpfqpd">, Intrinsic<[], [llvm_i8_ty, llvm_v8i64_ty, llvm_ptr_ty, - llvm_i32_ty, llvm_i32_ty], [IntrArgMemOnly]>; + llvm_i32_ty, llvm_i32_ty], [ImmArg<3>, ImmArg<4>]>; def int_x86_avx512_scatterpf_qps_512 : GCCBuiltin<"__builtin_ia32_scatterpfqps">, Intrinsic<[], [llvm_i8_ty, llvm_v8i64_ty, llvm_ptr_ty, - llvm_i32_ty, llvm_i32_ty], [IntrArgMemOnly]>; + llvm_i32_ty, llvm_i32_ty], [ImmArg<3>, ImmArg<4>]>; } // AVX512 gather/scatter intrinsics that use vXi1 masks. let TargetPrefix = "x86" in { + // NOTE: These can't be ArgMemOnly because you can put the address completely + // in the index register. def int_x86_avx512_mask_gather_dpd_512 : Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_v8i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_gather_dps_512 : Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_ptr_ty, llvm_v16i32_ty, llvm_v16i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_gather_qpd_512 : Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_ptr_ty, llvm_v8i64_ty, llvm_v8i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_gather_qps_512 : Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_ptr_ty, llvm_v8i64_ty, llvm_v8i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_gather_dpq_512 : Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_v8i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_gather_dpi_512 : Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_ptr_ty, llvm_v16i32_ty, llvm_v16i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_gather_qpq_512 : Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_ptr_ty, llvm_v8i64_ty, llvm_v8i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_gather_qpi_512 : Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_ptr_ty, llvm_v8i64_ty, llvm_v8i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_gather3div2_df : Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v2i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_gather3div2_di : Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v2i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_gather3div4_df : Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_gather3div4_di : Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_gather3div4_sf : Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v2i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_gather3div4_si : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_ptr_ty, llvm_v2i64_ty, llvm_v2i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_gather3div8_sf : Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_gather3div8_si : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i64_ty, llvm_v4i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_gather3siv2_df : Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v2i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_gather3siv2_di : Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v2i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_gather3siv4_df : Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_gather3siv4_di : Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_gather3siv4_sf : Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_gather3siv4_si : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_ptr_ty, llvm_v4i32_ty, llvm_v4i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_gather3siv8_sf : Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_v8i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_gather3siv8_si : Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_ptr_ty, llvm_v8i32_ty, llvm_v8i1_ty, llvm_i32_ty], - [IntrReadMem, IntrArgMemOnly]>; + [IntrReadMem, ImmArg<4>]>; def int_x86_avx512_mask_scatter_dpd_512 : Intrinsic<[], [llvm_ptr_ty, llvm_v8i1_ty, llvm_v8i32_ty, llvm_v8f64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_mask_scatter_dps_512 : Intrinsic<[], [llvm_ptr_ty, llvm_v16i1_ty, llvm_v16i32_ty, llvm_v16f32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_mask_scatter_qpd_512 : Intrinsic<[], [llvm_ptr_ty, llvm_v8i1_ty, llvm_v8i64_ty, llvm_v8f64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_mask_scatter_qps_512 : Intrinsic<[], [llvm_ptr_ty, llvm_v8i1_ty, llvm_v8i64_ty, llvm_v8f32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; + // NOTE: These can't be ArgMemOnly because you can put the address completely + // in the index register. def int_x86_avx512_mask_scatter_dpq_512 : Intrinsic<[], [llvm_ptr_ty, llvm_v8i1_ty, llvm_v8i32_ty, llvm_v8i64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_mask_scatter_dpi_512 : Intrinsic<[], [llvm_ptr_ty, llvm_v16i1_ty, llvm_v16i32_ty, llvm_v16i32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_mask_scatter_qpq_512 : Intrinsic<[], [llvm_ptr_ty, llvm_v8i1_ty,llvm_v8i64_ty, llvm_v8i64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_mask_scatter_qpi_512 : Intrinsic<[], [llvm_ptr_ty, llvm_v8i1_ty, llvm_v8i64_ty, llvm_v8i32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_mask_scatterdiv2_df : Intrinsic<[], [llvm_ptr_ty, llvm_v2i1_ty, llvm_v2i64_ty, llvm_v2f64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_mask_scatterdiv2_di : Intrinsic<[], [llvm_ptr_ty, llvm_v2i1_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_mask_scatterdiv4_df : Intrinsic<[], [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i64_ty, llvm_v4f64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_mask_scatterdiv4_di : Intrinsic<[], [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_mask_scatterdiv4_sf : Intrinsic<[], [llvm_ptr_ty, llvm_v2i1_ty, llvm_v2i64_ty, llvm_v4f32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_mask_scatterdiv4_si : Intrinsic<[], [llvm_ptr_ty, llvm_v2i1_ty, llvm_v2i64_ty, llvm_v4i32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_mask_scatterdiv8_sf : Intrinsic<[], [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i64_ty, llvm_v4f32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_mask_scatterdiv8_si : Intrinsic<[], [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i64_ty, llvm_v4i32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_mask_scattersiv2_df : Intrinsic<[], [llvm_ptr_ty, llvm_v2i1_ty, llvm_v4i32_ty, llvm_v2f64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_mask_scattersiv2_di : Intrinsic<[], [llvm_ptr_ty, llvm_v2i1_ty, llvm_v4i32_ty, llvm_v2i64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_mask_scattersiv4_df : Intrinsic<[], [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i32_ty, llvm_v4f64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_mask_scattersiv4_di : Intrinsic<[], [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i32_ty, llvm_v4i64_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_mask_scattersiv4_sf : Intrinsic<[], [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i32_ty, llvm_v4f32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_mask_scattersiv4_si : Intrinsic<[], [llvm_ptr_ty, llvm_v4i1_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_mask_scattersiv8_sf : Intrinsic<[], [llvm_ptr_ty, llvm_v8i1_ty, llvm_v8i32_ty, llvm_v8f32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; def int_x86_avx512_mask_scattersiv8_si : Intrinsic<[], [llvm_ptr_ty, llvm_v8i1_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty], - [IntrArgMemOnly]>; + [ImmArg<4>]>; } // AVX-512 conflict detection instruction // Instructions that count the number of leading zero bits let TargetPrefix = "x86" in { - def int_x86_avx512_mask_conflict_d_128 : - GCCBuiltin<"__builtin_ia32_vpconflictsi_128_mask">, - Intrinsic<[llvm_v4i32_ty], - [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_conflict_d_256 : - GCCBuiltin<"__builtin_ia32_vpconflictsi_256_mask">, - Intrinsic<[llvm_v8i32_ty], - [llvm_v8i32_ty, llvm_v8i32_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_conflict_d_512 : - GCCBuiltin<"__builtin_ia32_vpconflictsi_512_mask">, - Intrinsic<[llvm_v16i32_ty], - [llvm_v16i32_ty, llvm_v16i32_ty, llvm_i16_ty], - [IntrNoMem]>; - - def int_x86_avx512_mask_conflict_q_128 : - GCCBuiltin<"__builtin_ia32_vpconflictdi_128_mask">, - Intrinsic<[llvm_v2i64_ty], - [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_conflict_q_256 : - GCCBuiltin<"__builtin_ia32_vpconflictdi_256_mask">, - Intrinsic<[llvm_v4i64_ty], - [llvm_v4i64_ty, llvm_v4i64_ty, llvm_i8_ty], - [IntrNoMem]>; - def int_x86_avx512_mask_conflict_q_512 : - GCCBuiltin<"__builtin_ia32_vpconflictdi_512_mask">, - Intrinsic<[llvm_v8i64_ty], - [llvm_v8i64_ty, llvm_v8i64_ty, llvm_i8_ty], - [IntrNoMem]>; + def int_x86_avx512_conflict_d_128 : + GCCBuiltin<"__builtin_ia32_vpconflictsi_128">, + Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty], [IntrNoMem]>; + def int_x86_avx512_conflict_d_256 : + GCCBuiltin<"__builtin_ia32_vpconflictsi_256">, + Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty], [IntrNoMem]>; + def int_x86_avx512_conflict_d_512 : + GCCBuiltin<"__builtin_ia32_vpconflictsi_512">, + Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty], [IntrNoMem]>; + + def int_x86_avx512_conflict_q_128 : + GCCBuiltin<"__builtin_ia32_vpconflictdi_128">, + Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty], [IntrNoMem]>; + def int_x86_avx512_conflict_q_256 : + GCCBuiltin<"__builtin_ia32_vpconflictdi_256">, + Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty], [IntrNoMem]>; + def int_x86_avx512_conflict_q_512 : + GCCBuiltin<"__builtin_ia32_vpconflictdi_512">, + Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty], [IntrNoMem]>; } // Compares @@ -4131,164 +4123,26 @@ let TargetPrefix = "x86" in { // 512-bit def int_x86_avx512_vcomi_sd : GCCBuiltin<"__builtin_ia32_vcomisd">, Intrinsic<[llvm_i32_ty], [llvm_v2f64_ty, - llvm_v2f64_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_v2f64_ty, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<2>, ImmArg<3>]>; def int_x86_avx512_vcomi_ss : GCCBuiltin<"__builtin_ia32_vcomiss">, Intrinsic<[llvm_i32_ty], [llvm_v4f32_ty, - llvm_v4f32_ty, llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_v4f32_ty, llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<2>, ImmArg<3>]>; } // Compress, Expand let TargetPrefix = "x86" in { - def int_x86_avx512_mask_compress_ps_512 : - GCCBuiltin<"__builtin_ia32_compresssf512_mask">, - Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, - llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_compress_pd_512 : - GCCBuiltin<"__builtin_ia32_compressdf512_mask">, - Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_compress_ps_256 : - GCCBuiltin<"__builtin_ia32_compresssf256_mask">, - Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_compress_pd_256 : - GCCBuiltin<"__builtin_ia32_compressdf256_mask">, - Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_compress_ps_128 : - GCCBuiltin<"__builtin_ia32_compresssf128_mask">, - Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_compress_pd_128 : - GCCBuiltin<"__builtin_ia32_compressdf128_mask">, - Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, - llvm_i8_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_compress_d_512 : - GCCBuiltin<"__builtin_ia32_compresssi512_mask">, - Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, - llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_compress_q_512 : - GCCBuiltin<"__builtin_ia32_compressdi512_mask">, - Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_compress_d_256 : - GCCBuiltin<"__builtin_ia32_compresssi256_mask">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_compress_q_256 : - GCCBuiltin<"__builtin_ia32_compressdi256_mask">, - Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_compress_d_128 : - GCCBuiltin<"__builtin_ia32_compresssi128_mask">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_compress_q_128 : - GCCBuiltin<"__builtin_ia32_compressdi128_mask">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, - llvm_i8_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_compress_b_512 : - GCCBuiltin<"__builtin_ia32_compressqi512_mask">, - Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, - llvm_i64_ty], [IntrNoMem]>; - def int_x86_avx512_mask_compress_w_512 : - GCCBuiltin<"__builtin_ia32_compresshi512_mask">, - Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, - llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_compress_b_256 : - GCCBuiltin<"__builtin_ia32_compressqi256_mask">, - Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, - llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_compress_w_256 : - GCCBuiltin<"__builtin_ia32_compresshi256_mask">, - Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, - llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_compress_b_128 : - GCCBuiltin<"__builtin_ia32_compressqi128_mask">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, - llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_compress_w_128 : - GCCBuiltin<"__builtin_ia32_compresshi128_mask">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, - llvm_i8_ty], [IntrNoMem]>; - -// expand - def int_x86_avx512_mask_expand_ps_512 : - GCCBuiltin<"__builtin_ia32_expandsf512_mask">, - Intrinsic<[llvm_v16f32_ty], [llvm_v16f32_ty, llvm_v16f32_ty, - llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_expand_pd_512 : - GCCBuiltin<"__builtin_ia32_expanddf512_mask">, - Intrinsic<[llvm_v8f64_ty], [llvm_v8f64_ty, llvm_v8f64_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_expand_ps_256 : - GCCBuiltin<"__builtin_ia32_expandsf256_mask">, - Intrinsic<[llvm_v8f32_ty], [llvm_v8f32_ty, llvm_v8f32_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_expand_pd_256 : - GCCBuiltin<"__builtin_ia32_expanddf256_mask">, - Intrinsic<[llvm_v4f64_ty], [llvm_v4f64_ty, llvm_v4f64_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_expand_ps_128 : - GCCBuiltin<"__builtin_ia32_expandsf128_mask">, - Intrinsic<[llvm_v4f32_ty], [llvm_v4f32_ty, llvm_v4f32_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_expand_pd_128 : - GCCBuiltin<"__builtin_ia32_expanddf128_mask">, - Intrinsic<[llvm_v2f64_ty], [llvm_v2f64_ty, llvm_v2f64_ty, - llvm_i8_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_expand_d_512 : - GCCBuiltin<"__builtin_ia32_expandsi512_mask">, - Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, - llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_expand_q_512 : - GCCBuiltin<"__builtin_ia32_expanddi512_mask">, - Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_expand_d_256 : - GCCBuiltin<"__builtin_ia32_expandsi256_mask">, - Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_expand_q_256 : - GCCBuiltin<"__builtin_ia32_expanddi256_mask">, - Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_expand_d_128 : - GCCBuiltin<"__builtin_ia32_expandsi128_mask">, - Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, - llvm_i8_ty], [IntrNoMem]>; - def int_x86_avx512_mask_expand_q_128 : - GCCBuiltin<"__builtin_ia32_expanddi128_mask">, - Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, - llvm_i8_ty], [IntrNoMem]>; - - def int_x86_avx512_mask_expand_b_512 : - GCCBuiltin<"__builtin_ia32_expandqi512_mask">, - Intrinsic<[llvm_v64i8_ty], [llvm_v64i8_ty, llvm_v64i8_ty, - llvm_i64_ty], [IntrNoMem]>; - def int_x86_avx512_mask_expand_w_512 : - GCCBuiltin<"__builtin_ia32_expandhi512_mask">, - Intrinsic<[llvm_v32i16_ty], [llvm_v32i16_ty, llvm_v32i16_ty, - llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_expand_b_256 : - GCCBuiltin<"__builtin_ia32_expandqi256_mask">, - Intrinsic<[llvm_v32i8_ty], [llvm_v32i8_ty, llvm_v32i8_ty, - llvm_i32_ty], [IntrNoMem]>; - def int_x86_avx512_mask_expand_w_256 : - GCCBuiltin<"__builtin_ia32_expandhi256_mask">, - Intrinsic<[llvm_v16i16_ty], [llvm_v16i16_ty, llvm_v16i16_ty, - llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_expand_b_128 : - GCCBuiltin<"__builtin_ia32_expandqi128_mask">, - Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty, - llvm_i16_ty], [IntrNoMem]>; - def int_x86_avx512_mask_expand_w_128 : - GCCBuiltin<"__builtin_ia32_expandhi128_mask">, - Intrinsic<[llvm_v8i16_ty], [llvm_v8i16_ty, llvm_v8i16_ty, - llvm_i8_ty], [IntrNoMem]>; + def int_x86_avx512_mask_compress : + Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], + [IntrNoMem]>; + def int_x86_avx512_mask_expand : + Intrinsic<[llvm_anyvector_ty], + [LLVMMatchType<0>, LLVMMatchType<0>, + LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], + [IntrNoMem]>; } // truncate @@ -4502,10 +4356,6 @@ let TargetPrefix = "x86" in { Intrinsic<[], [llvm_ptr_ty, llvm_v2i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; - def int_x86_avx512_mask_pmov_qd_256 : // FIXME: Replace with trunc+select. - Intrinsic<[llvm_v4i32_ty], - [llvm_v4i64_ty, llvm_v4i32_ty, llvm_i8_ty], - [IntrNoMem]>; def int_x86_avx512_mask_pmov_qd_mem_256 : GCCBuiltin<"__builtin_ia32_pmovqd256mem_mask">, Intrinsic<[], @@ -4531,10 +4381,6 @@ let TargetPrefix = "x86" in { Intrinsic<[], [llvm_ptr_ty, llvm_v4i64_ty, llvm_i8_ty], [IntrArgMemOnly]>; - def int_x86_avx512_mask_pmov_qd_512 : // FIXME: Replace with trunc+select. - Intrinsic<[llvm_v8i32_ty], - [llvm_v8i64_ty, llvm_v8i32_ty, llvm_i8_ty], - [IntrNoMem]>; def int_x86_avx512_mask_pmov_qd_mem_512 : GCCBuiltin<"__builtin_ia32_pmovqd512mem_mask">, Intrinsic<[], @@ -4768,10 +4614,6 @@ let TargetPrefix = "x86" in { Intrinsic<[], [llvm_ptr_ty, llvm_v8i16_ty, llvm_i8_ty], [IntrArgMemOnly]>; - def int_x86_avx512_mask_pmov_wb_256 : // FIXME: Replace with trunc+select. - Intrinsic<[llvm_v16i8_ty], - [llvm_v16i16_ty, llvm_v16i8_ty, llvm_i16_ty], - [IntrNoMem]>; def int_x86_avx512_mask_pmov_wb_mem_256 : GCCBuiltin<"__builtin_ia32_pmovwb256mem_mask">, Intrinsic<[], @@ -4797,10 +4639,6 @@ let TargetPrefix = "x86" in { Intrinsic<[], [llvm_ptr_ty, llvm_v16i16_ty, llvm_i16_ty], [IntrArgMemOnly]>; - def int_x86_avx512_mask_pmov_wb_512 : // FIXME: Replace with trunc+select. - Intrinsic<[llvm_v32i8_ty], - [llvm_v32i16_ty, llvm_v32i8_ty, llvm_i32_ty], - [IntrNoMem]>; def int_x86_avx512_mask_pmov_wb_mem_512 : GCCBuiltin<"__builtin_ia32_pmovwb512mem_mask">, Intrinsic<[], @@ -4834,36 +4672,64 @@ let TargetPrefix = "x86" in { GCCBuiltin<"__builtin_ia32_pternlogd128">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_pternlog_d_256 : GCCBuiltin<"__builtin_ia32_pternlogd256">, Intrinsic<[llvm_v8i32_ty], [llvm_v8i32_ty, llvm_v8i32_ty, llvm_v8i32_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_pternlog_d_512 : GCCBuiltin<"__builtin_ia32_pternlogd512">, Intrinsic<[llvm_v16i32_ty], [llvm_v16i32_ty, llvm_v16i32_ty, llvm_v16i32_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_pternlog_q_128 : GCCBuiltin<"__builtin_ia32_pternlogq128">, Intrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_pternlog_q_256 : GCCBuiltin<"__builtin_ia32_pternlogq256">, Intrinsic<[llvm_v4i64_ty], [llvm_v4i64_ty, llvm_v4i64_ty, llvm_v4i64_ty, llvm_i32_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<3>]>; def int_x86_avx512_pternlog_q_512 : GCCBuiltin<"__builtin_ia32_pternlogq512">, Intrinsic<[llvm_v8i64_ty], [llvm_v8i64_ty, llvm_v8i64_ty, llvm_v8i64_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<3>]>; +} + +// vp2intersect +let TargetPrefix = "x86" in { + def int_x86_avx512_vp2intersect_q_512 : + Intrinsic<[llvm_v8i1_ty, llvm_v8i1_ty], + [llvm_v8i64_ty, llvm_v8i64_ty], + [IntrNoMem]>; + def int_x86_avx512_vp2intersect_q_256 : + Intrinsic<[llvm_v4i1_ty, llvm_v4i1_ty], + [llvm_v4i64_ty, llvm_v4i64_ty], + [IntrNoMem]>; + def int_x86_avx512_vp2intersect_q_128 : + Intrinsic<[llvm_v2i1_ty, llvm_v2i1_ty], + [llvm_v2i64_ty, llvm_v2i64_ty], + [IntrNoMem]>; + def int_x86_avx512_vp2intersect_d_512 : + Intrinsic<[llvm_v16i1_ty, llvm_v16i1_ty], + [llvm_v16i32_ty, llvm_v16i32_ty], + [IntrNoMem]>; + def int_x86_avx512_vp2intersect_d_256 : + Intrinsic<[llvm_v8i1_ty, llvm_v8i1_ty], + [llvm_v8i32_ty, llvm_v8i32_ty], + [IntrNoMem]>; + def int_x86_avx512_vp2intersect_d_128 : + Intrinsic<[llvm_v4i1_ty, llvm_v4i1_ty], + [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; } @@ -4873,31 +4739,35 @@ let TargetPrefix = "x86" in { // distinction in signaling behaviour is not implemented. def int_x86_avx512_cmp_ps_512 : Intrinsic<[llvm_v16i1_ty], [llvm_v16f32_ty, llvm_v16f32_ty, - llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<2>, ImmArg<3>]>; def int_x86_avx512_cmp_pd_512 : Intrinsic<[llvm_v8i1_ty], [llvm_v8f64_ty, llvm_v8f64_ty, - llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<2>, ImmArg<3>]>; def int_x86_avx512_cmp_ps_256 : Intrinsic<[llvm_v8i1_ty], [llvm_v8f32_ty, llvm_v8f32_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_cmp_pd_256 : Intrinsic<[llvm_v4i1_ty], [llvm_v4f64_ty, llvm_v4f64_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_cmp_ps_128 : Intrinsic<[llvm_v4i1_ty], [llvm_v4f32_ty, llvm_v4f32_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_cmp_pd_128 : Intrinsic<[llvm_v2i1_ty], [llvm_v2f64_ty, llvm_v2f64_ty, - llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty], [IntrNoMem, ImmArg<2>]>; def int_x86_avx512_mask_cmp_ss : GCCBuiltin<"__builtin_ia32_cmpss_mask">, Intrinsic<[llvm_i8_ty], [llvm_v4f32_ty, llvm_v4f32_ty, - llvm_i32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<2>, ImmArg<4>]>; def int_x86_avx512_mask_cmp_sd : GCCBuiltin<"__builtin_ia32_cmpsd_mask">, Intrinsic<[llvm_i8_ty], [llvm_v2f64_ty, llvm_v2f64_ty, - llvm_i32_ty, llvm_i8_ty, llvm_i32_ty], [IntrNoMem]>; + llvm_i32_ty, llvm_i8_ty, llvm_i32_ty], + [IntrNoMem, ImmArg<2>, ImmArg<4>]>; } //===----------------------------------------------------------------------===// @@ -4905,7 +4775,7 @@ let TargetPrefix = "x86" in { let TargetPrefix = "x86" in { def int_x86_sha1rnds4 : GCCBuiltin<"__builtin_ia32_sha1rnds4">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_i8_ty], - [IntrNoMem]>; + [IntrNoMem, ImmArg<2>]>; def int_x86_sha1nexte : GCCBuiltin<"__builtin_ia32_sha1nexte">, Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; def int_x86_sha1msg1 : GCCBuiltin<"__builtin_ia32_sha1msg1">, @@ -5000,3 +4870,51 @@ let TargetPrefix = "x86" in { def int_x86_invpcid : GCCBuiltin<"__builtin_ia32_invpcid">, Intrinsic<[], [llvm_i32_ty, llvm_ptr_ty], []>; } + +let TargetPrefix = "x86" in { + def int_x86_avx512bf16_cvtne2ps2bf16_128: + GCCBuiltin<"__builtin_ia32_cvtne2ps2bf16_128">, + Intrinsic<[llvm_v8i16_ty], [llvm_v4f32_ty, llvm_v4f32_ty], + [IntrNoMem]>; + def int_x86_avx512bf16_cvtne2ps2bf16_256: + GCCBuiltin<"__builtin_ia32_cvtne2ps2bf16_256">, + Intrinsic<[llvm_v16i16_ty], [llvm_v8f32_ty, llvm_v8f32_ty], + [IntrNoMem]>; + def int_x86_avx512bf16_cvtne2ps2bf16_512: + GCCBuiltin<"__builtin_ia32_cvtne2ps2bf16_512">, + Intrinsic<[llvm_v32i16_ty], [llvm_v16f32_ty, llvm_v16f32_ty], + [IntrNoMem]>; + // Intrinsic must be masked due to it producing less than 128 bits of results. + def int_x86_avx512bf16_mask_cvtneps2bf16_128: + Intrinsic<[llvm_v8i16_ty], + [llvm_v4f32_ty, llvm_v8i16_ty, llvm_v4i1_ty], + [IntrNoMem]>; + def int_x86_avx512bf16_cvtneps2bf16_256: + GCCBuiltin<"__builtin_ia32_cvtneps2bf16_256">, + Intrinsic<[llvm_v8i16_ty], [llvm_v8f32_ty], [IntrNoMem]>; + def int_x86_avx512bf16_cvtneps2bf16_512: + GCCBuiltin<"__builtin_ia32_cvtneps2bf16_512">, + Intrinsic<[llvm_v16i16_ty], [llvm_v16f32_ty], [IntrNoMem]>; + def int_x86_avx512bf16_dpbf16ps_128: + GCCBuiltin<"__builtin_ia32_dpbf16ps_128">, + Intrinsic<[llvm_v4f32_ty], + [llvm_v4f32_ty, llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; + def int_x86_avx512bf16_dpbf16ps_256: + GCCBuiltin<"__builtin_ia32_dpbf16ps_256">, + Intrinsic<[llvm_v8f32_ty], + [llvm_v8f32_ty, llvm_v8i32_ty, llvm_v8i32_ty], [IntrNoMem]>; + def int_x86_avx512bf16_dpbf16ps_512: + GCCBuiltin<"__builtin_ia32_dpbf16ps_512">, + Intrinsic<[llvm_v16f32_ty], + [llvm_v16f32_ty, llvm_v16i32_ty, llvm_v16i32_ty], [IntrNoMem]>; +} + +//===----------------------------------------------------------------------===// +// ENQCMD - Enqueue Stores Instructions + +let TargetPrefix = "x86" in { + def int_x86_enqcmd : GCCBuiltin<"__builtin_ia32_enqcmd">, + Intrinsic<[llvm_i8_ty], [llvm_ptr_ty, llvm_ptr_ty], []>; + def int_x86_enqcmds : GCCBuiltin<"__builtin_ia32_enqcmds">, + Intrinsic<[llvm_i8_ty], [llvm_ptr_ty, llvm_ptr_ty], []>; +} diff --git a/include/llvm/IR/IntrinsicsXCore.td b/include/llvm/IR/IntrinsicsXCore.td index b614e1ed6ec0..7fe8bdfd3bd0 100644 --- a/include/llvm/IR/IntrinsicsXCore.td +++ b/include/llvm/IR/IntrinsicsXCore.td @@ -1,9 +1,8 @@ //==- IntrinsicsXCore.td - XCore intrinsics -*- tablegen -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/LLVMContext.h b/include/llvm/IR/LLVMContext.h index bd7097b39a3e..c80504500418 100644 --- a/include/llvm/IR/LLVMContext.h +++ b/include/llvm/IR/LLVMContext.h @@ -1,9 +1,8 @@ //===- llvm/LLVMContext.h - Class for managing "global" state ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -36,12 +35,8 @@ template class SmallVectorImpl; class SMDiagnostic; class StringRef; class Twine; - -namespace yaml { - -class Output; - -} // end namespace yaml +class RemarkStreamer; +class raw_ostream; namespace SyncScope { @@ -103,6 +98,8 @@ public: MD_callees = 23, // "callees" MD_irr_loop = 24, // "irr_loop" MD_access_group = 25, // "llvm.access.group" + MD_callback = 26, // "callback" + MD_preserve_access_index = 27, // "llvm.preserve.*.access.index" }; /// Known operand bundle tag IDs, which always have the same value. All @@ -246,16 +243,23 @@ public: /// included in optimization diagnostics. void setDiagnosticsHotnessThreshold(uint64_t Threshold); - /// Return the YAML file used by the backend to save optimization - /// diagnostics. If null, diagnostics are not saved in a file but only - /// emitted via the diagnostic handler. - yaml::Output *getDiagnosticsOutputFile(); - /// Set the diagnostics output file used for optimization diagnostics. + /// Return the streamer used by the backend to save remark diagnostics. If it + /// does not exist, diagnostics are not saved in a file but only emitted via + /// the diagnostic handler. + RemarkStreamer *getRemarkStreamer(); + const RemarkStreamer *getRemarkStreamer() const; + + /// Set the diagnostics output used for optimization diagnostics. + /// This filename may be embedded in a section for tools to find the + /// diagnostics whenever they're needed. + /// + /// If a remark streamer is already set, it will be replaced with + /// \p RemarkStreamer. /// - /// By default or if invoked with null, diagnostics are not saved in a file - /// but only emitted via the diagnostic handler. Even if an output file is - /// set, the handler is invoked for each diagnostic message. - void setDiagnosticsOutputFile(std::unique_ptr F); + /// By default, diagnostics are not saved in a file but only emitted via the + /// diagnostic handler. Even if an output file is set, the handler is invoked + /// for each diagnostic message. + void setRemarkStreamer(std::unique_ptr RemarkStreamer); /// Get the prefix that should be printed in front of a diagnostic of /// the given \p Severity diff --git a/include/llvm/IR/LegacyPassManager.h b/include/llvm/IR/LegacyPassManager.h index 5257a0eed488..d6bb79ab6019 100644 --- a/include/llvm/IR/LegacyPassManager.h +++ b/include/llvm/IR/LegacyPassManager.h @@ -1,9 +1,8 @@ //===- LegacyPassManager.h - Legacy Container for Passes --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/LegacyPassManagers.h b/include/llvm/IR/LegacyPassManagers.h index 51a2eb2a146d..72bc80fb5381 100644 --- a/include/llvm/IR/LegacyPassManagers.h +++ b/include/llvm/IR/LegacyPassManagers.h @@ -1,9 +1,8 @@ //===- LegacyPassManagers.h - Legacy Pass Infrastructure --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/LegacyPassNameParser.h b/include/llvm/IR/LegacyPassNameParser.h index 4cec08196408..30820e750350 100644 --- a/include/llvm/IR/LegacyPassNameParser.h +++ b/include/llvm/IR/LegacyPassNameParser.h @@ -1,9 +1,8 @@ //===- LegacyPassNameParser.h -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/MDBuilder.h b/include/llvm/IR/MDBuilder.h index 174616c7ab1d..3a2b1bddf45d 100644 --- a/include/llvm/IR/MDBuilder.h +++ b/include/llvm/IR/MDBuilder.h @@ -1,9 +1,8 @@ //===---- llvm/MDBuilder.h - Builder for LLVM metadata ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -94,6 +93,17 @@ public: /// calls. MDNode *createCallees(ArrayRef Callees); + //===------------------------------------------------------------------===// + // Callback metadata. + //===------------------------------------------------------------------===// + + /// Return metadata describing a callback (see llvm::AbstractCallSite). + MDNode *createCallbackEncoding(unsigned CalleeArgNo, ArrayRef Arguments, + bool VarArgsArePassed); + + /// Merge the new callback encoding \p NewCB into \p ExistingCallbacks. + MDNode *mergeCallbackEncodings(MDNode *ExistingCallbacks, MDNode *NewCB); + //===------------------------------------------------------------------===// // AA metadata. //===------------------------------------------------------------------===// diff --git a/include/llvm/IR/Mangler.h b/include/llvm/IR/Mangler.h index 0261c00f524c..e4a05ab46a65 100644 --- a/include/llvm/IR/Mangler.h +++ b/include/llvm/IR/Mangler.h @@ -1,9 +1,8 @@ //===-- llvm/IR/Mangler.h - Self-contained name mangler ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/Metadata.def b/include/llvm/IR/Metadata.def index 70a03f28b488..1df60cadac08 100644 --- a/include/llvm/IR/Metadata.def +++ b/include/llvm/IR/Metadata.def @@ -1,9 +1,8 @@ //===- llvm/IR/Metadata.def - Metadata definitions --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -114,6 +113,7 @@ HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DIImportedEntity) HANDLE_SPECIALIZED_MDNODE_BRANCH(DIMacroNode) HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DIMacro) HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DIMacroFile) +HANDLE_SPECIALIZED_MDNODE_LEAF_UNIQUABLE(DICommonBlock) #undef HANDLE_METADATA #undef HANDLE_METADATA_LEAF diff --git a/include/llvm/IR/Metadata.h b/include/llvm/IR/Metadata.h index be82c4efc115..7ca2540181ba 100644 --- a/include/llvm/IR/Metadata.h +++ b/include/llvm/IR/Metadata.h @@ -1,9 +1,8 @@ //===- llvm/IR/Metadata.h - Metadata definitions ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/Module.h b/include/llvm/IR/Module.h index 9ef35f1f73cd..f458680cfe15 100644 --- a/include/llvm/IR/Module.h +++ b/include/llvm/IR/Module.h @@ -1,9 +1,8 @@ //===- llvm/Module.h - C++ class to represent a VM module -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -29,6 +28,7 @@ #include "llvm/IR/GlobalIFunc.h" #include "llvm/IR/GlobalVariable.h" #include "llvm/IR/Metadata.h" +#include "llvm/IR/ProfileSummary.h" #include "llvm/IR/SymbolTableListTraits.h" #include "llvm/Support/CBindingWrapping.h" #include "llvm/Support/CodeGen.h" @@ -333,16 +333,18 @@ public: /// Look up the specified function in the module symbol table. Four /// possibilities: /// 1. If it does not exist, add a prototype for the function and return it. - /// 2. If it exists, and has a local linkage, the existing function is - /// renamed and a new one is inserted. - /// 3. Otherwise, if the existing function has the correct prototype, return + /// 2. Otherwise, if the existing function has the correct prototype, return /// the existing function. - /// 4. Finally, the function exists but has the wrong prototype: return the + /// 3. Finally, the function exists but has the wrong prototype: return the /// function with a constantexpr cast to the right prototype. - Constant *getOrInsertFunction(StringRef Name, FunctionType *T, - AttributeList AttributeList); + /// + /// In all cases, the returned value is a FunctionCallee wrapper around the + /// 'FunctionType *T' passed in, as well as a 'Value*' either of the Function or + /// the bitcast to the function. + FunctionCallee getOrInsertFunction(StringRef Name, FunctionType *T, + AttributeList AttributeList); - Constant *getOrInsertFunction(StringRef Name, FunctionType *T); + FunctionCallee getOrInsertFunction(StringRef Name, FunctionType *T); /// Look up the specified function in the module symbol table. If it does not /// exist, add a prototype for the function and return it. This function @@ -350,11 +352,10 @@ public: /// or a ConstantExpr BitCast of that type if the named function has a /// different type. This version of the method takes a list of /// function arguments, which makes it easier for clients to use. - template - Constant *getOrInsertFunction(StringRef Name, - AttributeList AttributeList, - Type *RetTy, ArgsTy... Args) - { + template + FunctionCallee getOrInsertFunction(StringRef Name, + AttributeList AttributeList, Type *RetTy, + ArgsTy... Args) { SmallVector ArgTys{Args...}; return getOrInsertFunction(Name, FunctionType::get(RetTy, ArgTys, false), @@ -362,15 +363,17 @@ public: } /// Same as above, but without the attributes. - template - Constant *getOrInsertFunction(StringRef Name, Type *RetTy, ArgsTy... Args) { + template + FunctionCallee getOrInsertFunction(StringRef Name, Type *RetTy, + ArgsTy... Args) { return getOrInsertFunction(Name, AttributeList{}, RetTy, Args...); } // Avoid an incorrect ordering that'd otherwise compile incorrectly. template - Constant *getOrInsertFunction(StringRef Name, AttributeList AttributeList, - FunctionType *Invalid, ArgsTy... Args) = delete; + FunctionCallee + getOrInsertFunction(StringRef Name, AttributeList AttributeList, + FunctionType *Invalid, ArgsTy... Args) = delete; /// Look up the specified function in the module symbol table. If it does not /// exist, return null. @@ -866,10 +869,11 @@ public: /// @{ /// Attach profile summary metadata to this module. - void setProfileSummary(Metadata *M); + void setProfileSummary(Metadata *M, ProfileSummary::Kind Kind); - /// Returns profile summary metadata - Metadata *getProfileSummary(); + /// Returns profile summary metadata. When IsCS is true, use the context + /// sensitive profile summary. + Metadata *getProfileSummary(bool IsCS); /// @} /// Returns true if PLT should be avoided for RTLib calls. diff --git a/include/llvm/IR/ModuleSlotTracker.h b/include/llvm/IR/ModuleSlotTracker.h index eb26fba906ea..85f8ff938366 100644 --- a/include/llvm/IR/ModuleSlotTracker.h +++ b/include/llvm/IR/ModuleSlotTracker.h @@ -1,9 +1,8 @@ //===-- llvm/IR/ModuleSlotTracker.h -----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/IR/ModuleSummaryIndex.h b/include/llvm/IR/ModuleSummaryIndex.h index a1acee494475..aacf8cfc089f 100644 --- a/include/llvm/IR/ModuleSummaryIndex.h +++ b/include/llvm/IR/ModuleSummaryIndex.h @@ -1,9 +1,8 @@ //===- llvm/ModuleSummaryIndex.h - Module Summary Index ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -120,7 +119,7 @@ class GlobalValueSummary; using GlobalValueSummaryList = std::vector>; -struct GlobalValueSummaryInfo { +struct LLVM_ALIGNAS(8) GlobalValueSummaryInfo { union NameOrGV { NameOrGV(bool HaveGVs) { if (HaveGVs) @@ -163,7 +162,8 @@ using GlobalValueSummaryMapTy = /// Struct that holds a reference to a particular GUID in a global value /// summary. struct ValueInfo { - PointerIntPair + enum Flags { HaveGV = 1, ReadOnly = 2, WriteOnly = 4 }; + PointerIntPair RefAndFlags; ValueInfo() = default; @@ -189,15 +189,42 @@ struct ValueInfo { : getRef()->second.U.Name; } - bool haveGVs() const { return RefAndFlags.getInt() & 0x1; } - bool isReadOnly() const { return RefAndFlags.getInt() & 0x2; } - void setReadOnly() { RefAndFlags.setInt(RefAndFlags.getInt() | 0x2); } + bool haveGVs() const { return RefAndFlags.getInt() & HaveGV; } + bool isReadOnly() const { + assert(isValidAccessSpecifier()); + return RefAndFlags.getInt() & ReadOnly; + } + bool isWriteOnly() const { + assert(isValidAccessSpecifier()); + return RefAndFlags.getInt() & WriteOnly; + } + unsigned getAccessSpecifier() const { + assert(isValidAccessSpecifier()); + return RefAndFlags.getInt() & (ReadOnly | WriteOnly); + } + bool isValidAccessSpecifier() const { + unsigned BadAccessMask = ReadOnly | WriteOnly; + return (RefAndFlags.getInt() & BadAccessMask) != BadAccessMask; + } + void setReadOnly() { + // We expect ro/wo attribute to set only once during + // ValueInfo lifetime. + assert(getAccessSpecifier() == 0); + RefAndFlags.setInt(RefAndFlags.getInt() | ReadOnly); + } + void setWriteOnly() { + assert(getAccessSpecifier() == 0); + RefAndFlags.setInt(RefAndFlags.getInt() | WriteOnly); + } const GlobalValueSummaryMapTy::value_type *getRef() const { return RefAndFlags.getPointer(); } bool isDSOLocal() const; + + /// Checks if all copies are eligible for auto-hiding (have flag set). + bool canAutoHide() const; }; inline raw_ostream &operator<<(raw_ostream &OS, const ValueInfo &VI) { @@ -280,11 +307,23 @@ public: /// within the same linkage unit. unsigned DSOLocal : 1; + /// In the per-module summary, indicates that the global value is + /// linkonce_odr and global unnamed addr (so eligible for auto-hiding + /// via hidden visibility). In the combined summary, indicates that the + /// prevailing linkonce_odr copy can be auto-hidden via hidden visibility + /// when it is upgraded to weak_odr in the backend. This is legal when + /// all copies are eligible for auto-hiding (i.e. all copies were + /// linkonce_odr global unnamed addr. If any copy is not (e.g. it was + /// originally weak_odr, we cannot auto-hide the prevailing copy as it + /// means the symbol was externally visible. + unsigned CanAutoHide : 1; + /// Convenience Constructors explicit GVFlags(GlobalValue::LinkageTypes Linkage, - bool NotEligibleToImport, bool Live, bool IsLocal) + bool NotEligibleToImport, bool Live, bool IsLocal, + bool CanAutoHide) : Linkage(Linkage), NotEligibleToImport(NotEligibleToImport), - Live(Live), DSOLocal(IsLocal) {} + Live(Live), DSOLocal(IsLocal), CanAutoHide(CanAutoHide) {} }; private: @@ -365,6 +404,10 @@ public: bool isDSOLocal() const { return Flags.DSOLocal; } + void setCanAutoHide(bool CanAutoHide) { Flags.CanAutoHide = CanAutoHide; } + + bool canAutoHide() const { return Flags.CanAutoHide; } + /// Flag that this global value cannot be imported. void setNotEligibleToImport() { Flags.NotEligibleToImport = true; } @@ -381,25 +424,35 @@ public: /// Alias summary information. class AliasSummary : public GlobalValueSummary { + ValueInfo AliaseeValueInfo; + + /// This is the Aliasee in the same module as alias (could get from VI, trades + /// memory for time). Note that this pointer may be null (and the value info + /// empty) when we have a distributed index where the alias is being imported + /// (as a copy of the aliasee), but the aliasee is not. GlobalValueSummary *AliaseeSummary; - // AliaseeGUID is only set and accessed when we are building a combined index - // via the BitcodeReader. - GlobalValue::GUID AliaseeGUID; public: AliasSummary(GVFlags Flags) : GlobalValueSummary(AliasKind, Flags, ArrayRef{}), - AliaseeSummary(nullptr), AliaseeGUID(0) {} + AliaseeSummary(nullptr) {} /// Check if this is an alias summary. static bool classof(const GlobalValueSummary *GVS) { return GVS->getSummaryKind() == AliasKind; } - void setAliasee(GlobalValueSummary *Aliasee) { AliaseeSummary = Aliasee; } - void setAliaseeGUID(GlobalValue::GUID GUID) { AliaseeGUID = GUID; } + void setAliasee(ValueInfo &AliaseeVI, GlobalValueSummary *Aliasee) { + AliaseeValueInfo = AliaseeVI; + AliaseeSummary = Aliasee; + } - bool hasAliasee() const { return !!AliaseeSummary; } + bool hasAliasee() const { + assert(!!AliaseeSummary == (AliaseeValueInfo && + !AliaseeValueInfo.getSummaryList().empty()) && + "Expect to have both aliasee summary and summary list or neither"); + return !!AliaseeSummary; + } const GlobalValueSummary &getAliasee() const { assert(AliaseeSummary && "Unexpected missing aliasee summary"); @@ -410,10 +463,13 @@ public: return const_cast( static_cast(this)->getAliasee()); } - bool hasAliaseeGUID() const { return AliaseeGUID != 0; } - const GlobalValue::GUID &getAliaseeGUID() const { - assert(AliaseeGUID && "Unexpected missing aliasee GUID"); - return AliaseeGUID; + ValueInfo getAliaseeVI() const { + assert(AliaseeValueInfo && "Unexpected missing aliasee"); + return AliaseeValueInfo; + } + GlobalValue::GUID getAliaseeGUID() const { + assert(AliaseeValueInfo && "Unexpected missing aliasee"); + return AliaseeValueInfo.getGUID(); } }; @@ -500,7 +556,8 @@ public: return FunctionSummary( FunctionSummary::GVFlags( GlobalValue::LinkageTypes::AvailableExternallyLinkage, - /*NotEligibleToImport=*/true, /*Live=*/true, /*IsLocal=*/false), + /*NotEligibleToImport=*/true, /*Live=*/true, /*IsLocal=*/false, + /*CanAutoHide=*/false), /*InsCount=*/0, FunctionSummary::FFlags{}, /*EntryCount=*/0, std::vector(), std::move(Edges), std::vector(), @@ -552,8 +609,8 @@ public: std::move(TypeTestAssumeConstVCalls), std::move(TypeCheckedLoadConstVCalls)}); } - // Gets the number of immutable refs in RefEdgeList - unsigned immutableRefCount() const; + // Gets the number of readonly and writeonly refs in RefEdgeList + std::pair specialRefCounts() const; /// Check if this is a function summary. static bool classof(const GlobalValueSummary *GVS) { @@ -666,18 +723,43 @@ template <> struct DenseMapInfo { } }; +/// The ValueInfo and offset for a function within a vtable definition +/// initializer array. +struct VirtFuncOffset { + VirtFuncOffset(ValueInfo VI, uint64_t Offset) + : FuncVI(VI), VTableOffset(Offset) {} + + ValueInfo FuncVI; + uint64_t VTableOffset; +}; +/// List of functions referenced by a particular vtable definition. +using VTableFuncList = std::vector; + /// Global variable summary information to aid decisions and /// implementation of importing. /// -/// Global variable summary has extra flag, telling if it is -/// modified during the program run or not. This affects ThinLTO -/// internalization +/// Global variable summary has two extra flag, telling if it is +/// readonly or writeonly. Both readonly and writeonly variables +/// can be optimized in the backed: readonly variables can be +/// const-folded, while writeonly vars can be completely eliminated +/// together with corresponding stores. We let both things happen +/// by means of internalizing such variables after ThinLTO import. class GlobalVarSummary : public GlobalValueSummary { +private: + /// For vtable definitions this holds the list of functions and + /// their corresponding offsets within the initializer array. + std::unique_ptr VTableFuncs; + public: struct GVarFlags { - GVarFlags(bool ReadOnly = false) : ReadOnly(ReadOnly) {} - - unsigned ReadOnly : 1; + GVarFlags(bool ReadOnly, bool WriteOnly) + : MaybeReadOnly(ReadOnly), MaybeWriteOnly(WriteOnly) {} + + // In permodule summaries both MaybeReadOnly and MaybeWriteOnly + // bits are set, because attribute propagation occurs later on + // thin link phase. + unsigned MaybeReadOnly : 1; + unsigned MaybeWriteOnly : 1; } VarFlags; GlobalVarSummary(GVFlags Flags, GVarFlags VarFlags, @@ -691,8 +773,21 @@ public: } GVarFlags varflags() const { return VarFlags; } - void setReadOnly(bool RO) { VarFlags.ReadOnly = RO; } - bool isReadOnly() const { return VarFlags.ReadOnly; } + void setReadOnly(bool RO) { VarFlags.MaybeReadOnly = RO; } + void setWriteOnly(bool WO) { VarFlags.MaybeWriteOnly = WO; } + bool maybeReadOnly() const { return VarFlags.MaybeReadOnly; } + bool maybeWriteOnly() const { return VarFlags.MaybeWriteOnly; } + + void setVTableFuncs(VTableFuncList Funcs) { + assert(!VTableFuncs); + VTableFuncs = llvm::make_unique(std::move(Funcs)); + } + + ArrayRef vTableFuncs() const { + if (VTableFuncs) + return *VTableFuncs; + return {}; + } }; struct TypeTestResolution { @@ -791,6 +886,29 @@ using GVSummaryMapTy = DenseMap; using TypeIdSummaryMapTy = std::multimap>; +/// The following data structures summarize type metadata information. +/// For type metadata overview see https://llvm.org/docs/TypeMetadata.html. +/// Each type metadata includes both the type identifier and the offset of +/// the address point of the type (the address held by objects of that type +/// which may not be the beginning of the virtual table). Vtable definitions +/// are decorated with type metadata for the types they are compatible with. +/// +/// Holds information about vtable definitions decorated with type metadata: +/// the vtable definition value and its address point offset in a type +/// identifier metadata it is decorated (compatible) with. +struct TypeIdOffsetVtableInfo { + TypeIdOffsetVtableInfo(uint64_t Offset, ValueInfo VI) + : AddressPointOffset(Offset), VTableVI(VI) {} + + uint64_t AddressPointOffset; + ValueInfo VTableVI; +}; +/// List of vtable definitions decorated by a particular type identifier, +/// and their corresponding offsets in that type identifier's metadata. +/// Note that each type identifier may be compatible with multiple vtables, due +/// to inheritance, which is why this is a vector. +using TypeIdCompatibleVtableInfo = std::vector; + /// Class to hold module path string table and global value map, /// and encapsulate methods for operating on them. class ModuleSummaryIndex { @@ -803,9 +921,15 @@ private: ModulePathStringTableTy ModulePathStringTable; /// Mapping from type identifier GUIDs to type identifier and its summary - /// information. + /// information. Produced by thin link. TypeIdSummaryMapTy TypeIdMap; + /// Mapping from type identifier to information about vtables decorated + /// with that type identifier's metadata. Produced by per module summary + /// analysis and consumed by thin link. For more information, see description + /// above where TypeIdCompatibleVtableInfo is defined. + std::map TypeIdCompatibleVtableMap; + /// Mapping from original ID to GUID. If original ID can map to multiple /// GUIDs, it will be mapped to 0. std::map OidGuidMap; @@ -1044,24 +1168,30 @@ public: OidGuidMap[OrigGUID] = ValueGUID; } - /// Find the summary for global \p GUID in module \p ModuleId, or nullptr if + /// Find the summary for ValueInfo \p VI in module \p ModuleId, or nullptr if /// not found. - GlobalValueSummary *findSummaryInModule(GlobalValue::GUID ValueGUID, - StringRef ModuleId) const { - auto CalleeInfo = getValueInfo(ValueGUID); - if (!CalleeInfo) { - return nullptr; // This function does not have a summary - } + GlobalValueSummary *findSummaryInModule(ValueInfo VI, StringRef ModuleId) const { + auto SummaryList = VI.getSummaryList(); auto Summary = - llvm::find_if(CalleeInfo.getSummaryList(), + llvm::find_if(SummaryList, [&](const std::unique_ptr &Summary) { return Summary->modulePath() == ModuleId; }); - if (Summary == CalleeInfo.getSummaryList().end()) + if (Summary == SummaryList.end()) return nullptr; return Summary->get(); } + /// Find the summary for global \p GUID in module \p ModuleId, or nullptr if + /// not found. + GlobalValueSummary *findSummaryInModule(GlobalValue::GUID ValueGUID, + StringRef ModuleId) const { + auto CalleeInfo = getValueInfo(ValueGUID); + if (!CalleeInfo) + return nullptr; // This function does not have a summary + return findSummaryInModule(CalleeInfo, ModuleId); + } + /// Returns the first GlobalValueSummary for \p GV, asserting that there /// is only one if \p PerModuleIndex. GlobalValueSummary *getGlobalValueSummary(const GlobalValue &GV, @@ -1163,6 +1293,29 @@ public: return nullptr; } + const std::map & + typeIdCompatibleVtableMap() const { + return TypeIdCompatibleVtableMap; + } + + /// Return an existing or new TypeIdCompatibleVtableMap entry for \p TypeId. + /// This accessor can mutate the map and therefore should not be used in + /// the ThinLTO backends. + TypeIdCompatibleVtableInfo & + getOrInsertTypeIdCompatibleVtableSummary(StringRef TypeId) { + return TypeIdCompatibleVtableMap[TypeId]; + } + + /// For the given \p TypeId, this returns the TypeIdCompatibleVtableMap + /// entry if present in the summary map. This may be used when importing. + Optional + getTypeIdCompatibleVtableSummary(StringRef TypeId) const { + auto I = TypeIdCompatibleVtableMap.find(TypeId); + if (I == TypeIdCompatibleVtableMap.end()) + return None; + return I->second; + } + /// Collect for the given module the list of functions it defines /// (GUID -> Summary). void collectDefinedFunctionsForModule(StringRef ModulePath, @@ -1170,8 +1323,16 @@ public: /// Collect for each module the list of Summaries it defines (GUID -> /// Summary). - void collectDefinedGVSummariesPerModule( - StringMap &ModuleToDefinedGVSummaries) const; + template + void + collectDefinedGVSummariesPerModule(Map &ModuleToDefinedGVSummaries) const { + for (auto &GlobalList : *this) { + auto GUID = GlobalList.first; + for (auto &Summary : GlobalList.second.SummaryList) { + ModuleToDefinedGVSummaries[Summary->modulePath()][GUID] = Summary.get(); + } + } + } /// Print to an output stream. void print(raw_ostream &OS, bool IsForDebug = false) const; @@ -1186,7 +1347,7 @@ public: void dumpSCCs(raw_ostream &OS); /// Analyze index and detect unmodified globals - void propagateConstants(const DenseSet &PreservedSymbols); + void propagateAttributes(const DenseSet &PreservedSymbols); }; /// GraphTraits definition to build SCC for the index diff --git a/include/llvm/IR/ModuleSummaryIndexYAML.h b/include/llvm/IR/ModuleSummaryIndexYAML.h index a88ee26b51c3..26d9c43fabf1 100644 --- a/include/llvm/IR/ModuleSummaryIndexYAML.h +++ b/include/llvm/IR/ModuleSummaryIndexYAML.h @@ -1,9 +1,8 @@ //===-- llvm/ModuleSummaryIndexYAML.h - YAML I/O for summary ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -137,7 +136,7 @@ template <> struct MappingTraits { struct FunctionSummaryYaml { unsigned Linkage; - bool NotEligibleToImport, Live, IsLocal; + bool NotEligibleToImport, Live, IsLocal, CanAutoHide; std::vector Refs; std::vector TypeTests; std::vector TypeTestAssumeVCalls, @@ -181,6 +180,7 @@ template <> struct MappingTraits { io.mapOptional("NotEligibleToImport", summary.NotEligibleToImport); io.mapOptional("Live", summary.Live); io.mapOptional("Local", summary.IsLocal); + io.mapOptional("CanAutoHide", summary.CanAutoHide); io.mapOptional("Refs", summary.Refs); io.mapOptional("TypeTests", summary.TypeTests); io.mapOptional("TypeTestAssumeVCalls", summary.TypeTestAssumeVCalls); @@ -223,7 +223,7 @@ template <> struct CustomMappingTraits { Elem.SummaryList.push_back(llvm::make_unique( GlobalValueSummary::GVFlags( static_cast(FSum.Linkage), - FSum.NotEligibleToImport, FSum.Live, FSum.IsLocal), + FSum.NotEligibleToImport, FSum.Live, FSum.IsLocal, FSum.CanAutoHide), /*NumInsts=*/0, FunctionSummary::FFlags{}, /*EntryCount=*/0, Refs, ArrayRef{}, std::move(FSum.TypeTests), std::move(FSum.TypeTestAssumeVCalls), @@ -244,7 +244,8 @@ template <> struct CustomMappingTraits { FSum->flags().Linkage, static_cast(FSum->flags().NotEligibleToImport), static_cast(FSum->flags().Live), - static_cast(FSum->flags().DSOLocal), Refs, + static_cast(FSum->flags().DSOLocal), + static_cast(FSum->flags().CanAutoHide), Refs, FSum->type_tests(), FSum->type_test_assume_vcalls(), FSum->type_checked_load_vcalls(), FSum->type_test_assume_const_vcalls(), diff --git a/include/llvm/IR/NoFolder.h b/include/llvm/IR/NoFolder.h index def07ffe2ff6..0e3c19f4947f 100644 --- a/include/llvm/IR/NoFolder.h +++ b/include/llvm/IR/NoFolder.h @@ -1,9 +1,8 @@ //===- NoFolder.h - Constant folding helper ---------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -204,6 +203,10 @@ public: return BinaryOperator::CreateNot(C); } + Instruction *CreateUnOp(Instruction::UnaryOps Opc, Constant *C) const { + return UnaryOperator::Create(Opc, C); + } + //===--------------------------------------------------------------------===// // Memory Instructions //===--------------------------------------------------------------------===// diff --git a/include/llvm/IR/OperandTraits.h b/include/llvm/IR/OperandTraits.h index c618aff3df9a..979ad35019f8 100644 --- a/include/llvm/IR/OperandTraits.h +++ b/include/llvm/IR/OperandTraits.h @@ -1,9 +1,8 @@ //===-- llvm/OperandTraits.h - OperandTraits class definition ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/Operator.h b/include/llvm/IR/Operator.h index 6b387bbcccb1..8199c65ca8a0 100644 --- a/include/llvm/IR/Operator.h +++ b/include/llvm/IR/Operator.h @@ -1,9 +1,8 @@ //===-- llvm/Operator.h - Operator utility subclass -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -188,6 +187,12 @@ public: FastMathFlags() = default; + static FastMathFlags getFast() { + FastMathFlags FMF; + FMF.setFast(); + return FMF; + } + bool any() const { return Flags != 0; } bool none() const { return Flags == 0; } bool all() const { return Flags == ~0U; } @@ -380,6 +385,7 @@ public: case Instruction::ExtractElement: case Instruction::ShuffleVector: case Instruction::InsertElement: + case Instruction::PHI: return false; default: return V->getType()->isFPOrFPVectorTy(); diff --git a/include/llvm/IR/OptBisect.h b/include/llvm/IR/OptBisect.h index aa24c94c0130..1b2b0bd7acaa 100644 --- a/include/llvm/IR/OptBisect.h +++ b/include/llvm/IR/OptBisect.h @@ -1,9 +1,8 @@ //===- llvm/IR/OptBisect.h - LLVM Bisect support ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -20,12 +19,6 @@ namespace llvm { class Pass; -class Module; -class Function; -class BasicBlock; -class Region; -class Loop; -class CallGraphSCC; /// Extensions to this class implement mechanisms to disable passes and /// individual optimizations at compile time. @@ -33,12 +26,14 @@ class OptPassGate { public: virtual ~OptPassGate() = default; - virtual bool shouldRunPass(const Pass *P, const Module &U) { return true; } - virtual bool shouldRunPass(const Pass *P, const Function &U) {return true; } - virtual bool shouldRunPass(const Pass *P, const BasicBlock &U) { return true; } - virtual bool shouldRunPass(const Pass *P, const Region &U) { return true; } - virtual bool shouldRunPass(const Pass *P, const Loop &U) { return true; } - virtual bool shouldRunPass(const Pass *P, const CallGraphSCC &U) { return true; } + /// IRDescription is a textual description of the IR unit the pass is running + /// over. + virtual bool shouldRunPass(const Pass *P, StringRef IRDescription) { + return true; + } + + /// isEnabled should return true before calling shouldRunPass + virtual bool isEnabled() const { return false; } }; /// This class implements a mechanism to disable passes and individual @@ -60,23 +55,19 @@ public: /// Checks the bisect limit to determine if the specified pass should run. /// - /// These functions immediately return true if bisection is disabled. If the - /// bisect limit is set to -1, the functions print a message describing + /// If the bisect limit is set to -1, the function prints a message describing /// the pass and the bisect number assigned to it and return true. Otherwise, - /// the functions print a message with the bisect number assigned to the + /// the function prints a message with the bisect number assigned to the /// pass and indicating whether or not the pass will be run and return true if /// the bisect limit has not yet been exceeded or false if it has. /// - /// Most passes should not call these routines directly. Instead, they are + /// Most passes should not call this routine directly. Instead, they are /// called through helper routines provided by the pass base classes. For /// instance, function passes should call FunctionPass::skipFunction(). - bool shouldRunPass(const Pass *P, const Module &U) override; - bool shouldRunPass(const Pass *P, const Function &U) override; - bool shouldRunPass(const Pass *P, const BasicBlock &U) override; - bool shouldRunPass(const Pass *P, const Region &U) override; - bool shouldRunPass(const Pass *P, const Loop &U) override; - bool shouldRunPass(const Pass *P, const CallGraphSCC &U) override; + bool shouldRunPass(const Pass *P, StringRef IRDescription) override; + /// isEnabled should return true before calling shouldRunPass + bool isEnabled() const override { return BisectEnabled; } private: bool checkPass(const StringRef PassName, const StringRef TargetDesc); diff --git a/include/llvm/IR/PassInstrumentation.h b/include/llvm/IR/PassInstrumentation.h index 08dac1c4a274..f8a1196871cf 100644 --- a/include/llvm/IR/PassInstrumentation.h +++ b/include/llvm/IR/PassInstrumentation.h @@ -1,9 +1,8 @@ //===- llvm/IR/PassInstrumentation.h ----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/IR/PassManager.h b/include/llvm/IR/PassManager.h index 738a2242eea0..37fe2a5b01ad 100644 --- a/include/llvm/IR/PassManager.h +++ b/include/llvm/IR/PassManager.h @@ -1,9 +1,8 @@ //===- PassManager.h - Pass management infrastructure -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -287,6 +286,13 @@ public: PA.PreservedIDs.count(ID)); } + /// Return true if the checker's analysis was not abandoned, i.e. it was not + /// explicitly invalidated. Even if the analysis is not explicitly + /// preserved, if the analysis is known stateless, then it is preserved. + bool preservedWhenStateless() { + return !IsAbandoned; + } + /// Returns true if the checker's analysis was not abandoned and either /// - \p AnalysisSetT is explicitly preserved or /// - all analyses are preserved. diff --git a/include/llvm/IR/PassManagerInternal.h b/include/llvm/IR/PassManagerInternal.h index 5ad68be62742..58198bf67b11 100644 --- a/include/llvm/IR/PassManagerInternal.h +++ b/include/llvm/IR/PassManagerInternal.h @@ -1,9 +1,8 @@ //===- PassManager internal APIs and implementation details -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/IR/PassTimingInfo.h b/include/llvm/IR/PassTimingInfo.h index e9945f997f43..b8d8f117f73d 100644 --- a/include/llvm/IR/PassTimingInfo.h +++ b/include/llvm/IR/PassTimingInfo.h @@ -1,9 +1,8 @@ //===- PassTimingInfo.h - pass execution timing -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -27,10 +26,12 @@ namespace llvm { class Pass; class PassInstrumentationCallbacks; +class raw_ostream; /// If -time-passes has been specified, report the timings immediately and then -/// reset the timers to zero. -void reportAndResetTimings(); +/// reset the timers to zero. By default it uses the stream created by +/// CreateInfoOutputFile(). +void reportAndResetTimings(raw_ostream *OutStream = nullptr); /// Request the timer for this legacy-pass-manager's pass instance. Timer *getPassTimer(Pass *); @@ -63,18 +64,18 @@ class TimePassesHandler { /// Stack of currently active timers. SmallVector TimerStack; + /// Custom output stream to print timing information into. + /// By default (== nullptr) we emit time report into the stream created by + /// CreateInfoOutputFile(). + raw_ostream *OutStream = nullptr; + bool Enabled; public: TimePassesHandler(bool Enabled = TimePassesIsEnabled); /// Destructor handles the print action if it has not been handled before. - ~TimePassesHandler() { - // First destroying the timers from TimingData, which deploys all their - // collected data into the TG time group member, which later prints itself - // when being destroyed. - TimingData.clear(); - } + ~TimePassesHandler() { print(); } /// Prints out timing information and then resets the timers. void print(); @@ -85,6 +86,9 @@ public: void registerCallbacks(PassInstrumentationCallbacks &PIC); + /// Set a custom output stream for subsequent reporting. + void setOutStream(raw_ostream &OutStream); + private: /// Dumps information for running/triggered timers, useful for debugging LLVM_DUMP_METHOD void dump() const; diff --git a/include/llvm/IR/PatternMatch.h b/include/llvm/IR/PatternMatch.h index 120fc253b908..0f03d7cc56b8 100644 --- a/include/llvm/IR/PatternMatch.h +++ b/include/llvm/IR/PatternMatch.h @@ -1,9 +1,8 @@ //===- PatternMatch.h - Match on the LLVM IR --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -419,6 +418,46 @@ inline cst_pred_ty m_LowBitMask() { return cst_pred_ty(); } +struct icmp_pred_with_threshold { + ICmpInst::Predicate Pred; + const APInt *Thr; + bool isValue(const APInt &C) { + switch (Pred) { + case ICmpInst::Predicate::ICMP_EQ: + return C.eq(*Thr); + case ICmpInst::Predicate::ICMP_NE: + return C.ne(*Thr); + case ICmpInst::Predicate::ICMP_UGT: + return C.ugt(*Thr); + case ICmpInst::Predicate::ICMP_UGE: + return C.uge(*Thr); + case ICmpInst::Predicate::ICMP_ULT: + return C.ult(*Thr); + case ICmpInst::Predicate::ICMP_ULE: + return C.ule(*Thr); + case ICmpInst::Predicate::ICMP_SGT: + return C.sgt(*Thr); + case ICmpInst::Predicate::ICMP_SGE: + return C.sge(*Thr); + case ICmpInst::Predicate::ICMP_SLT: + return C.slt(*Thr); + case ICmpInst::Predicate::ICMP_SLE: + return C.sle(*Thr); + default: + llvm_unreachable("Unhandled ICmp predicate"); + } + } +}; +/// Match an integer or vector with every element comparing 'pred' (eg/ne/...) +/// to Threshold. For vectors, this includes constants with undefined elements. +inline cst_pred_ty +m_SpecificInt_ICMP(ICmpInst::Predicate Predicate, const APInt &Threshold) { + cst_pred_ty P; + P.Pred = Predicate; + P.Thr = &Threshold; + return P; +} + struct is_nan { bool isValue(const APFloat &C) { return C.isNaN(); } }; @@ -668,18 +707,26 @@ template struct FNeg_match { FNeg_match(const Op_t &Op) : X(Op) {} template bool match(OpTy *V) { auto *FPMO = dyn_cast(V); - if (!FPMO || FPMO->getOpcode() != Instruction::FSub) - return false; - if (FPMO->hasNoSignedZeros()) { - // With 'nsz', any zero goes. - if (!cstfp_pred_ty().match(FPMO->getOperand(0))) - return false; - } else { - // Without 'nsz', we need fsub -0.0, X exactly. - if (!cstfp_pred_ty().match(FPMO->getOperand(0))) - return false; + if (!FPMO) return false; + + if (FPMO->getOpcode() == Instruction::FNeg) + return X.match(FPMO->getOperand(0)); + + if (FPMO->getOpcode() == Instruction::FSub) { + if (FPMO->hasNoSignedZeros()) { + // With 'nsz', any zero goes. + if (!cstfp_pred_ty().match(FPMO->getOperand(0))) + return false; + } else { + // Without 'nsz', we need fsub -0.0, X exactly. + if (!cstfp_pred_ty().match(FPMO->getOperand(0))) + return false; + } + + return X.match(FPMO->getOperand(1)); } - return X.match(FPMO->getOperand(1)); + + return false; } }; @@ -1464,6 +1511,20 @@ struct UAddWithOverflow_match { if (AddExpr.match(ICmpRHS) && (ICmpLHS == AddLHS || ICmpLHS == AddRHS)) return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS); + // Match special-case for increment-by-1. + if (Pred == ICmpInst::ICMP_EQ) { + // (a + 1) == 0 + // (1 + a) == 0 + if (AddExpr.match(ICmpLHS) && m_ZeroInt().match(ICmpRHS) && + (m_One().match(AddLHS) || m_One().match(AddRHS))) + return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpLHS); + // 0 == (a + 1) + // 0 == (1 + a) + if (m_ZeroInt().match(ICmpLHS) && AddExpr.match(ICmpRHS) && + (m_One().match(AddLHS) || m_One().match(AddRHS))) + return L.match(AddLHS) && R.match(AddRHS) && S.match(ICmpRHS); + } + return false; } }; diff --git a/include/llvm/IR/PredIteratorCache.h b/include/llvm/IR/PredIteratorCache.h index 81f535311431..cc835277910b 100644 --- a/include/llvm/IR/PredIteratorCache.h +++ b/include/llvm/IR/PredIteratorCache.h @@ -1,9 +1,8 @@ //===- PredIteratorCache.h - pred_iterator Cache ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/ProfileSummary.h b/include/llvm/IR/ProfileSummary.h index e38663770a13..78635ec4386c 100644 --- a/include/llvm/IR/ProfileSummary.h +++ b/include/llvm/IR/ProfileSummary.h @@ -1,9 +1,8 @@ //===- ProfileSummary.h - Profile summary data structure. -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -43,11 +42,10 @@ using SummaryEntryVector = std::vector; class ProfileSummary { public: - enum Kind { PSK_Instr, PSK_Sample }; + enum Kind { PSK_Instr, PSK_CSInstr, PSK_Sample }; private: const Kind PSK; - static const char *KindStr[2]; SummaryEntryVector DetailedSummary; uint64_t TotalCount, MaxCount, MaxInternalCount, MaxFunctionCount; uint32_t NumCounts, NumFunctions; diff --git a/include/llvm/IR/RemarkStreamer.h b/include/llvm/IR/RemarkStreamer.h new file mode 100644 index 000000000000..f34cc660b2fb --- /dev/null +++ b/include/llvm/IR/RemarkStreamer.h @@ -0,0 +1,96 @@ +//===- llvm/IR/RemarkStreamer.h - Remark Streamer ---------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file declares the main interface for outputting remarks. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_IR_REMARKSTREAMER_H +#define LLVM_IR_REMARKSTREAMER_H + +#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/Remarks/RemarkSerializer.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/Regex.h" +#include "llvm/Support/ToolOutputFile.h" +#include "llvm/Support/raw_ostream.h" +#include +#include + +namespace llvm { +/// Streamer for remarks. +class RemarkStreamer { + /// The filename that the remark diagnostics are emitted to. + const std::string Filename; + /// The regex used to filter remarks based on the passes that emit them. + Optional PassFilter; + /// The object used to serialize the remarks to a specific format. + std::unique_ptr Serializer; + + /// Convert diagnostics into remark objects. + /// The lifetime of the members of the result is bound to the lifetime of + /// the LLVM diagnostics. + remarks::Remark toRemark(const DiagnosticInfoOptimizationBase &Diag); + +public: + RemarkStreamer(StringRef Filename, + std::unique_ptr Serializer); + /// Return the filename that the remark diagnostics are emitted to. + StringRef getFilename() const { return Filename; } + /// Return stream that the remark diagnostics are emitted to. + raw_ostream &getStream() { return Serializer->OS; } + /// Return the serializer used for this stream. + remarks::Serializer &getSerializer() { return *Serializer; } + /// Set a pass filter based on a regex \p Filter. + /// Returns an error if the regex is invalid. + Error setFilter(StringRef Filter); + /// Emit a diagnostic through the streamer. + void emit(const DiagnosticInfoOptimizationBase &Diag); +}; + +template +struct RemarkSetupErrorInfo : public ErrorInfo { + std::string Msg; + std::error_code EC; + + RemarkSetupErrorInfo(Error E) { + handleAllErrors(std::move(E), [&](const ErrorInfoBase &EIB) { + Msg = EIB.message(); + EC = EIB.convertToErrorCode(); + }); + } + + void log(raw_ostream &OS) const override { OS << Msg; } + std::error_code convertToErrorCode() const override { return EC; } +}; + +struct RemarkSetupFileError : RemarkSetupErrorInfo { + static char ID; + using RemarkSetupErrorInfo::RemarkSetupErrorInfo; +}; + +struct RemarkSetupPatternError : RemarkSetupErrorInfo { + static char ID; + using RemarkSetupErrorInfo::RemarkSetupErrorInfo; +}; + +struct RemarkSetupFormatError : RemarkSetupErrorInfo { + static char ID; + using RemarkSetupErrorInfo::RemarkSetupErrorInfo; +}; + +/// Setup optimization remarks. +Expected> +setupOptimizationRemarks(LLVMContext &Context, StringRef RemarksFilename, + StringRef RemarksPasses, StringRef RemarksFormat, + bool RemarksWithHotness, + unsigned RemarksHotnessThreshold = 0); + +} // end namespace llvm + +#endif // LLVM_IR_REMARKSTREAMER_H diff --git a/include/llvm/IR/RuntimeLibcalls.def b/include/llvm/IR/RuntimeLibcalls.def index 89005120cdc1..f6c74d497b18 100644 --- a/include/llvm/IR/RuntimeLibcalls.def +++ b/include/llvm/IR/RuntimeLibcalls.def @@ -1,9 +1,8 @@ //===-- llvm/RuntimeLibcalls.def - File that describes libcalls -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -255,6 +254,26 @@ HANDLE_LIBCALL(FMAX_F64, "fmax") HANDLE_LIBCALL(FMAX_F80, "fmaxl") HANDLE_LIBCALL(FMAX_F128, "fmaxl") HANDLE_LIBCALL(FMAX_PPCF128, "fmaxl") +HANDLE_LIBCALL(LROUND_F32, "lroundf") +HANDLE_LIBCALL(LROUND_F64, "lround") +HANDLE_LIBCALL(LROUND_F80, "lroundl") +HANDLE_LIBCALL(LROUND_F128, "lroundl") +HANDLE_LIBCALL(LROUND_PPCF128, "lroundl") +HANDLE_LIBCALL(LLROUND_F32, "llroundf") +HANDLE_LIBCALL(LLROUND_F64, "llround") +HANDLE_LIBCALL(LLROUND_F80, "llroundl") +HANDLE_LIBCALL(LLROUND_F128, "llroundl") +HANDLE_LIBCALL(LLROUND_PPCF128, "llroundl") +HANDLE_LIBCALL(LRINT_F32, "lrintf") +HANDLE_LIBCALL(LRINT_F64, "lrint") +HANDLE_LIBCALL(LRINT_F80, "lrintl") +HANDLE_LIBCALL(LRINT_F128, "lrintl") +HANDLE_LIBCALL(LRINT_PPCF128, "lrintl") +HANDLE_LIBCALL(LLRINT_F32, "llrintf") +HANDLE_LIBCALL(LLRINT_F64, "llrint") +HANDLE_LIBCALL(LLRINT_F80, "llrintl") +HANDLE_LIBCALL(LLRINT_F128, "llrintl") +HANDLE_LIBCALL(LLRINT_PPCF128, "llrintl") // Conversion HANDLE_LIBCALL(FPEXT_F32_PPCF128, "__gcc_stoq") @@ -530,6 +549,9 @@ HANDLE_LIBCALL(STACKPROTECTOR_CHECK_FAIL, "__stack_chk_fail") // Deoptimization HANDLE_LIBCALL(DEOPTIMIZE, "__llvm_deoptimize") +// Return address +HANDLE_LIBCALL(RETURN_ADDRESS, nullptr) + HANDLE_LIBCALL(UNKNOWN_LIBCALL, nullptr) #undef HANDLE_LIBCALL diff --git a/include/llvm/IR/SafepointIRVerifier.h b/include/llvm/IR/SafepointIRVerifier.h index 092050d1d207..ec5527954adc 100644 --- a/include/llvm/IR/SafepointIRVerifier.h +++ b/include/llvm/IR/SafepointIRVerifier.h @@ -1,9 +1,8 @@ //===- SafepointIRVerifier.h - Checks for GC relocation problems *- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,6 +18,8 @@ #ifndef LLVM_IR_SAFEPOINT_IR_VERIFIER #define LLVM_IR_SAFEPOINT_IR_VERIFIER +#include "llvm/IR/PassManager.h" + namespace llvm { class Function; @@ -30,6 +31,16 @@ void verifySafepointIR(Function &F); /// Create an instance of the safepoint verifier pass which can be added to /// a pass pipeline to check for relocation bugs. FunctionPass *createSafepointIRVerifierPass(); + +/// Create an instance of the safepoint verifier pass which can be added to +/// a pass pipeline to check for relocation bugs. +class SafepointIRVerifierPass : public PassInfoMixin { + +public: + explicit SafepointIRVerifierPass() {} + + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; } #endif // LLVM_IR_SAFEPOINT_IR_VERIFIER diff --git a/include/llvm/IR/Statepoint.h b/include/llvm/IR/Statepoint.h index 8908e1b0d090..89f130bc3351 100644 --- a/include/llvm/IR/Statepoint.h +++ b/include/llvm/IR/Statepoint.h @@ -1,14 +1,13 @@ //===- llvm/IR/Statepoint.h - gc.statepoint utilities -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This file contains utility functions and a wrapper class analogous to -// CallSite for accessing the fields of gc.statepoint, gc.relocate, +// CallBase for accessing the fields of gc.statepoint, gc.relocate, // gc.result intrinsics; and some general utilities helpful when dealing with // gc.statepoint. // @@ -21,7 +20,6 @@ #include "llvm/ADT/iterator_range.h" #include "llvm/IR/Attributes.h" #include "llvm/IR/BasicBlock.h" -#include "llvm/IR/CallSite.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Function.h" #include "llvm/IR/Instruction.h" @@ -57,42 +55,36 @@ enum class StatepointFlags { class GCRelocateInst; class GCResultInst; -bool isStatepoint(ImmutableCallSite CS); +bool isStatepoint(const CallBase *Call); bool isStatepoint(const Value *V); bool isStatepoint(const Value &V); -bool isGCRelocate(ImmutableCallSite CS); +bool isGCRelocate(const CallBase *Call); bool isGCRelocate(const Value *V); -bool isGCResult(ImmutableCallSite CS); +bool isGCResult(const CallBase *Call); bool isGCResult(const Value *V); -/// Analogous to CallSiteBase, this provides most of the actual +/// A wrapper around a GC intrinsic call, this provides most of the actual /// functionality for Statepoint and ImmutableStatepoint. It is /// templatized to allow easily specializing of const and non-const -/// concrete subtypes. This is structured analogous to CallSite -/// rather than the IntrinsicInst.h helpers since we need to support -/// invokable statepoints. +/// concrete subtypes. template + typename CallBaseTy> class StatepointBase { - CallSiteTy StatepointCS; + CallBaseTy *StatepointCall; protected: explicit StatepointBase(InstructionTy *I) { - if (isStatepoint(I)) { - StatepointCS = CallSiteTy(I); - assert(StatepointCS && "isStatepoint implies CallSite"); - } + StatepointCall = isStatepoint(I) ? cast(I) : nullptr; } - explicit StatepointBase(CallSiteTy CS) { - if (isStatepoint(CS)) - StatepointCS = CS; + explicit StatepointBase(CallBaseTy *Call) { + StatepointCall = isStatepoint(Call) ? Call : nullptr; } public: - using arg_iterator = typename CallSiteTy::arg_iterator; + using arg_iterator = typename CallBaseTy::const_op_iterator; enum { IDPos = 0, @@ -107,30 +99,30 @@ public: void *operator new(size_t s) = delete; explicit operator bool() const { - // We do not assign non-statepoint CallSites to StatepointCS. - return (bool)StatepointCS; + // We do not assign non-statepoint call instructions to StatepointCall. + return (bool)StatepointCall; } - /// Return the underlying CallSite. - CallSiteTy getCallSite() const { + /// Return the underlying call instruction. + CallBaseTy *getCall() const { assert(*this && "check validity first!"); - return StatepointCS; + return StatepointCall; } uint64_t getFlags() const { - return cast(getCallSite().getArgument(FlagsPos)) + return cast(getCall()->getArgOperand(FlagsPos)) ->getZExtValue(); } /// Return the ID associated with this statepoint. uint64_t getID() const { - const Value *IDVal = getCallSite().getArgument(IDPos); + const Value *IDVal = getCall()->getArgOperand(IDPos); return cast(IDVal)->getZExtValue(); } /// Return the number of patchable bytes associated with this statepoint. uint32_t getNumPatchBytes() const { - const Value *NumPatchBytesVal = getCallSite().getArgument(NumPatchBytesPos); + const Value *NumPatchBytesVal = getCall()->getArgOperand(NumPatchBytesPos); uint64_t NumPatchBytes = cast(NumPatchBytesVal)->getZExtValue(); assert(isInt<32>(NumPatchBytes) && "should fit in 32 bits!"); @@ -139,12 +131,11 @@ public: /// Return the value actually being called or invoked. ValueTy *getCalledValue() const { - return getCallSite().getArgument(CalledFunctionPos); + return getCall()->getArgOperand(CalledFunctionPos); } - InstructionTy *getInstruction() const { - return getCallSite().getInstruction(); - } + // FIXME: Migrate users of this to `getCall` and remove it. + InstructionTy *getInstruction() const { return getCall(); } /// Return the function being called if this is a direct call, otherwise /// return null (if it's an indirect call). @@ -153,12 +144,12 @@ public: } /// Return the caller function for this statepoint. - FunTy *getCaller() const { return getCallSite().getCaller(); } + FunTy *getCaller() const { return getCall()->getCaller(); } /// Determine if the statepoint cannot unwind. bool doesNotThrow() const { Function *F = getCalledFunction(); - return getCallSite().doesNotThrow() || (F ? F->doesNotThrow() : false); + return getCall()->doesNotThrow() || (F ? F->doesNotThrow() : false); } /// Return the type of the value returned by the call underlying the @@ -171,18 +162,18 @@ public: /// Number of arguments to be passed to the actual callee. int getNumCallArgs() const { - const Value *NumCallArgsVal = getCallSite().getArgument(NumCallArgsPos); + const Value *NumCallArgsVal = getCall()->getArgOperand(NumCallArgsPos); return cast(NumCallArgsVal)->getZExtValue(); } size_t arg_size() const { return getNumCallArgs(); } - typename CallSiteTy::arg_iterator arg_begin() const { - assert(CallArgsBeginPos <= (int)getCallSite().arg_size()); - return getCallSite().arg_begin() + CallArgsBeginPos; + arg_iterator arg_begin() const { + assert(CallArgsBeginPos <= (int)getCall()->arg_size()); + return getCall()->arg_begin() + CallArgsBeginPos; } - typename CallSiteTy::arg_iterator arg_end() const { + arg_iterator arg_end() const { auto I = arg_begin() + arg_size(); - assert((getCallSite().arg_end() - I) >= 0); + assert((getCall()->arg_end() - I) >= 0); return I; } @@ -199,8 +190,8 @@ public: /// Return true if the call or the callee has the given attribute. bool paramHasAttr(unsigned i, Attribute::AttrKind A) const { Function *F = getCalledFunction(); - return getCallSite().paramHasAttr(i + CallArgsBeginPos, A) || - (F ? F->getAttributes().hasAttribute(i, A) : false); + return getCall()->paramHasAttr(i + CallArgsBeginPos, A) || + (F ? F->getAttributes().hasAttribute(i, A) : false); } /// Number of GC transition args. @@ -208,14 +199,14 @@ public: const Value *NumGCTransitionArgs = *arg_end(); return cast(NumGCTransitionArgs)->getZExtValue(); } - typename CallSiteTy::arg_iterator gc_transition_args_begin() const { + arg_iterator gc_transition_args_begin() const { auto I = arg_end() + 1; - assert((getCallSite().arg_end() - I) >= 0); + assert((getCall()->arg_end() - I) >= 0); return I; } - typename CallSiteTy::arg_iterator gc_transition_args_end() const { + arg_iterator gc_transition_args_end() const { auto I = gc_transition_args_begin() + getNumTotalGCTransitionArgs(); - assert((getCallSite().arg_end() - I) >= 0); + assert((getCall()->arg_end() - I) >= 0); return I; } @@ -231,14 +222,14 @@ public: return cast(NumVMSArgs)->getZExtValue(); } - typename CallSiteTy::arg_iterator deopt_begin() const { + arg_iterator deopt_begin() const { auto I = gc_transition_args_end() + 1; - assert((getCallSite().arg_end() - I) >= 0); + assert((getCall()->arg_end() - I) >= 0); return I; } - typename CallSiteTy::arg_iterator deopt_end() const { + arg_iterator deopt_end() const { auto I = deopt_begin() + getNumTotalVMSArgs(); - assert((getCallSite().arg_end() - I) >= 0); + assert((getCall()->arg_end() - I) >= 0); return I; } @@ -247,15 +238,11 @@ public: return make_range(deopt_begin(), deopt_end()); } - typename CallSiteTy::arg_iterator gc_args_begin() const { - return deopt_end(); - } - typename CallSiteTy::arg_iterator gc_args_end() const { - return getCallSite().arg_end(); - } + arg_iterator gc_args_begin() const { return deopt_end(); } + arg_iterator gc_args_end() const { return getCall()->arg_end(); } unsigned gcArgsStartIdx() const { - return gc_args_begin() - getInstruction()->op_begin(); + return gc_args_begin() - getCall()->op_begin(); } /// range adapter for gc arguments @@ -304,25 +291,24 @@ public: /// to a gc.statepoint. class ImmutableStatepoint : public StatepointBase { - using Base = - StatepointBase; + const CallBase> { + using Base = StatepointBase; public: explicit ImmutableStatepoint(const Instruction *I) : Base(I) {} - explicit ImmutableStatepoint(ImmutableCallSite CS) : Base(CS) {} + explicit ImmutableStatepoint(const CallBase *Call) : Base(Call) {} }; /// A specialization of it's base class for read-write access /// to a gc.statepoint. class Statepoint - : public StatepointBase { - using Base = StatepointBase; + : public StatepointBase { + using Base = StatepointBase; public: explicit Statepoint(Instruction *I) : Base(I) {} - explicit Statepoint(CallSite CS) : Base(CS) {} + explicit Statepoint(CallBase *Call) : Base(Call) {} }; /// Common base class for representing values projected from a statepoint. @@ -347,14 +333,14 @@ public: } /// The statepoint with which this gc.relocate is associated. - const Instruction *getStatepoint() const { + const CallBase *getStatepoint() const { const Value *Token = getArgOperand(0); // This takes care both of relocates for call statepoints and relocates // on normal path of invoke statepoint. if (!isa(Token)) { assert(isStatepoint(Token)); - return cast(Token); + return cast(Token); } // This relocate is on exceptional path of an invoke statepoint @@ -366,7 +352,7 @@ public: "safepoint block should be well formed"); assert(isStatepoint(InvokeBB->getTerminator())); - return InvokeBB->getTerminator(); + return cast(InvokeBB->getTerminator()); } }; @@ -395,13 +381,11 @@ public: } Value *getBasePtr() const { - ImmutableCallSite CS(getStatepoint()); - return *(CS.arg_begin() + getBasePtrIndex()); + return *(getStatepoint()->arg_begin() + getBasePtrIndex()); } Value *getDerivedPtr() const { - ImmutableCallSite CS(getStatepoint()); - return *(CS.arg_begin() + getDerivedPtrIndex()); + return *(getStatepoint()->arg_begin() + getDerivedPtrIndex()); } }; @@ -418,28 +402,25 @@ public: }; template + typename CallBaseTy> std::vector -StatepointBase::getRelocates() +StatepointBase::getRelocates() const { - std::vector Result; - CallSiteTy StatepointCS = getCallSite(); - // Search for relocated pointers. Note that working backwards from the // gc_relocates ensures that we only get pairs which are actually relocated // and used after the statepoint. - for (const User *U : getInstruction()->users()) + for (const User *U : StatepointCall->users()) if (auto *Relocate = dyn_cast(U)) Result.push_back(Relocate); - if (!StatepointCS.isInvoke()) + auto *StatepointInvoke = dyn_cast(StatepointCall); + if (!StatepointInvoke) return Result; // We need to scan thorough exceptional relocations if it is invoke statepoint - LandingPadInst *LandingPad = - cast(getInstruction())->getLandingPadInst(); + LandingPadInst *LandingPad = StatepointInvoke->getLandingPadInst(); // Search for gc relocates that are attached to this landingpad. for (const User *LandingPadUser : LandingPad->users()) { diff --git a/include/llvm/IR/SymbolTableListTraits.h b/include/llvm/IR/SymbolTableListTraits.h index 87ce902c2811..5b793e5dbf28 100644 --- a/include/llvm/IR/SymbolTableListTraits.h +++ b/include/llvm/IR/SymbolTableListTraits.h @@ -1,9 +1,8 @@ //===- llvm/SymbolTableListTraits.h - Traits for iplist ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/TrackingMDRef.h b/include/llvm/IR/TrackingMDRef.h index 084efada221f..d7377398b91b 100644 --- a/include/llvm/IR/TrackingMDRef.h +++ b/include/llvm/IR/TrackingMDRef.h @@ -1,9 +1,8 @@ //===- llvm/IR/TrackingMDRef.h - Tracking Metadata references ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/Type.h b/include/llvm/IR/Type.h index 9c1f99d1b3a2..f2aa49030aaa 100644 --- a/include/llvm/IR/Type.h +++ b/include/llvm/IR/Type.h @@ -1,9 +1,8 @@ //===- llvm/Type.h - Classes for handling data types ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -367,6 +366,7 @@ public: return ContainedTys[0]; } + inline bool getVectorIsScalable() const; inline unsigned getVectorNumElements() const; Type *getVectorElementType() const { assert(getTypeID() == VectorTyID); @@ -467,28 +467,6 @@ template <> struct isa_impl { } }; -//===----------------------------------------------------------------------===// -// Provide specializations of GraphTraits to be able to treat a type as a -// graph of sub types. - -template <> struct GraphTraits { - using NodeRef = Type *; - using ChildIteratorType = Type::subtype_iterator; - - static NodeRef getEntryNode(Type *T) { return T; } - static ChildIteratorType child_begin(NodeRef N) { return N->subtype_begin(); } - static ChildIteratorType child_end(NodeRef N) { return N->subtype_end(); } -}; - -template <> struct GraphTraits { - using NodeRef = const Type *; - using ChildIteratorType = Type::subtype_iterator; - - static NodeRef getEntryNode(NodeRef T) { return T; } - static ChildIteratorType child_begin(NodeRef N) { return N->subtype_begin(); } - static ChildIteratorType child_end(NodeRef N) { return N->subtype_end(); } -}; - // Create wrappers for C Binding types (see CBindingWrapping.h). DEFINE_ISA_CONVERSION_FUNCTIONS(Type, LLVMTypeRef) diff --git a/include/llvm/IR/TypeFinder.h b/include/llvm/IR/TypeFinder.h index c050c388d398..a83f85ea84c3 100644 --- a/include/llvm/IR/TypeFinder.h +++ b/include/llvm/IR/TypeFinder.h @@ -1,9 +1,8 @@ //===- llvm/IR/TypeFinder.h - Class to find used struct types ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/Use.h b/include/llvm/IR/Use.h index 25c44e0871a9..034ca2c8ac23 100644 --- a/include/llvm/IR/Use.h +++ b/include/llvm/IR/Use.h @@ -1,9 +1,8 @@ //===- llvm/Use.h - Definition of the Use class -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -140,7 +139,7 @@ private: const Use *getImpliedUser() const LLVM_READONLY; Value *Val = nullptr; - Use *Next; + Use *Next = nullptr; PointerIntPair Prev; void setPrev(Use **NewPrev) { Prev.setPointer(NewPrev); } diff --git a/include/llvm/IR/UseListOrder.h b/include/llvm/IR/UseListOrder.h index b6bb0f19a0aa..a1f313e269b2 100644 --- a/include/llvm/IR/UseListOrder.h +++ b/include/llvm/IR/UseListOrder.h @@ -1,9 +1,8 @@ //===- llvm/IR/UseListOrder.h - LLVM Use List Order -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/User.h b/include/llvm/IR/User.h index aea31467f2fa..19d87c5c621d 100644 --- a/include/llvm/IR/User.h +++ b/include/llvm/IR/User.h @@ -1,9 +1,8 @@ //===- llvm/User.h - User class definition ----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/Value.def b/include/llvm/IR/Value.def index e2ddba0aa159..aaf1651979a9 100644 --- a/include/llvm/IR/Value.def +++ b/include/llvm/IR/Value.def @@ -1,9 +1,8 @@ //===-------- llvm/IR/Value.def - File that describes Values ---v-*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/Value.h b/include/llvm/IR/Value.h index 4f3a45c684fc..b2d8e7ac4741 100644 --- a/include/llvm/IR/Value.h +++ b/include/llvm/IR/Value.h @@ -1,9 +1,8 @@ //===- llvm/Value.h - Definition of the Value class -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -494,7 +493,7 @@ public: /// swifterror attribute. bool isSwiftError() const; - /// Strip off pointer casts, all-zero GEPs, and aliases. + /// Strip off pointer casts, all-zero GEPs, address space casts, and aliases. /// /// Returns the original uncasted value. If this is called on a non-pointer /// value, it returns 'this'. @@ -504,6 +503,17 @@ public: static_cast(this)->stripPointerCasts()); } + /// Strip off pointer casts, all-zero GEPs, address space casts, and aliases + /// but ensures the representation of the result stays the same. + /// + /// Returns the original uncasted value with the same representation. If this + /// is called on a non-pointer value, it returns 'this'. + const Value *stripPointerCastsSameRepresentation() const; + Value *stripPointerCastsSameRepresentation() { + return const_cast(static_cast(this) + ->stripPointerCastsSameRepresentation()); + } + /// Strip off pointer casts, all-zero GEPs, aliases and invariant group /// info. /// @@ -536,19 +546,48 @@ public: static_cast(this)->stripInBoundsConstantOffsets()); } - /// Accumulate offsets from \a stripInBoundsConstantOffsets(). - /// - /// Stores the resulting constant offset stripped into the APInt provided. - /// The provided APInt will be extended or truncated as needed to be the - /// correct bitwidth for an offset of this pointer type. - /// - /// If this is called on a non-pointer value, it returns 'this'. + /// Accumulate the constant offset this value has compared to a base pointer. + /// Only 'getelementptr' instructions (GEPs) with constant indices are + /// accumulated but other instructions, e.g., casts, are stripped away as + /// well. The accumulated constant offset is added to \p Offset and the base + /// pointer is returned. + /// + /// The APInt \p Offset has to have a bit-width equal to the IntPtr type for + /// the address space of 'this' pointer value, e.g., use + /// DataLayout::getIndexTypeSizeInBits(Ty). + /// + /// If \p AllowNonInbounds is true, constant offsets in GEPs are stripped and + /// accumulated even if the GEP is not "inbounds". + /// + /// If this is called on a non-pointer value, it returns 'this' and the + /// \p Offset is not modified. + /// + /// Note that this function will never return a nullptr. It will also never + /// manipulate the \p Offset in a way that would not match the difference + /// between the underlying value and the returned one. Thus, if no constant + /// offset was found, the returned value is the underlying one and \p Offset + /// is unchanged. + const Value *stripAndAccumulateConstantOffsets(const DataLayout &DL, + APInt &Offset, + bool AllowNonInbounds) const; + Value *stripAndAccumulateConstantOffsets(const DataLayout &DL, APInt &Offset, + bool AllowNonInbounds) { + return const_cast( + static_cast(this)->stripAndAccumulateConstantOffsets( + DL, Offset, AllowNonInbounds)); + } + + /// This is a wrapper around stripAndAccumulateConstantOffsets with the + /// in-bounds requirement set to false. const Value *stripAndAccumulateInBoundsConstantOffsets(const DataLayout &DL, - APInt &Offset) const; + APInt &Offset) const { + return stripAndAccumulateConstantOffsets(DL, Offset, + /* AllowNonInbounds */ false); + } Value *stripAndAccumulateInBoundsConstantOffsets(const DataLayout &DL, APInt &Offset) { - return const_cast(static_cast(this) - ->stripAndAccumulateInBoundsConstantOffsets(DL, Offset)); + return stripAndAccumulateConstantOffsets(DL, Offset, + /* AllowNonInbounds */ false); } /// Strip off pointer casts and inbounds GEPs. diff --git a/include/llvm/IR/ValueHandle.h b/include/llvm/IR/ValueHandle.h index d94472ce1be1..1135d796f7ed 100644 --- a/include/llvm/IR/ValueHandle.h +++ b/include/llvm/IR/ValueHandle.h @@ -1,9 +1,8 @@ //===- ValueHandle.h - Value Smart Pointer classes --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -309,15 +308,6 @@ struct DenseMapInfo> { } }; -template -struct isPodLike> { -#ifdef NDEBUG - static const bool value = true; -#else - static const bool value = false; -#endif -}; - /// Value handle that tracks a Value across RAUW. /// /// TrackingVH is designed for situations where a client needs to hold a handle @@ -549,14 +539,6 @@ template struct DenseMapInfo> { } }; -template struct isPodLike> { -#ifdef NDEBUG - static const bool value = true; -#else - static const bool value = false; -#endif -}; - } // end namespace llvm #endif // LLVM_IR_VALUEHANDLE_H diff --git a/include/llvm/IR/ValueMap.h b/include/llvm/IR/ValueMap.h index e7e33918a613..6a79b1d387f3 100644 --- a/include/llvm/IR/ValueMap.h +++ b/include/llvm/IR/ValueMap.h @@ -1,9 +1,8 @@ //===- ValueMap.h - Safe map from Values to data ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/ValueSymbolTable.h b/include/llvm/IR/ValueSymbolTable.h index 012e717c7470..105ea73857af 100644 --- a/include/llvm/IR/ValueSymbolTable.h +++ b/include/llvm/IR/ValueSymbolTable.h @@ -1,9 +1,8 @@ //===- llvm/ValueSymbolTable.h - Implement a Value Symtab -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IR/Verifier.h b/include/llvm/IR/Verifier.h index 7255132e1e65..62c33c8325eb 100644 --- a/include/llvm/IR/Verifier.h +++ b/include/llvm/IR/Verifier.h @@ -1,9 +1,8 @@ //===- Verifier.h - LLVM IR Verifier ----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/IRReader/IRReader.h b/include/llvm/IRReader/IRReader.h index bedde8954fbb..05171300b602 100644 --- a/include/llvm/IRReader/IRReader.h +++ b/include/llvm/IRReader/IRReader.h @@ -1,9 +1,8 @@ //===---- llvm/IRReader/IRReader.h - Reader for LLVM IR files ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -21,11 +20,22 @@ namespace llvm { class StringRef; +class MemoryBuffer; class MemoryBufferRef; class Module; class SMDiagnostic; class LLVMContext; +/// If the given MemoryBuffer holds a bitcode image, return a Module +/// for it which does lazy deserialization of function bodies. Otherwise, +/// attempt to parse it as LLVM Assembly and return a fully populated +/// Module. The ShouldLazyLoadMetadata flag is passed down to the bitcode +/// reader to optionally enable lazy metadata loading. This takes ownership +/// of \p Buffer. +std::unique_ptr getLazyIRModule(std::unique_ptr Buffer, + SMDiagnostic &Err, LLVMContext &Context, + bool ShouldLazyLoadMetadata = false); + /// If the given file holds a bitcode image, return a Module /// for it which does lazy deserialization of function bodies. Otherwise, /// attempt to parse it as LLVM Assembly and return a fully populated diff --git a/include/llvm/InitializePasses.h b/include/llvm/InitializePasses.h index 037c0dbb56ec..164d0be2855a 100644 --- a/include/llvm/InitializePasses.h +++ b/include/llvm/InitializePasses.h @@ -1,9 +1,8 @@ //===- llvm/InitializePasses.h - Initialize All Passes ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -65,8 +64,9 @@ void initializeAAEvalLegacyPassPass(PassRegistry&); void initializeAAResultsWrapperPassPass(PassRegistry&); void initializeADCELegacyPassPass(PassRegistry&); void initializeAddDiscriminatorsLegacyPassPass(PassRegistry&); -void initializeAddressSanitizerModulePass(PassRegistry&); -void initializeAddressSanitizerPass(PassRegistry&); +void initializeModuleAddressSanitizerLegacyPassPass(PassRegistry &); +void initializeASanGlobalsMetadataWrapperPassPass(PassRegistry &); +void initializeAddressSanitizerLegacyPassPass(PassRegistry &); void initializeAggressiveInstCombinerLegacyPassPass(PassRegistry&); void initializeAliasSetPrinterPass(PassRegistry&); void initializeAlignmentFromAssumptionsPass(PassRegistry&); @@ -74,6 +74,7 @@ void initializeAlwaysInlinerLegacyPassPass(PassRegistry&); void initializeArgPromotionPass(PassRegistry&); void initializeAssumptionCacheTrackerPass(PassRegistry&); void initializeAtomicExpandPass(PassRegistry&); +void initializeAttributorLegacyPassPass(PassRegistry&); void initializeBDCELegacyPassPass(PassRegistry&); void initializeBarrierNoopPass(PassRegistry&); void initializeBasicAAWrapperPassPass(PassRegistry&); @@ -134,16 +135,15 @@ void initializeEarlyIfConverterPass(PassRegistry&); void initializeEarlyMachineLICMPass(PassRegistry&); void initializeEarlyTailDuplicatePass(PassRegistry&); void initializeEdgeBundlesPass(PassRegistry&); -void initializeEfficiencySanitizerPass(PassRegistry&); void initializeEliminateAvailableExternallyLegacyPassPass(PassRegistry&); void initializeEntryExitInstrumenterPass(PassRegistry&); -void initializeExpandISelPseudosPass(PassRegistry&); void initializeExpandMemCmpPassPass(PassRegistry&); void initializeExpandPostRAPass(PassRegistry&); void initializeExpandReductionsPass(PassRegistry&); void initializeMakeGuardsExplicitLegacyPassPass(PassRegistry&); void initializeExternalAAWrapperPassPass(PassRegistry&); void initializeFEntryInserterPass(PassRegistry&); +void initializeFinalizeISelPass(PassRegistry&); void initializeFinalizeMachineBundlesPass(PassRegistry&); void initializeFlattenCFGPassPass(PassRegistry&); void initializeFloat2IntLegacyPassPass(PassRegistry&); @@ -163,8 +163,9 @@ void initializeGlobalOptLegacyPassPass(PassRegistry&); void initializeGlobalSplitPass(PassRegistry&); void initializeGlobalsAAWrapperPassPass(PassRegistry&); void initializeGuardWideningLegacyPassPass(PassRegistry&); +void initializeHardwareLoopsPass(PassRegistry&); void initializeHotColdSplittingLegacyPassPass(PassRegistry&); -void initializeHWAddressSanitizerPass(PassRegistry&); +void initializeHWAddressSanitizerLegacyPassPass(PassRegistry &); void initializeIPCPPass(PassRegistry&); void initializeIPSCCPLegacyPassPass(PassRegistry&); void initializeIRCELegacyPassPass(PassRegistry&); @@ -181,6 +182,7 @@ void initializeInstCountPass(PassRegistry&); void initializeInstNamerPass(PassRegistry&); void initializeInstSimplifyLegacyPassPass(PassRegistry &); void initializeInstrProfilingLegacyPassPass(PassRegistry&); +void initializeInstrOrderFileLegacyPassPass(PassRegistry&); void initializeInstructionCombiningPassPass(PassRegistry&); void initializeInstructionSelectPass(PassRegistry&); void initializeInterleavedAccessPass(PassRegistry&); @@ -219,6 +221,7 @@ void initializeLoopDeletionLegacyPassPass(PassRegistry&); void initializeLoopDistributeLegacyPass(PassRegistry&); void initializeLoopExtractorPass(PassRegistry&); void initializeLoopGuardWideningLegacyPassPass(PassRegistry&); +void initializeLoopFuseLegacyPass(PassRegistry&); void initializeLoopIdiomRecognizeLegacyPassPass(PassRegistry&); void initializeLoopInfoWrapperPassPass(PassRegistry&); void initializeLoopInstSimplifyLegacyPassPass(PassRegistry&); @@ -241,6 +244,7 @@ void initializeLowerAtomicLegacyPassPass(PassRegistry&); void initializeLowerEmuTLSPass(PassRegistry&); void initializeLowerExpectIntrinsicPass(PassRegistry&); void initializeLowerGuardIntrinsicLegacyPassPass(PassRegistry&); +void initializeLowerWidenableConditionLegacyPassPass(PassRegistry&); void initializeLowerIntrinsicsPass(PassRegistry&); void initializeLowerInvokeLegacyPassPass(PassRegistry&); void initializeLowerSwitchPass(PassRegistry&); @@ -277,7 +281,7 @@ void initializeMemorySSAPrinterLegacyPassPass(PassRegistry&); void initializeMemorySSAWrapperPassPass(PassRegistry&); void initializeMemorySanitizerLegacyPassPass(PassRegistry&); void initializeMergeFunctionsPass(PassRegistry&); -void initializeMergeICmpsPass(PassRegistry&); +void initializeMergeICmpsLegacyPassPass(PassRegistry &); void initializeMergedLoadStoreMotionLegacyPassPass(PassRegistry&); void initializeMetaRenamerPass(PassRegistry&); void initializeModuleDebugInfoPrinterPass(PassRegistry&); @@ -298,6 +302,7 @@ void initializePEIPass(PassRegistry&); void initializePGOIndirectCallPromotionLegacyPassPass(PassRegistry&); void initializePGOInstrumentationGenLegacyPassPass(PassRegistry&); void initializePGOInstrumentationUseLegacyPassPass(PassRegistry&); +void initializePGOInstrumentationGenCreateVarLegacyPassPass(PassRegistry&); void initializePGOMemOPSizeOptLegacyPassPass(PassRegistry&); void initializePHIEliminationPass(PassRegistry&); void initializePartialInlinerLegacyPassPass(PassRegistry&); diff --git a/include/llvm/LTO/Caching.h b/include/llvm/LTO/Caching.h index 7201ab31f5b0..4c4a708d24e9 100644 --- a/include/llvm/LTO/Caching.h +++ b/include/llvm/LTO/Caching.h @@ -1,9 +1,8 @@ //===- Caching.h - LLVM Link Time Optimizer Configuration -----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,8 +24,8 @@ namespace lto { /// (e.g. in a cache). /// /// Buffer callbacks must be thread safe. -typedef std::function MB)> - AddBufferFn; +using AddBufferFn = + std::function MB)>; /// Create a local file system cache which uses the given cache directory and /// file callback. This function also creates the cache directory if it does not diff --git a/include/llvm/LTO/Config.h b/include/llvm/LTO/Config.h index 7058602c3ee2..fb107e3fbe02 100644 --- a/include/llvm/LTO/Config.h +++ b/include/llvm/LTO/Config.h @@ -1,9 +1,8 @@ //===-Config.h - LLVM Link Time Optimizer Configuration -------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -56,6 +55,9 @@ struct Config { /// Disable entirely the optimizer, including importing for ThinLTO bool CodeGenOnly = false; + /// Run PGO context sensitive IR instrumentation. + bool RunCSIRInstr = false; + /// If this field is set, the set of passes run in the middle-end optimizer /// will be the one specified by the string. Only works with the new pass /// manager as the old one doesn't have this ability. @@ -74,6 +76,9 @@ struct Config { /// with this triple. std::string DefaultTriple; + /// Context Sensitive PGO profile path. + std::string CSIRProfile; + /// Sample PGO profile path. std::string SampleProfile; @@ -83,17 +88,29 @@ struct Config { /// The directory to store .dwo files. std::string DwoDir; + /// The name for the split debug info file used for the DW_AT_[GNU_]dwo_name + /// attribute in the skeleton CU. This should generally only be used when + /// running an individual backend directly via thinBackend(), as otherwise + /// all objects would use the same .dwo file. Not used as output path. + std::string SplitDwarfFile; + /// The path to write a .dwo file to. This should generally only be used when /// running an individual backend directly via thinBackend(), as otherwise - /// all .dwo files will be written to the same path. - std::string DwoPath; + /// all .dwo files will be written to the same path. Not used in skeleton CU. + std::string SplitDwarfOutput; /// Optimization remarks file path. std::string RemarksFilename = ""; + /// Optimization remarks pass filter. + std::string RemarksPasses = ""; + /// Whether to emit optimization remarks with hotness informations. bool RemarksWithHotness = false; + /// The format used for serializing remarks (default: YAML). + std::string RemarksFormat = ""; + /// Whether to emit the pass manager debuggging informations. bool DebugPassManager = false; @@ -133,7 +150,7 @@ struct Config { /// /// Note that in out-of-process backend scenarios, none of the hooks will be /// called for ThinLTO tasks. - typedef std::function ModuleHookFn; + using ModuleHookFn = std::function; /// This module hook is called after linking (regular LTO) or loading /// (ThinLTO) the module, before modifying it. @@ -166,8 +183,8 @@ struct Config { /// /// It is called regardless of whether the backend is in-process, although it /// is not called from individual backend processes. - typedef std::function - CombinedIndexHookFn; + using CombinedIndexHookFn = + std::function; CombinedIndexHookFn CombinedIndexHook; /// This is a convenience function that configures this Config object to write diff --git a/include/llvm/LTO/LTO.h b/include/llvm/LTO/LTO.h index 534d9b6f3f2a..ca0a8b64523a 100644 --- a/include/llvm/LTO/LTO.h +++ b/include/llvm/LTO/LTO.h @@ -1,9 +1,8 @@ //===-LTO.h - LLVM Link Time Optimizer ------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -21,6 +20,7 @@ #include "llvm/ADT/StringSet.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/ModuleSummaryIndex.h" +#include "llvm/IR/RemarkStreamer.h" #include "llvm/LTO/Config.h" #include "llvm/Linker/IRMover.h" #include "llvm/Object/IRSymtab.h" @@ -51,7 +51,8 @@ void thinLTOResolvePrevailingInIndex( function_ref isPrevailing, function_ref - recordNewLinkage); + recordNewLinkage, + const DenseSet &GUIDPreservedSymbols); /// Update the linkages in the given \p Index to mark exported values /// as external and non-exported values as internal. The ThinLTO backends @@ -84,8 +85,13 @@ std::string getThinLTOOutputFile(const std::string &Path, /// Setup optimization remarks. Expected> -setupOptimizationRemarks(LLVMContext &Context, StringRef LTORemarksFilename, - bool LTOPassRemarksWithHotness, int Count = -1); +setupOptimizationRemarks(LLVMContext &Context, StringRef RemarksFilename, + StringRef RemarksPasses, StringRef RemarksFormat, + bool RemarksWithHotness, int Count = -1); + +/// Setups the output file for saving statistics. +Expected> +setupStatsFile(StringRef StatsFilename); class LTO; struct SymbolResolution; @@ -110,6 +116,7 @@ private: std::vector> ModuleSymIndices; StringRef TargetTriple, SourceFileName, COFFLinkerOpts; + std::vector DependentLibraries; std::vector ComdatTable; public: @@ -131,6 +138,7 @@ public: using irsymtab::Symbol::isWeak; using irsymtab::Symbol::isIndirect; using irsymtab::Symbol::getName; + using irsymtab::Symbol::getIRName; using irsymtab::Symbol::getVisibility; using irsymtab::Symbol::canBeOmittedFromSymbolTable; using irsymtab::Symbol::isTLS; @@ -140,6 +148,7 @@ public: using irsymtab::Symbol::getCOFFWeakExternalFallback; using irsymtab::Symbol::getSectionName; using irsymtab::Symbol::isExecutable; + using irsymtab::Symbol::isUsed; }; /// A range over the symbols in this InputFile. @@ -148,6 +157,9 @@ public: /// Returns linker options specified in the input file. StringRef getCOFFLinkerOpts() const { return COFFLinkerOpts; } + /// Returns dependent library specifiers from the input file. + ArrayRef getDependentLibraries() const { return DependentLibraries; } + /// Returns the path to the InputFile. StringRef getName() const; @@ -160,6 +172,9 @@ public: // Returns a table with all the comdats used by this file. ArrayRef getComdatTable() const { return ComdatTable; } + // Returns the only BitcodeModule from InputFile. + BitcodeModule &getSingleBitcodeModule(); + private: ArrayRef module_symbols(unsigned I) const { const auto &Indices = ModuleSymIndices[I]; @@ -183,8 +198,8 @@ public: /// the fly. /// /// Stream callbacks must be thread safe. -typedef std::function(unsigned Task)> - AddStreamFn; +using AddStreamFn = + std::function(unsigned Task)>; /// This is the type of a native object cache. To request an item from the /// cache, pass a unique string as the Key. For hits, the cached file will be @@ -198,17 +213,16 @@ typedef std::function(unsigned Task)> /// /// if (AddStreamFn AddStream = Cache(Task, Key)) /// ProduceContent(AddStream); -typedef std::function - NativeObjectCache; +using NativeObjectCache = + std::function; /// A ThinBackend defines what happens after the thin-link phase during ThinLTO. /// The details of this type definition aren't important; clients can only /// create a ThinBackend using one of the create*ThinBackend() functions below. -typedef std::function( +using ThinBackend = std::function( Config &C, ModuleSummaryIndex &CombinedIndex, StringMap &ModuleToDefinedGVSummaries, - AddStreamFn AddStream, NativeObjectCache Cache)> - ThinBackend; + AddStreamFn AddStream, NativeObjectCache Cache)>; /// This ThinBackend runs the individual backend jobs in-process. ThinBackend createInProcessThinBackend(unsigned ParallelismLevel); @@ -397,7 +411,10 @@ private: const SymbolResolution *&ResI, const SymbolResolution *ResE); Error runRegularLTO(AddStreamFn AddStream); - Error runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache); + Error runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache, + const DenseSet &GUIDPreservedSymbols); + + Error checkPartiallySplit(); mutable bool CalledGetMaxTasks = false; diff --git a/include/llvm/LTO/LTOBackend.h b/include/llvm/LTO/LTOBackend.h index d4743f6940ff..4ff8a1993d49 100644 --- a/include/llvm/LTO/LTOBackend.h +++ b/include/llvm/LTO/LTOBackend.h @@ -1,9 +1,8 @@ //===-LTOBackend.h - LLVM Link Time Optimizer Backend ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/LTO/SummaryBasedOptimizations.h b/include/llvm/LTO/SummaryBasedOptimizations.h index ad3a8e7dc77b..6697c821a5ea 100644 --- a/include/llvm/LTO/SummaryBasedOptimizations.h +++ b/include/llvm/LTO/SummaryBasedOptimizations.h @@ -1,9 +1,8 @@ //=- llvm/LTO/SummaryBasedOptimizations.h -Link time optimizations-*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/LTO/legacy/LTOCodeGenerator.h b/include/llvm/LTO/legacy/LTOCodeGenerator.h index 8f23b7cb4574..d3cb4c8b79a0 100644 --- a/include/llvm/LTO/legacy/LTOCodeGenerator.h +++ b/include/llvm/LTO/legacy/LTOCodeGenerator.h @@ -1,9 +1,8 @@ //===-LTOCodeGenerator.h - LLVM Link Time Optimizer -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -242,6 +241,7 @@ private: TargetMachine::CodeGenFileType FileType = TargetMachine::CGFT_ObjectFile; std::unique_ptr DiagnosticOutputFile; bool Freestanding = false; + std::unique_ptr StatsFile = nullptr; }; } #endif diff --git a/include/llvm/LTO/legacy/LTOModule.h b/include/llvm/LTO/legacy/LTOModule.h index 017e223ed8a6..84b9b8c02942 100644 --- a/include/llvm/LTO/legacy/LTOModule.h +++ b/include/llvm/LTO/legacy/LTOModule.h @@ -1,9 +1,8 @@ //===-LTOModule.h - LLVM Link Time Optimizer ------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,6 +17,7 @@ #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringSet.h" #include "llvm/IR/Module.h" +#include "llvm/LTO/LTO.h" #include "llvm/Object/IRObjectFile.h" #include "llvm/Object/ModuleSymbolTable.h" #include "llvm/Target/TargetMachine.h" @@ -48,6 +48,8 @@ private: std::string LinkerOpts; + std::string DependentLibraries; + std::unique_ptr Mod; MemoryBufferRef MBRef; ModuleSymbolTable SymTab; @@ -156,9 +158,17 @@ public: const std::vector &getAsmUndefinedRefs() { return _asm_undefines; } + static lto::InputFile *createInputFile(const void *buffer, size_t buffer_size, + const char *path, std::string &out_error); + + static size_t getDependentLibraryCount(lto::InputFile *input); + + static const char *getDependentLibrary(lto::InputFile *input, size_t index, size_t *size); + private: /// Parse metadata from the module // FIXME: it only parses "llvm.linker.options" metadata at the moment + // FIXME: can't access metadata in lazily loaded modules void parseMetadata(); /// Parse the symbols from the module and model-level ASM and add them to diff --git a/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h b/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h index d4c69a1ce260..210a2ce00bdf 100644 --- a/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h +++ b/include/llvm/LTO/legacy/ThinLTOCodeGenerator.h @@ -1,9 +1,8 @@ //===-ThinLTOCodeGenerator.h - LLVM Link Time Optimizer -------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -20,6 +19,7 @@ #include "llvm/ADT/StringSet.h" #include "llvm/ADT/Triple.h" #include "llvm/IR/ModuleSummaryIndex.h" +#include "llvm/LTO/LTO.h" #include "llvm/Support/CachePruning.h" #include "llvm/Support/CodeGen.h" #include "llvm/Support/MemoryBuffer.h" @@ -32,23 +32,6 @@ class StringRef; class LLVMContext; class TargetMachine; -/// Wrapper around MemoryBufferRef, owning the identifier -class ThinLTOBuffer { - std::string OwnedIdentifier; - StringRef Buffer; - -public: - ThinLTOBuffer(StringRef Buffer, StringRef Identifier) - : OwnedIdentifier(Identifier), Buffer(Buffer) {} - - MemoryBufferRef getMemBuffer() const { - return MemoryBufferRef(Buffer, - {OwnedIdentifier.c_str(), OwnedIdentifier.size()}); - } - StringRef getBuffer() const { return Buffer; } - StringRef getBufferIdentifier() const { return OwnedIdentifier; } -}; - /// Helper to gather options relevant to the target machine creation struct TargetMachineBuilder { Triple TheTriple; @@ -268,37 +251,49 @@ public: * and additionally resolve weak and linkonce symbols. * Index is updated to reflect linkage changes from weak resolution. */ - void promote(Module &Module, ModuleSummaryIndex &Index); + void promote(Module &Module, ModuleSummaryIndex &Index, + const lto::InputFile &File); /** * Compute and emit the imported files for module at \p ModulePath. */ void emitImports(Module &Module, StringRef OutputName, - ModuleSummaryIndex &Index); + ModuleSummaryIndex &Index, + const lto::InputFile &File); /** * Perform cross-module importing for the module identified by * ModuleIdentifier. */ - void crossModuleImport(Module &Module, ModuleSummaryIndex &Index); + void crossModuleImport(Module &Module, ModuleSummaryIndex &Index, + const lto::InputFile &File); /** * Compute the list of summaries needed for importing into module. */ void gatherImportedSummariesForModule( Module &Module, ModuleSummaryIndex &Index, - std::map &ModuleToSummariesForIndex); + std::map &ModuleToSummariesForIndex, + const lto::InputFile &File); /** * Perform internalization. Index is updated to reflect linkage changes. */ - void internalize(Module &Module, ModuleSummaryIndex &Index); + void internalize(Module &Module, ModuleSummaryIndex &Index, + const lto::InputFile &File); /** * Perform post-importing ThinLTO optimizations. */ void optimize(Module &Module); + /** + * Write temporary object file to SavedObjectDirectoryPath, write symlink + * to Cache directory if needed. Returns the path to the generated file in + * SavedObjectsDirectoryPath. + */ + std::string writeGeneratedObject(int count, StringRef CacheEntryPath, + const MemoryBuffer &OutputBuffer); /**@}*/ private: @@ -314,7 +309,7 @@ private: /// Vector holding the input buffers containing the bitcode modules to /// process. - std::vector Modules; + std::vector> Modules; /// Set of symbols that need to be preserved outside of the set of bitcode /// files. diff --git a/include/llvm/LTO/legacy/UpdateCompilerUsed.h b/include/llvm/LTO/legacy/UpdateCompilerUsed.h index 4be0027e97d7..af9d62b4af30 100644 --- a/include/llvm/LTO/legacy/UpdateCompilerUsed.h +++ b/include/llvm/LTO/legacy/UpdateCompilerUsed.h @@ -1,9 +1,8 @@ //==------ UpdateCompilerUsed.h - LLVM Link Time Optimizer Utility --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/LineEditor/LineEditor.h b/include/llvm/LineEditor/LineEditor.h index 68995d0633ad..0beaf1bb23a9 100644 --- a/include/llvm/LineEditor/LineEditor.h +++ b/include/llvm/LineEditor/LineEditor.h @@ -1,9 +1,8 @@ //===-- llvm/LineEditor/LineEditor.h - line editor --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/LinkAllIR.h b/include/llvm/LinkAllIR.h index 4f4af7187be4..4b0aabeee701 100644 --- a/include/llvm/LinkAllIR.h +++ b/include/llvm/LinkAllIR.h @@ -1,9 +1,8 @@ //===----- LinkAllIR.h - Reference All VMCore Code --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/LinkAllPasses.h b/include/llvm/LinkAllPasses.h index 0851c2f8d265..675d179eb22a 100644 --- a/include/llvm/LinkAllPasses.h +++ b/include/llvm/LinkAllPasses.h @@ -1,9 +1,8 @@ //===- llvm/LinkAllPasses.h ------------ Reference All Passes ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -42,6 +41,7 @@ #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h" #include "llvm/Transforms/IPO.h" #include "llvm/Transforms/IPO/AlwaysInliner.h" +#include "llvm/Transforms/IPO/Attributor.h" #include "llvm/Transforms/IPO/FunctionAttrs.h" #include "llvm/Transforms/InstCombine/InstCombine.h" #include "llvm/Transforms/Instrumentation.h" @@ -103,6 +103,7 @@ namespace { (void) llvm::createGCOVProfilerPass(); (void) llvm::createPGOInstrumentationGenLegacyPass(); (void) llvm::createPGOInstrumentationUseLegacyPass(); + (void) llvm::createPGOInstrumentationGenCreateVarLegacyPass(); (void) llvm::createPGOIndirectCallPromotionLegacyPass(); (void) llvm::createPGOMemOPSizeOptLegacyPass(); (void) llvm::createInstrProfilingLegacyPass(); @@ -188,10 +189,11 @@ namespace { (void) llvm::createPostDomTree(); (void) llvm::createInstructionNamerPass(); (void) llvm::createMetaRenamerPass(); + (void) llvm::createAttributorLegacyPass(); (void) llvm::createPostOrderFunctionAttrsLegacyPass(); (void) llvm::createReversePostOrderFunctionAttrsPass(); (void) llvm::createMergeFunctionsPass(); - (void) llvm::createMergeICmpsPass(); + (void) llvm::createMergeICmpsLegacyPass(); (void) llvm::createExpandMemCmpPass(); std::string buf; llvm::raw_string_ostream os(buf); @@ -221,6 +223,7 @@ namespace { (void) llvm::createEliminateAvailableExternallyPass(); (void) llvm::createScalarizeMaskedMemIntrinPass(); (void) llvm::createWarnMissedTransformationsPass(); + (void) llvm::createHardwareLoopsPass(); (void)new llvm::IntervalPartition(); (void)new llvm::ScalarEvolutionWrapperPass(); diff --git a/include/llvm/Linker/IRMover.h b/include/llvm/Linker/IRMover.h index 235ada47cef4..e5df83f01fe3 100644 --- a/include/llvm/Linker/IRMover.h +++ b/include/llvm/Linker/IRMover.h @@ -1,9 +1,8 @@ //===- IRMover.h ------------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Linker/Linker.h b/include/llvm/Linker/Linker.h index 7776c720ec53..c9b1d42b3903 100644 --- a/include/llvm/Linker/Linker.h +++ b/include/llvm/Linker/Linker.h @@ -1,9 +1,8 @@ //===- Linker.h - Module Linker Interface -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/ConstantPools.h b/include/llvm/MC/ConstantPools.h index ef33250204ec..2fe5ce252c94 100644 --- a/include/llvm/MC/ConstantPools.h +++ b/include/llvm/MC/ConstantPools.h @@ -1,9 +1,8 @@ -//===- ConstantPool.h - Keep track of assembler-generated ------*- C++ -*-===// +//===- ConstantPools.h - Keep track of assembler-generated ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/MC/LaneBitmask.h b/include/llvm/MC/LaneBitmask.h index 8c0b4ecb8fd4..d5f69287a265 100644 --- a/include/llvm/MC/LaneBitmask.h +++ b/include/llvm/MC/LaneBitmask.h @@ -1,9 +1,8 @@ //===- llvm/MC/LaneBitmask.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/include/llvm/MC/MCAsmBackend.h b/include/llvm/MC/MCAsmBackend.h index 07835c21fced..1f3ad6c1e547 100644 --- a/include/llvm/MC/MCAsmBackend.h +++ b/include/llvm/MC/MCAsmBackend.h @@ -1,9 +1,8 @@ //===- llvm/MC/MCAsmBackend.h - MC Asm Backend ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -88,6 +87,22 @@ public: return false; } + /// Hook to check if extra nop bytes must be inserted for alignment directive. + /// For some targets this may be necessary in order to support linker + /// relaxation. The number of bytes to insert are returned in Size. + virtual bool shouldInsertExtraNopBytesForCodeAlign(const MCAlignFragment &AF, + unsigned &Size) { + return false; + } + + /// Hook which indicates if the target requires a fixup to be generated when + /// handling an align directive in an executable section + virtual bool shouldInsertFixupForCodeAlign(MCAssembler &Asm, + const MCAsmLayout &Layout, + MCAlignFragment &AF) { + return false; + } + /// Apply the \p Value for given \p Fixup into the provided data fragment, at /// the offset specified by the fixup and following the fixup kind as /// appropriate. Errors (such as an out of range fixup value) should be diff --git a/include/llvm/MC/MCAsmInfo.h b/include/llvm/MC/MCAsmInfo.h index 120fb8fa7492..971e9354da8c 100644 --- a/include/llvm/MC/MCAsmInfo.h +++ b/include/llvm/MC/MCAsmInfo.h @@ -1,9 +1,8 @@ //===-- llvm/MC/MCAsmInfo.h - Asm info --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,16 +17,17 @@ #include "llvm/ADT/StringRef.h" #include "llvm/MC/MCDirectives.h" -#include "llvm/MC/MCDwarf.h" #include "llvm/MC/MCTargetOptions.h" #include namespace llvm { class MCContext; +class MCCFIInstruction; class MCExpr; class MCSection; class MCStreamer; +class MCSubtargetInfo; class MCSymbol; namespace WinEH { @@ -474,7 +474,13 @@ public: bool hasMachoTBSSDirective() const { return HasMachoTBSSDirective; } bool hasCOFFAssociativeComdats() const { return HasCOFFAssociativeComdats; } bool hasCOFFComdatConstants() const { return HasCOFFComdatConstants; } - unsigned getMaxInstLength() const { return MaxInstLength; } + + /// Returns the maximum possible encoded instruction size in bytes. If \p STI + /// is null, this should be the maximum size for any subtarget. + virtual unsigned getMaxInstLength(const MCSubtargetInfo *STI = nullptr) const { + return MaxInstLength; + } + unsigned getMinInstAlignment() const { return MinInstAlignment; } bool getDollarIsPC() const { return DollarIsPC; } const char *getSeparatorString() const { return SeparatorString; } @@ -492,7 +498,7 @@ public: StringRef getPrivateLabelPrefix() const { return PrivateLabelPrefix; } bool hasLinkerPrivateGlobalPrefix() const { - return LinkerPrivateGlobalPrefix[0] != '\0'; + return !LinkerPrivateGlobalPrefix.empty(); } StringRef getLinkerPrivateGlobalPrefix() const { @@ -598,9 +604,7 @@ public: return SupportsExtendedDwarfLocDirective; } - void addInitialFrameState(const MCCFIInstruction &Inst) { - InitialFrameState.push_back(Inst); - } + void addInitialFrameState(const MCCFIInstruction &Inst); const std::vector &getInitialFrameState() const { return InitialFrameState; diff --git a/include/llvm/MC/MCAsmInfoCOFF.h b/include/llvm/MC/MCAsmInfoCOFF.h index 01c8ae49a6fc..1dfb4750af66 100644 --- a/include/llvm/MC/MCAsmInfoCOFF.h +++ b/include/llvm/MC/MCAsmInfoCOFF.h @@ -1,9 +1,8 @@ //===- MCAsmInfoCOFF.h - COFF asm properties --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCAsmInfoDarwin.h b/include/llvm/MC/MCAsmInfoDarwin.h index a533d604a89e..c889ce99cebe 100644 --- a/include/llvm/MC/MCAsmInfoDarwin.h +++ b/include/llvm/MC/MCAsmInfoDarwin.h @@ -1,9 +1,8 @@ //===- MCAsmInfoDarwin.h - Darwin asm properties ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/MC/MCAsmInfoELF.h b/include/llvm/MC/MCAsmInfoELF.h index f113afc9885e..aa2e5873e2c6 100644 --- a/include/llvm/MC/MCAsmInfoELF.h +++ b/include/llvm/MC/MCAsmInfoELF.h @@ -1,9 +1,8 @@ //===- llvm/MC/MCAsmInfoELF.h - ELF Asm info --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCAsmInfoWasm.h b/include/llvm/MC/MCAsmInfoWasm.h index 71c6ee28df70..3afc610b8b07 100644 --- a/include/llvm/MC/MCAsmInfoWasm.h +++ b/include/llvm/MC/MCAsmInfoWasm.h @@ -1,9 +1,8 @@ //===-- llvm/MC/MCAsmInfoWasm.h - Wasm Asm info -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCAsmInfoXCOFF.h b/include/llvm/MC/MCAsmInfoXCOFF.h new file mode 100644 index 000000000000..2a72ba7398a7 --- /dev/null +++ b/include/llvm/MC/MCAsmInfoXCOFF.h @@ -0,0 +1,25 @@ +//===- MCAsmInfoXCOFF.h - XCOFF asm properties ----------------- *- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_MC_MCASMINFOXCOFF_H +#define LLVM_MC_MCASMINFOXCOFF_H + +#include "llvm/MC/MCAsmInfo.h" + +namespace llvm { + +class MCAsmInfoXCOFF : public MCAsmInfo { + virtual void anchor(); + +protected: + MCAsmInfoXCOFF(); +}; + +} // end namespace llvm + +#endif // LLVM_MC_MCASMINFOXCOFF_H diff --git a/include/llvm/MC/MCAsmLayout.h b/include/llvm/MC/MCAsmLayout.h index b711db319302..45ac96f0b81e 100644 --- a/include/llvm/MC/MCAsmLayout.h +++ b/include/llvm/MC/MCAsmLayout.h @@ -1,9 +1,8 @@ //===- MCAsmLayout.h - Assembly Layout Object -------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCAsmMacro.h b/include/llvm/MC/MCAsmMacro.h index 135fa4f2e33d..364d3b5f3666 100644 --- a/include/llvm/MC/MCAsmMacro.h +++ b/include/llvm/MC/MCAsmMacro.h @@ -1,9 +1,8 @@ //===- MCAsmMacro.h - Assembly Macros ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCAssembler.h b/include/llvm/MC/MCAssembler.h index 986c6e17548f..4543018901a4 100644 --- a/include/llvm/MC/MCAssembler.h +++ b/include/llvm/MC/MCAssembler.h @@ -1,9 +1,8 @@ //===- MCAssembler.h - Object File Generation -------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCCodeEmitter.h b/include/llvm/MC/MCCodeEmitter.h index f1b0b784a2df..04b4367ada7b 100644 --- a/include/llvm/MC/MCCodeEmitter.h +++ b/include/llvm/MC/MCCodeEmitter.h @@ -1,9 +1,8 @@ //===- llvm/MC/MCCodeEmitter.h - Instruction Encoding -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCCodePadder.h b/include/llvm/MC/MCCodePadder.h index 4dde6bf59272..f7b1a2113a9a 100644 --- a/include/llvm/MC/MCCodePadder.h +++ b/include/llvm/MC/MCCodePadder.h @@ -1,9 +1,8 @@ -//===- llvm/MC/CodePadder.h - MC Code Padder --------------------*- C++ -*-===// +//===- llvm/MC/MCCodePadder.h - MC Code Padder ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCCodeView.h b/include/llvm/MC/MCCodeView.h index cef03a409f95..2126354cded6 100644 --- a/include/llvm/MC/MCCodeView.h +++ b/include/llvm/MC/MCCodeView.h @@ -1,9 +1,8 @@ //===- MCCodeView.h - Machine Code CodeView support -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/MC/MCContext.h b/include/llvm/MC/MCContext.h index 3b8ac8b79e21..5c2124cc0d15 100644 --- a/include/llvm/MC/MCContext.h +++ b/include/llvm/MC/MCContext.h @@ -1,9 +1,8 @@ //===- MCContext.h - Machine Code Context -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -19,6 +18,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/BinaryFormat/Dwarf.h" +#include "llvm/BinaryFormat/XCOFF.h" #include "llvm/MC/MCAsmMacro.h" #include "llvm/MC/MCDwarf.h" #include "llvm/MC/MCSubtargetInfo.h" @@ -50,6 +50,7 @@ namespace llvm { class MCSectionELF; class MCSectionMachO; class MCSectionWasm; + class MCSectionXCOFF; class MCStreamer; class MCSymbol; class MCSymbolELF; @@ -92,6 +93,7 @@ namespace llvm { SpecificBumpPtrAllocator ELFAllocator; SpecificBumpPtrAllocator MachOAllocator; SpecificBumpPtrAllocator WasmAllocator; + SpecificBumpPtrAllocator XCOFFAllocator; /// Bindings of names to symbols. SymbolTable Symbols; @@ -247,10 +249,25 @@ namespace llvm { } }; + struct XCOFFSectionKey { + std::string SectionName; + XCOFF::StorageMappingClass MappingClass; + + XCOFFSectionKey(StringRef SectionName, + XCOFF::StorageMappingClass MappingClass) + : SectionName(SectionName), MappingClass(MappingClass) {} + + bool operator<(const XCOFFSectionKey &Other) const { + return std::tie(SectionName, MappingClass) < + std::tie(Other.SectionName, Other.MappingClass); + } + }; + StringMap MachOUniquingMap; std::map ELFUniquingMap; std::map COFFUniquingMap; std::map WasmUniquingMap; + std::map XCOFFUniquingMap; StringMap RelSecNames; SpecificBumpPtrAllocator MCSubtargetAllocator; @@ -441,8 +458,6 @@ namespace llvm { SectionKind Kind, const char *BeginSymName = nullptr); - MCSectionCOFF *getCOFFSection(StringRef Section); - /// Gets or creates a section equivalent to Sec that is associated with the /// section containing KeySym. For example, to create a debug info section /// associated with an inline function, pass the normal debug info section @@ -473,6 +488,11 @@ namespace llvm { const MCSymbolWasm *Group, unsigned UniqueID, const char *BeginSymName); + MCSectionXCOFF *getXCOFFSection(StringRef Section, + XCOFF::StorageMappingClass MappingClass, + SectionKind K, + const char *BeginSymName = nullptr); + // Create and save a copy of STI and return a reference to the copy. MCSubtargetInfo &getSubtargetCopy(const MCSubtargetInfo &STI); @@ -489,12 +509,6 @@ namespace llvm { /// Set the compilation directory for DW_AT_comp_dir void setCompilationDir(StringRef S) { CompilationDir = S.str(); } - /// Get the debug prefix map. - const std::map & - getDebugPrefixMap() const { - return DebugPrefixMap; - } - /// Add an entry to the debug prefix map. void addDebugPrefixMapEntry(const std::string &From, const std::string &To); @@ -512,7 +526,7 @@ namespace llvm { /// Creates an entry in the dwarf file and directory tables. Expected getDwarfFile(StringRef Directory, StringRef FileName, unsigned FileNumber, - MD5::MD5Result *Checksum, + Optional Checksum, Optional Source, unsigned CUID); bool isValidDwarfFileNumber(unsigned FileNumber, unsigned CUID = 0); @@ -539,13 +553,6 @@ namespace llvm { return getMCDwarfLineTable(CUID).getMCDwarfDirs(); } - bool hasMCLineSections() const { - for (const auto &Table : MCDwarfLineTablesCUMap) - if (!Table.second.getMCDwarfFiles().empty() || Table.second.getLabel()) - return true; - return false; - } - unsigned getDwarfCompileUnitID() { return DwarfCompileUnitID; } void setDwarfCompileUnitID(unsigned CUIndex) { @@ -555,7 +562,8 @@ namespace llvm { /// Specifies the "root" file and directory of the compilation unit. /// These are "file 0" and "directory 0" in DWARF v5. void setMCLineTableRootFile(unsigned CUID, StringRef CompilationDir, - StringRef Filename, MD5::MD5Result *Checksum, + StringRef Filename, + Optional Checksum, Optional Source) { getMCDwarfLineTable(CUID).setRootFile(CompilationDir, Filename, Checksum, Source); @@ -595,6 +603,10 @@ namespace llvm { GenDwarfFileNumber = FileNumber; } + /// Specifies information about the "root file" for assembler clients + /// (e.g., llvm-mc). Assumes compilation dir etc. have been set up. + void setGenDwarfRootFile(StringRef FileName, StringRef Buffer); + const SetVector &getGenDwarfSectionSyms() { return SectionsForRanges; } diff --git a/include/llvm/MC/MCDirectives.h b/include/llvm/MC/MCDirectives.h index 8c74b169135b..4029264c2026 100644 --- a/include/llvm/MC/MCDirectives.h +++ b/include/llvm/MC/MCDirectives.h @@ -1,9 +1,8 @@ //===- MCDirectives.h - Enums for directives on various targets -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -20,6 +19,7 @@ enum MCSymbolAttr { MCSA_Invalid = 0, ///< Not a valid directive. // Various directives in alphabetical order. + MCSA_Cold, ///< .cold (MachO) MCSA_ELF_TypeFunction, ///< .type _foo, STT_FUNC # aka @function MCSA_ELF_TypeIndFunction, ///< .type _foo, STT_GNU_IFUNC MCSA_ELF_TypeObject, ///< .type _foo, STT_OBJECT # aka @object diff --git a/include/llvm/MC/MCDisassembler/MCDisassembler.h b/include/llvm/MC/MCDisassembler/MCDisassembler.h index 7f09c05ccf2a..268f3ccad889 100644 --- a/include/llvm/MC/MCDisassembler/MCDisassembler.h +++ b/include/llvm/MC/MCDisassembler/MCDisassembler.h @@ -1,9 +1,8 @@ //===- llvm/MC/MCDisassembler.h - Disassembler interface --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -17,6 +16,7 @@ namespace llvm { template class ArrayRef; +class StringRef; class MCContext; class MCInst; class MCSubtargetInfo; @@ -80,6 +80,23 @@ public: raw_ostream &VStream, raw_ostream &CStream) const = 0; + /// May parse any prelude that precedes instructions after the start of a + /// symbol. Needed for some targets, e.g. WebAssembly. + /// + /// \param Name - The name of the symbol. + /// \param Size - The number of bytes consumed. + /// \param Address - The address, in the memory space of region, of the first + /// byte of the symbol. + /// \param Bytes - A reference to the actual bytes at the symbol location. + /// \param VStream - The stream to print warnings and diagnostic messages on. + /// \param CStream - The stream to print comments and annotations on. + /// \return - MCDisassembler::Success if the bytes are valid, + /// MCDisassembler::Fail if the bytes were invalid. + virtual DecodeStatus onSymbolStart(StringRef Name, uint64_t &Size, + ArrayRef Bytes, uint64_t Address, + raw_ostream &VStream, + raw_ostream &CStream) const; + private: MCContext &Ctx; diff --git a/include/llvm/MC/MCDisassembler/MCExternalSymbolizer.h b/include/llvm/MC/MCDisassembler/MCExternalSymbolizer.h index df909a0dccd3..ffac5ee5cb1f 100644 --- a/include/llvm/MC/MCDisassembler/MCExternalSymbolizer.h +++ b/include/llvm/MC/MCDisassembler/MCExternalSymbolizer.h @@ -1,9 +1,8 @@ //===-- llvm/MC/MCExternalSymbolizer.h - ------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/MC/MCDisassembler/MCRelocationInfo.h b/include/llvm/MC/MCDisassembler/MCRelocationInfo.h index 6030ae660d38..efc59da19335 100644 --- a/include/llvm/MC/MCDisassembler/MCRelocationInfo.h +++ b/include/llvm/MC/MCDisassembler/MCRelocationInfo.h @@ -1,9 +1,8 @@ //===- llvm/MC/MCRelocationInfo.h -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/MC/MCDisassembler/MCSymbolizer.h b/include/llvm/MC/MCDisassembler/MCSymbolizer.h index 0bfa569474ec..b7ca83a5f16c 100644 --- a/include/llvm/MC/MCDisassembler/MCSymbolizer.h +++ b/include/llvm/MC/MCDisassembler/MCSymbolizer.h @@ -1,9 +1,8 @@ //===- llvm/MC/MCSymbolizer.h - MCSymbolizer class --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/MC/MCDwarf.h b/include/llvm/MC/MCDwarf.h index 7b96e9aaca89..1a37aafd0654 100644 --- a/include/llvm/MC/MCDwarf.h +++ b/include/llvm/MC/MCDwarf.h @@ -1,9 +1,8 @@ //===- MCDwarf.h - Machine Code Dwarf support -------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -42,11 +41,14 @@ class raw_ostream; class SMLoc; class SourceMgr; -/// Instances of this class represent the name of the dwarf -/// .file directive and its associated dwarf file number in the MC file, -/// and MCDwarfFile's are created and uniqued by the MCContext class where -/// the file number for each is its index into the vector of DwarfFiles (note -/// index 0 is not used and not a valid dwarf file number). +/// Instances of this class represent the name of the dwarf .file directive and +/// its associated dwarf file number in the MC file. MCDwarfFile's are created +/// and uniqued by the MCContext class. In Dwarf 4 file numbers start from 1; +/// i.e. the entry with file number 1 is the first element in the vector of +/// DwarfFiles and there is no MCDwarfFile with file number 0. In Dwarf 5 file +/// numbers start from 0, with the MCDwarfFile with file number 0 being the +/// primary source file, and file numbers correspond to their index in the +/// vector. struct MCDwarfFile { // The base name of the file without its directory path. std::string Name; @@ -56,7 +58,7 @@ struct MCDwarfFile { /// The MD5 checksum, if there is one. Non-owning pointer to data allocated /// in MCContext. - MD5::MD5Result *Checksum = nullptr; + Optional Checksum; /// The source code of the file. Non-owning reference to data allocated in /// MCContext. @@ -224,8 +226,9 @@ public: MCDwarfLineTableHeader() = default; Expected tryGetFile(StringRef &Directory, StringRef &FileName, - MD5::MD5Result *Checksum, - Optional &Source, + Optional Checksum, + Optional Source, + uint16_t DwarfVersion, unsigned FileNumber = 0); std::pair Emit(MCStreamer *MCOS, MCDwarfLineTableParams Params, @@ -246,32 +249,50 @@ public: return MCDwarfFiles.empty() || (HasAllMD5 == HasAnyMD5); } + void setRootFile(StringRef Directory, StringRef FileName, + Optional Checksum, + Optional Source) { + CompilationDir = Directory; + RootFile.Name = FileName; + RootFile.DirIndex = 0; + RootFile.Checksum = Checksum; + RootFile.Source = Source; + trackMD5Usage(Checksum.hasValue()); + HasSource = Source.hasValue(); + } + + void resetFileTable() { + MCDwarfDirs.clear(); + MCDwarfFiles.clear(); + RootFile.Name.clear(); + resetMD5Usage(); + HasSource = false; + } + private: void emitV2FileDirTables(MCStreamer *MCOS) const; - void emitV5FileDirTables(MCStreamer *MCOS, Optional &LineStr, - StringRef CtxCompilationDir) const; + void emitV5FileDirTables(MCStreamer *MCOS, Optional &LineStr) const; }; class MCDwarfDwoLineTable { MCDwarfLineTableHeader Header; + bool HasSplitLineTable = false; public: void maybeSetRootFile(StringRef Directory, StringRef FileName, - MD5::MD5Result *Checksum, Optional Source) { + Optional Checksum, + Optional Source) { if (!Header.RootFile.Name.empty()) return; - Header.CompilationDir = Directory; - Header.RootFile.Name = FileName; - Header.RootFile.DirIndex = 0; - Header.RootFile.Checksum = Checksum; - Header.RootFile.Source = Source; - Header.trackMD5Usage(Checksum); - Header.HasSource = Source.hasValue(); + Header.setRootFile(Directory, FileName, Checksum, Source); } unsigned getFile(StringRef Directory, StringRef FileName, - MD5::MD5Result *Checksum, Optional Source) { - return cantFail(Header.tryGetFile(Directory, FileName, Checksum, Source)); + Optional Checksum, uint16_t DwarfVersion, + Optional Source) { + HasSplitLineTable = true; + return cantFail(Header.tryGetFile(Directory, FileName, Checksum, Source, + DwarfVersion)); } void Emit(MCStreamer &MCOS, MCDwarfLineTableParams Params, @@ -291,36 +312,34 @@ public: Optional &LineStr) const; Expected tryGetFile(StringRef &Directory, StringRef &FileName, - MD5::MD5Result *Checksum, + Optional Checksum, Optional Source, + uint16_t DwarfVersion, unsigned FileNumber = 0); unsigned getFile(StringRef &Directory, StringRef &FileName, - MD5::MD5Result *Checksum, Optional &Source, - unsigned FileNumber = 0) { + Optional Checksum, Optional Source, + uint16_t DwarfVersion, unsigned FileNumber = 0) { return cantFail(tryGetFile(Directory, FileName, Checksum, Source, - FileNumber)); + DwarfVersion, FileNumber)); } void setRootFile(StringRef Directory, StringRef FileName, - MD5::MD5Result *Checksum, Optional Source) { + Optional Checksum, Optional Source) { Header.CompilationDir = Directory; Header.RootFile.Name = FileName; Header.RootFile.DirIndex = 0; Header.RootFile.Checksum = Checksum; Header.RootFile.Source = Source; - Header.trackMD5Usage(Checksum); + Header.trackMD5Usage(Checksum.hasValue()); Header.HasSource = Source.hasValue(); } - void resetRootFile() { - assert(Header.MCDwarfFiles.empty()); - Header.RootFile.Name.clear(); - Header.resetMD5Usage(); - Header.HasSource = false; - } + void resetFileTable() { Header.resetFileTable(); } bool hasRootFile() const { return !Header.RootFile.Name.empty(); } + const MCDwarfFile &getRootFile() const { return Header.RootFile; } + // Report whether MD5 usage has been consistent (all-or-none). bool isMD5UsageConsistent() const { return Header.isMD5UsageConsistent(); } diff --git a/include/llvm/MC/MCELFObjectWriter.h b/include/llvm/MC/MCELFObjectWriter.h index f226d6a45a5a..2d441fdeee28 100644 --- a/include/llvm/MC/MCELFObjectWriter.h +++ b/include/llvm/MC/MCELFObjectWriter.h @@ -1,9 +1,8 @@ //===- llvm/MC/MCELFObjectWriter.h - ELF Object Writer ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -54,13 +53,14 @@ struct ELFRelocationEntry { class MCELFObjectTargetWriter : public MCObjectTargetWriter { const uint8_t OSABI; + const uint8_t ABIVersion; const uint16_t EMachine; const unsigned HasRelocationAddend : 1; const unsigned Is64Bit : 1; protected: MCELFObjectTargetWriter(bool Is64Bit_, uint8_t OSABI_, uint16_t EMachine_, - bool HasRelocationAddend); + bool HasRelocationAddend_, uint8_t ABIVersion_ = 0); public: virtual ~MCELFObjectTargetWriter() = default; @@ -98,6 +98,7 @@ public: /// \name Accessors /// @{ uint8_t getOSABI() const { return OSABI; } + uint8_t getABIVersion() const { return ABIVersion; } uint16_t getEMachine() const { return EMachine; } bool hasRelocationAddend() const { return HasRelocationAddend; } bool is64Bit() const { return Is64Bit; } diff --git a/include/llvm/MC/MCELFStreamer.h b/include/llvm/MC/MCELFStreamer.h index 3797079661e4..8838d53d75b5 100644 --- a/include/llvm/MC/MCELFStreamer.h +++ b/include/llvm/MC/MCELFStreamer.h @@ -1,9 +1,8 @@ //===- MCELFStreamer.h - MCStreamer ELF Object File Interface ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCExpr.h b/include/llvm/MC/MCExpr.h index 8cb6b86fd672..fb23c0114c76 100644 --- a/include/llvm/MC/MCExpr.h +++ b/include/llvm/MC/MCExpr.h @@ -1,9 +1,8 @@ //===- MCExpr.h - Assembly Level Expressions --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -135,15 +134,21 @@ inline raw_ostream &operator<<(raw_ostream &OS, const MCExpr &E) { //// Represent a constant integer expression. class MCConstantExpr : public MCExpr { int64_t Value; + bool PrintInHex = false; - explicit MCConstantExpr(int64_t Value) + MCConstantExpr(int64_t Value) : MCExpr(MCExpr::Constant, SMLoc()), Value(Value) {} + MCConstantExpr(int64_t Value, bool PrintInHex) + : MCExpr(MCExpr::Constant, SMLoc()), Value(Value), + PrintInHex(PrintInHex) {} + public: /// \name Construction /// @{ - static const MCConstantExpr *create(int64_t Value, MCContext &Ctx); + static const MCConstantExpr *create(int64_t Value, MCContext &Ctx, + bool PrintInHex = false); /// @} /// \name Accessors @@ -151,6 +156,8 @@ public: int64_t getValue() const { return Value; } + bool useHexFormat() const { return PrintInHex; } + /// @} static bool classof(const MCExpr *E) { @@ -285,16 +292,17 @@ public: VK_Hexagon_IE, VK_Hexagon_IE_GOT, - VK_WebAssembly_FUNCTION, // Function table index, rather than virtual addr - VK_WebAssembly_GLOBAL, // Global object index - VK_WebAssembly_TYPEINDEX,// Type table index - VK_WebAssembly_EVENT, // Event index + VK_WASM_TYPEINDEX, // Reference to a symbol's type (signature) + VK_WASM_MBREL, // Memory address relative to memory base + VK_WASM_TBREL, // Table index relative to table bare VK_AMDGPU_GOTPCREL32_LO, // symbol@gotpcrel32@lo VK_AMDGPU_GOTPCREL32_HI, // symbol@gotpcrel32@hi VK_AMDGPU_REL32_LO, // symbol@rel32@lo VK_AMDGPU_REL32_HI, // symbol@rel32@hi VK_AMDGPU_REL64, // symbol@rel64 + VK_AMDGPU_ABS32_LO, // symbol@abs32@lo + VK_AMDGPU_ABS32_HI, // symbol@abs32@hi VK_TPREL, VK_DTPREL diff --git a/include/llvm/MC/MCFixedLenDisassembler.h b/include/llvm/MC/MCFixedLenDisassembler.h index ad34d9494bb9..218ae0d13189 100644 --- a/include/llvm/MC/MCFixedLenDisassembler.h +++ b/include/llvm/MC/MCFixedLenDisassembler.h @@ -1,9 +1,8 @@ //===-- llvm/MC/MCFixedLenDisassembler.h - Decoder driver -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Fixed length disassembler decoder state machine driver. diff --git a/include/llvm/MC/MCFixup.h b/include/llvm/MC/MCFixup.h index 5f301eafc556..accffb7f2247 100644 --- a/include/llvm/MC/MCFixup.h +++ b/include/llvm/MC/MCFixup.h @@ -1,9 +1,8 @@ //===-- llvm/MC/MCFixup.h - Instruction Relocation and Patching -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -21,7 +20,8 @@ class MCExpr; /// Extensible enumeration to represent the type of a fixup. enum MCFixupKind { - FK_Data_1 = 0, ///< A one-byte fixup. + FK_NONE = 0, ///< A no-op fixup. + FK_Data_1, ///< A one-byte fixup. FK_Data_2, ///< A two-byte fixup. FK_Data_4, ///< A four-byte fixup. FK_Data_8, ///< A eight-byte fixup. diff --git a/include/llvm/MC/MCFixupKindInfo.h b/include/llvm/MC/MCFixupKindInfo.h index 483abb39403f..0ea34866db6a 100644 --- a/include/llvm/MC/MCFixupKindInfo.h +++ b/include/llvm/MC/MCFixupKindInfo.h @@ -1,9 +1,8 @@ //===-- llvm/MC/MCFixupKindInfo.h - Fixup Descriptors -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCFragment.h b/include/llvm/MC/MCFragment.h index c999c9fc4f17..aadf2ce725ea 100644 --- a/include/llvm/MC/MCFragment.h +++ b/include/llvm/MC/MCFragment.h @@ -1,9 +1,8 @@ //===- MCFragment.h - Fragment type hierarchy -------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCInst.h b/include/llvm/MC/MCInst.h index d501b686bb2e..8df8096bba94 100644 --- a/include/llvm/MC/MCInst.h +++ b/include/llvm/MC/MCInst.h @@ -1,9 +1,8 @@ //===- llvm/MC/MCInst.h - MCInst class --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -154,8 +153,6 @@ public: bool evaluateAsConstantImm(int64_t &Imm) const; }; -template <> struct isPodLike { static const bool value = true; }; - /// Instances of this class represent a single low-level machine /// instruction. class MCInst { @@ -190,6 +187,7 @@ public: void clear() { Operands.clear(); } void erase(iterator I) { Operands.erase(I); } + void erase(iterator First, iterator Last) { Operands.erase(First, Last); } size_t size() const { return Operands.size(); } iterator begin() { return Operands.begin(); } const_iterator begin() const { return Operands.begin(); } diff --git a/include/llvm/MC/MCInstBuilder.h b/include/llvm/MC/MCInstBuilder.h index c5c4f481e7df..0c8e01fdc412 100644 --- a/include/llvm/MC/MCInstBuilder.h +++ b/include/llvm/MC/MCInstBuilder.h @@ -1,9 +1,8 @@ //===-- llvm/MC/MCInstBuilder.h - Simplify creation of MCInsts --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/MC/MCInstPrinter.h b/include/llvm/MC/MCInstPrinter.h index df221e1db0e7..6bbc4bc2903b 100644 --- a/include/llvm/MC/MCInstPrinter.h +++ b/include/llvm/MC/MCInstPrinter.h @@ -1,9 +1,8 @@ //===- MCInstPrinter.h - MCInst to target assembly syntax -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -65,6 +64,10 @@ public: virtual ~MCInstPrinter(); + /// Customize the printer according to a command line option. + /// @return true if the option is recognized and applied. + virtual bool applyTargetSpecificCLOption(StringRef Opt) { return false; } + /// Specify a stream to emit comments to. void setCommentStream(raw_ostream &OS) { CommentStream = &OS; } diff --git a/include/llvm/MC/MCInstrAnalysis.h b/include/llvm/MC/MCInstrAnalysis.h index 200f10f7d64b..dfefd7e72777 100644 --- a/include/llvm/MC/MCInstrAnalysis.h +++ b/include/llvm/MC/MCInstrAnalysis.h @@ -1,9 +1,8 @@ //===- llvm/MC/MCInstrAnalysis.h - InstrDesc target hooks -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/MC/MCInstrDesc.h b/include/llvm/MC/MCInstrDesc.h index 61e7d09afbcb..0aa586dfc901 100644 --- a/include/llvm/MC/MCInstrDesc.h +++ b/include/llvm/MC/MCInstrDesc.h @@ -1,9 +1,8 @@ //===-- llvm/MC/MCInstrDesc.h - Instruction Descriptors -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -135,6 +134,7 @@ enum Flag { FoldableAsLoad, MayLoad, MayStore, + MayRaiseFPException, Predicable, NotDuplicable, UnmodeledSideEffects, @@ -404,6 +404,11 @@ public: /// may not actually modify anything, for example. bool mayStore() const { return Flags & (1ULL << MCID::MayStore); } + /// Return true if this instruction may raise a floating-point exception. + bool mayRaiseFPException() const { + return Flags & (1ULL << MCID::MayRaiseFPException); + } + /// Return true if this instruction has side /// effects that are not modeled by other flags. This does not return true /// for instructions whose effects are captured by: diff --git a/include/llvm/MC/MCInstrInfo.h b/include/llvm/MC/MCInstrInfo.h index 18da87cf8929..874b1e46795b 100644 --- a/include/llvm/MC/MCInstrInfo.h +++ b/include/llvm/MC/MCInstrInfo.h @@ -1,9 +1,8 @@ //===-- llvm/MC/MCInstrInfo.h - Target Instruction Info ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/MC/MCInstrItineraries.h b/include/llvm/MC/MCInstrItineraries.h index fe81376e0db7..485aa663272e 100644 --- a/include/llvm/MC/MCInstrItineraries.h +++ b/include/llvm/MC/MCInstrItineraries.h @@ -1,9 +1,8 @@ //===- llvm/MC/MCInstrItineraries.h - Scheduling ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/MC/MCLabel.h b/include/llvm/MC/MCLabel.h index aaf70691fc01..0b8afac8f754 100644 --- a/include/llvm/MC/MCLabel.h +++ b/include/llvm/MC/MCLabel.h @@ -1,9 +1,8 @@ //===- MCLabel.h - Machine Code Directional Local Labels --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/MC/MCLinkerOptimizationHint.h b/include/llvm/MC/MCLinkerOptimizationHint.h index f0fd07f43cf3..f2a1364ad884 100644 --- a/include/llvm/MC/MCLinkerOptimizationHint.h +++ b/include/llvm/MC/MCLinkerOptimizationHint.h @@ -1,10 +1,9 @@ //===- MCLinkerOptimizationHint.h - LOH interface ---------------*- C++ -*-===// // // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/MC/MCMachObjectWriter.h b/include/llvm/MC/MCMachObjectWriter.h index 22fbeb72a4ec..278aebee99ac 100644 --- a/include/llvm/MC/MCMachObjectWriter.h +++ b/include/llvm/MC/MCMachObjectWriter.h @@ -1,9 +1,8 @@ //===- llvm/MC/MCMachObjectWriter.h - Mach Object Writer --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCObjectFileInfo.h b/include/llvm/MC/MCObjectFileInfo.h index f8142ccd8ac5..abc87bf27748 100644 --- a/include/llvm/MC/MCObjectFileInfo.h +++ b/include/llvm/MC/MCObjectFileInfo.h @@ -1,9 +1,8 @@ //===-- llvm/MC/MCObjectFileInfo.h - Object File Info -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -160,6 +159,9 @@ protected: /// FaultMap section. MCSection *FaultMapSection; + /// Remarks section. + MCSection *RemarksSection; + /// EH frame section. /// /// It is initialized on demand so it can be overwritten (with uniquing). @@ -315,6 +317,7 @@ public: MCSection *getStackMapSection() const { return StackMapSection; } MCSection *getFaultMapSection() const { return FaultMapSection; } + MCSection *getRemarksSection() const { return RemarksSection; } MCSection *getStackSizesSection(const MCSection &TextSec) const; @@ -381,7 +384,7 @@ public: return EHFrameSection; } - enum Environment { IsMachO, IsELF, IsCOFF, IsWasm }; + enum Environment { IsMachO, IsELF, IsCOFF, IsWasm, IsXCOFF }; Environment getObjectFileType() const { return Env; } bool isPositionIndependent() const { return PositionIndependent; } @@ -397,6 +400,7 @@ private: void initELFMCObjectFileInfo(const Triple &T, bool Large); void initCOFFMCObjectFileInfo(const Triple &T); void initWasmMCObjectFileInfo(const Triple &T); + void initXCOFFMCObjectFileInfo(const Triple &T); MCSection *getDwarfComdatSection(const char *Name, uint64_t Hash) const; public: diff --git a/include/llvm/MC/MCObjectStreamer.h b/include/llvm/MC/MCObjectStreamer.h index 892909656c15..8affca49490f 100644 --- a/include/llvm/MC/MCObjectStreamer.h +++ b/include/llvm/MC/MCObjectStreamer.h @@ -1,9 +1,8 @@ //===- MCObjectStreamer.h - MCStreamer Object File Interface ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -116,8 +115,7 @@ public: void EmitSLEB128Value(const MCExpr *Value) override; void EmitWeakReference(MCSymbol *Alias, const MCSymbol *Symbol) override; void ChangeSection(MCSection *Section, const MCExpr *Subsection) override; - void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI, - bool = false) override; + void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI) override; /// Emit an instruction to a special fragment, because this instruction /// can change its size during relaxation. diff --git a/include/llvm/MC/MCObjectWriter.h b/include/llvm/MC/MCObjectWriter.h index 8bae2bf20083..2547b2b7c9c1 100644 --- a/include/llvm/MC/MCObjectWriter.h +++ b/include/llvm/MC/MCObjectWriter.h @@ -1,9 +1,8 @@ //===- llvm/MC/MCObjectWriter.h - Object File Writer Interface --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCParser/AsmCond.h b/include/llvm/MC/MCParser/AsmCond.h index a6e0fbd7f337..ea2155010081 100644 --- a/include/llvm/MC/MCParser/AsmCond.h +++ b/include/llvm/MC/MCParser/AsmCond.h @@ -1,9 +1,8 @@ //===- AsmCond.h - Assembly file conditional assembly ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCParser/AsmLexer.h b/include/llvm/MC/MCParser/AsmLexer.h index 2e9b8dfa3b26..b7294493b2f8 100644 --- a/include/llvm/MC/MCParser/AsmLexer.h +++ b/include/llvm/MC/MCParser/AsmLexer.h @@ -1,9 +1,8 @@ //===- AsmLexer.h - Lexer for Assembly Files --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/MC/MCParser/MCAsmLexer.h b/include/llvm/MC/MCParser/MCAsmLexer.h index ea13d1cdc09f..e89abeaac94c 100644 --- a/include/llvm/MC/MCParser/MCAsmLexer.h +++ b/include/llvm/MC/MCParser/MCAsmLexer.h @@ -1,9 +1,8 @@ //===- llvm/MC/MCAsmLexer.h - Abstract Asm Lexer Interface ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCParser/MCAsmParser.h b/include/llvm/MC/MCParser/MCAsmParser.h index b80289878e6e..da5653ee71d3 100644 --- a/include/llvm/MC/MCParser/MCAsmParser.h +++ b/include/llvm/MC/MCParser/MCAsmParser.h @@ -1,9 +1,8 @@ //===- llvm/MC/MCAsmParser.h - Abstract Asm Parser Interface ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -130,9 +129,6 @@ protected: // Can only create subclasses. /// Flag tracking whether any errors have been encountered. bool HadError = false; - /// Enable print [latency:throughput] in output file. - bool EnablePrintSchedInfo = false; - bool ShowParsedOperands = false; public: @@ -166,9 +162,6 @@ public: bool getShowParsedOperands() const { return ShowParsedOperands; } void setShowParsedOperands(bool Value) { ShowParsedOperands = Value; } - void setEnablePrintSchedInfo(bool Value) { EnablePrintSchedInfo = Value; } - bool shouldPrintSchedInfo() const { return EnablePrintSchedInfo; } - /// Run the parser on the input source buffer. virtual bool Run(bool NoInitialTextSection, bool NoFinalize = false) = 0; diff --git a/include/llvm/MC/MCParser/MCAsmParserExtension.h b/include/llvm/MC/MCParser/MCAsmParserExtension.h index 1a132bceddc5..5d2afe81a54b 100644 --- a/include/llvm/MC/MCParser/MCAsmParserExtension.h +++ b/include/llvm/MC/MCParser/MCAsmParserExtension.h @@ -1,9 +1,8 @@ //===- llvm/MC/MCAsmParserExtension.h - Asm Parser Hooks --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCParser/MCAsmParserUtils.h b/include/llvm/MC/MCParser/MCAsmParserUtils.h index 84173bb9cb8e..d692da7402fe 100644 --- a/include/llvm/MC/MCParser/MCAsmParserUtils.h +++ b/include/llvm/MC/MCParser/MCAsmParserUtils.h @@ -1,9 +1,8 @@ //===- llvm/MC/MCAsmParserUtils.h - Asm Parser Utilities --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCParser/MCParsedAsmOperand.h b/include/llvm/MC/MCParser/MCParsedAsmOperand.h index 4af76ac2a858..2b6e2aa48b8f 100644 --- a/include/llvm/MC/MCParser/MCParsedAsmOperand.h +++ b/include/llvm/MC/MCParser/MCParsedAsmOperand.h @@ -1,9 +1,8 @@ //===- llvm/MC/MCParsedAsmOperand.h - Asm Parser Operand --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCParser/MCTargetAsmParser.h b/include/llvm/MC/MCParser/MCTargetAsmParser.h index ccf13a6a4fb4..849dbd57f1aa 100644 --- a/include/llvm/MC/MCParser/MCTargetAsmParser.h +++ b/include/llvm/MC/MCParser/MCTargetAsmParser.h @@ -1,9 +1,8 @@ //===- llvm/MC/MCTargetAsmParser.h - Target Assembly Parser -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -17,6 +16,7 @@ #include "llvm/MC/MCParser/MCParsedAsmOperand.h" #include "llvm/MC/MCParser/MCAsmParserExtension.h" #include "llvm/MC/MCTargetOptions.h" +#include "llvm/MC/SubtargetFeature.h" #include "llvm/Support/SMLoc.h" #include #include @@ -203,7 +203,7 @@ public: // The instruction encoding is not valid because it requires some target // features that are not currently enabled. MissingFeatures has a bit set for // each feature that the encoding needs but which is not enabled. - static NearMissInfo getMissedFeature(uint64_t MissingFeatures) { + static NearMissInfo getMissedFeature(const FeatureBitset &MissingFeatures) { NearMissInfo Result; Result.Kind = NearMissFeature; Result.Features = MissingFeatures; @@ -255,7 +255,7 @@ public: // Feature flags required by the instruction, that the current target does // not have. - uint64_t getFeatures() const { + const FeatureBitset& getFeatures() const { assert(Kind == NearMissFeature); return Features; } @@ -305,7 +305,7 @@ private: }; union { - uint64_t Features; + FeatureBitset Features; unsigned PredicateError; MissedOpInfo MissedOperand; TooFewOperandsInfo TooFewOperands; @@ -335,7 +335,7 @@ protected: // Can only create subclasses. MCSubtargetInfo ©STI(); /// AvailableFeatures - The current set of available features. - uint64_t AvailableFeatures = 0; + FeatureBitset AvailableFeatures; /// ParsingInlineAsm - Are we parsing ms-style inline assembly? bool ParsingInlineAsm = false; @@ -360,8 +360,12 @@ public: const MCSubtargetInfo &getSTI() const; - uint64_t getAvailableFeatures() const { return AvailableFeatures; } - void setAvailableFeatures(uint64_t Value) { AvailableFeatures = Value; } + const FeatureBitset& getAvailableFeatures() const { + return AvailableFeatures; + } + void setAvailableFeatures(const FeatureBitset& Value) { + AvailableFeatures = Value; + } bool isParsingInlineAsm () { return ParsingInlineAsm; } void setParsingInlineAsm (bool Value) { ParsingInlineAsm = Value; } @@ -380,9 +384,6 @@ public: virtual bool ParseRegister(unsigned &RegNo, SMLoc &StartLoc, SMLoc &EndLoc) = 0; - /// Sets frame register corresponding to the current MachineFunction. - virtual void SetFrameRegister(unsigned RegNo) {} - /// ParseInstruction - Parse one assembly instruction. /// /// The parser is positioned following the instruction name. The target diff --git a/include/llvm/MC/MCRegisterInfo.h b/include/llvm/MC/MCRegisterInfo.h index 8d8c677c77ea..92d39c3fcfb7 100644 --- a/include/llvm/MC/MCRegisterInfo.h +++ b/include/llvm/MC/MCRegisterInfo.h @@ -1,9 +1,8 @@ //===- MC/MCRegisterInfo.h - Target Register Description --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/MC/MCSchedule.h b/include/llvm/MC/MCSchedule.h index 689ac73cbdd1..df3248ee6e86 100644 --- a/include/llvm/MC/MCSchedule.h +++ b/include/llvm/MC/MCSchedule.h @@ -1,9 +1,8 @@ //===-- llvm/MC/MCSchedule.h - Scheduling -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,6 +14,7 @@ #ifndef LLVM_MC_MCSCHEDULE_H #define LLVM_MC_MCSCHEDULE_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/Optional.h" #include "llvm/Config/llvm-config.h" #include "llvm/Support/DataTypes.h" @@ -370,6 +370,11 @@ struct MCSchedModel { getReciprocalThroughput(const MCSubtargetInfo &STI, const MCInstrInfo &MCII, const MCInst &Inst) const; + /// Returns the maximum forwarding delay for register reads dependent on + /// writes of scheduling class WriteResourceIdx. + static unsigned getForwardingDelayCycles(ArrayRef Entries, + unsigned WriteResourceIdx = 0); + /// Returns the default initialized model. static const MCSchedModel &GetDefaultSchedModel() { return Default; } static const MCSchedModel Default; diff --git a/include/llvm/MC/MCSection.h b/include/llvm/MC/MCSection.h index eb210b4e9dfa..6fad1ec2069c 100644 --- a/include/llvm/MC/MCSection.h +++ b/include/llvm/MC/MCSection.h @@ -1,9 +1,8 @@ //===- MCSection.h - Machine Code Sections ----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -38,7 +37,7 @@ template <> struct ilist_alloc_traits { /// current translation unit. The MCContext class uniques and creates these. class MCSection { public: - enum SectionVariant { SV_COFF = 0, SV_ELF, SV_MachO, SV_Wasm }; + enum SectionVariant { SV_COFF = 0, SV_ELF, SV_MachO, SV_Wasm, SV_XCOFF }; /// Express the state of bundle locked groups while emitting code. enum BundleLockStateType { diff --git a/include/llvm/MC/MCSectionCOFF.h b/include/llvm/MC/MCSectionCOFF.h index 24b9f8898ebb..8be95e0f1de5 100644 --- a/include/llvm/MC/MCSectionCOFF.h +++ b/include/llvm/MC/MCSectionCOFF.h @@ -1,9 +1,8 @@ //===- MCSectionCOFF.h - COFF Machine Code Sections -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -63,8 +62,6 @@ private: } public: - ~MCSectionCOFF(); - /// Decides whether a '.section' directive should be printed before the /// section name bool ShouldOmitSectionDirective(StringRef Name, const MCAsmInfo &MAI) const; diff --git a/include/llvm/MC/MCSectionELF.h b/include/llvm/MC/MCSectionELF.h index 00c289c6bd6e..fe6b2d7afc79 100644 --- a/include/llvm/MC/MCSectionELF.h +++ b/include/llvm/MC/MCSectionELF.h @@ -1,9 +1,8 @@ //===- MCSectionELF.h - ELF Machine Code Sections ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -64,8 +63,6 @@ private: void setSectionName(StringRef Name) { SectionName = Name; } public: - ~MCSectionELF(); - /// Decides whether a '.section' directive should be printed before the /// section name bool ShouldOmitSectionDirective(StringRef Name, const MCAsmInfo &MAI) const; diff --git a/include/llvm/MC/MCSectionMachO.h b/include/llvm/MC/MCSectionMachO.h index 89db09cbdbdc..2c73661fb1fd 100644 --- a/include/llvm/MC/MCSectionMachO.h +++ b/include/llvm/MC/MCSectionMachO.h @@ -1,9 +1,8 @@ //===- MCSectionMachO.h - MachO Machine Code Sections -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/MC/MCSectionWasm.h b/include/llvm/MC/MCSectionWasm.h index ab4cd7b007ec..2941a40f3b8c 100644 --- a/include/llvm/MC/MCSectionWasm.h +++ b/include/llvm/MC/MCSectionWasm.h @@ -1,9 +1,8 @@ //===- MCSectionWasm.h - Wasm Machine Code Sections -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -43,20 +42,19 @@ class MCSectionWasm final : public MCSection { // segment uint32_t SegmentIndex = 0; + // Whether this data segment is passive + bool IsPassive = false; + friend class MCContext; MCSectionWasm(StringRef Section, SectionKind K, const MCSymbolWasm *group, unsigned UniqueID, MCSymbol *Begin) : MCSection(SV_Wasm, K, Begin), SectionName(Section), UniqueID(UniqueID), Group(group) {} - void setSectionName(StringRef Name) { SectionName = Name; } - public: - ~MCSectionWasm(); - /// Decides whether a '.section' directive should be printed before the /// section name - bool ShouldOmitSectionDirective(StringRef Name, const MCAsmInfo &MAI) const; + bool shouldOmitSectionDirective(StringRef Name, const MCAsmInfo &MAI) const; StringRef getSectionName() const { return SectionName; } const MCSymbolWasm *getGroup() const { return Group; } @@ -68,7 +66,8 @@ public: bool isVirtualSection() const override; bool isWasmData() const { - return Kind.isGlobalWriteableData() || Kind.isReadOnly(); + return Kind.isGlobalWriteableData() || Kind.isReadOnly() || + Kind.isThreadLocal(); } bool isUnique() const { return UniqueID != ~0U; } @@ -80,6 +79,14 @@ public: uint32_t getSegmentIndex() const { return SegmentIndex; } void setSegmentIndex(uint32_t Index) { SegmentIndex = Index; } + bool getPassive() const { + assert(isWasmData()); + return IsPassive; + } + void setPassive(bool V = true) { + assert(isWasmData()); + IsPassive = V; + } static bool classof(const MCSection *S) { return S->getVariant() == SV_Wasm; } }; diff --git a/include/llvm/MC/MCSectionXCOFF.h b/include/llvm/MC/MCSectionXCOFF.h new file mode 100644 index 000000000000..2a3f391fd3e2 --- /dev/null +++ b/include/llvm/MC/MCSectionXCOFF.h @@ -0,0 +1,56 @@ +//===- MCSectionXCOFF.h - XCOFF Machine Code Sections -----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file declares the MCSectionXCOFF class. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_MC_MCSECTIONXCOFF_H +#define LLVM_MC_MCSECTIONXCOFF_H + +#include "llvm/ADT/Twine.h" +#include "llvm/BinaryFormat/XCOFF.h" +#include "llvm/MC/MCSection.h" + +namespace llvm { + +class MCSymbol; + +// This class represents an XCOFF `Control Section`, more commonly referred to +// as a csect. A csect represents the smallest possible unit of data/code which +// will be relocated as a single block. +class MCSectionXCOFF final : public MCSection { + friend class MCContext; + + StringRef Name; + XCOFF::StorageMappingClass MappingClass; + + MCSectionXCOFF(StringRef Section, XCOFF::StorageMappingClass SMC, + SectionKind K, MCSymbol *Begin) + : MCSection(SV_XCOFF, K, Begin), Name(Section), MappingClass(SMC) {} + +public: + ~MCSectionXCOFF(); + + static bool classof(const MCSection *S) { + return S->getVariant() == SV_XCOFF; + } + + StringRef getSectionName() const { return Name; } + XCOFF::StorageMappingClass getMappingClass() const { return MappingClass; } + + void PrintSwitchToSection(const MCAsmInfo &MAI, const Triple &T, + raw_ostream &OS, + const MCExpr *Subsection) const override; + bool UseCodeAlign() const override; + bool isVirtualSection() const override; +}; + +} // end namespace llvm + +#endif diff --git a/include/llvm/MC/MCStreamer.h b/include/llvm/MC/MCStreamer.h index f613d3a1943f..731e7515448c 100644 --- a/include/llvm/MC/MCStreamer.h +++ b/include/llvm/MC/MCStreamer.h @@ -1,9 +1,8 @@ //===- MCStreamer.h - High-level Streaming Machine Code Output --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -20,7 +19,6 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/MC/MCDirectives.h" -#include "llvm/MC/MCDwarf.h" #include "llvm/MC/MCLinkerOptimizationHint.h" #include "llvm/MC/MCSymbol.h" #include "llvm/MC/MCWinEH.h" @@ -44,6 +42,7 @@ class MCAsmBackend; class MCCodeEmitter; struct MCCodePaddingContext; class MCContext; +struct MCDwarfFrameInfo; class MCExpr; class MCInst; class MCInstPrinter; @@ -267,10 +266,8 @@ public: /// closed. Otherwise, issue an error and return null. WinEH::FrameInfo *EnsureValidWinFrameInfo(SMLoc Loc); - unsigned getNumFrameInfos() { return DwarfFrameInfos.size(); } - ArrayRef getDwarfFrameInfos() const { - return DwarfFrameInfos; - } + unsigned getNumFrameInfos(); + ArrayRef getDwarfFrameInfos() const; bool hasUnfinishedDwarfFrameInfo(); @@ -629,13 +626,20 @@ public: /// to pass in a MCExpr for constant integers. virtual void EmitIntValue(uint64_t Value, unsigned Size); + /// Special case of EmitValue that avoids the client having to pass + /// in a MCExpr for constant integers & prints in Hex format for certain + /// modes. + virtual void EmitIntValueInHex(uint64_t Value, unsigned Size) { + EmitIntValue(Value, Size); + } + virtual void EmitULEB128Value(const MCExpr *Value); virtual void EmitSLEB128Value(const MCExpr *Value); /// Special case of EmitULEB128Value that avoids the client having to /// pass in a MCExpr for constant integers. - void EmitULEB128IntValue(uint64_t Value); + void EmitULEB128IntValue(uint64_t Value, unsigned PadTo = 0); /// Special case of EmitSLEB128Value that avoids the client having to /// pass in a MCExpr for constant integers. @@ -782,7 +786,7 @@ public: /// implements the DWARF2 '.file 4 "foo.c"' assembler directive. unsigned EmitDwarfFileDirective(unsigned FileNo, StringRef Directory, StringRef Filename, - MD5::MD5Result *Checksum = nullptr, + Optional Checksum = None, Optional Source = None, unsigned CUID = 0) { return cantFail( @@ -797,12 +801,12 @@ public: /// '.file 4 "dir/foo.c" md5 "..." source "..."' assembler directive. virtual Expected tryEmitDwarfFileDirective( unsigned FileNo, StringRef Directory, StringRef Filename, - MD5::MD5Result *Checksum = nullptr, Optional Source = None, + Optional Checksum = None, Optional Source = None, unsigned CUID = 0); /// Specify the "root" file of the compilation, using the ".file 0" extension. virtual void emitDwarfFile0Directive(StringRef Directory, StringRef Filename, - MD5::MD5Result *Checksum, + Optional Checksum, Optional Source, unsigned CUID = 0); @@ -953,9 +957,7 @@ public: virtual void EmitAddrsigSym(const MCSymbol *Sym) {} /// Emit the given \p Instruction into the current section. - /// PrintSchedInfo == true then schedul comment should be added to output - virtual void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI, - bool PrintSchedInfo = false); + virtual void EmitInstruction(const MCInst &Inst, const MCSubtargetInfo &STI); /// Set the bundle alignment mode from now on in the section. /// The argument is the power of 2 to which the alignment is set. The diff --git a/include/llvm/MC/MCSubtargetInfo.h b/include/llvm/MC/MCSubtargetInfo.h index b3ce523d9c0c..9490a6ecedad 100644 --- a/include/llvm/MC/MCSubtargetInfo.h +++ b/include/llvm/MC/MCSubtargetInfo.h @@ -1,9 +1,8 @@ //===- llvm/MC/MCSubtargetInfo.h - Subtarget Information --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -29,6 +28,45 @@ namespace llvm { class MCInst; +//===----------------------------------------------------------------------===// + +/// Used to provide key value pairs for feature and CPU bit flags. +struct SubtargetFeatureKV { + const char *Key; ///< K-V key string + const char *Desc; ///< Help descriptor + unsigned Value; ///< K-V integer value + FeatureBitArray Implies; ///< K-V bit mask + + /// Compare routine for std::lower_bound + bool operator<(StringRef S) const { + return StringRef(Key) < S; + } + + /// Compare routine for std::is_sorted. + bool operator<(const SubtargetFeatureKV &Other) const { + return StringRef(Key) < StringRef(Other.Key); + } +}; + +//===----------------------------------------------------------------------===// + +/// Used to provide key value pairs for feature and CPU bit flags. +struct SubtargetSubTypeKV { + const char *Key; ///< K-V key string + FeatureBitArray Implies; ///< K-V bit mask + const MCSchedModel *SchedModel; + + /// Compare routine for std::lower_bound + bool operator<(StringRef S) const { + return StringRef(Key) < S; + } + + /// Compare routine for std::is_sorted. + bool operator<(const SubtargetSubTypeKV &Other) const { + return StringRef(Key) < StringRef(Other.Key); + } +}; + //===----------------------------------------------------------------------===// /// /// Generic base class for all target subtargets. @@ -37,10 +75,9 @@ class MCSubtargetInfo { Triple TargetTriple; std::string CPU; // CPU being targeted. ArrayRef ProcFeatures; // Processor feature list - ArrayRef ProcDesc; // Processor descriptions + ArrayRef ProcDesc; // Processor descriptions // Scheduler machine model - const SubtargetInfoKV *ProcSchedModels; const MCWriteProcResEntry *WriteProcResTable; const MCWriteLatencyEntry *WriteLatencyTable; const MCReadAdvanceEntry *ReadAdvanceTable; @@ -55,8 +92,7 @@ public: MCSubtargetInfo(const MCSubtargetInfo &) = default; MCSubtargetInfo(const Triple &TT, StringRef CPU, StringRef FS, ArrayRef PF, - ArrayRef PD, - const SubtargetInfoKV *ProcSched, + ArrayRef PD, const MCWriteProcResEntry *WPR, const MCWriteLatencyEntry *WL, const MCReadAdvanceEntry *RA, const InstrStage *IS, const unsigned *OC, const unsigned *FP); @@ -105,6 +141,10 @@ public: /// all feature bits implied by the flag. FeatureBitset ApplyFeatureFlag(StringRef FS); + /// Set/clear additional feature bits, including all other bits they imply. + FeatureBitset SetFeatureBitsTransitively(const FeatureBitset& FB); + FeatureBitset ClearFeatureBitsTransitively(const FeatureBitset &FB); + /// Check whether the subtarget features are enabled/disabled as per /// the provided string, ignoring all other features. bool checkFeatures(StringRef FS) const; @@ -153,6 +193,16 @@ public: return 0; } + /// Return the set of ReadAdvance entries declared by the scheduling class + /// descriptor in input. + ArrayRef + getReadAdvanceEntries(const MCSchedClassDesc &SC) const { + if (!SC.NumReadAdvanceEntries) + return ArrayRef(); + return ArrayRef(&ReadAdvanceTable[SC.ReadAdvanceIdx], + SC.NumReadAdvanceEntries); + } + /// Get scheduling itinerary of a CPU. InstrItineraryData getInstrItineraryForCPU(StringRef CPU) const; @@ -171,11 +221,6 @@ public: auto Found = std::lower_bound(ProcDesc.begin(), ProcDesc.end(), CPU); return Found != ProcDesc.end() && StringRef(Found->Key) == CPU; } - - /// Returns string representation of scheduler comment - virtual std::string getSchedInfoStr(MCInst const &MCI) const { - return {}; - } }; } // end namespace llvm diff --git a/include/llvm/MC/MCSymbol.h b/include/llvm/MC/MCSymbol.h index 4681a1be60c4..189484deac7e 100644 --- a/include/llvm/MC/MCSymbol.h +++ b/include/llvm/MC/MCSymbol.h @@ -1,9 +1,8 @@ //===- MCSymbol.h - Machine Code Symbols ------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -49,6 +48,7 @@ protected: SymbolKindELF, SymbolKindMachO, SymbolKindWasm, + SymbolKindXCOFF, }; /// A symbol can contain an Offset, or Value, or be Common, but never more @@ -58,6 +58,7 @@ protected: SymContentsOffset, SymContentsVariable, SymContentsCommon, + SymContentsTargetCommon, // Index stores the section index }; // Special sentinal value for the absolute pseudo fragment. @@ -108,7 +109,7 @@ protected: /// This is actually a Contents enumerator, but is unsigned to avoid sign /// extension and achieve better bitpacking with MSVC. - unsigned SymbolContents : 2; + unsigned SymbolContents : 3; /// The alignment of the symbol, if it is 'common', or -1. /// @@ -286,6 +287,8 @@ public: bool isWasm() const { return Kind == SymbolKindWasm; } + bool isXCOFF() const { return Kind == SymbolKindXCOFF; } + /// @} /// \name Variable Symbols /// @{ @@ -342,10 +345,11 @@ public: /// /// \param Size - The size of the symbol. /// \param Align - The alignment of the symbol. - void setCommon(uint64_t Size, unsigned Align) { + /// \param Target - Is the symbol a target-specific common-like symbol. + void setCommon(uint64_t Size, unsigned Align, bool Target = false) { assert(getOffset() == 0); CommonSize = Size; - SymbolContents = SymContentsCommon; + SymbolContents = Target ? SymContentsTargetCommon : SymContentsCommon; assert((!Align || isPowerOf2_32(Align)) && "Alignment must be a power of 2"); @@ -365,20 +369,28 @@ public: /// /// \param Size - The size of the symbol. /// \param Align - The alignment of the symbol. + /// \param Target - Is the symbol a target-specific common-like symbol. /// \return True if symbol was already declared as a different type - bool declareCommon(uint64_t Size, unsigned Align) { + bool declareCommon(uint64_t Size, unsigned Align, bool Target = false) { assert(isCommon() || getOffset() == 0); if(isCommon()) { - if(CommonSize != Size || getCommonAlignment() != Align) - return true; + if (CommonSize != Size || getCommonAlignment() != Align || + isTargetCommon() != Target) + return true; } else - setCommon(Size, Align); + setCommon(Size, Align, Target); return false; } /// Is this a 'common' symbol. bool isCommon() const { - return SymbolContents == SymContentsCommon; + return SymbolContents == SymContentsCommon || + SymbolContents == SymContentsTargetCommon; + } + + /// Is this a target-specific common-like symbol. + bool isTargetCommon() const { + return SymbolContents == SymContentsTargetCommon; } MCFragment *getFragment(bool SetUsed = true) const { diff --git a/include/llvm/MC/MCSymbolCOFF.h b/include/llvm/MC/MCSymbolCOFF.h index 7918c353dc15..94087ce871ae 100644 --- a/include/llvm/MC/MCSymbolCOFF.h +++ b/include/llvm/MC/MCSymbolCOFF.h @@ -1,9 +1,8 @@ //===- MCSymbolCOFF.h - ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCSymbolELF.h b/include/llvm/MC/MCSymbolELF.h index bbcd22e8e7db..34e5c4344aff 100644 --- a/include/llvm/MC/MCSymbolELF.h +++ b/include/llvm/MC/MCSymbolELF.h @@ -1,9 +1,8 @@ //===- MCSymbolELF.h - -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef LLVM_MC_MCSYMBOLELF_H diff --git a/include/llvm/MC/MCSymbolMachO.h b/include/llvm/MC/MCSymbolMachO.h index 6125c2050976..8f9ff56470a7 100644 --- a/include/llvm/MC/MCSymbolMachO.h +++ b/include/llvm/MC/MCSymbolMachO.h @@ -1,9 +1,8 @@ //===- MCSymbolMachO.h - ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef LLVM_MC_MCSYMBOLMACHO_H @@ -35,6 +34,7 @@ class MCSymbolMachO : public MCSymbol { SF_WeakDefinition = 0x0080, SF_SymbolResolver = 0x0100, SF_AltEntry = 0x0200, + SF_Cold = 0x0400, // Common alignment SF_CommonAlignmentMask = 0xF0FF, @@ -98,6 +98,10 @@ public: return getFlags() & SF_AltEntry; } + void setCold() const { modifyFlags(SF_Cold, SF_Cold); } + + bool isCold() const { return getFlags() & SF_Cold; } + void setDesc(unsigned Value) const { assert(Value == (Value & SF_DescFlagsMask) && "Invalid .desc value!"); diff --git a/include/llvm/MC/MCSymbolWasm.h b/include/llvm/MC/MCSymbolWasm.h index 8e66dc881d0f..c50cd0ee4709 100644 --- a/include/llvm/MC/MCSymbolWasm.h +++ b/include/llvm/MC/MCSymbolWasm.h @@ -1,9 +1,8 @@ //===- MCSymbolWasm.h - ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef LLVM_MC_MCSYMBOLWASM_H @@ -19,7 +18,9 @@ class MCSymbolWasm : public MCSymbol { bool IsWeak = false; bool IsHidden = false; bool IsComdat = false; - std::string ModuleName; + mutable bool IsUsedInGOT = false; + Optional ImportModule; + Optional ImportName; wasm::WasmSignature *Signature = nullptr; Optional GlobalType; Optional EventType; @@ -32,7 +33,7 @@ public: // Use a module name of "env" for now, for compatibility with existing tools. // This is temporary, and may change, as the ABI is not yet stable. MCSymbolWasm(const StringMapEntry *Name, bool isTemporary) - : MCSymbol(SymbolKindWasm, Name, isTemporary), ModuleName("env") {} + : MCSymbol(SymbolKindWasm, Name, isTemporary) {} static bool classof(const MCSymbol *S) { return S->isWasm(); } const MCExpr *getSize() const { return SymbolSize; } @@ -46,6 +47,13 @@ public: wasm::WasmSymbolType getType() const { return Type; } void setType(wasm::WasmSymbolType type) { Type = type; } + bool isExported() const { + return getFlags() & wasm::WASM_SYMBOL_EXPORTED; + } + void setExported() const { + modifyFlags(wasm::WASM_SYMBOL_EXPORTED, wasm::WASM_SYMBOL_EXPORTED); + } + bool isWeak() const { return IsWeak; } void setWeak(bool isWeak) { IsWeak = isWeak; } @@ -55,8 +63,24 @@ public: bool isComdat() const { return IsComdat; } void setComdat(bool isComdat) { IsComdat = isComdat; } - const StringRef getModuleName() const { return ModuleName; } - void setModuleName(StringRef Name) { ModuleName = Name; } + const StringRef getImportModule() const { + if (ImportModule.hasValue()) { + return ImportModule.getValue(); + } + return "env"; + } + void setImportModule(StringRef Name) { ImportModule = Name; } + + const StringRef getImportName() const { + if (ImportName.hasValue()) { + return ImportName.getValue(); + } + return getName(); + } + void setImportName(StringRef Name) { ImportName = Name; } + + void setUsedInGOT() const { IsUsedInGOT = true; } + bool isUsedInGOT() const { return IsUsedInGOT; } const wasm::WasmSignature *getSignature() const { return Signature; } void setSignature(wasm::WasmSignature *Sig) { Signature = Sig; } diff --git a/include/llvm/MC/MCSymbolXCOFF.h b/include/llvm/MC/MCSymbolXCOFF.h new file mode 100644 index 000000000000..0a1fe1475138 --- /dev/null +++ b/include/llvm/MC/MCSymbolXCOFF.h @@ -0,0 +1,26 @@ +//===- MCSymbolXCOFF.h - ----------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_MC_MCSYMBOLXCOFF_H +#define LLVM_MC_MCSYMBOLXCOFF_H + +#include "llvm/BinaryFormat/XCOFF.h" +#include "llvm/MC/MCSymbol.h" + +namespace llvm { + +class MCSymbolXCOFF : public MCSymbol { +public: + MCSymbolXCOFF(const StringMapEntry *Name, bool isTemporary) + : MCSymbol(SymbolKindXCOFF, Name, isTemporary) {} + + static bool classof(const MCSymbol *S) { return S->isXCOFF(); } +}; + +} // end namespace llvm + +#endif // LLVM_MC_MCSYMBOLXCOFF_H diff --git a/include/llvm/MC/MCTargetOptions.h b/include/llvm/MC/MCTargetOptions.h index f5d330fbeb22..f184620ff047 100644 --- a/include/llvm/MC/MCTargetOptions.h +++ b/include/llvm/MC/MCTargetOptions.h @@ -1,9 +1,8 @@ //===- MCTargetOptions.h - MC Target Options --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -16,18 +15,18 @@ namespace llvm { enum class ExceptionHandling { - None, /// No exception support - DwarfCFI, /// DWARF-like instruction based exceptions - SjLj, /// setjmp/longjmp based exceptions - ARM, /// ARM EHABI - WinEH, /// Windows Exception Handling - Wasm, /// WebAssembly Exception Handling + None, ///< No exception support + DwarfCFI, ///< DWARF-like instruction based exceptions + SjLj, ///< setjmp/longjmp based exceptions + ARM, ///< ARM EHABI + WinEH, ///< Windows Exception Handling + Wasm, ///< WebAssembly Exception Handling }; enum class DebugCompressionType { - None, /// No compression - GNU, /// zlib-gnu style compression - Z, /// zlib style complession + None, ///< No compression + GNU, ///< zlib-gnu style compression + Z, ///< zlib style complession }; class StringRef; @@ -39,9 +38,6 @@ public: AsmInstrumentationAddress }; - /// Enables AddressSanitizer instrumentation at machine level. - bool SanitizeAddress : 1; - bool MCRelaxAll : 1; bool MCNoExecStack : 1; bool MCFatalWarnings : 1; diff --git a/include/llvm/MC/MCTargetOptionsCommandFlags.inc b/include/llvm/MC/MCTargetOptionsCommandFlags.inc index 5172fa44511f..9f1177f470b9 100644 --- a/include/llvm/MC/MCTargetOptionsCommandFlags.inc +++ b/include/llvm/MC/MCTargetOptionsCommandFlags.inc @@ -1,9 +1,8 @@ //===-- MCTargetOptionsCommandFlags.h --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,15 +18,6 @@ #include "llvm/Support/CommandLine.h" using namespace llvm; -static cl::opt AsmInstrumentation( - "asm-instrumentation", cl::desc("Instrumentation of inline assembly and " - "assembly source files"), - cl::init(MCTargetOptions::AsmInstrumentationNone), - cl::values(clEnumValN(MCTargetOptions::AsmInstrumentationNone, "none", - "no instrumentation at all"), - clEnumValN(MCTargetOptions::AsmInstrumentationAddress, "address", - "instrument instructions with memory arguments"))); - static cl::opt RelaxAll("mc-relax-all", cl::desc("When used with filetype=obj, " "relax all fixups in the emitted object file")); @@ -63,8 +53,6 @@ ABIName("target-abi", cl::Hidden, static MCTargetOptions InitMCTargetOptionsFromFlags() { MCTargetOptions Options; - Options.SanitizeAddress = - (AsmInstrumentation == MCTargetOptions::AsmInstrumentationAddress); Options.MCRelaxAll = RelaxAll; Options.MCIncrementalLinkerCompatible = IncrementalLinkerCompatible; Options.MCPIECopyRelocations = PIECopyRelocations; diff --git a/include/llvm/MC/MCValue.h b/include/llvm/MC/MCValue.h index 11f5082ed3f4..0be7ce7055c5 100644 --- a/include/llvm/MC/MCValue.h +++ b/include/llvm/MC/MCValue.h @@ -1,9 +1,8 @@ //===-- llvm/MC/MCValue.h - MCValue class -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/MC/MCWasmObjectWriter.h b/include/llvm/MC/MCWasmObjectWriter.h index 6b788cfe96b9..4adbca28f116 100644 --- a/include/llvm/MC/MCWasmObjectWriter.h +++ b/include/llvm/MC/MCWasmObjectWriter.h @@ -1,9 +1,8 @@ //===-- llvm/MC/MCWasmObjectWriter.h - Wasm Object Writer -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCWasmStreamer.h b/include/llvm/MC/MCWasmStreamer.h index 01e6a4379287..2d7f2b9975c9 100644 --- a/include/llvm/MC/MCWasmStreamer.h +++ b/include/llvm/MC/MCWasmStreamer.h @@ -1,9 +1,8 @@ //===- MCWasmStreamer.h - MCStreamer Wasm Object File Interface -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCWin64EH.h b/include/llvm/MC/MCWin64EH.h index 1a9f6f403d7c..60ec06e61b7c 100644 --- a/include/llvm/MC/MCWin64EH.h +++ b/include/llvm/MC/MCWin64EH.h @@ -1,9 +1,8 @@ //===- MCWin64EH.h - Machine Code Win64 EH support --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/MC/MCWinCOFFObjectWriter.h b/include/llvm/MC/MCWinCOFFObjectWriter.h index c1d35ea1f6ba..3fe124fd7f1c 100644 --- a/include/llvm/MC/MCWinCOFFObjectWriter.h +++ b/include/llvm/MC/MCWinCOFFObjectWriter.h @@ -1,9 +1,8 @@ //===- llvm/MC/MCWinCOFFObjectWriter.h - Win COFF Object Writer -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCWinCOFFStreamer.h b/include/llvm/MC/MCWinCOFFStreamer.h index 0049d04b4b3f..c1c1ec56cb48 100644 --- a/include/llvm/MC/MCWinCOFFStreamer.h +++ b/include/llvm/MC/MCWinCOFFStreamer.h @@ -1,9 +1,8 @@ //===- MCWinCOFFStreamer.h - COFF Object File Interface ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCWinEH.h b/include/llvm/MC/MCWinEH.h index 98ef0367a11d..b1c28c0ecae7 100644 --- a/include/llvm/MC/MCWinEH.h +++ b/include/llvm/MC/MCWinEH.h @@ -1,9 +1,8 @@ //===- MCWinEH.h - Windows Unwinding Support --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/MCXCOFFObjectWriter.h b/include/llvm/MC/MCXCOFFObjectWriter.h new file mode 100644 index 000000000000..fe4087f70614 --- /dev/null +++ b/include/llvm/MC/MCXCOFFObjectWriter.h @@ -0,0 +1,41 @@ +//===-- llvm/MC/MCXCOFFObjectWriter.h - XCOFF Object Writer ---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_MC_MCXCOFFOBJECTWRITER_H +#define LLVM_MC_MCXCOFFOBJECTWRITER_H + +#include "llvm/MC/MCObjectWriter.h" + +namespace llvm { + +class raw_pwrite_stream; + +class MCXCOFFObjectTargetWriter : public MCObjectTargetWriter { +protected: + MCXCOFFObjectTargetWriter(bool Is64Bit); + +public: + ~MCXCOFFObjectTargetWriter() override; + + Triple::ObjectFormatType getFormat() const override { return Triple::XCOFF; } + static bool classof(const MCObjectTargetWriter *W) { + return W->getFormat() == Triple::XCOFF; + } + bool is64Bit() const { return Is64Bit; } + +private: + bool Is64Bit; +}; + +std::unique_ptr +createXCOFFObjectWriter(std::unique_ptr MOTW, + raw_pwrite_stream &OS); + +} // end namespace llvm + +#endif // LLVM_MC_MCXCOFFOBJECTWRITER_H diff --git a/include/llvm/MC/MCXCOFFStreamer.h b/include/llvm/MC/MCXCOFFStreamer.h new file mode 100644 index 000000000000..159ae4818749 --- /dev/null +++ b/include/llvm/MC/MCXCOFFStreamer.h @@ -0,0 +1,33 @@ +//===- MCXCOFFObjectStreamer.h - MCStreamer XCOFF Object File Interface ---===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_MC_MCXCOFFSTREAMER_H +#define LLVM_MC_MCXCOFFSTREAMER_H + +#include "llvm/MC/MCObjectStreamer.h" + +namespace llvm { + +class MCXCOFFStreamer : public MCObjectStreamer { +public: + MCXCOFFStreamer(MCContext &Context, std::unique_ptr MAB, + std::unique_ptr OW, + std::unique_ptr Emitter); + + bool EmitSymbolAttribute(MCSymbol *Symbol, MCSymbolAttr Attribute) override; + void EmitCommonSymbol(MCSymbol *Symbol, uint64_t Size, + unsigned ByteAlignment) override; + void EmitZerofill(MCSection *Section, MCSymbol *Symbol = nullptr, + uint64_t Size = 0, unsigned ByteAlignment = 0, + SMLoc Loc = SMLoc()) override; + void EmitInstToData(const MCInst &Inst, const MCSubtargetInfo &) override; +}; + +} // end namespace llvm + +#endif // LLVM_MC_MCXCOFFSTREAMER_H diff --git a/include/llvm/MC/MachineLocation.h b/include/llvm/MC/MachineLocation.h index 91ed661ebeab..5872540e6104 100644 --- a/include/llvm/MC/MachineLocation.h +++ b/include/llvm/MC/MachineLocation.h @@ -1,9 +1,8 @@ //===- llvm/MC/MachineLocation.h --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // The MachineLocation class is used to represent a simple location in a machine diff --git a/include/llvm/MC/SectionKind.h b/include/llvm/MC/SectionKind.h index 66eb9ec56d14..0342c4cfbbde 100644 --- a/include/llvm/MC/SectionKind.h +++ b/include/llvm/MC/SectionKind.h @@ -1,9 +1,8 @@ //===-- llvm/MC/SectionKind.h - Classification of sections ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/StringTableBuilder.h b/include/llvm/MC/StringTableBuilder.h index 265260fcee4d..c83eca4e512d 100644 --- a/include/llvm/MC/StringTableBuilder.h +++ b/include/llvm/MC/StringTableBuilder.h @@ -1,9 +1,8 @@ //===- StringTableBuilder.h - String table building utility -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/MC/SubtargetFeature.h b/include/llvm/MC/SubtargetFeature.h index 76c7dd560800..fc9565ceafad 100644 --- a/include/llvm/MC/SubtargetFeature.h +++ b/include/llvm/MC/SubtargetFeature.h @@ -1,9 +1,8 @@ //===- llvm/MC/SubtargetFeature.h - CPU characteristics ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,6 +18,7 @@ #define LLVM_MC_SUBTARGETFEATURE_H #include "llvm/ADT/StringRef.h" +#include #include #include #include @@ -26,11 +26,12 @@ namespace llvm { -template class ArrayRef; class raw_ostream; class Triple; -const unsigned MAX_SUBTARGET_FEATURES = 192; +const unsigned MAX_SUBTARGET_WORDS = 3; +const unsigned MAX_SUBTARGET_FEATURES = MAX_SUBTARGET_WORDS * 64; + /// Container class for subtarget features. /// This is convenient because std::bitset does not have a constructor /// with an initializer list of set bits. @@ -45,38 +46,34 @@ public: for (auto I : Init) set(I); } -}; - -//===----------------------------------------------------------------------===// - -/// Used to provide key value pairs for feature and CPU bit flags. -struct SubtargetFeatureKV { - const char *Key; ///< K-V key string - const char *Desc; ///< Help descriptor - FeatureBitset Value; ///< K-V integer value - FeatureBitset Implies; ///< K-V bit mask - /// Compare routine for std::lower_bound - bool operator<(StringRef S) const { - return StringRef(Key) < S; - } - - /// Compare routine for std::is_sorted. - bool operator<(const SubtargetFeatureKV &Other) const { - return StringRef(Key) < StringRef(Other.Key); + bool operator < (const FeatureBitset &Other) const { + for (unsigned I = 0, E = size(); I != E; ++I) { + bool LHS = test(I), RHS = Other.test(I); + if (LHS != RHS) + return LHS < RHS; + } + return false; } }; -//===----------------------------------------------------------------------===// +/// Class used to store the subtarget bits in the tables created by tablegen. +/// The std::initializer_list constructor of FeatureBitset can't be done at +/// compile time and requires a static constructor to run at startup. +class FeatureBitArray { + std::array Bits; + +public: + constexpr FeatureBitArray(const std::array &B) + : Bits(B) {} -/// Used to provide key value pairs for CPU and arbitrary pointers. -struct SubtargetInfoKV { - const char *Key; ///< K-V key string - const void *Value; ///< K-V pointer value + FeatureBitset getAsBitset() const { + FeatureBitset Result; - /// Compare routine for std::lower_bound - bool operator<(StringRef S) const { - return StringRef(Key) < S; + for (unsigned i = 0, e = Bits.size(); i != e; ++i) + Result |= FeatureBitset(Bits[i]) << (64 * i); + + return Result; } }; @@ -102,19 +99,6 @@ public: /// Adds Features. void AddFeature(StringRef String, bool Enable = true); - /// Toggles a feature and update the feature bits. - static void ToggleFeature(FeatureBitset &Bits, StringRef String, - ArrayRef FeatureTable); - - /// Applies the feature flag and update the feature bits. - static void ApplyFeatureFlag(FeatureBitset &Bits, StringRef Feature, - ArrayRef FeatureTable); - - /// Returns feature bits of a CPU. - FeatureBitset getFeatureBits(StringRef CPU, - ArrayRef CPUTable, - ArrayRef FeatureTable); - /// Returns the vector of individual subtarget features. const std::vector &getFeatures() const { return Features; } @@ -126,6 +110,32 @@ public: /// Adds the default features for the specified target triple. void getDefaultSubtargetFeatures(const Triple& Triple); + + /// Determine if a feature has a flag; '+' or '-' + static bool hasFlag(StringRef Feature) { + assert(!Feature.empty() && "Empty string"); + // Get first character + char Ch = Feature[0]; + // Check if first character is '+' or '-' flag + return Ch == '+' || Ch =='-'; + } + + /// Return string stripped of flag. + static std::string StripFlag(StringRef Feature) { + return hasFlag(Feature) ? Feature.substr(1) : Feature; + } + + /// Return true if enable flag; '+'. + static inline bool isEnabled(StringRef Feature) { + assert(!Feature.empty() && "Empty string"); + // Get first character + char Ch = Feature[0]; + // Check if first character is '+' for enabled + return Ch == '+'; + } + + /// Splits a string of comma separated items in to a vector of strings. + static void Split(std::vector &V, StringRef S); }; } // end namespace llvm diff --git a/include/llvm/MCA/Context.h b/include/llvm/MCA/Context.h index 6b2bee0fdc42..503d780d4947 100644 --- a/include/llvm/MCA/Context.h +++ b/include/llvm/MCA/Context.h @@ -1,9 +1,8 @@ //===---------------------------- Context.h ---------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -32,15 +31,21 @@ namespace mca { /// This is a convenience struct to hold the parameters necessary for creating /// the pre-built "default" out-of-order pipeline. struct PipelineOptions { - PipelineOptions(unsigned DW, unsigned RFS, unsigned LQS, unsigned SQS, - bool NoAlias) - : DispatchWidth(DW), RegisterFileSize(RFS), LoadQueueSize(LQS), - StoreQueueSize(SQS), AssumeNoAlias(NoAlias) {} + PipelineOptions(unsigned UOPQSize, unsigned DecThr, unsigned DW, unsigned RFS, + unsigned LQS, unsigned SQS, bool NoAlias, + bool ShouldEnableBottleneckAnalysis = false) + : MicroOpQueueSize(UOPQSize), DecodersThroughput(DecThr), + DispatchWidth(DW), RegisterFileSize(RFS), LoadQueueSize(LQS), + StoreQueueSize(SQS), AssumeNoAlias(NoAlias), + EnableBottleneckAnalysis(ShouldEnableBottleneckAnalysis) {} + unsigned MicroOpQueueSize; + unsigned DecodersThroughput; // Instructions per cycle. unsigned DispatchWidth; unsigned RegisterFileSize; unsigned LoadQueueSize; unsigned StoreQueueSize; bool AssumeNoAlias; + bool EnableBottleneckAnalysis; }; class Context { diff --git a/include/llvm/MCA/HWEventListener.h b/include/llvm/MCA/HWEventListener.h index 3b32b2cd6577..e11d06de2b2e 100644 --- a/include/llvm/MCA/HWEventListener.h +++ b/include/llvm/MCA/HWEventListener.h @@ -1,9 +1,8 @@ //===----------------------- HWEventListener.h ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -40,6 +39,7 @@ public: // Events generated by the Retire Control Unit. Retired, // Events generated by the Scheduler. + Pending, Ready, Issued, Executed, @@ -126,6 +126,35 @@ public: const InstRef &IR; }; +// A HWPressureEvent describes an increase in backend pressure caused by +// the presence of data dependencies or unavailability of pipeline resources. +class HWPressureEvent { +public: + enum GenericReason { + INVALID = 0, + // Scheduler was unable to issue all the ready instructions because some + // pipeline resources were unavailable. + RESOURCES, + // Instructions could not be issued because of register data dependencies. + REGISTER_DEPS, + // Instructions could not be issued because of memory dependencies. + MEMORY_DEPS + }; + + HWPressureEvent(GenericReason reason, ArrayRef Insts, + uint64_t Mask = 0) + : Reason(reason), AffectedInstructions(Insts), ResourceMask(Mask) {} + + // Reason for this increase in backend pressure. + GenericReason Reason; + + // Instructions affected (i.e. delayed) by this increase in backend pressure. + ArrayRef AffectedInstructions; + + // A mask of unavailable processor resources. + const uint64_t ResourceMask; +}; + class HWEventListener { public: // Generic events generated by the pipeline. @@ -134,6 +163,7 @@ public: virtual void onEvent(const HWInstructionEvent &Event) {} virtual void onEvent(const HWStallEvent &Event) {} + virtual void onEvent(const HWPressureEvent &Event) {} using ResourceRef = std::pair; virtual void onResourceAvailable(const ResourceRef &RRef) {} diff --git a/include/llvm/MCA/HardwareUnits/HardwareUnit.h b/include/llvm/MCA/HardwareUnits/HardwareUnit.h index 104a2009f219..f6e178bcff10 100644 --- a/include/llvm/MCA/HardwareUnits/HardwareUnit.h +++ b/include/llvm/MCA/HardwareUnits/HardwareUnit.h @@ -1,9 +1,8 @@ //===-------------------------- HardwareUnit.h ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/MCA/HardwareUnits/LSUnit.h b/include/llvm/MCA/HardwareUnits/LSUnit.h index e217fc50f780..ae9a49c64855 100644 --- a/include/llvm/MCA/HardwareUnits/LSUnit.h +++ b/include/llvm/MCA/HardwareUnits/LSUnit.h @@ -1,9 +1,8 @@ //===------------------------- LSUnit.h --------------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -16,21 +15,298 @@ #ifndef LLVM_MCA_LSUNIT_H #define LLVM_MCA_LSUNIT_H -#include "llvm/ADT/SmallSet.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallVector.h" #include "llvm/MC/MCSchedule.h" #include "llvm/MCA/HardwareUnits/HardwareUnit.h" +#include "llvm/MCA/Instruction.h" namespace llvm { namespace mca { -class InstRef; class Scheduler; -/// A Load/Store Unit implementing a load and store queues. +/// A node of a memory dependency graph. A MemoryGroup describes a set of +/// instructions with same memory dependencies. /// -/// This class implements a load queue and a store queue to emulate the -/// out-of-order execution of memory operations. -/// Each load (or store) consumes an entry in the load (or store) queue. +/// By construction, instructions of a MemoryGroup don't depend on each other. +/// At dispatch stage, instructions are mapped by the LSUnit to MemoryGroups. +/// A Memory group identifier is then stored as a "token" in field +/// Instruction::LSUTokenID of each dispatched instructions. That token is used +/// internally by the LSUnit to track memory dependencies. +class MemoryGroup { + unsigned NumPredecessors; + unsigned NumExecutingPredecessors; + unsigned NumExecutedPredecessors; + + unsigned NumInstructions; + unsigned NumExecuting; + unsigned NumExecuted; + SmallVector Succ; + + CriticalDependency CriticalPredecessor; + InstRef CriticalMemoryInstruction; + + MemoryGroup(const MemoryGroup &) = delete; + MemoryGroup &operator=(const MemoryGroup &) = delete; + +public: + MemoryGroup() + : NumPredecessors(0), NumExecutingPredecessors(0), + NumExecutedPredecessors(0), NumInstructions(0), NumExecuting(0), + NumExecuted(0), CriticalPredecessor(), CriticalMemoryInstruction() {} + MemoryGroup(MemoryGroup &&) = default; + + ArrayRef getSuccessors() const { return Succ; } + unsigned getNumSuccessors() const { return Succ.size(); } + unsigned getNumPredecessors() const { return NumPredecessors; } + unsigned getNumExecutingPredecessors() const { + return NumExecutingPredecessors; + } + unsigned getNumExecutedPredecessors() const { + return NumExecutedPredecessors; + } + unsigned getNumInstructions() const { return NumInstructions; } + unsigned getNumExecuting() const { return NumExecuting; } + unsigned getNumExecuted() const { return NumExecuted; } + + const InstRef &getCriticalMemoryInstruction() const { + return CriticalMemoryInstruction; + } + const CriticalDependency &getCriticalPredecessor() const { + return CriticalPredecessor; + } + + void addSuccessor(MemoryGroup *Group) { + Group->NumPredecessors++; + assert(!isExecuted() && "Should have been removed!"); + if (isExecuting()) + Group->onGroupIssued(CriticalMemoryInstruction); + Succ.emplace_back(Group); + } + + bool isWaiting() const { + return NumPredecessors > + (NumExecutingPredecessors + NumExecutedPredecessors); + } + bool isPending() const { + return NumExecutingPredecessors && + ((NumExecutedPredecessors + NumExecutingPredecessors) == + NumPredecessors); + } + bool isReady() const { return NumExecutedPredecessors == NumPredecessors; } + bool isExecuting() const { + return NumExecuting && (NumExecuting == (NumInstructions - NumExecuted)); + } + bool isExecuted() const { return NumInstructions == NumExecuted; } + + void onGroupIssued(const InstRef &IR) { + assert(!isReady() && "Unexpected group-start event!"); + NumExecutingPredecessors++; + + unsigned Cycles = IR.getInstruction()->getCyclesLeft(); + if (CriticalPredecessor.Cycles < Cycles) { + CriticalPredecessor.IID = IR.getSourceIndex(); + CriticalPredecessor.Cycles = Cycles; + } + } + + void onGroupExecuted() { + assert(!isReady() && "Inconsistent state found!"); + NumExecutingPredecessors--; + NumExecutedPredecessors++; + } + + void onInstructionIssued(const InstRef &IR) { + assert(!isExecuting() && "Invalid internal state!"); + ++NumExecuting; + + // update the CriticalMemDep. + const Instruction &IS = *IR.getInstruction(); + if ((bool)CriticalMemoryInstruction) { + const Instruction &OtherIS = *CriticalMemoryInstruction.getInstruction(); + if (OtherIS.getCyclesLeft() < IS.getCyclesLeft()) + CriticalMemoryInstruction = IR; + } else { + CriticalMemoryInstruction = IR; + } + + if (!isExecuting()) + return; + + // Notify successors that this group started execution. + for (MemoryGroup *MG : Succ) + MG->onGroupIssued(CriticalMemoryInstruction); + } + + void onInstructionExecuted() { + assert(isReady() && !isExecuted() && "Invalid internal state!"); + --NumExecuting; + ++NumExecuted; + + if (!isExecuted()) + return; + + // Notify successors that this group has finished execution. + for (MemoryGroup *MG : Succ) + MG->onGroupExecuted(); + } + + void addInstruction() { + assert(!getNumSuccessors() && "Cannot add instructions to this group!"); + ++NumInstructions; + } + + void cycleEvent() { + if (isWaiting() && CriticalPredecessor.Cycles) + CriticalPredecessor.Cycles--; + } +}; + +/// Abstract base interface for LS (load/store) units in llvm-mca. +class LSUnitBase : public HardwareUnit { + /// Load queue size. + /// + /// A value of zero for this field means that the load queue is unbounded. + /// Processor models can declare the size of a load queue via tablegen (see + /// the definition of tablegen class LoadQueue in + /// llvm/Target/TargetSchedule.td). + unsigned LQSize; + + /// Load queue size. + /// + /// A value of zero for this field means that the store queue is unbounded. + /// Processor models can declare the size of a store queue via tablegen (see + /// the definition of tablegen class StoreQueue in + /// llvm/Target/TargetSchedule.td). + unsigned SQSize; + + unsigned UsedLQEntries; + unsigned UsedSQEntries; + + /// True if loads don't alias with stores. + /// + /// By default, the LS unit assumes that loads and stores don't alias with + /// eachother. If this field is set to false, then loads are always assumed to + /// alias with stores. + const bool NoAlias; + + /// Used to map group identifiers to MemoryGroups. + DenseMap> Groups; + unsigned NextGroupID; + +public: + LSUnitBase(const MCSchedModel &SM, unsigned LoadQueueSize, + unsigned StoreQueueSize, bool AssumeNoAlias); + + virtual ~LSUnitBase(); + + /// Returns the total number of entries in the load queue. + unsigned getLoadQueueSize() const { return LQSize; } + + /// Returns the total number of entries in the store queue. + unsigned getStoreQueueSize() const { return SQSize; } + + unsigned getUsedLQEntries() const { return UsedLQEntries; } + unsigned getUsedSQEntries() const { return UsedSQEntries; } + unsigned assignLQSlot() { return UsedLQEntries++; } + unsigned assignSQSlot() { return UsedSQEntries++; } + + bool assumeNoAlias() const { return NoAlias; } + + enum Status { + LSU_AVAILABLE = 0, + LSU_LQUEUE_FULL, // Load Queue unavailable + LSU_SQUEUE_FULL // Store Queue unavailable + }; + + /// This method checks the availability of the load/store buffers. + /// + /// Returns LSU_AVAILABLE if there are enough load/store queue entries to + /// accomodate instruction IR. By default, LSU_AVAILABLE is returned if IR is + /// not a memory operation. + virtual Status isAvailable(const InstRef &IR) const = 0; + + /// Allocates LS resources for instruction IR. + /// + /// This method assumes that a previous call to `isAvailable(IR)` succeeded + /// with a LSUnitBase::Status value of LSU_AVAILABLE. + /// Returns the GroupID associated with this instruction. That value will be + /// used to set the LSUTokenID field in class Instruction. + virtual unsigned dispatch(const InstRef &IR) = 0; + + bool isSQEmpty() const { return !UsedSQEntries; } + bool isLQEmpty() const { return !UsedLQEntries; } + bool isSQFull() const { return SQSize && SQSize == UsedSQEntries; } + bool isLQFull() const { return LQSize && LQSize == UsedLQEntries; } + + bool isValidGroupID(unsigned Index) const { + return Index && (Groups.find(Index) != Groups.end()); + } + + /// Check if a peviously dispatched instruction IR is now ready for execution. + bool isReady(const InstRef &IR) const { + unsigned GroupID = IR.getInstruction()->getLSUTokenID(); + const MemoryGroup &Group = getGroup(GroupID); + return Group.isReady(); + } + + /// Check if instruction IR only depends on memory instructions that are + /// currently executing. + bool isPending(const InstRef &IR) const { + unsigned GroupID = IR.getInstruction()->getLSUTokenID(); + const MemoryGroup &Group = getGroup(GroupID); + return Group.isPending(); + } + + /// Check if instruction IR is still waiting on memory operations, and the + /// wait time is still unknown. + bool isWaiting(const InstRef &IR) const { + unsigned GroupID = IR.getInstruction()->getLSUTokenID(); + const MemoryGroup &Group = getGroup(GroupID); + return Group.isWaiting(); + } + + bool hasDependentUsers(const InstRef &IR) const { + unsigned GroupID = IR.getInstruction()->getLSUTokenID(); + const MemoryGroup &Group = getGroup(GroupID); + return !Group.isExecuted() && Group.getNumSuccessors(); + } + + const MemoryGroup &getGroup(unsigned Index) const { + assert(isValidGroupID(Index) && "Group doesn't exist!"); + return *Groups.find(Index)->second; + } + + MemoryGroup &getGroup(unsigned Index) { + assert(isValidGroupID(Index) && "Group doesn't exist!"); + return *Groups.find(Index)->second; + } + + unsigned createMemoryGroup() { + Groups.insert( + std::make_pair(NextGroupID, llvm::make_unique())); + return NextGroupID++; + } + + // Instruction executed event handlers. + virtual void onInstructionExecuted(const InstRef &IR); + + virtual void onInstructionIssued(const InstRef &IR) { + unsigned GroupID = IR.getInstruction()->getLSUTokenID(); + Groups[GroupID]->onInstructionIssued(IR); + } + + virtual void cycleEvent(); + +#ifndef NDEBUG + void dump() const; +#endif +}; + +/// Default Load/Store Unit (LS Unit) for simulated processors. +/// +/// Each load (or store) consumes one entry in the load (or store) queue. /// /// Rules are: /// 1) A younger load is allowed to pass an older load only if there are no @@ -89,26 +365,7 @@ class Scheduler; /// A load/store barrier is "executed" when it becomes the oldest entry in /// the load/store queue(s). That also means, all the older loads/stores have /// already been executed. -class LSUnit : public HardwareUnit { - // Load queue size. - // LQ_Size == 0 means that there are infinite slots in the load queue. - unsigned LQ_Size; - - // Store queue size. - // SQ_Size == 0 means that there are infinite slots in the store queue. - unsigned SQ_Size; - - // If true, loads will never alias with stores. This is the default. - bool NoAlias; - - // When a `MayLoad` instruction is dispatched to the schedulers for execution, - // the LSUnit reserves an entry in the `LoadQueue` for it. - // - // LoadQueue keeps track of all the loads that are in-flight. A load - // instruction is eventually removed from the LoadQueue when it reaches - // completion stage. That means, a load leaves the queue whe it is 'executed', - // and its value can be forwarded on the data path to outside units. - // +class LSUnit : public LSUnitBase { // This class doesn't know about the latency of a load instruction. So, it // conservatively/pessimistically assumes that the latency of a load opcode // matches the instruction latency. @@ -139,66 +396,50 @@ class LSUnit : public HardwareUnit { // alternative approaches that let instructions specify the number of // load/store queue entries which they consume at dispatch stage (See // PR39830). - SmallSet LoadQueue; - SmallSet StoreQueue; - - void assignLQSlot(unsigned Index); - void assignSQSlot(unsigned Index); - bool isReadyNoAlias(unsigned Index) const; - + // // An instruction that both 'mayStore' and 'HasUnmodeledSideEffects' is // conservatively treated as a store barrier. It forces older store to be // executed before newer stores are issued. - SmallSet StoreBarriers; - + // // An instruction that both 'MayLoad' and 'HasUnmodeledSideEffects' is // conservatively treated as a load barrier. It forces older loads to execute // before newer loads are issued. - SmallSet LoadBarriers; - - bool isSQEmpty() const { return StoreQueue.empty(); } - bool isLQEmpty() const { return LoadQueue.empty(); } - bool isSQFull() const { return SQ_Size != 0 && StoreQueue.size() == SQ_Size; } - bool isLQFull() const { return LQ_Size != 0 && LoadQueue.size() == LQ_Size; } + unsigned CurrentLoadGroupID; + unsigned CurrentLoadBarrierGroupID; + unsigned CurrentStoreGroupID; public: - LSUnit(const MCSchedModel &SM, unsigned LQ = 0, unsigned SQ = 0, - bool AssumeNoAlias = false); + LSUnit(const MCSchedModel &SM) + : LSUnit(SM, /* LQSize */ 0, /* SQSize */ 0, /* NoAlias */ false) {} + LSUnit(const MCSchedModel &SM, unsigned LQ, unsigned SQ) + : LSUnit(SM, LQ, SQ, /* NoAlias */ false) {} + LSUnit(const MCSchedModel &SM, unsigned LQ, unsigned SQ, bool AssumeNoAlias) + : LSUnitBase(SM, LQ, SQ, AssumeNoAlias), CurrentLoadGroupID(0), + CurrentLoadBarrierGroupID(0), CurrentStoreGroupID(0) {} -#ifndef NDEBUG - void dump() const; -#endif + /// Returns LSU_AVAILABLE if there are enough load/store queue entries to + /// accomodate instruction IR. + Status isAvailable(const InstRef &IR) const override; - enum Status { LSU_AVAILABLE = 0, LSU_LQUEUE_FULL, LSU_SQUEUE_FULL }; + /// Allocates LS resources for instruction IR. + /// + /// This method assumes that a previous call to `isAvailable(IR)` succeeded + /// returning LSU_AVAILABLE. + /// + /// Rules are: + /// By default, rules are: + /// 1. A store may not pass a previous store. + /// 2. A load may not pass a previous store unless flag 'NoAlias' is set. + /// 3. A load may pass a previous load. + /// 4. A store may not pass a previous load (regardless of flag 'NoAlias'). + /// 5. A load has to wait until an older load barrier is fully executed. + /// 6. A store has to wait until an older store barrier is fully executed. + unsigned dispatch(const InstRef &IR) override; - // Returns LSU_AVAILABLE if there are enough load/store queue entries to serve - // IR. It also returns LSU_AVAILABLE if IR is not a memory operation. - Status isAvailable(const InstRef &IR) const; - - // Allocates load/store queue resources for IR. - // - // This method assumes that a previous call to `isAvailable(IR)` returned - // LSU_AVAILABLE, and that IR is a memory operation. - void dispatch(const InstRef &IR); - - // By default, rules are: - // 1. A store may not pass a previous store. - // 2. A load may not pass a previous store unless flag 'NoAlias' is set. - // 3. A load may pass a previous load. - // 4. A store may not pass a previous load (regardless of flag 'NoAlias'). - // 5. A load has to wait until an older load barrier is fully executed. - // 6. A store has to wait until an older store barrier is fully executed. - virtual bool isReady(const InstRef &IR) const; - - // Load and store instructions are tracked by their corresponding queues from - // dispatch until the "instruction executed" event. - // Only when a load instruction reaches the 'Executed' stage, its value - // becomes available to the users. At that point, the load no longer needs to - // be tracked by the load queue. // FIXME: For simplicity, we optimistically assume a similar behavior for // store instructions. In practice, store operations don't tend to leave the // store queue until they reach the 'Retired' stage (See PR39830). - void onInstructionExecuted(const InstRef &IR); + void onInstructionExecuted(const InstRef &IR) override; }; } // namespace mca diff --git a/include/llvm/MCA/HardwareUnits/RegisterFile.h b/include/llvm/MCA/HardwareUnits/RegisterFile.h index c23ab0389234..36506327bd29 100644 --- a/include/llvm/MCA/HardwareUnits/RegisterFile.h +++ b/include/llvm/MCA/HardwareUnits/RegisterFile.h @@ -1,9 +1,8 @@ //===--------------------- RegisterFile.h -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -21,6 +20,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCSchedule.h" +#include "llvm/MC/MCSubtargetInfo.h" #include "llvm/MCA/HardwareUnits/HardwareUnit.h" #include "llvm/Support/Error.h" @@ -196,7 +196,7 @@ public: // Collect writes that are in a data dependency with RS, and update RS // internal state. - void addRegisterRead(ReadState &RS, SmallVectorImpl &Writes) const; + void addRegisterRead(ReadState &RS, const MCSubtargetInfo &STI) const; // Removes write \param WS from the register mappings. // Physical registers may be released to reflect this update. diff --git a/include/llvm/MCA/HardwareUnits/ResourceManager.h b/include/llvm/MCA/HardwareUnits/ResourceManager.h index 549a46c247fe..2f91185516fb 100644 --- a/include/llvm/MCA/HardwareUnits/ResourceManager.h +++ b/include/llvm/MCA/HardwareUnits/ResourceManager.h @@ -1,9 +1,8 @@ //===--------------------- ResourceManager.h --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -335,13 +334,26 @@ class ResourceManager { // Used to quickly identify groups that own a particular resource unit. std::vector Resource2Groups; - // A table to map processor resource IDs to processor resource masks. + // A table that maps processor resource IDs to processor resource masks. SmallVector ProcResID2Mask; + // A table that maps resource indices to actual processor resource IDs in the + // scheduling model. + SmallVector ResIndex2ProcResID; + // Keeps track of which resources are busy, and how many cycles are left // before those become usable again. SmallDenseMap BusyResources; + // Set of processor resource units available on the target. + uint64_t ProcResUnitMask; + + // Set of processor resource units that are available during this cycle. + uint64_t AvailableProcResUnits; + + // Set of processor resource groups that are currently reserved. + uint64_t ReservedResourceGroups; + // Returns the actual resource unit that will be used. ResourceRef selectPipe(uint64_t ResourceID); @@ -389,7 +401,14 @@ public: // Release a previously reserved processor resource. void releaseResource(uint64_t ResourceID); - bool canBeIssued(const InstrDesc &Desc) const; + // Returns a zero mask if resources requested by Desc are all available during + // this cycle. It returns a non-zero mask value only if there are unavailable + // processor resources; each bit set in the mask represents a busy processor + // resource unit or a reserved processor resource group. + uint64_t checkAvailability(const InstrDesc &Desc) const; + + uint64_t getProcResUnitMask() const { return ProcResUnitMask; } + uint64_t getAvailableProcResUnits() const { return AvailableProcResUnits; } void issueInstruction( const InstrDesc &Desc, diff --git a/include/llvm/MCA/HardwareUnits/RetireControlUnit.h b/include/llvm/MCA/HardwareUnits/RetireControlUnit.h index 71360e984ade..06290141739e 100644 --- a/include/llvm/MCA/HardwareUnits/RetireControlUnit.h +++ b/include/llvm/MCA/HardwareUnits/RetireControlUnit.h @@ -1,9 +1,8 @@ //===---------------------- RetireControlUnit.h -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/MCA/HardwareUnits/Scheduler.h b/include/llvm/MCA/HardwareUnits/Scheduler.h index 351ea4827df9..27beb842dfd2 100644 --- a/include/llvm/MCA/HardwareUnits/Scheduler.h +++ b/include/llvm/MCA/HardwareUnits/Scheduler.h @@ -1,9 +1,8 @@ //===--------------------- Scheduler.h ------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -68,22 +67,6 @@ public: /// resources. This class is also responsible for tracking the progress of /// instructions from the dispatch stage, until the write-back stage. /// -/// An instruction dispatched to the Scheduler is initially placed into either -/// the 'WaitSet' or the 'ReadySet' depending on the availability of the input -/// operands. -/// -/// An instruction is moved from the WaitSet to the ReadySet when register -/// operands become available, and all memory dependencies are met. -/// Instructions that are moved from the WaitSet to the ReadySet transition -/// in state from 'IS_AVAILABLE' to 'IS_READY'. -/// -/// On every cycle, the Scheduler checks if it can promote instructions from the -/// WaitSet to the ReadySet. -/// -/// An Instruction is moved from the ReadySet the `IssuedSet` when it is issued -/// to a (one or more) pipeline(s). This event also causes an instruction state -/// transition (i.e. from state IS_READY, to state IS_EXECUTING). An Instruction -/// leaves the IssuedSet when it reaches the write-back stage. class Scheduler : public HardwareUnit { LSUnit &LSU; @@ -93,10 +76,58 @@ class Scheduler : public HardwareUnit { // Hardware resources that are managed by this scheduler. std::unique_ptr Resources; + // Instructions dispatched to the Scheduler are internally classified based on + // the instruction stage (see Instruction::InstrStage). + // + // An Instruction dispatched to the Scheduler is added to the WaitSet if not + // all its register operands are available, and at least one latency is + // unknown. By construction, the WaitSet only contains instructions that are + // in the IS_DISPATCHED stage. + // + // An Instruction transitions from the WaitSet to the PendingSet if the + // instruction is not ready yet, but the latency of every register read is + // known. Instructions in the PendingSet can only be in the IS_PENDING or + // IS_READY stage. Only IS_READY instructions that are waiting on memory + // dependencies can be added to the PendingSet. + // + // Instructions in the PendingSet are immediately dominated only by + // instructions that have already been issued to the underlying pipelines. In + // the presence of bottlenecks caused by data dependencies, the PendingSet can + // be inspected to identify problematic data dependencies between + // instructions. + // + // An instruction is moved to the ReadySet when all register operands become + // available, and all memory dependencies are met. Instructions that are + // moved from the PendingSet to the ReadySet must transition to the 'IS_READY' + // stage. + // + // On every cycle, the Scheduler checks if it can promote instructions from the + // PendingSet to the ReadySet. + // + // An Instruction is moved from the ReadySet to the `IssuedSet` when it starts + // exection. This event also causes an instruction state transition (i.e. from + // state IS_READY, to state IS_EXECUTING). An Instruction leaves the IssuedSet + // only when it reaches the write-back stage. std::vector WaitSet; + std::vector PendingSet; std::vector ReadySet; std::vector IssuedSet; + // A mask of busy resource units. It defaults to the empty set (i.e. a zero + // mask), and it is cleared at the beginning of every cycle. + // It is updated every time the scheduler fails to issue an instruction from + // the ready set due to unavailable pipeline resources. + // Each bit of the mask represents an unavailable resource. + uint64_t BusyResourceUnits; + + // Counts the number of instructions in the pending set that were dispatched + // during this cycle. + unsigned NumDispatchedToThePendingSet; + + // True if the previous pipeline Stage was unable to dispatch a full group of + // opcodes because scheduler buffers (or LS queues) were unavailable. + bool HadTokenStall; + /// Verify the given selection strategy and set the Strategy member /// accordingly. If no strategy is provided, the DefaultSchedulerStrategy is /// used. @@ -112,9 +143,15 @@ class Scheduler : public HardwareUnit { // vector 'Executed'. void updateIssuedSet(SmallVectorImpl &Executed); - // Try to promote instructions from WaitSet to ReadySet. + // Try to promote instructions from the PendingSet to the ReadySet. // Add promoted instructions to the 'Ready' vector in input. - void promoteToReadySet(SmallVectorImpl &Ready); + // Returns true if at least one instruction was promoted. + bool promoteToReadySet(SmallVectorImpl &Ready); + + // Try to promote instructions from the WaitSet to the PendingSet. + // Add promoted instructions to the 'Pending' vector in input. + // Returns true if at least one instruction was promoted. + bool promoteToPendingSet(SmallVectorImpl &Pending); public: Scheduler(const MCSchedModel &Model, LSUnit &Lsu) @@ -127,7 +164,8 @@ public: Scheduler(std::unique_ptr RM, LSUnit &Lsu, std::unique_ptr SelectStrategy) - : LSU(Lsu), Resources(std::move(RM)) { + : LSU(Lsu), Resources(std::move(RM)), BusyResourceUnits(0), + NumDispatchedToThePendingSet(0), HadTokenStall(false) { initializeStrategy(std::move(SelectStrategy)); } @@ -140,15 +178,12 @@ public: SC_DISPATCH_GROUP_STALL, }; - /// Check if the instruction in 'IR' can be dispatched and returns an answer - /// in the form of a Status value. + /// Check if the instruction in 'IR' can be dispatched during this cycle. + /// Return SC_AVAILABLE if both scheduler and LS resources are available. /// - /// The DispatchStage is responsible for querying the Scheduler before - /// dispatching new instructions. This routine is used for performing such - /// a query. If the instruction 'IR' can be dispatched, then true is - /// returned, otherwise false is returned with Event set to the stall type. - /// Internally, it also checks if the load/store unit is available. - Status isAvailable(const InstRef &IR) const; + /// This method is also responsible for setting field HadTokenStall if + /// IR cannot be dispatched to the Scheduler due to unavailable resources. + Status isAvailable(const InstRef &IR); /// Reserves buffer and LSUnit queue resources that are necessary to issue /// this instruction. @@ -156,11 +191,11 @@ public: /// Returns true if instruction IR is ready to be issued to the underlying /// pipelines. Note that this operation cannot fail; it assumes that a /// previous call to method `isAvailable(IR)` returned `SC_AVAILABLE`. - void dispatch(const InstRef &IR); - - /// Returns true if IR is ready to be executed by the underlying pipelines. - /// This method assumes that IR has been previously dispatched. - bool isReady(const InstRef &IR) const; + /// + /// If IR is a memory operation, then the Scheduler queries the LS unit to + /// obtain a LS token. An LS token is used internally to track memory + /// dependencies. + bool dispatch(InstRef &IR); /// Issue an instruction and populates a vector of used pipeline resources, /// and a vector of instructions that transitioned to the ready state as a @@ -168,6 +203,7 @@ public: void issueInstruction( InstRef &IR, SmallVectorImpl> &Used, + SmallVectorImpl &Pending, SmallVectorImpl &Ready); /// Returns true if IR has to be issued immediately, or if IR is a zero @@ -181,9 +217,15 @@ public: /// have changed in state, and that are now available to new instructions. /// Instructions executed are added to vector Executed, while vector Ready is /// populated with instructions that have become ready in this new cycle. + /// Vector Pending is popluated by instructions that have transitioned through + /// the pending stat during this cycle. The Pending and Ready sets may not be + /// disjoint. An instruction is allowed to transition from the WAIT state to + /// the READY state (going through the PENDING state) within a single cycle. + /// That means, instructions may appear in both the Pending and Ready set. void cycleEvent(SmallVectorImpl &Freed, - SmallVectorImpl &Ready, - SmallVectorImpl &Executed); + SmallVectorImpl &Executed, + SmallVectorImpl &Pending, + SmallVectorImpl &Ready); /// Convert a resource mask into a valid llvm processor resource identifier. unsigned getResourceID(uint64_t Mask) const { @@ -195,6 +237,26 @@ public: /// resources are not available. InstRef select(); + bool isReadySetEmpty() const { return ReadySet.empty(); } + bool isWaitSetEmpty() const { return WaitSet.empty(); } + + /// This method is called by the ExecuteStage at the end of each cycle to + /// identify bottlenecks caused by data dependencies. Vector RegDeps is + /// populated by instructions that were not issued because of unsolved + /// register dependencies. Vector MemDeps is populated by instructions that + /// were not issued because of unsolved memory dependencies. + void analyzeDataDependencies(SmallVectorImpl &RegDeps, + SmallVectorImpl &MemDeps); + + /// Returns a mask of busy resources, and populates vector Insts with + /// instructions that could not be issued to the underlying pipelines because + /// not all pipeline resources were available. + uint64_t analyzeResourcePressure(SmallVectorImpl &Insts); + + // Returns true if the dispatch logic couldn't dispatch a full group due to + // unavailable scheduler and/or LS resources. + bool hadTokenStall() const { return HadTokenStall; } + #ifndef NDEBUG // Update the ready queues. void dump() const; diff --git a/include/llvm/MCA/InstrBuilder.h b/include/llvm/MCA/InstrBuilder.h index 5f998db5e4ce..690016354f7a 100644 --- a/include/llvm/MCA/InstrBuilder.h +++ b/include/llvm/MCA/InstrBuilder.h @@ -1,9 +1,8 @@ //===--------------------- InstrBuilder.h -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/MCA/Instruction.h b/include/llvm/MCA/Instruction.h index b91610c64d85..d4d3f22797f7 100644 --- a/include/llvm/MCA/Instruction.h +++ b/include/llvm/MCA/Instruction.h @@ -1,9 +1,8 @@ //===--------------------- Instruction.h ------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -81,6 +80,15 @@ struct ReadDescriptor { class ReadState; +/// A critical data dependency descriptor. +/// +/// Field RegID is set to the invalid register for memory dependencies. +struct CriticalDependency { + unsigned IID; + unsigned RegID; + unsigned Cycles; +}; + /// Tracks uses of a register definition (e.g. register write). /// /// Each implicit/explicit register write is associated with an instance of @@ -124,9 +132,11 @@ class WriteState { // A partial write that is in a false dependency with this write. WriteState *PartialWrite; - unsigned DependentWriteCyclesLeft; + // Critical register dependency for this write. + CriticalDependency CRD; + // A list of dependent reads. Users is a set of dependent // reads. A dependent read is added to the set only if CyclesLeft // is "unknown". As soon as CyclesLeft is 'known', each user in the set @@ -141,7 +151,7 @@ public: : WD(&Desc), CyclesLeft(UNKNOWN_CYCLES), RegisterID(RegID), PRFID(0), ClearsSuperRegs(clearsSuperRegs), WritesZero(writesZero), IsEliminated(false), DependentWrite(nullptr), PartialWrite(nullptr), - DependentWriteCyclesLeft(0) {} + DependentWriteCyclesLeft(0), CRD() {} WriteState(const WriteState &Other) = default; WriteState &operator=(const WriteState &Other) = default; @@ -151,13 +161,21 @@ public: unsigned getRegisterID() const { return RegisterID; } unsigned getRegisterFileID() const { return PRFID; } unsigned getLatency() const { return WD->Latency; } - - void addUser(ReadState *Use, int ReadAdvance); - void addUser(WriteState *Use); - unsigned getDependentWriteCyclesLeft() const { return DependentWriteCyclesLeft; } + const WriteState *getDependentWrite() const { return DependentWrite; } + const CriticalDependency &getCriticalRegDep() const { return CRD; } + + // This method adds Use to the set of data dependent reads. IID is the + // instruction identifier associated with this write. ReadAdvance is the + // number of cycles to subtract from the latency of this data dependency. + // Use is in a RAW dependency with this write. + void addUser(unsigned IID, ReadState *Use, int ReadAdvance); + + // Use is a younger register write that is in a false dependency with this + // write. IID is the instruction identifier associated with this write. + void addUser(unsigned IID, WriteState *Use); unsigned getNumUsers() const { unsigned NumUsers = Users.size(); @@ -169,17 +187,20 @@ public: bool clearsSuperRegisters() const { return ClearsSuperRegs; } bool isWriteZero() const { return WritesZero; } bool isEliminated() const { return IsEliminated; } - bool isExecuted() const { - return CyclesLeft != UNKNOWN_CYCLES && CyclesLeft <= 0; + + bool isReady() const { + if (DependentWrite) + return false; + unsigned CyclesLeft = getDependentWriteCyclesLeft(); + return !CyclesLeft || CyclesLeft < getLatency(); } - const WriteState *getDependentWrite() const { return DependentWrite; } - void setDependentWrite(WriteState *Other) { DependentWrite = Other; } - void writeStartEvent(unsigned Cycles) { - DependentWriteCyclesLeft = Cycles; - DependentWrite = nullptr; + bool isExecuted() const { + return CyclesLeft != UNKNOWN_CYCLES && CyclesLeft <= 0; } + void setDependentWrite(const WriteState *Other) { DependentWrite = Other; } + void writeStartEvent(unsigned IID, unsigned RegID, unsigned Cycles); void setWriteZero() { WritesZero = true; } void setEliminated() { assert(Users.empty() && "Write is in an inconsistent state."); @@ -191,7 +212,7 @@ public: // On every cycle, update CyclesLeft and notify dependent users. void cycleEvent(); - void onInstructionIssued(); + void onInstructionIssued(unsigned IID); #ifndef NDEBUG void dump() const; @@ -221,6 +242,8 @@ class ReadState { // dependent writes (i.e. field DependentWrite) is zero, this value is // propagated to field CyclesLeft. unsigned TotalCycles; + // Longest register dependency. + CriticalDependency CRD; // This field is set to true only if there are no dependent writes, and // there are no `CyclesLeft' to wait. bool IsReady; @@ -232,14 +255,16 @@ class ReadState { public: ReadState(const ReadDescriptor &Desc, unsigned RegID) : RD(&Desc), RegisterID(RegID), PRFID(0), DependentWrites(0), - CyclesLeft(UNKNOWN_CYCLES), TotalCycles(0), IsReady(true), + CyclesLeft(UNKNOWN_CYCLES), TotalCycles(0), CRD(), IsReady(true), IsZero(false), IndependentFromDef(false) {} const ReadDescriptor &getDescriptor() const { return *RD; } unsigned getSchedClass() const { return RD->SchedClassID; } unsigned getRegisterID() const { return RegisterID; } unsigned getRegisterFileID() const { return PRFID; } + const CriticalDependency &getCriticalRegDep() const { return CRD; } + bool isPending() const { return !IndependentFromDef && CyclesLeft > 0; } bool isReady() const { return IsReady; } bool isImplicitRead() const { return RD->isImplicitRead(); } @@ -247,7 +272,7 @@ public: void setIndependentFromDef() { IndependentFromDef = true; } void cycleEvent(); - void writeStartEvent(unsigned Cycles); + void writeStartEvent(unsigned IID, unsigned RegID, unsigned Cycles); void setDependentWrites(unsigned Writes) { DependentWrites = Writes; IsReady = !Writes; @@ -330,9 +355,16 @@ struct InstrDesc { // A list of buffered resources consumed by this instruction. SmallVector Buffers; + unsigned UsedProcResUnits; + unsigned UsedProcResGroups; + unsigned MaxLatency; // Number of MicroOps for this instruction. unsigned NumMicroOps; + // SchedClassID used to construct this InstrDesc. + // This information is currently used by views to do fast queries on the + // subtarget when computing the reciprocal throughput. + unsigned SchedClassID; bool MayLoad; bool MayStore; @@ -398,6 +430,7 @@ public: // Returns true if this instruction is a candidate for move elimination. bool isOptimizableMove() const { return IsOptimizableMove; } void setOptimizableMove() { IsOptimizableMove = true; } + bool isMemOp() const { return Desc.MayLoad || Desc.MayStore; } }; /// An instruction propagated through the simulated instruction pipeline. @@ -406,12 +439,13 @@ public: /// that are sent to the various components of the simulated hardware pipeline. class Instruction : public InstructionBase { enum InstrStage { - IS_INVALID, // Instruction in an invalid state. - IS_AVAILABLE, // Instruction dispatched but operands are not ready. - IS_READY, // Instruction dispatched and operands ready. - IS_EXECUTING, // Instruction issued. - IS_EXECUTED, // Instruction executed. Values are written back. - IS_RETIRED // Instruction retired. + IS_INVALID, // Instruction in an invalid state. + IS_DISPATCHED, // Instruction dispatched but operands are not ready. + IS_PENDING, // Instruction is not ready, but operand latency is known. + IS_READY, // Instruction dispatched and operands ready. + IS_EXECUTING, // Instruction issued. + IS_EXECUTED, // Instruction executed. Values are written back. + IS_RETIRED // Instruction retired. }; // The current instruction stage. @@ -424,12 +458,34 @@ class Instruction : public InstructionBase { // Retire Unit token ID for this instruction. unsigned RCUTokenID; + // LS token ID for this instruction. + // This field is set to the invalid null token if this is not a memory + // operation. + unsigned LSUTokenID; + + // Critical register dependency. + CriticalDependency CriticalRegDep; + + // Critical memory dependency. + CriticalDependency CriticalMemDep; + + // A bitmask of busy processor resource units. + // This field is set to zero only if execution is not delayed during this + // cycle because of unavailable pipeline resources. + uint64_t CriticalResourceMask; + + // True if this instruction has been optimized at register renaming stage. + bool IsEliminated; + public: Instruction(const InstrDesc &D) : InstructionBase(D), Stage(IS_INVALID), CyclesLeft(UNKNOWN_CYCLES), - RCUTokenID(0) {} + RCUTokenID(0), LSUTokenID(0), CriticalRegDep(), CriticalMemDep(), + CriticalResourceMask(0), IsEliminated(false) {} unsigned getRCUTokenID() const { return RCUTokenID; } + unsigned getLSUTokenID() const { return LSUTokenID; } + void setLSUTokenID(unsigned LSUTok) { LSUTokenID = LSUTok; } int getCyclesLeft() const { return CyclesLeft; } // Transition to the dispatch stage, and assign a RCUToken to this @@ -438,37 +494,48 @@ public: void dispatch(unsigned RCUTokenID); // Instruction issued. Transition to the IS_EXECUTING state, and update - // all the definitions. - void execute(); - - // Force a transition from the IS_AVAILABLE state to the IS_READY state if - // input operands are all ready. State transitions normally occur at the - // beginning of a new cycle (see method cycleEvent()). However, the scheduler - // may decide to promote instructions from the wait queue to the ready queue - // as the result of another issue event. This method is called every time the - // instruction might have changed in state. + // all the register definitions. + void execute(unsigned IID); + + // Force a transition from the IS_DISPATCHED state to the IS_READY or + // IS_PENDING state. State transitions normally occur either at the beginning + // of a new cycle (see method cycleEvent()), or as a result of another issue + // event. This method is called every time the instruction might have changed + // in state. It internally delegates to method updateDispatched() and + // updateWaiting(). void update(); + bool updateDispatched(); + bool updatePending(); - bool isDispatched() const { return Stage == IS_AVAILABLE; } + bool isDispatched() const { return Stage == IS_DISPATCHED; } + bool isPending() const { return Stage == IS_PENDING; } bool isReady() const { return Stage == IS_READY; } bool isExecuting() const { return Stage == IS_EXECUTING; } bool isExecuted() const { return Stage == IS_EXECUTED; } bool isRetired() const { return Stage == IS_RETIRED; } + bool isEliminated() const { return IsEliminated; } - bool isEliminated() const { - return isReady() && getDefs().size() && - all_of(getDefs(), - [](const WriteState &W) { return W.isEliminated(); }); - } - - // Forces a transition from state IS_AVAILABLE to state IS_EXECUTED. + // Forces a transition from state IS_DISPATCHED to state IS_EXECUTED. void forceExecuted(); + void setEliminated() { IsEliminated = true; } void retire() { assert(isExecuted() && "Instruction is in an invalid state!"); Stage = IS_RETIRED; } + const CriticalDependency &getCriticalRegDep() const { return CriticalRegDep; } + const CriticalDependency &getCriticalMemDep() const { return CriticalMemDep; } + const CriticalDependency &computeCriticalRegDep(); + void setCriticalMemDep(const CriticalDependency &MemDep) { + CriticalMemDep = MemDep; + } + + uint64_t getCriticalResourceMask() const { return CriticalResourceMask; } + void setCriticalResourceMask(uint64_t ResourceMask) { + CriticalResourceMask = ResourceMask; + } + void cycleEvent(); }; @@ -483,13 +550,17 @@ public: InstRef(unsigned Index, Instruction *I) : Data(std::make_pair(Index, I)) {} bool operator==(const InstRef &Other) const { return Data == Other.Data; } + bool operator!=(const InstRef &Other) const { return Data != Other.Data; } + bool operator<(const InstRef &Other) const { + return Data.first < Other.Data.first; + } unsigned getSourceIndex() const { return Data.first; } Instruction *getInstruction() { return Data.second; } const Instruction *getInstruction() const { return Data.second; } /// Returns true if this references a valid instruction. - operator bool() const { return Data.second != nullptr; } + explicit operator bool() const { return Data.second != nullptr; } /// Invalidate this reference. void invalidate() { Data.second = nullptr; } @@ -537,7 +608,7 @@ public: return !WS || WS->isExecuted(); } - bool isValid() const { return Data.first != INVALID_IID && Data.second; } + bool isValid() const { return Data.second && Data.first != INVALID_IID; } bool operator==(const WriteRef &Other) const { return Data == Other.Data; } #ifndef NDEBUG diff --git a/include/llvm/MCA/Pipeline.h b/include/llvm/MCA/Pipeline.h index acd256060bdd..935033f67f8b 100644 --- a/include/llvm/MCA/Pipeline.h +++ b/include/llvm/MCA/Pipeline.h @@ -1,9 +1,8 @@ //===--------------------- Pipeline.h ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/MCA/SourceMgr.h b/include/llvm/MCA/SourceMgr.h index 5e0ca6419f5d..dbe31db1b1dd 100644 --- a/include/llvm/MCA/SourceMgr.h +++ b/include/llvm/MCA/SourceMgr.h @@ -1,9 +1,8 @@ //===--------------------- SourceMgr.h --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/MCA/Stages/DispatchStage.h b/include/llvm/MCA/Stages/DispatchStage.h index f015cd7522eb..d80ededeaca1 100644 --- a/include/llvm/MCA/Stages/DispatchStage.h +++ b/include/llvm/MCA/Stages/DispatchStage.h @@ -1,9 +1,8 @@ //===----------------------- DispatchStage.h --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -62,8 +61,6 @@ class DispatchStage final : public Stage { bool canDispatch(const InstRef &IR) const; Error dispatch(InstRef IR); - void updateRAWDependencies(ReadState &RS, const MCSubtargetInfo &STI); - void notifyInstructionDispatched(const InstRef &IR, ArrayRef UsedPhysRegs, unsigned uOps) const; @@ -71,9 +68,7 @@ class DispatchStage final : public Stage { public: DispatchStage(const MCSubtargetInfo &Subtarget, const MCRegisterInfo &MRI, unsigned MaxDispatchWidth, RetireControlUnit &R, - RegisterFile &F) - : DispatchWidth(MaxDispatchWidth), AvailableEntries(MaxDispatchWidth), - CarryOver(0U), CarriedOver(), STI(Subtarget), RCU(R), PRF(F) {} + RegisterFile &F); bool isAvailable(const InstRef &IR) const override; diff --git a/include/llvm/MCA/Stages/EntryStage.h b/include/llvm/MCA/Stages/EntryStage.h index cd9a65b8cc2b..59a2daff886e 100644 --- a/include/llvm/MCA/Stages/EntryStage.h +++ b/include/llvm/MCA/Stages/EntryStage.h @@ -1,9 +1,8 @@ //===---------------------- EntryStage.h ------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/MCA/Stages/ExecuteStage.h b/include/llvm/MCA/Stages/ExecuteStage.h index 8cb287e06d9f..03737e0220eb 100644 --- a/include/llvm/MCA/Stages/ExecuteStage.h +++ b/include/llvm/MCA/Stages/ExecuteStage.h @@ -1,9 +1,8 @@ //===---------------------- ExecuteStage.h ----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -29,6 +28,12 @@ namespace mca { class ExecuteStage final : public Stage { Scheduler &HWS; + unsigned NumDispatchedOpcodes; + unsigned NumIssuedOpcodes; + + // True if this stage should notify listeners of HWPressureEvents. + bool EnablePressureEvents; + Error issueInstruction(InstRef &IR); // Called at the beginning of each cycle to issue already dispatched @@ -42,7 +47,10 @@ class ExecuteStage final : public Stage { ExecuteStage &operator=(const ExecuteStage &Other) = delete; public: - ExecuteStage(Scheduler &S) : Stage(), HWS(S) {} + ExecuteStage(Scheduler &S) : ExecuteStage(S, false) {} + ExecuteStage(Scheduler &S, bool ShouldPerformBottleneckAnalysis) + : Stage(), HWS(S), NumDispatchedOpcodes(0), NumIssuedOpcodes(0), + EnablePressureEvents(ShouldPerformBottleneckAnalysis) {} // This stage works under the assumption that the Pipeline will eventually // execute a retire stage. We don't need to check if pipelines and/or @@ -61,12 +69,14 @@ public: // Instructions that transitioned to the 'Executed' state are automatically // moved to the next stage (i.e. RetireStage). Error cycleStart() override; + Error cycleEnd() override; Error execute(InstRef &IR) override; void notifyInstructionIssued( const InstRef &IR, MutableArrayRef> Used) const; void notifyInstructionExecuted(const InstRef &IR) const; + void notifyInstructionPending(const InstRef &IR) const; void notifyInstructionReady(const InstRef &IR) const; void notifyResourceAvailable(const ResourceRef &RR) const; diff --git a/include/llvm/MCA/Stages/InstructionTables.h b/include/llvm/MCA/Stages/InstructionTables.h index 34e338f0ce6b..4b463c9b51c1 100644 --- a/include/llvm/MCA/Stages/InstructionTables.h +++ b/include/llvm/MCA/Stages/InstructionTables.h @@ -1,9 +1,8 @@ //===--------------------- InstructionTables.h ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/MCA/Stages/MicroOpQueueStage.h b/include/llvm/MCA/Stages/MicroOpQueueStage.h new file mode 100644 index 000000000000..50a5ef87b2d2 --- /dev/null +++ b/include/llvm/MCA/Stages/MicroOpQueueStage.h @@ -0,0 +1,88 @@ +//===---------------------- MicroOpQueueStage.h -----------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// +/// This file defines a stage that implements a queue of micro opcodes. +/// It can be used to simulate a hardware micro-op queue that serves opcodes to +/// the out of order backend. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_MCA_MICRO_OP_QUEUE_STAGE_H +#define LLVM_MCA_MICRO_OP_QUEUE_STAGE_H + +#include "llvm/ADT/SmallVector.h" +#include "llvm/MCA/Stages/Stage.h" + +namespace llvm { +namespace mca { + +/// A stage that simulates a queue of instruction opcodes. +class MicroOpQueueStage : public Stage { + SmallVector Buffer; + unsigned NextAvailableSlotIdx; + unsigned CurrentInstructionSlotIdx; + + // Limits the number of instructions that can be written to this buffer every + // cycle. A value of zero means that there is no limit to the instruction + // throughput in input. + const unsigned MaxIPC; + unsigned CurrentIPC; + + // Number of entries that are available during this cycle. + unsigned AvailableEntries; + + // True if instructions dispatched to this stage don't need to wait for the + // next cycle before moving to the next stage. + // False if this buffer acts as a one cycle delay in the execution pipeline. + bool IsZeroLatencyStage; + + MicroOpQueueStage(const MicroOpQueueStage &Other) = delete; + MicroOpQueueStage &operator=(const MicroOpQueueStage &Other) = delete; + + // By default, an instruction consumes a number of buffer entries equal to its + // number of micro opcodes (see field `InstrDesc::NumMicroOpcodes`). The + // number of entries consumed by an instruction is normalized to the + // minimum value between NumMicroOpcodes and the buffer size. This is to avoid + // problems with (microcoded) instructions that generate a number of micro + // opcodes than doesn't fit in the buffer. + unsigned getNormalizedOpcodes(const InstRef &IR) const { + unsigned NormalizedOpcodes = + std::min(static_cast(Buffer.size()), + IR.getInstruction()->getDesc().NumMicroOps); + return NormalizedOpcodes ? NormalizedOpcodes : 1U; + } + + Error moveInstructions(); + +public: + MicroOpQueueStage(unsigned Size, unsigned IPC = 0, + bool ZeroLatencyStage = true); + + bool isAvailable(const InstRef &IR) const override { + if (MaxIPC && CurrentIPC == MaxIPC) + return false; + unsigned NormalizedOpcodes = getNormalizedOpcodes(IR); + if (NormalizedOpcodes > AvailableEntries) + return false; + return true; + } + + bool hasWorkToComplete() const override { + return AvailableEntries != Buffer.size(); + } + + Error execute(InstRef &IR) override; + Error cycleStart() override; + Error cycleEnd() override; +}; + +} // namespace mca +} // namespace llvm + +#endif // LLVM_MCA_MICRO_OP_QUEUE_STAGE_H diff --git a/include/llvm/MCA/Stages/RetireStage.h b/include/llvm/MCA/Stages/RetireStage.h index 2051ce5c86ad..08c216ac7bf4 100644 --- a/include/llvm/MCA/Stages/RetireStage.h +++ b/include/llvm/MCA/Stages/RetireStage.h @@ -1,9 +1,8 @@ //===---------------------- RetireStage.h -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/MCA/Stages/Stage.h b/include/llvm/MCA/Stages/Stage.h index fc7ab569bb0f..46b242caa6cf 100644 --- a/include/llvm/MCA/Stages/Stage.h +++ b/include/llvm/MCA/Stages/Stage.h @@ -1,9 +1,8 @@ //===---------------------- Stage.h -----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/MCA/Support.h b/include/llvm/MCA/Support.h index 7b0c5bf3a486..1da097c90922 100644 --- a/include/llvm/MCA/Support.h +++ b/include/llvm/MCA/Support.h @@ -1,9 +1,8 @@ //===--------------------- Support.h ----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -61,24 +60,13 @@ public: return (Denominator == 1) ? Numerator : (double)Numerator / Denominator; } + unsigned getNumerator() const { return Numerator; } + unsigned getDenominator() const { return Denominator; } + // Add the components of RHS to this instance. Instead of calculating // the final value here, we keep track of the numerator and denominator // separately, to reduce floating point error. - ResourceCycles &operator+=(const ResourceCycles &RHS) { - if (Denominator == RHS.Denominator) - Numerator += RHS.Numerator; - else { - // Create a common denominator for LHS and RHS by calculating the least - // common multiple from the GCD. - unsigned GCD = GreatestCommonDivisor64(Denominator, RHS.Denominator); - unsigned LCM = (Denominator * RHS.Denominator) / GCD; - unsigned LHSNumerator = Numerator * (LCM / Denominator); - unsigned RHSNumerator = RHS.Numerator * (LCM / RHS.Denominator); - Numerator = LHSNumerator + RHSNumerator; - Denominator = LCM; - } - return *this; - } + ResourceCycles &operator+=(const ResourceCycles &RHS); }; /// Populates vector Masks with processor resource masks. @@ -106,6 +94,13 @@ public: void computeProcResourceMasks(const MCSchedModel &SM, MutableArrayRef Masks); +// Returns the index of the highest bit set. For resource masks, the position of +// the highest bit set can be used to construct a resource mask identifier. +inline unsigned getResourceStateIndex(uint64_t Mask) { + assert(Mask && "Processor Resource Mask cannot be zero!"); + return (std::numeric_limits::digits - countLeadingZeros(Mask)) - 1; +} + /// Compute the reciprocal block throughput from a set of processor resource /// cycles. The reciprocal block throughput is computed as the MAX between: /// - NumMicroOps / DispatchWidth diff --git a/include/llvm/Object/Archive.h b/include/llvm/Object/Archive.h index 9ef1e4875191..c40278a4f923 100644 --- a/include/llvm/Object/Archive.h +++ b/include/llvm/Object/Archive.h @@ -1,9 +1,8 @@ //===- Archive.h - ar archive file format -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -16,6 +15,7 @@ #include "llvm/ADT/Optional.h" #include "llvm/ADT/StringRef.h" +#include "llvm/ADT/fallible_iterator.h" #include "llvm/ADT/iterator_range.h" #include "llvm/Object/Binary.h" #include "llvm/Support/Chrono.h" @@ -143,44 +143,38 @@ public: getAsBinary(LLVMContext *Context = nullptr) const; }; - class child_iterator { + class ChildFallibleIterator { Child C; - Error *E = nullptr; public: - child_iterator() : C(Child(nullptr, nullptr, nullptr)) {} - child_iterator(const Child &C, Error *E) : C(C), E(E) {} + ChildFallibleIterator() : C(Child(nullptr, nullptr, nullptr)) {} + ChildFallibleIterator(const Child &C) : C(C) {} const Child *operator->() const { return &C; } const Child &operator*() const { return C; } - bool operator==(const child_iterator &other) const { + bool operator==(const ChildFallibleIterator &other) const { // Ignore errors here: If an error occurred during increment then getNext // will have been set to child_end(), and the following comparison should // do the right thing. return C == other.C; } - bool operator!=(const child_iterator &other) const { + bool operator!=(const ChildFallibleIterator &other) const { return !(*this == other); } - // Code in loops with child_iterators must check for errors on each loop - // iteration. And if there is an error break out of the loop. - child_iterator &operator++() { // Preincrement - assert(E && "Can't increment iterator with no Error attached"); - ErrorAsOutParameter ErrAsOutParam(E); - if (auto ChildOrErr = C.getNext()) - C = *ChildOrErr; - else { - C = C.getParent()->child_end().C; - *E = ChildOrErr.takeError(); - E = nullptr; - } - return *this; + Error inc() { + auto NextChild = C.getNext(); + if (!NextChild) + return NextChild.takeError(); + C = std::move(*NextChild); + return Error::success(); } }; + using child_iterator = fallible_iterator; + class Symbol { const Archive *Parent; uint32_t SymbolIndex; diff --git a/include/llvm/Object/ArchiveWriter.h b/include/llvm/Object/ArchiveWriter.h index 495b943d04c0..9e6daf2da36e 100644 --- a/include/llvm/Object/ArchiveWriter.h +++ b/include/llvm/Object/ArchiveWriter.h @@ -1,9 +1,8 @@ //===- ArchiveWriter.h - ar archive file format writer ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -27,7 +26,6 @@ struct NewArchiveMember { sys::TimePoint ModTime; unsigned UID = 0, GID = 0, Perms = 0644; - bool IsNew = false; NewArchiveMember() = default; NewArchiveMember(MemoryBufferRef BufRef); @@ -38,6 +36,8 @@ struct NewArchiveMember { bool Deterministic); }; +Expected computeArchiveRelativePath(StringRef From, StringRef To); + Error writeArchive(StringRef ArcName, ArrayRef NewMembers, bool WriteSymtab, object::Archive::Kind Kind, bool Deterministic, bool Thin, diff --git a/include/llvm/Object/Binary.h b/include/llvm/Object/Binary.h index 99745e24b8c8..3c3e977baff4 100644 --- a/include/llvm/Object/Binary.h +++ b/include/llvm/Object/Binary.h @@ -1,9 +1,8 @@ //===- Binary.h - A generic binary file -------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,6 +13,7 @@ #ifndef LLVM_OBJECT_BINARY_H #define LLVM_OBJECT_BINARY_H +#include "llvm-c/Types.h" #include "llvm/ADT/Triple.h" #include "llvm/Object/Error.h" #include "llvm/Support/Error.h" @@ -42,7 +42,9 @@ protected: ID_Archive, ID_MachOUniversalBinary, ID_COFFImportFile, - ID_IR, // LLVM IR + ID_IR, // LLVM IR + + ID_Minidump, ID_WinRes, // Windows resource (.res) file. @@ -50,6 +52,9 @@ protected: ID_StartObjects, ID_COFF, + ID_XCOFF32, // AIX XCOFF 32-bit + ID_XCOFF64, // AIX XCOFF 64-bit + ID_ELF32L, // ELF 32-bit, little endian ID_ELF32B, // ELF 32-bit, big endian ID_ELF64L, // ELF 64-bit, little endian @@ -118,6 +123,8 @@ public: return TypeID == ID_COFF; } + bool isXCOFF() const { return TypeID == ID_XCOFF32 || TypeID == ID_XCOFF64; } + bool isWasm() const { return TypeID == ID_Wasm; } bool isCOFFImportFile() const { @@ -128,6 +135,8 @@ public: return TypeID == ID_IR; } + bool isMinidump() const { return TypeID == ID_Minidump; } + bool isLittleEndian() const { return !(TypeID == ID_ELF32B || TypeID == ID_ELF64B || TypeID == ID_MachO32B || TypeID == ID_MachO64B); @@ -156,6 +165,9 @@ public: } }; +// Create wrappers for C Binding types (see CBindingWrapping.h). +DEFINE_ISA_CONVERSION_FUNCTIONS(Binary, LLVMBinaryRef) + /// Create a Binary from Source, autodetecting the file type. /// /// @param Source The data to create the Binary from. diff --git a/include/llvm/Object/COFF.h b/include/llvm/Object/COFF.h index b753d261a0fc..c53cbc46c747 100644 --- a/include/llvm/Object/COFF.h +++ b/include/llvm/Object/COFF.h @@ -1,9 +1,8 @@ //===- COFF.h - COFF object file implementation -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -898,13 +897,12 @@ protected: Expected getSymbolType(DataRefImpl Symb) const override; Expected getSymbolSection(DataRefImpl Symb) const override; void moveSectionNext(DataRefImpl &Sec) const override; - std::error_code getSectionName(DataRefImpl Sec, - StringRef &Res) const override; + Expected getSectionName(DataRefImpl Sec) const override; uint64_t getSectionAddress(DataRefImpl Sec) const override; uint64_t getSectionIndex(DataRefImpl Sec) const override; uint64_t getSectionSize(DataRefImpl Sec) const override; - std::error_code getSectionContents(DataRefImpl Sec, - StringRef &Res) const override; + Expected> + getSectionContents(DataRefImpl Sec) const override; uint64_t getSectionAlignment(DataRefImpl Sec) const override; bool isSectionCompressed(DataRefImpl Sec) const override; bool isSectionText(DataRefImpl Sec) const override; @@ -1034,10 +1032,10 @@ public: ArrayRef getRelocations(const coff_section *Sec) const; - std::error_code getSectionName(const coff_section *Sec, StringRef &Res) const; + Expected getSectionName(const coff_section *Sec) const; uint64_t getSectionSize(const coff_section *Sec) const; - std::error_code getSectionContents(const coff_section *Sec, - ArrayRef &Res) const; + Error getSectionContents(const coff_section *Sec, + ArrayRef &Res) const; uint64_t getImageBase() const; std::error_code getVaPtr(uint64_t VA, uintptr_t &Res) const; diff --git a/include/llvm/Object/COFFImportFile.h b/include/llvm/Object/COFFImportFile.h index 0a4556ad8884..5aa836411118 100644 --- a/include/llvm/Object/COFFImportFile.h +++ b/include/llvm/Object/COFFImportFile.h @@ -1,9 +1,8 @@ //===- COFFImportFile.h - COFF short import file implementation -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -37,12 +36,11 @@ public: void moveSymbolNext(DataRefImpl &Symb) const override { ++Symb.p; } - std::error_code printSymbolName(raw_ostream &OS, - DataRefImpl Symb) const override { + Error printSymbolName(raw_ostream &OS, DataRefImpl Symb) const override { if (Symb.p == 0) OS << "__imp_"; OS << StringRef(Data.getBufferStart() + sizeof(coff_import_header)); - return std::error_code(); + return Error::success(); } uint32_t getSymbolFlags(DataRefImpl Symb) const override { @@ -71,9 +69,21 @@ private: }; struct COFFShortExport { + /// The name of the export as specified in the .def file or on the command + /// line, i.e. "foo" in "/EXPORT:foo", and "bar" in "/EXPORT:foo=bar". This + /// may lack mangling, such as underscore prefixing and stdcall suffixing. std::string Name; + + /// The external, exported name. Only non-empty when export renaming is in + /// effect, i.e. "foo" in "/EXPORT:foo=bar". std::string ExtName; + + /// The real, mangled symbol name from the object file. Given + /// "/export:foo=bar", this could be "_bar@8" if bar is stdcall. std::string SymbolName; + + /// Creates a weak alias. This is the name of the weak aliasee. In a .def + /// file, this is "baz" in "EXPORTS\nfoo = bar == baz". std::string AliasTarget; uint16_t Ordinal = 0; diff --git a/include/llvm/Object/COFFModuleDefinition.h b/include/llvm/Object/COFFModuleDefinition.h index be139a2833b0..ab52259fea1a 100644 --- a/include/llvm/Object/COFFModuleDefinition.h +++ b/include/llvm/Object/COFFModuleDefinition.h @@ -1,9 +1,8 @@ //===--- COFFModuleDefinition.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Object/CVDebugRecord.h b/include/llvm/Object/CVDebugRecord.h index faad72c0df29..d41c7391f701 100644 --- a/include/llvm/Object/CVDebugRecord.h +++ b/include/llvm/Object/CVDebugRecord.h @@ -1,9 +1,8 @@ //===- CVDebugRecord.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Object/Decompressor.h b/include/llvm/Object/Decompressor.h index 2a77d2ffbf68..cc918481b308 100644 --- a/include/llvm/Object/Decompressor.h +++ b/include/llvm/Object/Decompressor.h @@ -1,9 +1,8 @@ //===-- Decompressor.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===/ diff --git a/include/llvm/Object/ELF.h b/include/llvm/Object/ELF.h index bcdc190cc7dc..cf8e4529bad9 100644 --- a/include/llvm/Object/ELF.h +++ b/include/llvm/Object/ELF.h @@ -1,9 +1,8 @@ //===- ELF.h - ELF object file implementation -------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -45,10 +44,26 @@ getElfArchType(StringRef Object) { (uint8_t)Object[ELF::EI_DATA]); } -static inline Error createError(StringRef Err) { +static inline Error createError(const Twine &Err) { return make_error(Err, object_error::parse_failed); } +template class ELFFile; + +template +std::string getSecIndexForError(const ELFFile *Obj, + const typename ELFT::Shdr *Sec) { + auto TableOrErr = Obj->sections(); + if (TableOrErr) + return "[index " + std::to_string(Sec - &TableOrErr->front()) + "]"; + // To make this helper be more convenient for error reporting purposes we + // drop the error. But really it should never be triggered. Before this point, + // our code should have called 'sections()' and reported a proper error on + // failure. + llvm::consumeError(TableOrErr.takeError()); + return "[unknown index]"; +} + template class ELFFile { public: @@ -80,9 +95,7 @@ public: using Elf_Relr_Range = typename ELFT::RelrRange; using Elf_Phdr_Range = typename ELFT::PhdrRange; - const uint8_t *base() const { - return reinterpret_cast(Buf.data()); - } + const uint8_t *base() const { return Buf.bytes_begin(); } size_t getBufSize() const { return Buf.size(); } @@ -115,8 +128,8 @@ public: SmallVectorImpl &Result) const; uint32_t getRelativeRelocationType() const; - const char *getDynamicTagAsString(unsigned Arch, uint64_t Type) const; - const char *getDynamicTagAsString(uint64_t Type) const; + std::string getDynamicTagAsString(unsigned Arch, uint64_t Type) const; + std::string getDynamicTagAsString(uint64_t Type) const; /// Get the symbol for a given relocation. Expected getRelocationSymbol(const Elf_Rel *Rel, @@ -165,11 +178,16 @@ public: /// Iterate over program header table. Expected program_headers() const { if (getHeader()->e_phnum && getHeader()->e_phentsize != sizeof(Elf_Phdr)) - return createError("invalid e_phentsize"); + return createError("invalid e_phentsize: " + + Twine(getHeader()->e_phentsize)); if (getHeader()->e_phoff + (getHeader()->e_phnum * getHeader()->e_phentsize) > getBufSize()) - return createError("program headers longer than binary"); + return createError("program headers are longer than binary of size " + + Twine(getBufSize()) + ": e_phoff = 0x" + + Twine::utohexstr(getHeader()->e_phoff) + + ", e_phnum = " + Twine(getHeader()->e_phnum) + + ", e_phentsize = " + Twine(getHeader()->e_phentsize)); auto *Begin = reinterpret_cast(base() + getHeader()->e_phoff); return makeArrayRef(Begin, Begin + getHeader()->e_phnum); @@ -183,12 +201,12 @@ public: /// \param Err [out] an error to support fallible iteration, which should /// be checked after iteration ends. Elf_Note_Iterator notes_begin(const Elf_Phdr &Phdr, Error &Err) const { - if (Phdr.p_type != ELF::PT_NOTE) { - Err = createError("attempt to iterate notes of non-note program header"); - return Elf_Note_Iterator(Err); - } + assert(Phdr.p_type == ELF::PT_NOTE && "Phdr is not of type PT_NOTE"); + ErrorAsOutParameter ErrAsOutParam(&Err); if (Phdr.p_offset + Phdr.p_filesz > getBufSize()) { - Err = createError("invalid program header offset/size"); + Err = createError("PT_NOTE header has invalid offset (0x" + + Twine::utohexstr(Phdr.p_offset) + ") or size (0x" + + Twine::utohexstr(Phdr.p_filesz) + ")"); return Elf_Note_Iterator(Err); } return Elf_Note_Iterator(base() + Phdr.p_offset, Phdr.p_filesz, Err); @@ -202,12 +220,13 @@ public: /// \param Err [out] an error to support fallible iteration, which should /// be checked after iteration ends. Elf_Note_Iterator notes_begin(const Elf_Shdr &Shdr, Error &Err) const { - if (Shdr.sh_type != ELF::SHT_NOTE) { - Err = createError("attempt to iterate notes of non-note section"); - return Elf_Note_Iterator(Err); - } + assert(Shdr.sh_type == ELF::SHT_NOTE && "Shdr is not of type SHT_NOTE"); + ErrorAsOutParameter ErrAsOutParam(&Err); if (Shdr.sh_offset + Shdr.sh_size > getBufSize()) { - Err = createError("invalid section offset/size"); + Err = createError("SHT_NOTE section " + getSecIndexForError(this, &Shdr) + + " has invalid offset (0x" + + Twine::utohexstr(Shdr.sh_offset) + ") or size (0x" + + Twine::utohexstr(Shdr.sh_size) + ")"); return Elf_Note_Iterator(Err); } return Elf_Note_Iterator(base() + Shdr.sh_offset, Shdr.sh_size, Err); @@ -274,7 +293,7 @@ template inline Expected getSection(typename ELFT::ShdrRange Sections, uint32_t Index) { if (Index >= Sections.size()) - return createError("invalid section index"); + return createError("invalid section index: " + Twine(Index)); return &Sections[Index]; } @@ -286,7 +305,10 @@ getExtendedSymbolTableIndex(const typename ELFT::Sym *Sym, assert(Sym->st_shndx == ELF::SHN_XINDEX); unsigned Index = Sym - FirstSym; if (Index >= ShndxTable.size()) - return createError("index past the end of the symbol table"); + return createError( + "extended symbol index (" + Twine(Index) + + ") is past the end of the SHT_SYMTAB_SHNDX section of size " + + Twine(ShndxTable.size())); // The size of the table was checked in getSHNDXTable. return ShndxTable[Index]; @@ -332,21 +354,19 @@ ELFFile::getSection(const Elf_Sym *Sym, Elf_Sym_Range Symbols, return getSection(Index); } -template -inline Expected -getSymbol(typename ELFT::SymRange Symbols, uint32_t Index) { - if (Index >= Symbols.size()) - return createError("invalid symbol index"); - return &Symbols[Index]; -} - template Expected ELFFile::getSymbol(const Elf_Shdr *Sec, uint32_t Index) const { - auto SymtabOrErr = symbols(Sec); - if (!SymtabOrErr) - return SymtabOrErr.takeError(); - return object::getSymbol(*SymtabOrErr, Index); + auto SymsOrErr = symbols(Sec); + if (!SymsOrErr) + return SymsOrErr.takeError(); + + Elf_Sym_Range Symbols = *SymsOrErr; + if (Index >= Symbols.size()) + return createError("unable to get symbol from section " + + getSecIndexForError(this, Sec) + + ": invalid symbol index (" + Twine(Index) + ")"); + return &Symbols[Index]; } template @@ -354,18 +374,26 @@ template Expected> ELFFile::getSectionContentsAsArray(const Elf_Shdr *Sec) const { if (Sec->sh_entsize != sizeof(T) && sizeof(T) != 1) - return createError("invalid sh_entsize"); + return createError("section " + getSecIndexForError(this, Sec) + + " has an invalid sh_entsize: " + Twine(Sec->sh_entsize)); uintX_t Offset = Sec->sh_offset; uintX_t Size = Sec->sh_size; if (Size % sizeof(T)) - return createError("size is not a multiple of sh_entsize"); + return createError("section " + getSecIndexForError(this, Sec) + + " has an invalid sh_size (" + Twine(Size) + + ") which is not a multiple of its sh_entsize (" + + Twine(Sec->sh_entsize) + ")"); if ((std::numeric_limits::max() - Offset < Size) || Offset + Size > Buf.size()) - return createError("invalid section offset"); + return createError("section " + getSecIndexForError(this, Sec) + + " has a sh_offset (0x" + Twine::utohexstr(Offset) + + ") + sh_size (0x" + Twine(Size) + + ") that cannot be represented"); if (Offset % alignof(T)) + // TODO: this error is untested. return createError("unaligned data"); const T *Start = reinterpret_cast(base() + Offset); @@ -438,8 +466,10 @@ ELFFile::getSectionStringTable(Elf_Shdr_Range Sections) const { if (!Index) // no section string table. return ""; + // TODO: Test a case when the sh_link of the section with index 0 is broken. if (Index >= Sections.size()) - return createError("invalid section index"); + return createError("section header string table index " + Twine(Index) + + " does not exist"); return getStringTable(&Sections[Index]); } @@ -448,7 +478,9 @@ template ELFFile::ELFFile(StringRef Object) : Buf(Object) {} template Expected> ELFFile::create(StringRef Object) { if (sizeof(Elf_Ehdr) > Object.size()) - return createError("Invalid buffer"); + return createError("invalid buffer: the size (" + Twine(Object.size()) + + ") is smaller than an ELF header (" + + Twine(sizeof(Elf_Ehdr)) + ")"); return ELFFile(Object); } @@ -459,16 +491,18 @@ Expected ELFFile::sections() const { return ArrayRef(); if (getHeader()->e_shentsize != sizeof(Elf_Shdr)) - return createError( - "invalid section header entry size (e_shentsize) in ELF header"); + return createError("invalid e_shentsize in ELF header: " + + Twine(getHeader()->e_shentsize)); const uint64_t FileSize = Buf.size(); - if (SectionTableOffset + sizeof(Elf_Shdr) > FileSize) - return createError("section header table goes past the end of the file"); + return createError( + "section header table goes past the end of the file: e_shoff = 0x" + + Twine::utohexstr(SectionTableOffset)); // Invalid address alignment of section headers if (SectionTableOffset & (alignof(Elf_Shdr) - 1)) + // TODO: this error is untested. return createError("invalid alignment of section headers"); const Elf_Shdr *First = @@ -479,6 +513,7 @@ Expected ELFFile::sections() const { NumSections = First->sh_size; if (NumSections > UINT64_MAX / sizeof(Elf_Shdr)) + // TODO: this error is untested. return createError("section table goes past the end of file"); const uint64_t SectionTableSize = NumSections * sizeof(Elf_Shdr); @@ -505,10 +540,14 @@ template Expected ELFFile::getEntry(const Elf_Shdr *Section, uint32_t Entry) const { if (sizeof(T) != Section->sh_entsize) + // TODO: this error is untested. return createError("invalid sh_entsize"); size_t Pos = Section->sh_offset + Entry * sizeof(T); if (Pos + sizeof(T) > Buf.size()) - return createError("invalid section offset"); + return createError("unable to access section " + + getSecIndexForError(this, Section) + " data at 0x" + + Twine::utohexstr(Pos) + + ": offset goes past the end of file"); return reinterpret_cast(base() + Pos); } @@ -534,6 +573,7 @@ ELFFile::getSection(const StringRef SectionName) const { if (*SecNameOrErr == SectionName) return &Sec; } + // TODO: this error is untested. return createError("invalid section name"); } @@ -541,15 +581,24 @@ template Expected ELFFile::getStringTable(const Elf_Shdr *Section) const { if (Section->sh_type != ELF::SHT_STRTAB) - return createError("invalid sh_type for string table, expected SHT_STRTAB"); + return createError("invalid sh_type for string table section " + + getSecIndexForError(this, Section) + + ": expected SHT_STRTAB, but got " + + object::getELFSectionTypeName(getHeader()->e_machine, + Section->sh_type)); auto V = getSectionContentsAsArray(Section); if (!V) return V.takeError(); ArrayRef Data = *V; if (Data.empty()) + // TODO: this error is untested. return createError("empty string table"); if (Data.back() != '\0') - return createError("string table non-null terminated"); + return createError(object::getELFSectionTypeName(getHeader()->e_machine, + Section->sh_type) + + " string table section " + + getSecIndexForError(this, Section) + + " is non-null terminated"); return StringRef(Data.begin(), Data.size()); } @@ -577,9 +626,13 @@ ELFFile::getSHNDXTable(const Elf_Shdr &Section, const Elf_Shdr &SymTable = **SymTableOrErr; if (SymTable.sh_type != ELF::SHT_SYMTAB && SymTable.sh_type != ELF::SHT_DYNSYM) + // TODO: this error is untested. return createError("invalid sh_type"); if (V.size() != (SymTable.sh_size / sizeof(Elf_Sym))) - return createError("invalid section contents size"); + return createError("SHT_SYMTAB_SHNDX section has sh_size (" + + Twine(SymTable.sh_size) + + ") which is not equal to the number of symbols (" + + Twine(V.size()) + ")"); return V; } @@ -598,6 +651,7 @@ ELFFile::getStringTableForSymtab(const Elf_Shdr &Sec, Elf_Shdr_Range Sections) const { if (Sec.sh_type != ELF::SHT_SYMTAB && Sec.sh_type != ELF::SHT_DYNSYM) + // TODO: this error is untested. return createError( "invalid sh_type for symbol table, expected SHT_SYMTAB or SHT_DYNSYM"); auto SectionOrErr = object::getSection(Sections, Sec.sh_link); @@ -625,7 +679,11 @@ Expected ELFFile::getSectionName(const Elf_Shdr *Section, if (Offset == 0) return StringRef(); if (Offset >= DotShstrtab.size()) - return createError("invalid string offset"); + return createError("a section " + getSecIndexForError(this, Section) + + " has an invalid sh_name (0x" + + Twine::utohexstr(Offset) + + ") offset which goes past the end of the " + "section name string table"); return StringRef(DotShstrtab.data() + Offset); } diff --git a/include/llvm/Object/ELFObjectFile.h b/include/llvm/Object/ELFObjectFile.h index 0f620681cd99..86c015efd704 100644 --- a/include/llvm/Object/ELFObjectFile.h +++ b/include/llvm/Object/ELFObjectFile.h @@ -1,9 +1,8 @@ //===- ELFObjectFile.h - ELF object file implementation ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -42,6 +41,9 @@ namespace llvm { namespace object { +constexpr int NumElfSymbolTypes = 8; +extern const llvm::EnumEntry ElfSymbolTypes[NumElfSymbolTypes]; + class elf_symbol_iterator; class ELFObjectFileBase : public ObjectFile { @@ -52,8 +54,8 @@ class ELFObjectFileBase : public ObjectFile { protected: ELFObjectFileBase(unsigned int Type, MemoryBufferRef Source); - virtual uint16_t getEMachine() const = 0; virtual uint64_t getSymbolSize(DataRefImpl Symb) const = 0; + virtual uint8_t getSymbolBinding(DataRefImpl Symb) const = 0; virtual uint8_t getSymbolOther(DataRefImpl Symb) const = 0; virtual uint8_t getSymbolELFType(DataRefImpl Symb) const = 0; @@ -62,6 +64,7 @@ protected: virtual uint64_t getSectionOffset(DataRefImpl Sec) const = 0; virtual Expected getRelocationAddend(DataRefImpl Rel) const = 0; + virtual Error getBuildAttributes(ARMAttributeParser &Attributes) const = 0; public: using elf_symbol_iterator_range = iterator_range; @@ -87,6 +90,8 @@ public: virtual uint16_t getEType() const = 0; + virtual uint16_t getEMachine() const = 0; + std::vector> getPltAddresses() const; }; @@ -142,6 +147,10 @@ public: return getObject()->getSymbolSize(getRawDataRefImpl()); } + uint8_t getBinding() const { + return getObject()->getSymbolBinding(getRawDataRefImpl()); + } + uint8_t getOther() const { return getObject()->getSymbolOther(getRawDataRefImpl()); } @@ -149,6 +158,16 @@ public: uint8_t getELFType() const { return getObject()->getSymbolELFType(getRawDataRefImpl()); } + + StringRef getELFTypeName() const { + uint8_t Type = getELFType(); + for (auto &EE : ElfSymbolTypes) { + if (EE.Value == Type) { + return EE.AltName; + } + } + return ""; + } }; class elf_symbol_iterator : public symbol_iterator { @@ -239,6 +258,7 @@ protected: uint32_t getSymbolAlignment(DataRefImpl Symb) const override; uint64_t getCommonSymbolSizeImpl(DataRefImpl Symb) const override; uint32_t getSymbolFlags(DataRefImpl Symb) const override; + uint8_t getSymbolBinding(DataRefImpl Symb) const override; uint8_t getSymbolOther(DataRefImpl Symb) const override; uint8_t getSymbolELFType(DataRefImpl Symb) const override; Expected getSymbolType(DataRefImpl Symb) const override; @@ -247,13 +267,12 @@ protected: Expected getSymbolSection(DataRefImpl Symb) const override; void moveSectionNext(DataRefImpl &Sec) const override; - std::error_code getSectionName(DataRefImpl Sec, - StringRef &Res) const override; + Expected getSectionName(DataRefImpl Sec) const override; uint64_t getSectionAddress(DataRefImpl Sec) const override; uint64_t getSectionIndex(DataRefImpl Sec) const override; uint64_t getSectionSize(DataRefImpl Sec) const override; - std::error_code getSectionContents(DataRefImpl Sec, - StringRef &Res) const override; + Expected> + getSectionContents(DataRefImpl Sec) const override; uint64_t getSectionAlignment(DataRefImpl Sec) const override; bool isSectionCompressed(DataRefImpl Sec) const override; bool isSectionText(DataRefImpl Sec) const override; @@ -341,6 +360,28 @@ protected: (Visibility == ELF::STV_DEFAULT || Visibility == ELF::STV_PROTECTED)); } + Error getBuildAttributes(ARMAttributeParser &Attributes) const override { + auto SectionsOrErr = EF.sections(); + if (!SectionsOrErr) + return SectionsOrErr.takeError(); + + for (const Elf_Shdr &Sec : *SectionsOrErr) { + if (Sec.sh_type == ELF::SHT_ARM_ATTRIBUTES) { + auto ErrorOrContents = EF.getSectionContents(&Sec); + if (!ErrorOrContents) + return ErrorOrContents.takeError(); + + auto Contents = ErrorOrContents.get(); + if (Contents[0] != ARMBuildAttrs::Format_Version || Contents.size() == 1) + return Error::success(); + + Attributes.Parse(Contents, ELFT::TargetEndianness == support::little); + break; + } + } + return Error::success(); + } + // This flag is used for classof, to distinguish ELFObjectFile from // its subclass. If more subclasses will be created, this flag will // have to become an enum. @@ -382,28 +423,6 @@ public: unsigned getPlatformFlags() const override { return EF.getHeader()->e_flags; } - std::error_code getBuildAttributes(ARMAttributeParser &Attributes) const override { - auto SectionsOrErr = EF.sections(); - if (!SectionsOrErr) - return errorToErrorCode(SectionsOrErr.takeError()); - - for (const Elf_Shdr &Sec : *SectionsOrErr) { - if (Sec.sh_type == ELF::SHT_ARM_ATTRIBUTES) { - auto ErrorOrContents = EF.getSectionContents(&Sec); - if (!ErrorOrContents) - return errorToErrorCode(ErrorOrContents.takeError()); - - auto Contents = ErrorOrContents.get(); - if (Contents[0] != ARMBuildAttrs::Format_Version || Contents.size() == 1) - return std::error_code(); - - Attributes.Parse(Contents, ELFT::TargetEndianness == support::little); - break; - } - } - return std::error_code(); - } - const ELFFile *getELFFile() const { return &EF; } bool isDyldType() const { return isDyldELFObject; } @@ -441,7 +460,16 @@ Expected ELFObjectFile::getSymbolName(DataRefImpl Sym) const { auto SymStrTabOrErr = EF.getStringTable(StringTableSec); if (!SymStrTabOrErr) return SymStrTabOrErr.takeError(); - return ESym->getName(*SymStrTabOrErr); + Expected Name = ESym->getName(*SymStrTabOrErr); + + // If the symbol name is empty use the section name. + if ((!Name || Name->empty()) && ESym->getType() == ELF::STT_SECTION) { + StringRef SecName; + Expected Sec = getSymbolSection(Sym); + if (Sec && !(*Sec)->getName(SecName)) + return SecName; + } + return Name; } template @@ -532,6 +560,11 @@ uint64_t ELFObjectFile::getCommonSymbolSizeImpl(DataRefImpl Symb) const { return getSymbol(Symb)->st_size; } +template +uint8_t ELFObjectFile::getSymbolBinding(DataRefImpl Symb) const { + return getSymbol(Symb)->getBinding(); +} + template uint8_t ELFObjectFile::getSymbolOther(DataRefImpl Symb) const { return getSymbol(Symb)->st_other; @@ -654,13 +687,8 @@ void ELFObjectFile::moveSectionNext(DataRefImpl &Sec) const { } template -std::error_code ELFObjectFile::getSectionName(DataRefImpl Sec, - StringRef &Result) const { - auto Name = EF.getSectionName(&*getSection(Sec)); - if (!Name) - return errorToErrorCode(Name.takeError()); - Result = *Name; - return std::error_code(); +Expected ELFObjectFile::getSectionName(DataRefImpl Sec) const { + return EF.getSectionName(&*getSection(Sec)); } template @@ -685,16 +713,15 @@ uint64_t ELFObjectFile::getSectionSize(DataRefImpl Sec) const { } template -std::error_code -ELFObjectFile::getSectionContents(DataRefImpl Sec, - StringRef &Result) const { +Expected> +ELFObjectFile::getSectionContents(DataRefImpl Sec) const { const Elf_Shdr *EShdr = getSection(Sec); if (std::error_code EC = checkOffset(getMemoryBufferRef(), (uintptr_t)base() + EShdr->sh_offset, EShdr->sh_size)) - return EC; - Result = StringRef((const char *)base() + EShdr->sh_offset, EShdr->sh_size); - return std::error_code(); + return errorCodeToError(EC); + return makeArrayRef((const uint8_t *)base() + EShdr->sh_offset, + EShdr->sh_size); } template @@ -750,7 +777,7 @@ ELFObjectFile::dynamic_relocation_sections() const { } } for (const Elf_Shdr &Sec : *SectionsOrErr) { - if (is_contained(Offsets, Sec.sh_offset)) + if (is_contained(Offsets, Sec.sh_addr)) Res.emplace_back(toDRI(&Sec), this); } return Res; @@ -925,15 +952,13 @@ ELFObjectFile::create(MemoryBufferRef Object) { for (const Elf_Shdr &Sec : *SectionsOrErr) { switch (Sec.sh_type) { case ELF::SHT_DYNSYM: { - if (DotDynSymSec) - return createError("More than one dynamic symbol table!"); - DotDynSymSec = &Sec; + if (!DotDynSymSec) + DotDynSymSec = &Sec; break; } case ELF::SHT_SYMTAB: { - if (DotSymtabSec) - return createError("More than one static symbol table!"); - DotSymtabSec = &Sec; + if (!DotSymtabSec) + DotSymtabSec = &Sec; break; } case ELF::SHT_SYMTAB_SHNDX: { @@ -967,7 +992,9 @@ ELFObjectFile::ELFObjectFile(ELFObjectFile &&Other) template basic_symbol_iterator ELFObjectFile::symbol_begin() const { - DataRefImpl Sym = toDRI(DotSymtabSec, 0); + DataRefImpl Sym = + toDRI(DotSymtabSec, + DotSymtabSec && DotSymtabSec->sh_size >= sizeof(Elf_Sym) ? 1 : 0); return basic_symbol_iterator(SymbolRef(Sym, this)); } diff --git a/include/llvm/Object/ELFTypes.h b/include/llvm/Object/ELFTypes.h index ec3c8e7bae46..5552208b1f8a 100644 --- a/include/llvm/Object/ELFTypes.h +++ b/include/llvm/Object/ELFTypes.h @@ -1,9 +1,8 @@ //===- ELFTypes.h - Endian specific types for ELF ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -593,9 +592,9 @@ class Elf_Note_Impl { template friend class Elf_Note_Iterator_Impl; +public: Elf_Note_Impl(const Elf_Nhdr_Impl &Nhdr) : Nhdr(Nhdr) {} -public: /// Get the note's name, excluding the terminating null byte. StringRef getName() const { if (!Nhdr.n_namesz) diff --git a/include/llvm/Object/Error.h b/include/llvm/Object/Error.h index a15f8b9236eb..b7bbf06fc86d 100644 --- a/include/llvm/Object/Error.h +++ b/include/llvm/Object/Error.h @@ -1,9 +1,8 @@ //===- Error.h - system_error extensions for Object -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Object/IRObjectFile.h b/include/llvm/Object/IRObjectFile.h index 993359b766a1..08b92f1bae50 100644 --- a/include/llvm/Object/IRObjectFile.h +++ b/include/llvm/Object/IRObjectFile.h @@ -1,9 +1,8 @@ //===- IRObjectFile.h - LLVM IR object file implementation ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -38,8 +37,7 @@ class IRObjectFile : public SymbolicFile { public: ~IRObjectFile() override; void moveSymbolNext(DataRefImpl &Symb) const override; - std::error_code printSymbolName(raw_ostream &OS, - DataRefImpl Symb) const override; + Error printSymbolName(raw_ostream &OS, DataRefImpl Symb) const override; uint32_t getSymbolFlags(DataRefImpl Symb) const override; basic_symbol_iterator symbol_begin() const override; basic_symbol_iterator symbol_end() const override; diff --git a/include/llvm/Object/IRSymtab.h b/include/llvm/Object/IRSymtab.h index 5f6a024cd132..0bbfc932493c 100644 --- a/include/llvm/Object/IRSymtab.h +++ b/include/llvm/Object/IRSymtab.h @@ -1,9 +1,8 @@ //===- IRSymtab.h - data definitions for IR symbol tables -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -126,12 +125,13 @@ struct Uncommon { Str SectionName; }; + struct Header { /// Version number of the symtab format. This number should be incremented /// when the format changes, but it does not need to be incremented if a /// change to LLVM would cause it to create a different symbol table. Word Version; - enum { kCurrentVersion = 1 }; + enum { kCurrentVersion = 2 }; /// The producer's version string (LLVM_VERSION_STRING " " LLVM_REVISION). /// Consumers should rebuild the symbol table from IR if the producer's @@ -148,6 +148,9 @@ struct Header { /// COFF-specific: linker directives. Str COFFLinkerOpts; + + /// Dependent Library Specifiers + Range DependentLibraries; }; } // end namespace storage @@ -232,6 +235,7 @@ class Reader { ArrayRef Comdats; ArrayRef Symbols; ArrayRef Uncommons; + ArrayRef DependentLibraries; StringRef str(storage::Str S) const { return S.get(Strtab); } @@ -252,6 +256,7 @@ public: Comdats = range(header().Comdats); Symbols = range(header().Symbols); Uncommons = range(header().Uncommons); + DependentLibraries = range(header().DependentLibraries); } using symbol_range = iterator_range>; @@ -284,6 +289,16 @@ public: /// COFF-specific: returns linker options specified in the input file. StringRef getCOFFLinkerOpts() const { return str(header().COFFLinkerOpts); } + + /// Returns dependent library specifiers + std::vector getDependentLibraries() const { + std::vector Specifiers; + Specifiers.reserve(DependentLibraries.size()); + for (auto S : DependentLibraries) { + Specifiers.push_back(str(S)); + } + return Specifiers; + } }; /// Ephemeral symbols produced by Reader::symbols() and diff --git a/include/llvm/Object/MachO.h b/include/llvm/Object/MachO.h index c2f4f4062934..ca9512f21706 100644 --- a/include/llvm/Object/MachO.h +++ b/include/llvm/Object/MachO.h @@ -1,9 +1,8 @@ //===- MachO.h - MachO object file implementation ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -134,11 +133,9 @@ public: BindRebaseSegInfo(const MachOObjectFile *Obj); // Used to check a Mach-O Bind or Rebase entry for errors when iterating. - const char *checkSegAndOffset(int32_t SegIndex, uint64_t SegOffset, - bool endInvalid); - const char *checkCountAndSkip(uint32_t Count, uint32_t Skip, - uint8_t PointerSize, int32_t SegIndex, - uint64_t SegOffset); + const char* checkSegAndOffsets(int32_t SegIndex, uint64_t SegOffset, + uint8_t PointerSize, uint32_t Count=1, + uint32_t Skip=0); // Used with valid SegIndex/SegOffset values from checked entries. StringRef segmentName(int32_t SegIndex); StringRef sectionName(int32_t SegIndex, uint64_t SegOffset); @@ -296,13 +293,12 @@ public: unsigned getSectionID(SectionRef Sec) const; void moveSectionNext(DataRefImpl &Sec) const override; - std::error_code getSectionName(DataRefImpl Sec, - StringRef &Res) const override; + Expected getSectionName(DataRefImpl Sec) const override; uint64_t getSectionAddress(DataRefImpl Sec) const override; uint64_t getSectionIndex(DataRefImpl Sec) const override; uint64_t getSectionSize(DataRefImpl Sec) const override; - std::error_code getSectionContents(DataRefImpl Sec, - StringRef &Res) const override; + Expected> + getSectionContents(DataRefImpl Sec) const override; uint64_t getSectionAlignment(DataRefImpl Sec) const override; Expected getSection(unsigned SectionIndex) const; Expected getSection(StringRef SectionName) const; @@ -413,36 +409,32 @@ public: bool is64, MachOBindEntry::Kind); - /// For use with a SegIndex,SegOffset pair in MachOBindEntry::moveNext() to - /// validate a MachOBindEntry. - const char *BindEntryCheckSegAndOffset(int32_t SegIndex, uint64_t SegOffset, - bool endInvalid) const { - return BindRebaseSectionTable->checkSegAndOffset(SegIndex, SegOffset, - endInvalid); - } - /// For use in MachOBindEntry::moveNext() to validate a MachOBindEntry for - /// the BIND_OPCODE_DO_BIND_ULEB_TIMES_SKIPPING_ULEB opcode. - const char *BindEntryCheckCountAndSkip(uint32_t Count, uint32_t Skip, - uint8_t PointerSize, int32_t SegIndex, - uint64_t SegOffset) const { - return BindRebaseSectionTable->checkCountAndSkip(Count, Skip, PointerSize, - SegIndex, SegOffset); + // Given a SegIndex, SegOffset, and PointerSize, verify a valid section exists + // that fully contains a pointer at that location. Multiple fixups in a bind + // (such as with the BIND_OPCODE_DO_BIND_ULEB_TIMES_SKIPPING_ULEB opcode) can + // be tested via the Count and Skip parameters. + // + // This is used by MachOBindEntry::moveNext() to validate a MachOBindEntry. + const char *BindEntryCheckSegAndOffsets(int32_t SegIndex, uint64_t SegOffset, + uint8_t PointerSize, uint32_t Count=1, + uint32_t Skip=0) const { + return BindRebaseSectionTable->checkSegAndOffsets(SegIndex, SegOffset, + PointerSize, Count, Skip); } - /// For use with a SegIndex,SegOffset pair in MachORebaseEntry::moveNext() to - /// validate a MachORebaseEntry. - const char *RebaseEntryCheckSegAndOffset(int32_t SegIndex, uint64_t SegOffset, - bool endInvalid) const { - return BindRebaseSectionTable->checkSegAndOffset(SegIndex, SegOffset, - endInvalid); - } - /// For use in MachORebaseEntry::moveNext() to validate a MachORebaseEntry for - /// the REBASE_OPCODE_DO_*_TIMES* opcodes. - const char *RebaseEntryCheckCountAndSkip(uint32_t Count, uint32_t Skip, - uint8_t PointerSize, int32_t SegIndex, - uint64_t SegOffset) const { - return BindRebaseSectionTable->checkCountAndSkip(Count, Skip, PointerSize, - SegIndex, SegOffset); + // Given a SegIndex, SegOffset, and PointerSize, verify a valid section exists + // that fully contains a pointer at that location. Multiple fixups in a rebase + // (such as with the REBASE_OPCODE_DO_*_TIMES* opcodes) can be tested via the + // Count and Skip parameters. + // + // This is used by MachORebaseEntry::moveNext() to validate a MachORebaseEntry + const char *RebaseEntryCheckSegAndOffsets(int32_t SegIndex, + uint64_t SegOffset, + uint8_t PointerSize, + uint32_t Count=1, + uint32_t Skip=0) const { + return BindRebaseSectionTable->checkSegAndOffsets(SegIndex, SegOffset, + PointerSize, Count, Skip); } /// For use with the SegIndex of a checked Mach-O Bind or Rebase entry to @@ -579,6 +571,7 @@ public: const char **McpuDefault = nullptr, const char **ArchFlag = nullptr); static bool isValidArch(StringRef ArchFlag); + static ArrayRef getValidArchs(); static Triple getHostArch(); bool isRelocatableObject() const override; @@ -616,6 +609,7 @@ public: case MachO::PLATFORM_TVOS: return "tvos"; case MachO::PLATFORM_WATCHOS: return "watchos"; case MachO::PLATFORM_BRIDGEOS: return "bridgeos"; + case MachO::PLATFORM_MACCATALYST: return "macCatalyst"; case MachO::PLATFORM_IOSSIMULATOR: return "iossimulator"; case MachO::PLATFORM_TVOSSIMULATOR: return "tvossimulator"; case MachO::PLATFORM_WATCHOSSIMULATOR: return "watchossimulator"; diff --git a/include/llvm/Object/MachOUniversal.h b/include/llvm/Object/MachOUniversal.h index 9e70b0bc30c0..5bf724f2c8b2 100644 --- a/include/llvm/Object/MachOUniversal.h +++ b/include/llvm/Object/MachOUniversal.h @@ -1,9 +1,8 @@ //===- MachOUniversal.h - Mach-O universal binaries -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Object/Minidump.h b/include/llvm/Object/Minidump.h new file mode 100644 index 000000000000..470008d552e7 --- /dev/null +++ b/include/llvm/Object/Minidump.h @@ -0,0 +1,165 @@ +//===- Minidump.h - Minidump object file implementation ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJECT_MINIDUMP_H +#define LLVM_OBJECT_MINIDUMP_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringExtras.h" +#include "llvm/BinaryFormat/Minidump.h" +#include "llvm/Object/Binary.h" +#include "llvm/Support/Error.h" + +namespace llvm { +namespace object { + +/// A class providing access to the contents of a minidump file. +class MinidumpFile : public Binary { +public: + /// Construct a new MinidumpFile object from the given memory buffer. Returns + /// an error if this file cannot be identified as a minidump file, or if its + /// contents are badly corrupted (i.e. we cannot read the stream directory). + static Expected> create(MemoryBufferRef Source); + + static bool classof(const Binary *B) { return B->isMinidump(); } + + /// Returns the contents of the minidump header. + const minidump::Header &header() const { return Header; } + + /// Returns the list of streams (stream directory entries) in this file. + ArrayRef streams() const { return Streams; } + + /// Returns the raw contents of the stream given by the directory entry. + ArrayRef getRawStream(const minidump::Directory &Stream) const { + return getData().slice(Stream.Location.RVA, Stream.Location.DataSize); + } + + /// Returns the raw contents of the stream of the given type, or None if the + /// file does not contain a stream of this type. + Optional> getRawStream(minidump::StreamType Type) const; + + /// Returns the raw contents of an object given by the LocationDescriptor. An + /// error is returned if the descriptor points outside of the minidump file. + Expected> + getRawData(minidump::LocationDescriptor Desc) const { + return getDataSlice(getData(), Desc.RVA, Desc.DataSize); + } + + /// Returns the minidump string at the given offset. An error is returned if + /// we fail to parse the string, or the string is invalid UTF16. + Expected getString(size_t Offset) const; + + /// Returns the contents of the SystemInfo stream, cast to the appropriate + /// type. An error is returned if the file does not contain this stream, or + /// the stream is smaller than the size of the SystemInfo structure. The + /// internal consistency of the stream is not checked in any way. + Expected getSystemInfo() const { + return getStream(minidump::StreamType::SystemInfo); + } + + /// Returns the module list embedded in the ModuleList stream. An error is + /// returned if the file does not contain this stream, or if the stream is + /// not large enough to contain the number of modules declared in the stream + /// header. The consistency of the Module entries themselves is not checked in + /// any way. + Expected> getModuleList() const { + return getListStream(minidump::StreamType::ModuleList); + } + + /// Returns the thread list embedded in the ThreadList stream. An error is + /// returned if the file does not contain this stream, or if the stream is + /// not large enough to contain the number of threads declared in the stream + /// header. The consistency of the Thread entries themselves is not checked in + /// any way. + Expected> getThreadList() const { + return getListStream(minidump::StreamType::ThreadList); + } + + /// Returns the list of memory ranges embedded in the MemoryList stream. An + /// error is returned if the file does not contain this stream, or if the + /// stream is not large enough to contain the number of memory descriptors + /// declared in the stream header. The consistency of the MemoryDescriptor + /// entries themselves is not checked in any way. + Expected> getMemoryList() const { + return getListStream( + minidump::StreamType::MemoryList); + } + +private: + static Error createError(StringRef Str) { + return make_error(Str, object_error::parse_failed); + } + + static Error createEOFError() { + return make_error("Unexpected EOF", + object_error::unexpected_eof); + } + + /// Return a slice of the given data array, with bounds checking. + static Expected> getDataSlice(ArrayRef Data, + size_t Offset, size_t Size); + + /// Return the slice of the given data array as an array of objects of the + /// given type. The function checks that the input array is large enough to + /// contain the correct number of objects of the given type. + template + static Expected> getDataSliceAs(ArrayRef Data, + size_t Offset, size_t Count); + + MinidumpFile(MemoryBufferRef Source, const minidump::Header &Header, + ArrayRef Streams, + DenseMap StreamMap) + : Binary(ID_Minidump, Source), Header(Header), Streams(Streams), + StreamMap(std::move(StreamMap)) {} + + ArrayRef getData() const { + return arrayRefFromStringRef(Data.getBuffer()); + } + + /// Return the stream of the given type, cast to the appropriate type. Checks + /// that the stream is large enough to hold an object of this type. + template + Expected getStream(minidump::StreamType Stream) const; + + /// Return the contents of a stream which contains a list of fixed-size items, + /// prefixed by the list size. + template + Expected> getListStream(minidump::StreamType Stream) const; + + const minidump::Header &Header; + ArrayRef Streams; + DenseMap StreamMap; +}; + +template +Expected MinidumpFile::getStream(minidump::StreamType Stream) const { + if (auto OptionalStream = getRawStream(Stream)) { + if (OptionalStream->size() >= sizeof(T)) + return *reinterpret_cast(OptionalStream->data()); + return createEOFError(); + } + return createError("No such stream"); +} + +template +Expected> MinidumpFile::getDataSliceAs(ArrayRef Data, + size_t Offset, + size_t Count) { + // Check for overflow. + if (Count > std::numeric_limits::max() / sizeof(T)) + return createEOFError(); + auto ExpectedArray = getDataSlice(Data, Offset, sizeof(T) * Count); + if (!ExpectedArray) + return ExpectedArray.takeError(); + return ArrayRef(reinterpret_cast(ExpectedArray->data()), Count); +} + +} // end namespace object +} // end namespace llvm + +#endif // LLVM_OBJECT_MINIDUMP_H diff --git a/include/llvm/Object/ModuleSymbolTable.h b/include/llvm/Object/ModuleSymbolTable.h index c3cbc27998e5..4c582fbcda81 100644 --- a/include/llvm/Object/ModuleSymbolTable.h +++ b/include/llvm/Object/ModuleSymbolTable.h @@ -1,9 +1,8 @@ //===- ModuleSymbolTable.h - symbol table for in-memory IR ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Object/ObjectFile.h b/include/llvm/Object/ObjectFile.h index 036c99cb6baf..483a3486bd72 100644 --- a/include/llvm/Object/ObjectFile.h +++ b/include/llvm/Object/ObjectFile.h @@ -1,9 +1,8 @@ //===- ObjectFile.h - File format independent object file -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,6 +13,7 @@ #ifndef LLVM_OBJECT_OBJECTFILE_H #define LLVM_OBJECT_OBJECTFILE_H +#include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" #include "llvm/ADT/iterator_range.h" @@ -98,7 +98,7 @@ public: uint64_t getAddress() const; uint64_t getIndex() const; uint64_t getSize() const; - std::error_code getContents(StringRef &Result) const; + Expected getContents() const; /// Get the alignment of this section as the actual value (not log 2). uint64_t getAlignment() const; @@ -136,6 +136,30 @@ public: const ObjectFile *getObject() const; }; +struct SectionedAddress { + // TODO: constructors could be removed when C++14 would be adopted. + SectionedAddress() {} + SectionedAddress(uint64_t Addr, uint64_t SectIdx) + : Address(Addr), SectionIndex(SectIdx) {} + + const static uint64_t UndefSection = UINT64_MAX; + + uint64_t Address = 0; + uint64_t SectionIndex = UndefSection; +}; + +inline bool operator<(const SectionedAddress &LHS, + const SectionedAddress &RHS) { + return std::tie(LHS.SectionIndex, LHS.Address) < + std::tie(RHS.SectionIndex, RHS.Address); +} + +inline bool operator==(const SectionedAddress &LHS, + const SectionedAddress &RHS) { + return std::tie(LHS.SectionIndex, LHS.Address) == + std::tie(RHS.SectionIndex, RHS.Address); +} + /// This is a value type class that represents a single symbol in the list of /// symbols in the object file. class SymbolRef : public BasicSymbolRef { @@ -220,7 +244,7 @@ protected: friend class SymbolRef; virtual Expected getSymbolName(DataRefImpl Symb) const = 0; - std::error_code printSymbolName(raw_ostream &OS, + Error printSymbolName(raw_ostream &OS, DataRefImpl Symb) const override; virtual Expected getSymbolAddress(DataRefImpl Symb) const = 0; virtual uint64_t getSymbolValueImpl(DataRefImpl Symb) const = 0; @@ -234,13 +258,12 @@ protected: friend class SectionRef; virtual void moveSectionNext(DataRefImpl &Sec) const = 0; - virtual std::error_code getSectionName(DataRefImpl Sec, - StringRef &Res) const = 0; + virtual Expected getSectionName(DataRefImpl Sec) const = 0; virtual uint64_t getSectionAddress(DataRefImpl Sec) const = 0; virtual uint64_t getSectionIndex(DataRefImpl Sec) const = 0; virtual uint64_t getSectionSize(DataRefImpl Sec) const = 0; - virtual std::error_code getSectionContents(DataRefImpl Sec, - StringRef &Res) const = 0; + virtual Expected> + getSectionContents(DataRefImpl Sec) const = 0; virtual uint64_t getSectionAlignment(DataRefImpl Sec) const = 0; virtual bool isSectionCompressed(DataRefImpl Sec) const = 0; virtual bool isSectionText(DataRefImpl Sec) const = 0; @@ -308,11 +331,6 @@ public: /// Create a triple from the data in this object file. Triple makeTriple() const; - virtual std::error_code - getBuildAttributes(ARMAttributeParser &Attributes) const { - return std::error_code(); - } - /// Maps a debug section name to a standard DWARF section name. virtual StringRef mapDebugSectionName(StringRef Name) const { return Name; } @@ -340,6 +358,9 @@ public: static Expected> createCOFFObjectFile(MemoryBufferRef Object); + static Expected> + createXCOFFObjectFile(MemoryBufferRef Object, unsigned FileType); + static Expected> createELFObjectFile(MemoryBufferRef Object); @@ -396,14 +417,16 @@ inline SectionRef::SectionRef(DataRefImpl SectionP, , OwningObject(Owner) {} inline bool SectionRef::operator==(const SectionRef &Other) const { - return SectionPimpl == Other.SectionPimpl; + return OwningObject == Other.OwningObject && + SectionPimpl == Other.SectionPimpl; } inline bool SectionRef::operator!=(const SectionRef &Other) const { - return SectionPimpl != Other.SectionPimpl; + return !(*this == Other); } inline bool SectionRef::operator<(const SectionRef &Other) const { + assert(OwningObject == Other.OwningObject); return SectionPimpl < Other.SectionPimpl; } @@ -412,7 +435,11 @@ inline void SectionRef::moveNext() { } inline std::error_code SectionRef::getName(StringRef &Result) const { - return OwningObject->getSectionName(SectionPimpl, Result); + Expected NameOrErr = OwningObject->getSectionName(SectionPimpl); + if (!NameOrErr) + return errorToErrorCode(NameOrErr.takeError()); + Result = *NameOrErr; + return std::error_code(); } inline uint64_t SectionRef::getAddress() const { @@ -427,8 +454,12 @@ inline uint64_t SectionRef::getSize() const { return OwningObject->getSectionSize(SectionPimpl); } -inline std::error_code SectionRef::getContents(StringRef &Result) const { - return OwningObject->getSectionContents(SectionPimpl, Result); +inline Expected SectionRef::getContents() const { + Expected> Res = + OwningObject->getSectionContents(SectionPimpl); + if (!Res) + return Res.takeError(); + return StringRef(reinterpret_cast(Res->data()), Res->size()); } inline uint64_t SectionRef::getAlignment() const { @@ -531,6 +562,25 @@ inline const ObjectFile *RelocationRef::getObject() const { } // end namespace object +template <> struct DenseMapInfo { + static bool isEqual(const object::SectionRef &A, + const object::SectionRef &B) { + return A == B; + } + static object::SectionRef getEmptyKey() { + return object::SectionRef({}, nullptr); + } + static object::SectionRef getTombstoneKey() { + object::DataRefImpl TS; + TS.p = (uintptr_t)-1; + return object::SectionRef(TS, nullptr); + } + static unsigned getHashValue(const object::SectionRef &Sec) { + object::DataRefImpl Raw = Sec.getRawDataRefImpl(); + return hash_combine(Raw.p, Raw.d.a, Raw.d.b); + } +}; + } // end namespace llvm #endif // LLVM_OBJECT_OBJECTFILE_H diff --git a/include/llvm/Object/RelocVisitor.h b/include/llvm/Object/RelocVisitor.h deleted file mode 100644 index 9a978de2e599..000000000000 --- a/include/llvm/Object/RelocVisitor.h +++ /dev/null @@ -1,351 +0,0 @@ -//===- RelocVisitor.h - Visitor for object file relocations -----*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file provides a wrapper around all the different types of relocations -// in different file formats, such that a client can handle them in a unified -// manner by only implementing a minimal number of functions. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_OBJECT_RELOCVISITOR_H -#define LLVM_OBJECT_RELOCVISITOR_H - -#include "llvm/ADT/Triple.h" -#include "llvm/BinaryFormat/ELF.h" -#include "llvm/BinaryFormat/MachO.h" -#include "llvm/Object/COFF.h" -#include "llvm/Object/ELFObjectFile.h" -#include "llvm/Object/MachO.h" -#include "llvm/Object/ObjectFile.h" -#include "llvm/Object/Wasm.h" -#include "llvm/Support/Casting.h" -#include "llvm/Support/ErrorHandling.h" -#include -#include - -namespace llvm { -namespace object { - -/// Base class for object file relocation visitors. -class RelocVisitor { -public: - explicit RelocVisitor(const ObjectFile &Obj) : ObjToVisit(Obj) {} - - // TODO: Should handle multiple applied relocations via either passing in the - // previously computed value or just count paired relocations as a single - // visit. - uint64_t visit(uint32_t Rel, RelocationRef R, uint64_t Value = 0) { - if (isa(ObjToVisit)) - return visitELF(Rel, R, Value); - if (isa(ObjToVisit)) - return visitCOFF(Rel, R, Value); - if (isa(ObjToVisit)) - return visitMachO(Rel, R, Value); - if (isa(ObjToVisit)) - return visitWasm(Rel, R, Value); - - HasError = true; - return 0; - } - - bool error() { return HasError; } - -private: - const ObjectFile &ObjToVisit; - bool HasError = false; - - uint64_t visitELF(uint32_t Rel, RelocationRef R, uint64_t Value) { - if (ObjToVisit.getBytesInAddress() == 8) { // 64-bit object file - switch (ObjToVisit.getArch()) { - case Triple::x86_64: - return visitX86_64(Rel, R, Value); - case Triple::aarch64: - case Triple::aarch64_be: - return visitAarch64(Rel, R, Value); - case Triple::bpfel: - case Triple::bpfeb: - return visitBpf(Rel, R, Value); - case Triple::mips64el: - case Triple::mips64: - return visitMips64(Rel, R, Value); - case Triple::ppc64le: - case Triple::ppc64: - return visitPPC64(Rel, R, Value); - case Triple::systemz: - return visitSystemz(Rel, R, Value); - case Triple::sparcv9: - return visitSparc64(Rel, R, Value); - case Triple::amdgcn: - return visitAmdgpu(Rel, R, Value); - default: - HasError = true; - return 0; - } - } - - // 32-bit object file - assert(ObjToVisit.getBytesInAddress() == 4 && - "Invalid word size in object file"); - - switch (ObjToVisit.getArch()) { - case Triple::x86: - return visitX86(Rel, R, Value); - case Triple::ppc: - return visitPPC32(Rel, R, Value); - case Triple::arm: - case Triple::armeb: - return visitARM(Rel, R, Value); - case Triple::lanai: - return visitLanai(Rel, R, Value); - case Triple::mipsel: - case Triple::mips: - return visitMips32(Rel, R, Value); - case Triple::sparc: - return visitSparc32(Rel, R, Value); - case Triple::hexagon: - return visitHexagon(Rel, R, Value); - default: - HasError = true; - return 0; - } - } - - int64_t getELFAddend(RelocationRef R) { - Expected AddendOrErr = ELFRelocationRef(R).getAddend(); - handleAllErrors(AddendOrErr.takeError(), [](const ErrorInfoBase &EI) { - report_fatal_error(EI.message()); - }); - return *AddendOrErr; - } - - uint64_t visitX86_64(uint32_t Rel, RelocationRef R, uint64_t Value) { - switch (Rel) { - case ELF::R_X86_64_NONE: - return 0; - case ELF::R_X86_64_64: - case ELF::R_X86_64_DTPOFF32: - case ELF::R_X86_64_DTPOFF64: - return Value + getELFAddend(R); - case ELF::R_X86_64_PC32: - return Value + getELFAddend(R) - R.getOffset(); - case ELF::R_X86_64_32: - case ELF::R_X86_64_32S: - return (Value + getELFAddend(R)) & 0xFFFFFFFF; - } - HasError = true; - return 0; - } - - uint64_t visitAarch64(uint32_t Rel, RelocationRef R, uint64_t Value) { - switch (Rel) { - case ELF::R_AARCH64_ABS32: { - int64_t Res = Value + getELFAddend(R); - if (Res < INT32_MIN || Res > UINT32_MAX) - HasError = true; - return static_cast(Res); - } - case ELF::R_AARCH64_ABS64: - return Value + getELFAddend(R); - } - HasError = true; - return 0; - } - - uint64_t visitBpf(uint32_t Rel, RelocationRef R, uint64_t Value) { - switch (Rel) { - case ELF::R_BPF_64_32: - return Value & 0xFFFFFFFF; - case ELF::R_BPF_64_64: - return Value; - } - HasError = true; - return 0; - } - - uint64_t visitMips64(uint32_t Rel, RelocationRef R, uint64_t Value) { - switch (Rel) { - case ELF::R_MIPS_32: - return (Value + getELFAddend(R)) & 0xFFFFFFFF; - case ELF::R_MIPS_64: - return Value + getELFAddend(R); - case ELF::R_MIPS_TLS_DTPREL64: - return Value + getELFAddend(R) - 0x8000; - } - HasError = true; - return 0; - } - - uint64_t visitPPC64(uint32_t Rel, RelocationRef R, uint64_t Value) { - switch (Rel) { - case ELF::R_PPC64_ADDR32: - return (Value + getELFAddend(R)) & 0xFFFFFFFF; - case ELF::R_PPC64_ADDR64: - return Value + getELFAddend(R); - } - HasError = true; - return 0; - } - - uint64_t visitSystemz(uint32_t Rel, RelocationRef R, uint64_t Value) { - switch (Rel) { - case ELF::R_390_32: { - int64_t Res = Value + getELFAddend(R); - if (Res < INT32_MIN || Res > UINT32_MAX) - HasError = true; - return static_cast(Res); - } - case ELF::R_390_64: - return Value + getELFAddend(R); - } - HasError = true; - return 0; - } - - uint64_t visitSparc64(uint32_t Rel, RelocationRef R, uint64_t Value) { - switch (Rel) { - case ELF::R_SPARC_32: - case ELF::R_SPARC_64: - case ELF::R_SPARC_UA32: - case ELF::R_SPARC_UA64: - return Value + getELFAddend(R); - } - HasError = true; - return 0; - } - - uint64_t visitAmdgpu(uint32_t Rel, RelocationRef R, uint64_t Value) { - switch (Rel) { - case ELF::R_AMDGPU_ABS32: - case ELF::R_AMDGPU_ABS64: - return Value + getELFAddend(R); - } - HasError = true; - return 0; - } - - uint64_t visitX86(uint32_t Rel, RelocationRef R, uint64_t Value) { - switch (Rel) { - case ELF::R_386_NONE: - return 0; - case ELF::R_386_32: - return Value; - case ELF::R_386_PC32: - return Value - R.getOffset(); - } - HasError = true; - return 0; - } - - uint64_t visitPPC32(uint32_t Rel, RelocationRef R, uint64_t Value) { - if (Rel == ELF::R_PPC_ADDR32) - return (Value + getELFAddend(R)) & 0xFFFFFFFF; - HasError = true; - return 0; - } - - uint64_t visitARM(uint32_t Rel, RelocationRef R, uint64_t Value) { - if (Rel == ELF::R_ARM_ABS32) { - if ((int64_t)Value < INT32_MIN || (int64_t)Value > UINT32_MAX) - HasError = true; - return static_cast(Value); - } - HasError = true; - return 0; - } - - uint64_t visitLanai(uint32_t Rel, RelocationRef R, uint64_t Value) { - if (Rel == ELF::R_LANAI_32) - return (Value + getELFAddend(R)) & 0xFFFFFFFF; - HasError = true; - return 0; - } - - uint64_t visitMips32(uint32_t Rel, RelocationRef R, uint64_t Value) { - // FIXME: Take in account implicit addends to get correct results. - if (Rel == ELF::R_MIPS_32) - return Value & 0xFFFFFFFF; - if (Rel == ELF::R_MIPS_TLS_DTPREL32) - return Value & 0xFFFFFFFF; - HasError = true; - return 0; - } - - uint64_t visitSparc32(uint32_t Rel, RelocationRef R, uint64_t Value) { - if (Rel == ELF::R_SPARC_32 || Rel == ELF::R_SPARC_UA32) - return Value + getELFAddend(R); - HasError = true; - return 0; - } - - uint64_t visitHexagon(uint32_t Rel, RelocationRef R, uint64_t Value) { - if (Rel == ELF::R_HEX_32) - return Value + getELFAddend(R); - HasError = true; - return 0; - } - - uint64_t visitCOFF(uint32_t Rel, RelocationRef R, uint64_t Value) { - switch (ObjToVisit.getArch()) { - case Triple::x86: - switch (Rel) { - case COFF::IMAGE_REL_I386_SECREL: - case COFF::IMAGE_REL_I386_DIR32: - return static_cast(Value); - } - break; - case Triple::x86_64: - switch (Rel) { - case COFF::IMAGE_REL_AMD64_SECREL: - return static_cast(Value); - case COFF::IMAGE_REL_AMD64_ADDR64: - return Value; - } - break; - default: - break; - } - HasError = true; - return 0; - } - - uint64_t visitMachO(uint32_t Rel, RelocationRef R, uint64_t Value) { - if (ObjToVisit.getArch() == Triple::x86_64 && - Rel == MachO::X86_64_RELOC_UNSIGNED) - return Value; - HasError = true; - return 0; - } - - uint64_t visitWasm(uint32_t Rel, RelocationRef R, uint64_t Value) { - if (ObjToVisit.getArch() == Triple::wasm32) { - switch (Rel) { - case wasm::R_WEBASSEMBLY_FUNCTION_INDEX_LEB: - case wasm::R_WEBASSEMBLY_TABLE_INDEX_SLEB: - case wasm::R_WEBASSEMBLY_TABLE_INDEX_I32: - case wasm::R_WEBASSEMBLY_MEMORY_ADDR_LEB: - case wasm::R_WEBASSEMBLY_MEMORY_ADDR_SLEB: - case wasm::R_WEBASSEMBLY_MEMORY_ADDR_I32: - case wasm::R_WEBASSEMBLY_TYPE_INDEX_LEB: - case wasm::R_WEBASSEMBLY_GLOBAL_INDEX_LEB: - case wasm::R_WEBASSEMBLY_FUNCTION_OFFSET_I32: - case wasm::R_WEBASSEMBLY_SECTION_OFFSET_I32: - case wasm::R_WEBASSEMBLY_EVENT_INDEX_LEB: - // For wasm section, its offset at 0 -- ignoring Value - return 0; - } - } - HasError = true; - return 0; - } -}; - -} // end namespace object -} // end namespace llvm - -#endif // LLVM_OBJECT_RELOCVISITOR_H diff --git a/include/llvm/Object/RelocationResolver.h b/include/llvm/Object/RelocationResolver.h new file mode 100644 index 000000000000..1246dcc5ec73 --- /dev/null +++ b/include/llvm/Object/RelocationResolver.h @@ -0,0 +1,42 @@ +//===- RelocVisitor.h - Visitor for object file relocations -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides a wrapper around all the different types of relocations +// in different file formats, such that a client can handle them in a unified +// manner by only implementing a minimal number of functions. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJECT_RELOCVISITOR_H +#define LLVM_OBJECT_RELOCVISITOR_H + +#include "llvm/ADT/Triple.h" +#include "llvm/BinaryFormat/ELF.h" +#include "llvm/BinaryFormat/MachO.h" +#include "llvm/Object/COFF.h" +#include "llvm/Object/ELFObjectFile.h" +#include "llvm/Object/MachO.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Object/Wasm.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/ErrorHandling.h" +#include +#include + +namespace llvm { +namespace object { + +using RelocationResolver = uint64_t (*)(RelocationRef R, uint64_t S, uint64_t A); + +std::pair +getRelocationResolver(const ObjectFile &Obj); + +} // end namespace object +} // end namespace llvm + +#endif // LLVM_OBJECT_RELOCVISITOR_H diff --git a/include/llvm/Object/StackMapParser.h b/include/llvm/Object/StackMapParser.h index 557db5afa825..ed44efbf80b9 100644 --- a/include/llvm/Object/StackMapParser.h +++ b/include/llvm/Object/StackMapParser.h @@ -1,9 +1,8 @@ //===- StackMapParser.h - StackMap Parsing Support --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -20,8 +19,9 @@ namespace llvm { +/// A parser for the latest stackmap format. At the moment, latest=V2. template -class StackMapV2Parser { +class StackMapParser { public: template class AccessorIterator { @@ -50,7 +50,7 @@ public: /// Accessor for function records. class FunctionAccessor { - friend class StackMapV2Parser; + friend class StackMapParser; public: /// Get the function address. @@ -82,7 +82,7 @@ public: /// Accessor for constants. class ConstantAccessor { - friend class StackMapV2Parser; + friend class StackMapParser; public: /// Return the value of this constant. @@ -106,7 +106,7 @@ public: /// Accessor for location records. class LocationAccessor { - friend class StackMapV2Parser; + friend class StackMapParser; friend class RecordAccessor; public: @@ -115,6 +115,12 @@ public: return LocationKind(P[KindOffset]); } + /// Get the Size for this location. + unsigned getSizeInBytes() const { + return read(P + SizeOffset); + + } + /// Get the Dwarf register number for this location. uint16_t getDwarfRegNum() const { return read(P + DwarfRegNumOffset); @@ -149,16 +155,17 @@ public: } static const int KindOffset = 0; - static const int DwarfRegNumOffset = KindOffset + sizeof(uint16_t); - static const int SmallConstantOffset = DwarfRegNumOffset + sizeof(uint16_t); - static const int LocationAccessorSize = sizeof(uint64_t); + static const int SizeOffset = KindOffset + sizeof(uint16_t); + static const int DwarfRegNumOffset = SizeOffset + sizeof(uint16_t); + static const int SmallConstantOffset = DwarfRegNumOffset + sizeof(uint32_t); + static const int LocationAccessorSize = sizeof(uint64_t) + sizeof(uint32_t); const uint8_t *P; }; /// Accessor for stackmap live-out fields. class LiveOutAccessor { - friend class StackMapV2Parser; + friend class StackMapParser; friend class RecordAccessor; public: @@ -189,7 +196,7 @@ public: /// Accessor for stackmap records. class RecordAccessor { - friend class StackMapV2Parser; + friend class StackMapParser; public: using location_iterator = AccessorIterator; @@ -264,8 +271,9 @@ public: RecordAccessor(const uint8_t *P) : P(P) {} unsigned getNumLiveOutsOffset() const { - return LocationListOffset + LocationSize * getNumLocations() + - sizeof(uint16_t); + unsigned LocOffset = + ((LocationListOffset + LocationSize * getNumLocations()) + 7) & ~0x7; + return LocOffset + sizeof(uint16_t); } unsigned getSizeInBytes() const { @@ -285,7 +293,7 @@ public: InstructionOffsetOffset + sizeof(uint32_t) + sizeof(uint16_t); static const unsigned LocationListOffset = NumLocationsOffset + sizeof(uint16_t); - static const unsigned LocationSize = sizeof(uint64_t); + static const unsigned LocationSize = sizeof(uint64_t) + sizeof(uint32_t); static const unsigned LiveOutSize = sizeof(uint32_t); const uint8_t *P; @@ -293,12 +301,12 @@ public: /// Construct a parser for a version-2 stackmap. StackMap data will be read /// from the given array. - StackMapV2Parser(ArrayRef StackMapSection) + StackMapParser(ArrayRef StackMapSection) : StackMapSection(StackMapSection) { ConstantsListOffset = FunctionListOffset + getNumFunctions() * FunctionSize; - assert(StackMapSection[0] == 2 && - "StackMapV2Parser can only parse version 2 stackmaps"); + assert(StackMapSection[0] == 3 && + "StackMapParser can only parse version 3 stackmaps"); unsigned CurrentRecordOffset = ConstantsListOffset + getNumConstants() * ConstantSize; @@ -314,8 +322,8 @@ public: using constant_iterator = AccessorIterator; using record_iterator = AccessorIterator; - /// Get the version number of this stackmap. (Always returns 2). - unsigned getVersion() const { return 2; } + /// Get the version number of this stackmap. (Always returns 3). + unsigned getVersion() const { return 3; } /// Get the number of functions in the stack map. uint32_t getNumFunctions() const { diff --git a/include/llvm/Object/SymbolSize.h b/include/llvm/Object/SymbolSize.h index 1a1dc8752943..085623e35907 100644 --- a/include/llvm/Object/SymbolSize.h +++ b/include/llvm/Object/SymbolSize.h @@ -1,9 +1,8 @@ //===- SymbolSize.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Object/SymbolicFile.h b/include/llvm/Object/SymbolicFile.h index 5b9549bc3449..1398fa134c81 100644 --- a/include/llvm/Object/SymbolicFile.h +++ b/include/llvm/Object/SymbolicFile.h @@ -1,9 +1,8 @@ //===- SymbolicFile.h - Interface that only provides symbols ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -127,7 +126,7 @@ public: void moveNext(); - std::error_code printName(raw_ostream &OS) const; + Error printName(raw_ostream &OS) const; /// Get symbol flags (bitwise OR of SymbolRef::Flags) uint32_t getFlags() const; @@ -146,8 +145,7 @@ public: // virtual interface. virtual void moveSymbolNext(DataRefImpl &Symb) const = 0; - virtual std::error_code printSymbolName(raw_ostream &OS, - DataRefImpl Symb) const = 0; + virtual Error printSymbolName(raw_ostream &OS, DataRefImpl Symb) const = 0; virtual uint32_t getSymbolFlags(DataRefImpl Symb) const = 0; @@ -194,7 +192,7 @@ inline void BasicSymbolRef::moveNext() { return OwningObject->moveSymbolNext(SymbolPimpl); } -inline std::error_code BasicSymbolRef::printName(raw_ostream &OS) const { +inline Error BasicSymbolRef::printName(raw_ostream &OS) const { return OwningObject->printSymbolName(OS, SymbolPimpl); } diff --git a/include/llvm/Object/Wasm.h b/include/llvm/Object/Wasm.h index ed857652a048..e130ea32ed21 100644 --- a/include/llvm/Object/Wasm.h +++ b/include/llvm/Object/Wasm.h @@ -1,9 +1,8 @@ -//===- WasmObjectFile.h - Wasm object file implementation -------*- C++ -*-===// +//===- Wasm.h - Wasm object file implementation -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -130,6 +129,10 @@ public: static bool classof(const Binary *v) { return v->isWasm(); } const wasm::WasmDylinkInfo &dylinkInfo() const { return DylinkInfo; } + const wasm::WasmProducerInfo &getProducerInfo() const { return ProducerInfo; } + ArrayRef getTargetFeatures() const { + return TargetFeatures; + } ArrayRef types() const { return Signatures; } ArrayRef functionTypes() const { return FunctionTypes; } ArrayRef imports() const { return Imports; } @@ -149,7 +152,6 @@ public: uint32_t getNumImportedGlobals() const { return NumImportedGlobals; } uint32_t getNumImportedFunctions() const { return NumImportedFunctions; } uint32_t getNumImportedEvents() const { return NumImportedEvents; } - void moveSymbolNext(DataRefImpl &Symb) const override; uint32_t getSymbolFlags(DataRefImpl Symb) const override; @@ -169,13 +171,12 @@ public: // Overrides from SectionRef. void moveSectionNext(DataRefImpl &Sec) const override; - std::error_code getSectionName(DataRefImpl Sec, - StringRef &Res) const override; + Expected getSectionName(DataRefImpl Sec) const override; uint64_t getSectionAddress(DataRefImpl Sec) const override; uint64_t getSectionIndex(DataRefImpl Sec) const override; uint64_t getSectionSize(DataRefImpl Sec) const override; - std::error_code getSectionContents(DataRefImpl Sec, - StringRef &Res) const override; + Expected> + getSectionContents(DataRefImpl Sec) const override; uint64_t getSectionAlignment(DataRefImpl Sec) const override; bool isSectionCompressed(DataRefImpl Sec) const override; bool isSectionText(DataRefImpl Sec) const override; @@ -222,13 +223,13 @@ private: bool isValidDataSymbol(uint32_t Index) const; bool isValidSectionSymbol(uint32_t Index) const; wasm::WasmFunction &getDefinedFunction(uint32_t Index); + const wasm::WasmFunction &getDefinedFunction(uint32_t Index) const; wasm::WasmGlobal &getDefinedGlobal(uint32_t Index); wasm::WasmEvent &getDefinedEvent(uint32_t Index); const WasmSection &getWasmSection(DataRefImpl Ref) const; const wasm::WasmRelocation &getWasmRelocation(DataRefImpl Ref) const; - const uint8_t *getPtr(size_t Offset) const; Error parseSection(WasmSection &Sec); Error parseCustomSection(WasmSection &Sec, ReadContext &Ctx); @@ -245,6 +246,7 @@ private: Error parseElemSection(ReadContext &Ctx); Error parseCodeSection(ReadContext &Ctx); Error parseDataSection(ReadContext &Ctx); + Error parseDataCountSection(ReadContext &Ctx); // Custom section types Error parseDylinkSection(ReadContext &Ctx); @@ -252,11 +254,15 @@ private: Error parseLinkingSection(ReadContext &Ctx); Error parseLinkingSectionSymtab(ReadContext &Ctx); Error parseLinkingSectionComdat(ReadContext &Ctx); + Error parseProducersSection(ReadContext &Ctx); + Error parseTargetFeaturesSection(ReadContext &Ctx); Error parseRelocSection(StringRef Name, ReadContext &Ctx); wasm::WasmObjectHeader Header; std::vector Sections; wasm::WasmDylinkInfo DylinkInfo; + wasm::WasmProducerInfo ProducerInfo; + std::vector TargetFeatures; std::vector Signatures; std::vector FunctionTypes; std::vector Tables; @@ -267,6 +273,7 @@ private: std::vector Exports; std::vector ElemSegments; std::vector DataSegments; + llvm::Optional DataCount; std::vector Functions; std::vector Symbols; std::vector DebugNames; @@ -287,40 +294,51 @@ class WasmSectionOrderChecker { public: // We define orders for all core wasm sections and known custom sections. enum : int { + // Sentinel, must be zero + WASM_SEC_ORDER_NONE = 0, + // Core sections - // The order of standard sections is precisely given by the spec. - WASM_SEC_ORDER_TYPE = 1, - WASM_SEC_ORDER_IMPORT = 2, - WASM_SEC_ORDER_FUNCTION = 3, - WASM_SEC_ORDER_TABLE = 4, - WASM_SEC_ORDER_MEMORY = 5, - WASM_SEC_ORDER_GLOBAL = 6, - WASM_SEC_ORDER_EVENT = 7, - WASM_SEC_ORDER_EXPORT = 8, - WASM_SEC_ORDER_START = 9, - WASM_SEC_ORDER_ELEM = 10, - WASM_SEC_ORDER_DATACOUNT = 11, - WASM_SEC_ORDER_CODE = 12, - WASM_SEC_ORDER_DATA = 13, + WASM_SEC_ORDER_TYPE, + WASM_SEC_ORDER_IMPORT, + WASM_SEC_ORDER_FUNCTION, + WASM_SEC_ORDER_TABLE, + WASM_SEC_ORDER_MEMORY, + WASM_SEC_ORDER_GLOBAL, + WASM_SEC_ORDER_EVENT, + WASM_SEC_ORDER_EXPORT, + WASM_SEC_ORDER_START, + WASM_SEC_ORDER_ELEM, + WASM_SEC_ORDER_DATACOUNT, + WASM_SEC_ORDER_CODE, + WASM_SEC_ORDER_DATA, // Custom sections // "dylink" should be the very first section in the module - WASM_SEC_ORDER_DYLINK = 0, + WASM_SEC_ORDER_DYLINK, // "linking" section requires DATA section in order to validate data symbols - WASM_SEC_ORDER_LINKING = 100, + WASM_SEC_ORDER_LINKING, // Must come after "linking" section in order to validate reloc indexes. - WASM_SEC_ORDER_RELOC = 101, + WASM_SEC_ORDER_RELOC, // "name" section must appear after DATA. Comes after "linking" to allow // symbol table to set default function name. - WASM_SEC_ORDER_NAME = 102, + WASM_SEC_ORDER_NAME, // "producers" section must appear after "name" section. - WASM_SEC_ORDER_PRODUCERS = 103 + WASM_SEC_ORDER_PRODUCERS, + // "target_features" section must appear after producers section + WASM_SEC_ORDER_TARGET_FEATURES, + + // Must be last + WASM_NUM_SEC_ORDERS + }; + // Sections that may or may not be present, but cannot be predecessors + static int DisallowedPredecessors[WASM_NUM_SEC_ORDERS][WASM_NUM_SEC_ORDERS]; + bool isValidSectionOrder(unsigned ID, StringRef CustomSectionName = ""); private: - int LastOrder = -1; // Lastly seen known section's order + bool Seen[WASM_NUM_SEC_ORDERS] = {}; // Sections that have been seen already // Returns -1 for unknown sections. int getSectionOrder(unsigned ID, StringRef CustomSectionName = ""); diff --git a/include/llvm/Object/WasmTraits.h b/include/llvm/Object/WasmTraits.h index 049d72f79e41..3eee8e71b187 100644 --- a/include/llvm/Object/WasmTraits.h +++ b/include/llvm/Object/WasmTraits.h @@ -1,9 +1,8 @@ //===- WasmTraits.h - DenseMap traits for the Wasm structures ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Object/WindowsMachineFlag.h b/include/llvm/Object/WindowsMachineFlag.h new file mode 100644 index 000000000000..acc6afc0329c --- /dev/null +++ b/include/llvm/Object/WindowsMachineFlag.h @@ -0,0 +1,33 @@ +//===- WindowsMachineFlag.h -------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Functions for implementing the /machine: flag. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TOOLDRIVERS_MACHINEFLAG_MACHINEFLAG_H +#define LLVM_TOOLDRIVERS_MACHINEFLAG_MACHINEFLAG_H + +namespace llvm { + +class StringRef; +namespace COFF { +enum MachineTypes : unsigned; +} + +// Returns a user-readable string for ARMNT, ARM64, AMD64, I386. +// Other MachineTypes values must not be passed in. +StringRef machineToStr(COFF::MachineTypes MT); + +// Maps /machine: arguments to a MachineTypes value. +// Only returns ARMNT, ARM64, AMD64, I386, or IMAGE_FILE_MACHINE_UNKNOWN. +COFF::MachineTypes getMachineType(StringRef S); + +} + +#endif diff --git a/include/llvm/Object/WindowsResource.h b/include/llvm/Object/WindowsResource.h index a077c82871bf..356dcb03abba 100644 --- a/include/llvm/Object/WindowsResource.h +++ b/include/llvm/Object/WindowsResource.h @@ -1,9 +1,8 @@ //===-- WindowsResource.h ---------------------------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===---------------------------------------------------------------------===// // @@ -38,11 +37,14 @@ #include "llvm/Support/ConvertUTF.h" #include "llvm/Support/Endian.h" #include "llvm/Support/Error.h" -#include "llvm/Support/ScopedPrinter.h" #include namespace llvm { + +class raw_ostream; +class ScopedPrinter; + namespace object { class WindowsResource; @@ -118,6 +120,7 @@ private: const WindowsResource *Owner); BinaryStreamReader Reader; + const WindowsResource *Owner; bool IsStringType; ArrayRef Type; uint16_t TypeID; @@ -149,7 +152,7 @@ class WindowsResourceParser { public: class TreeNode; WindowsResourceParser(); - Error parse(WindowsResource *WR); + Error parse(WindowsResource *WR, std::vector &Duplicates); void printTree(raw_ostream &OS) const; const TreeNode &getTree() const { return Root; } const ArrayRef> getData() const { return Data; } @@ -185,21 +188,25 @@ public: static std::unique_ptr createIDNode(); static std::unique_ptr createDataNode(uint16_t MajorVersion, uint16_t MinorVersion, - uint32_t Characteristics); + uint32_t Characteristics, + uint32_t Origin); explicit TreeNode(bool IsStringNode); TreeNode(uint16_t MajorVersion, uint16_t MinorVersion, - uint32_t Characteristics); + uint32_t Characteristics, uint32_t Origin); - void addEntry(const ResourceEntryRef &Entry, bool &IsNewTypeString, - bool &IsNewNameString); + bool addEntry(const ResourceEntryRef &Entry, uint32_t Origin, + bool &IsNewTypeString, bool &IsNewNameString, + TreeNode *&Result); TreeNode &addTypeNode(const ResourceEntryRef &Entry, bool &IsNewTypeString); TreeNode &addNameNode(const ResourceEntryRef &Entry, bool &IsNewNameString); - TreeNode &addLanguageNode(const ResourceEntryRef &Entry); - TreeNode &addChild(uint32_t ID, bool IsDataNode = false, - uint16_t MajorVersion = 0, uint16_t MinorVersion = 0, - uint32_t Characteristics = 0); - TreeNode &addChild(ArrayRef NameRef, bool &IsNewString); + bool addLanguageNode(const ResourceEntryRef &Entry, uint32_t Origin, + TreeNode *&Result); + bool addDataChild(uint32_t ID, uint16_t MajorVersion, uint16_t MinorVersion, + uint32_t Characteristics, uint32_t Origin, + TreeNode *&Result); + TreeNode &addIDChild(uint32_t ID); + TreeNode &addNameChild(ArrayRef NameRef, bool &IsNewString); bool IsDataNode = false; uint32_t StringIndex; @@ -209,18 +216,26 @@ public: uint16_t MajorVersion = 0; uint16_t MinorVersion = 0; uint32_t Characteristics = 0; + + // The .res file that defined this TreeNode, for diagnostics. + // Index into InputFilenames. + uint32_t Origin; }; private: TreeNode Root; std::vector> Data; std::vector> StringTable; + + std::vector InputFilenames; }; Expected> writeWindowsResourceCOFF(llvm::COFF::MachineTypes MachineType, - const WindowsResourceParser &Parser); + const WindowsResourceParser &Parser, + uint32_t TimeDateStamp); +void printResourceTypeName(uint16_t TypeID, raw_ostream &OS); } // namespace object } // namespace llvm diff --git a/include/llvm/Object/XCOFFObjectFile.h b/include/llvm/Object/XCOFFObjectFile.h new file mode 100644 index 000000000000..cdee7129a2ab --- /dev/null +++ b/include/llvm/Object/XCOFFObjectFile.h @@ -0,0 +1,268 @@ +//===- XCOFFObjectFile.h - XCOFF object file implementation -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file declares the XCOFFObjectFile class. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJECT_XCOFFOBJECTFILE_H +#define LLVM_OBJECT_XCOFFOBJECTFILE_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/iterator_range.h" +#include "llvm/BinaryFormat/Magic.h" +#include "llvm/BinaryFormat/XCOFF.h" +#include "llvm/MC/SubtargetFeature.h" +#include "llvm/Object/Binary.h" +#include "llvm/Object/Error.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Object/SymbolicFile.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/MemoryBuffer.h" +#include +#include +#include +#include + +namespace llvm { +namespace object { + +struct XCOFFFileHeader32 { + support::ubig16_t Magic; + support::ubig16_t NumberOfSections; + + // Unix time value, value of 0 indicates no timestamp. + // Negative values are reserved. + support::big32_t TimeStamp; + + support::ubig32_t SymbolTableOffset; // File offset to symbol table. + support::big32_t NumberOfSymTableEntries; + support::ubig16_t AuxHeaderSize; + support::ubig16_t Flags; +}; + +struct XCOFFFileHeader64 { + support::ubig16_t Magic; + support::ubig16_t NumberOfSections; + + // Unix time value, value of 0 indicates no timestamp. + // Negative values are reserved. + support::big32_t TimeStamp; + + support::ubig64_t SymbolTableOffset; // File offset to symbol table. + support::ubig16_t AuxHeaderSize; + support::ubig16_t Flags; + support::ubig32_t NumberOfSymTableEntries; +}; + +struct XCOFFSectionHeader32 { + char Name[XCOFF::SectionNameSize]; + support::ubig32_t PhysicalAddress; + support::ubig32_t VirtualAddress; + support::ubig32_t SectionSize; + support::ubig32_t FileOffsetToRawData; + support::ubig32_t FileOffsetToRelocationInfo; + support::ubig32_t FileOffsetToLineNumberInfo; + support::ubig16_t NumberOfRelocations; + support::ubig16_t NumberOfLineNumbers; + support::big32_t Flags; + + StringRef getName() const; +}; + +struct XCOFFSectionHeader64 { + char Name[XCOFF::SectionNameSize]; + support::ubig64_t PhysicalAddress; + support::ubig64_t VirtualAddress; + support::ubig64_t SectionSize; + support::big64_t FileOffsetToRawData; + support::big64_t FileOffsetToRelocationInfo; + support::big64_t FileOffsetToLineNumberInfo; + support::ubig32_t NumberOfRelocations; + support::ubig32_t NumberOfLineNumbers; + support::big32_t Flags; + char Padding[4]; + + StringRef getName() const; +}; + +struct XCOFFSymbolEntry { + enum { NAME_IN_STR_TBL_MAGIC = 0x0 }; + typedef struct { + support::big32_t Magic; // Zero indicates name in string table. + support::ubig32_t Offset; + } NameInStrTblType; + + typedef struct { + uint8_t LanguageId; + uint8_t CpuTypeId; + } CFileLanguageIdAndTypeIdType; + + union { + char SymbolName[XCOFF::SymbolNameSize]; + NameInStrTblType NameInStrTbl; + }; + + support::ubig32_t Value; // Symbol value; storage class-dependent. + support::big16_t SectionNumber; + + union { + support::ubig16_t SymbolType; + CFileLanguageIdAndTypeIdType CFileLanguageIdAndTypeId; + }; + + XCOFF::StorageClass StorageClass; + uint8_t NumberOfAuxEntries; +}; + +struct XCOFFStringTable { + uint32_t Size; + const char *Data; +}; + +class XCOFFObjectFile : public ObjectFile { +private: + const void *FileHeader = nullptr; + const void *SectionHeaderTable = nullptr; + + const XCOFFSymbolEntry *SymbolTblPtr = nullptr; + XCOFFStringTable StringTable = {0, nullptr}; + + const XCOFFFileHeader32 *fileHeader32() const; + const XCOFFFileHeader64 *fileHeader64() const; + + const XCOFFSectionHeader32 *sectionHeaderTable32() const; + const XCOFFSectionHeader64 *sectionHeaderTable64() const; + + size_t getFileHeaderSize() const; + size_t getSectionHeaderSize() const; + + const XCOFFSectionHeader32 *toSection32(DataRefImpl Ref) const; + const XCOFFSectionHeader64 *toSection64(DataRefImpl Ref) const; + void checkSectionAddress(uintptr_t Addr, uintptr_t TableAddr) const; + uintptr_t getSectionHeaderTableAddress() const; + + // This returns a pointer to the start of the storage for the name field of + // the 32-bit or 64-bit SectionHeader struct. This string is *not* necessarily + // null-terminated. + const char *getSectionNameInternal(DataRefImpl Sec) const; + + int32_t getSectionFlags(DataRefImpl Sec) const; + + static bool isReservedSectionNumber(int16_t SectionNumber); + Expected getSectionByNum(int16_t Num) const; + + // Constructor and "create" factory function. The constructor is only a thin + // wrapper around the base constructor. The "create" function fills out the + // XCOFF-specific information and performs the error checking along the way. + XCOFFObjectFile(unsigned Type, MemoryBufferRef Object); + static Expected> create(unsigned Type, + MemoryBufferRef MBR); + + // Helper for parsing the StringTable. Returns an 'Error' if parsing failed + // and an XCOFFStringTable if parsing succeeded. + static Expected parseStringTable(const XCOFFObjectFile *Obj, + uint64_t Offset); + + // Make a friend so it can call the private 'create' function. + friend Expected> + ObjectFile::createXCOFFObjectFile(MemoryBufferRef Object, unsigned FileType); + +public: + // Interface inherited from base classes. + void moveSymbolNext(DataRefImpl &Symb) const override; + uint32_t getSymbolFlags(DataRefImpl Symb) const override; + basic_symbol_iterator symbol_begin() const override; + basic_symbol_iterator symbol_end() const override; + + Expected getSymbolName(DataRefImpl Symb) const override; + Expected getSymbolAddress(DataRefImpl Symb) const override; + uint64_t getSymbolValueImpl(DataRefImpl Symb) const override; + uint64_t getCommonSymbolSizeImpl(DataRefImpl Symb) const override; + Expected getSymbolType(DataRefImpl Symb) const override; + Expected getSymbolSection(DataRefImpl Symb) const override; + + void moveSectionNext(DataRefImpl &Sec) const override; + Expected getSectionName(DataRefImpl Sec) const override; + uint64_t getSectionAddress(DataRefImpl Sec) const override; + uint64_t getSectionIndex(DataRefImpl Sec) const override; + uint64_t getSectionSize(DataRefImpl Sec) const override; + Expected> + getSectionContents(DataRefImpl Sec) const override; + uint64_t getSectionAlignment(DataRefImpl Sec) const override; + bool isSectionCompressed(DataRefImpl Sec) const override; + bool isSectionText(DataRefImpl Sec) const override; + bool isSectionData(DataRefImpl Sec) const override; + bool isSectionBSS(DataRefImpl Sec) const override; + + bool isSectionVirtual(DataRefImpl Sec) const override; + relocation_iterator section_rel_begin(DataRefImpl Sec) const override; + relocation_iterator section_rel_end(DataRefImpl Sec) const override; + + void moveRelocationNext(DataRefImpl &Rel) const override; + uint64_t getRelocationOffset(DataRefImpl Rel) const override; + symbol_iterator getRelocationSymbol(DataRefImpl Rel) const override; + uint64_t getRelocationType(DataRefImpl Rel) const override; + void getRelocationTypeName(DataRefImpl Rel, + SmallVectorImpl &Result) const override; + + section_iterator section_begin() const override; + section_iterator section_end() const override; + uint8_t getBytesInAddress() const override; + StringRef getFileFormatName() const override; + Triple::ArchType getArch() const override; + SubtargetFeatures getFeatures() const override; + Expected getStartAddress() const override; + bool isRelocatableObject() const override; + + // Below here is the non-inherited interface. + bool is64Bit() const; + + const XCOFFSymbolEntry *getPointerToSymbolTable() const { + assert(!is64Bit() && "Symbol table handling not supported yet."); + return SymbolTblPtr; + } + + Expected + getSymbolSectionName(const XCOFFSymbolEntry *SymEntPtr) const; + + const XCOFFSymbolEntry *toSymbolEntry(DataRefImpl Ref) const; + + // File header related interfaces. + uint16_t getMagic() const; + uint16_t getNumberOfSections() const; + int32_t getTimeStamp() const; + + // Symbol table offset and entry count are handled differently between + // XCOFF32 and XCOFF64. + uint32_t getSymbolTableOffset32() const; + uint64_t getSymbolTableOffset64() const; + + // Note that this value is signed and might return a negative value. Negative + // values are reserved for future use. + int32_t getRawNumberOfSymbolTableEntries32() const; + + // The sanitized value appropriate to use as an index into the symbol table. + uint32_t getLogicalNumberOfSymbolTableEntries32() const; + + uint32_t getNumberOfSymbolTableEntries64() const; + + uint16_t getOptionalHeaderSize() const; + uint16_t getFlags() const; + + // Section header table related interfaces. + ArrayRef sections32() const; + ArrayRef sections64() const; +}; // XCOFFObjectFile + +} // namespace object +} // namespace llvm + +#endif // LLVM_OBJECT_XCOFFOBJECTFILE_H diff --git a/include/llvm/ObjectYAML/COFFYAML.h b/include/llvm/ObjectYAML/COFFYAML.h index 253c627dd683..eec5af928f6d 100644 --- a/include/llvm/ObjectYAML/COFFYAML.h +++ b/include/llvm/ObjectYAML/COFFYAML.h @@ -1,9 +1,8 @@ //===- COFFYAML.h - COFF YAMLIO implementation ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ObjectYAML/CodeViewYAMLDebugSections.h b/include/llvm/ObjectYAML/CodeViewYAMLDebugSections.h index d620008e22d2..9cbacb88b518 100644 --- a/include/llvm/ObjectYAML/CodeViewYAMLDebugSections.h +++ b/include/llvm/ObjectYAML/CodeViewYAMLDebugSections.h @@ -1,9 +1,8 @@ //=- CodeViewYAMLDebugSections.h - CodeView YAMLIO debug sections -*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ObjectYAML/CodeViewYAMLSymbols.h b/include/llvm/ObjectYAML/CodeViewYAMLSymbols.h index 791193c78f19..7c05c9eea05e 100644 --- a/include/llvm/ObjectYAML/CodeViewYAMLSymbols.h +++ b/include/llvm/ObjectYAML/CodeViewYAMLSymbols.h @@ -1,9 +1,8 @@ //===- CodeViewYAMLSymbols.h - CodeView YAMLIO Symbol implementation ------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ObjectYAML/CodeViewYAMLTypeHashing.h b/include/llvm/ObjectYAML/CodeViewYAMLTypeHashing.h index 344966fe6891..d6cec8d310eb 100644 --- a/include/llvm/ObjectYAML/CodeViewYAMLTypeHashing.h +++ b/include/llvm/ObjectYAML/CodeViewYAMLTypeHashing.h @@ -1,9 +1,8 @@ //==- CodeViewYAMLTypeHashing.h - CodeView YAMLIO Type hashing ----*- C++-*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ObjectYAML/CodeViewYAMLTypes.h b/include/llvm/ObjectYAML/CodeViewYAMLTypes.h index 1b1306df4f53..04b5e0ba3aa1 100644 --- a/include/llvm/ObjectYAML/CodeViewYAMLTypes.h +++ b/include/llvm/ObjectYAML/CodeViewYAMLTypes.h @@ -1,9 +1,8 @@ //==- CodeViewYAMLTypes.h - CodeView YAMLIO Type implementation --*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ObjectYAML/DWARFEmitter.h b/include/llvm/ObjectYAML/DWARFEmitter.h index ce3227421930..2ccc876d5023 100644 --- a/include/llvm/ObjectYAML/DWARFEmitter.h +++ b/include/llvm/ObjectYAML/DWARFEmitter.h @@ -1,9 +1,8 @@ //===--- DWARFEmitter.h - ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/ObjectYAML/DWARFYAML.h b/include/llvm/ObjectYAML/DWARFYAML.h index 705c88778945..78d736c3ef05 100644 --- a/include/llvm/ObjectYAML/DWARFYAML.h +++ b/include/llvm/ObjectYAML/DWARFYAML.h @@ -1,9 +1,8 @@ //===- DWARFYAML.h - DWARF YAMLIO implementation ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/include/llvm/ObjectYAML/ELFYAML.h b/include/llvm/ObjectYAML/ELFYAML.h index f2b0c35521f0..f4212516f486 100644 --- a/include/llvm/ObjectYAML/ELFYAML.h +++ b/include/llvm/ObjectYAML/ELFYAML.h @@ -1,9 +1,8 @@ //===- ELFYAML.h - ELF YAMLIO implementation --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -44,6 +43,8 @@ LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_ELFDATA) LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_ELFOSABI) // Just use 64, since it can hold 32-bit values too. LLVM_YAML_STRONG_TYPEDEF(uint64_t, ELF_EF) +// Just use 64, since it can hold 32-bit values too. +LLVM_YAML_STRONG_TYPEDEF(uint64_t, ELF_DYNTAG) LLVM_YAML_STRONG_TYPEDEF(uint32_t, ELF_PF) LLVM_YAML_STRONG_TYPEDEF(uint32_t, ELF_SHT) LLVM_YAML_STRONG_TYPEDEF(uint32_t, ELF_REL) @@ -51,6 +52,7 @@ LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_RSS) // Just use 64, since it can hold 32-bit values too. LLVM_YAML_STRONG_TYPEDEF(uint64_t, ELF_SHF) LLVM_YAML_STRONG_TYPEDEF(uint16_t, ELF_SHN) +LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_STB) LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_STT) LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_STV) LLVM_YAML_STRONG_TYPEDEF(uint8_t, ELF_STO) @@ -73,6 +75,11 @@ struct FileHeader { ELF_EM Machine; ELF_EF Flags; llvm::yaml::Hex64 Entry; + + Optional SHEntSize; + Optional SHOffset; + Optional SHNum; + Optional SHStrNdx; }; struct SectionName { @@ -85,53 +92,81 @@ struct ProgramHeader { llvm::yaml::Hex64 VAddr; llvm::yaml::Hex64 PAddr; Optional Align; + Optional FileSize; + Optional MemSize; + Optional Offset; std::vector Sections; }; struct Symbol { StringRef Name; + Optional NameIndex; ELF_STT Type; StringRef Section; Optional Index; + ELF_STB Binding; llvm::yaml::Hex64 Value; llvm::yaml::Hex64 Size; uint8_t Other; }; -struct LocalGlobalWeakSymbols { - std::vector Local; - std::vector Global; - std::vector Weak; -}; - struct SectionOrType { StringRef sectionNameOrType; }; +struct DynamicEntry { + ELF_DYNTAG Tag; + llvm::yaml::Hex64 Val; +}; + struct Section { enum class SectionKind { + Dynamic, Group, RawContent, Relocation, NoBits, + Verdef, + Verneed, + Symver, MipsABIFlags }; SectionKind Kind; StringRef Name; ELF_SHT Type; - ELF_SHF Flags; + Optional Flags; llvm::yaml::Hex64 Address; StringRef Link; - StringRef Info; llvm::yaml::Hex64 AddressAlign; Optional EntSize; + // This can be used to override the sh_offset field. It does not place the + // section data at the offset specified. Useful for creating invalid objects. + Optional ShOffset; + + // This can be used to override the sh_size field. It does not affect the + // content written. + Optional ShSize; + Section(SectionKind Kind) : Kind(Kind) {} virtual ~Section(); }; + +struct DynamicSection : Section { + std::vector Entries; + Optional Content; + + DynamicSection() : Section(SectionKind::Dynamic) {} + + static bool classof(const Section *S) { + return S->Kind == SectionKind::Dynamic; + } +}; + struct RawContentSection : Section { - yaml::BinaryRef Content; - llvm::yaml::Hex64 Size; + Optional Content; + Optional Size; + Optional Info; RawContentSection() : Section(SectionKind::RawContent) {} @@ -150,10 +185,64 @@ struct NoBitsSection : Section { } }; +struct VernauxEntry { + uint32_t Hash; + uint16_t Flags; + uint16_t Other; + StringRef Name; +}; + +struct VerneedEntry { + uint16_t Version; + StringRef File; + std::vector AuxV; +}; + +struct VerneedSection : Section { + std::vector VerneedV; + llvm::yaml::Hex64 Info; + + VerneedSection() : Section(SectionKind::Verneed) {} + + static bool classof(const Section *S) { + return S->Kind == SectionKind::Verneed; + } +}; + +struct SymverSection : Section { + std::vector Entries; + + SymverSection() : Section(SectionKind::Symver) {} + + static bool classof(const Section *S) { + return S->Kind == SectionKind::Symver; + } +}; + +struct VerdefEntry { + uint16_t Version; + uint16_t Flags; + uint16_t VersionNdx; + uint32_t Hash; + std::vector VerNames; +}; + +struct VerdefSection : Section { + std::vector Entries; + llvm::yaml::Hex64 Info; + + VerdefSection() : Section(SectionKind::Verdef) {} + + static bool classof(const Section *S) { + return S->Kind == SectionKind::Verdef; + } +}; + struct Group : Section { // Members of a group contain a flag and a list of section indices // that are part of the group. std::vector Members; + StringRef Signature; /* Info */ Group() : Section(SectionKind::Group) {} @@ -171,6 +260,7 @@ struct Relocation { struct RelocationSection : Section { std::vector Relocations; + StringRef RelocatableSec; /* Info */ RelocationSection() : Section(SectionKind::Relocation) {} @@ -208,16 +298,20 @@ struct Object { // cleaner and nicer if we read them from the YAML as a separate // top-level key, which automatically ensures that invariants like there // being a single SHT_SYMTAB section are upheld. - LocalGlobalWeakSymbols Symbols; - LocalGlobalWeakSymbols DynamicSymbols; + std::vector Symbols; + std::vector DynamicSymbols; }; } // end namespace ELFYAML } // end namespace llvm +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::DynamicEntry) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::ProgramHeader) LLVM_YAML_IS_SEQUENCE_VECTOR(std::unique_ptr) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::Symbol) +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::VerdefEntry) +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::VernauxEntry) +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::VerneedEntry) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::Relocation) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::SectionOrType) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::ELFYAML::SectionName) @@ -277,6 +371,10 @@ template <> struct ScalarEnumerationTraits { static void enumeration(IO &IO, ELFYAML::ELF_SHN &Value); }; +template <> struct ScalarEnumerationTraits { + static void enumeration(IO &IO, ELFYAML::ELF_STB &Value); +}; + template <> struct ScalarEnumerationTraits { static void enumeration(IO &IO, ELFYAML::ELF_STT &Value); @@ -297,6 +395,11 @@ struct ScalarEnumerationTraits { static void enumeration(IO &IO, ELFYAML::ELF_REL &Value); }; +template <> +struct ScalarEnumerationTraits { + static void enumeration(IO &IO, ELFYAML::ELF_DYNTAG &Value); +}; + template <> struct ScalarEnumerationTraits { static void enumeration(IO &IO, ELFYAML::ELF_RSS &Value); @@ -347,9 +450,20 @@ struct MappingTraits { static StringRef validate(IO &IO, ELFYAML::Symbol &Symbol); }; -template <> -struct MappingTraits { - static void mapping(IO &IO, ELFYAML::LocalGlobalWeakSymbols &Symbols); +template <> struct MappingTraits { + static void mapping(IO &IO, ELFYAML::DynamicEntry &Rel); +}; + +template <> struct MappingTraits { + static void mapping(IO &IO, ELFYAML::VerdefEntry &E); +}; + +template <> struct MappingTraits { + static void mapping(IO &IO, ELFYAML::VerneedEntry &E); +}; + +template <> struct MappingTraits { + static void mapping(IO &IO, ELFYAML::VernauxEntry &E); }; template <> struct MappingTraits { diff --git a/include/llvm/ObjectYAML/MachOYAML.h b/include/llvm/ObjectYAML/MachOYAML.h index cec4f86185f0..d7e1c033f43b 100644 --- a/include/llvm/ObjectYAML/MachOYAML.h +++ b/include/llvm/ObjectYAML/MachOYAML.h @@ -1,9 +1,8 @@ //===- MachOYAML.h - Mach-O YAMLIO implementation ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/include/llvm/ObjectYAML/MinidumpYAML.h b/include/llvm/ObjectYAML/MinidumpYAML.h new file mode 100644 index 000000000000..39fdd62e017b --- /dev/null +++ b/include/llvm/ObjectYAML/MinidumpYAML.h @@ -0,0 +1,239 @@ +//===- MinidumpYAML.h - Minidump YAMLIO implementation ----------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_OBJECTYAML_MINIDUMPYAML_H +#define LLVM_OBJECTYAML_MINIDUMPYAML_H + +#include "llvm/BinaryFormat/Minidump.h" +#include "llvm/Object/Minidump.h" +#include "llvm/ObjectYAML/YAML.h" +#include "llvm/Support/YAMLTraits.h" + +namespace llvm { +namespace MinidumpYAML { + +/// The base class for all minidump streams. The "Type" of the stream +/// corresponds to the Stream Type field in the minidump file. The "Kind" field +/// specifies how are we going to treat it. For highly specialized streams (e.g. +/// SystemInfo), there is a 1:1 mapping between Types and Kinds, but in general +/// one stream Kind can be used to represent multiple stream Types (e.g. any +/// unrecognised stream Type will be handled via RawContentStream). The mapping +/// from Types to Kinds is fixed and given by the static getKind function. +struct Stream { + enum class StreamKind { + MemoryList, + ModuleList, + RawContent, + SystemInfo, + TextContent, + ThreadList, + }; + + Stream(StreamKind Kind, minidump::StreamType Type) : Kind(Kind), Type(Type) {} + virtual ~Stream(); // anchor + + const StreamKind Kind; + const minidump::StreamType Type; + + /// Get the stream Kind used for representing streams of a given Type. + static StreamKind getKind(minidump::StreamType Type); + + /// Create an empty stream of the given Type. + static std::unique_ptr create(minidump::StreamType Type); + + /// Create a stream from the given stream directory entry. + static Expected> + create(const minidump::Directory &StreamDesc, + const object::MinidumpFile &File); +}; + +namespace detail { +/// A stream representing a list of abstract entries in a minidump stream. Its +/// instantiations can be used to represent the ModuleList stream and other +/// streams with a similar structure. +template struct ListStream : public Stream { + using entry_type = EntryT; + + std::vector Entries; + + explicit ListStream(std::vector Entries = {}) + : Stream(EntryT::Kind, EntryT::Type), Entries(std::move(Entries)) {} + + static bool classof(const Stream *S) { return S->Kind == EntryT::Kind; } +}; + +/// A structure containing all data belonging to a single minidump module. +struct ParsedModule { + static constexpr Stream::StreamKind Kind = Stream::StreamKind::ModuleList; + static constexpr minidump::StreamType Type = minidump::StreamType::ModuleList; + + minidump::Module Entry; + std::string Name; + yaml::BinaryRef CvRecord; + yaml::BinaryRef MiscRecord; +}; + +/// A structure containing all data belonging to a single minidump thread. +struct ParsedThread { + static constexpr Stream::StreamKind Kind = Stream::StreamKind::ThreadList; + static constexpr minidump::StreamType Type = minidump::StreamType::ThreadList; + + minidump::Thread Entry; + yaml::BinaryRef Stack; + yaml::BinaryRef Context; +}; + +/// A structure containing all data describing a single memory region. +struct ParsedMemoryDescriptor { + static constexpr Stream::StreamKind Kind = Stream::StreamKind::MemoryList; + static constexpr minidump::StreamType Type = minidump::StreamType::MemoryList; + + minidump::MemoryDescriptor Entry; + yaml::BinaryRef Content; +}; +} // namespace detail + +using ModuleListStream = detail::ListStream; +using ThreadListStream = detail::ListStream; +using MemoryListStream = detail::ListStream; + +/// A minidump stream represented as a sequence of hex bytes. This is used as a +/// fallback when no other stream kind is suitable. +struct RawContentStream : public Stream { + yaml::BinaryRef Content; + yaml::Hex32 Size; + + RawContentStream(minidump::StreamType Type, ArrayRef Content = {}) + : Stream(StreamKind::RawContent, Type), Content(Content), + Size(Content.size()) {} + + static bool classof(const Stream *S) { + return S->Kind == StreamKind::RawContent; + } +}; + +/// SystemInfo minidump stream. +struct SystemInfoStream : public Stream { + minidump::SystemInfo Info; + std::string CSDVersion; + + explicit SystemInfoStream(const minidump::SystemInfo &Info, + std::string CSDVersion) + : Stream(StreamKind::SystemInfo, minidump::StreamType::SystemInfo), + Info(Info), CSDVersion(std::move(CSDVersion)) {} + + SystemInfoStream() + : Stream(StreamKind::SystemInfo, minidump::StreamType::SystemInfo) { + memset(&Info, 0, sizeof(Info)); + } + + static bool classof(const Stream *S) { + return S->Kind == StreamKind::SystemInfo; + } +}; + +/// A StringRef, which is printed using YAML block notation. +LLVM_YAML_STRONG_TYPEDEF(StringRef, BlockStringRef) + +/// A minidump stream containing textual data (typically, the contents of a +/// /proc/ file on linux). +struct TextContentStream : public Stream { + BlockStringRef Text; + + TextContentStream(minidump::StreamType Type, StringRef Text = {}) + : Stream(StreamKind::TextContent, Type), Text(Text) {} + + static bool classof(const Stream *S) { + return S->Kind == StreamKind::TextContent; + } +}; + +/// The top level structure representing a minidump object, consisting of a +/// minidump header, and zero or more streams. To construct an Object from a +/// minidump file, use the static create function. To serialize to/from yaml, +/// use the appropriate streaming operator on a yaml stream. +struct Object { + Object() = default; + Object(const Object &) = delete; + Object &operator=(const Object &) = delete; + Object(Object &&) = default; + Object &operator=(Object &&) = default; + + Object(const minidump::Header &Header, + std::vector> Streams) + : Header(Header), Streams(std::move(Streams)) {} + + /// The minidump header. + minidump::Header Header; + + /// The list of streams in this minidump object. + std::vector> Streams; + + static Expected create(const object::MinidumpFile &File); +}; + +/// Serialize the minidump file represented by Obj to OS in binary form. +void writeAsBinary(Object &Obj, raw_ostream &OS); + +/// Serialize the yaml string as a minidump file to OS in binary form. +Error writeAsBinary(StringRef Yaml, raw_ostream &OS); + +} // namespace MinidumpYAML + +namespace yaml { +template <> struct BlockScalarTraits { + static void output(const MinidumpYAML::BlockStringRef &Text, void *, + raw_ostream &OS) { + OS << Text; + } + + static StringRef input(StringRef Scalar, void *, + MinidumpYAML::BlockStringRef &Text) { + Text = Scalar; + return ""; + } +}; + +template <> struct MappingTraits> { + static void mapping(IO &IO, std::unique_ptr &S); + static StringRef validate(IO &IO, std::unique_ptr &S); +}; + +template <> struct MappingContextTraits { + static void mapping(IO &IO, minidump::MemoryDescriptor &Memory, + BinaryRef &Content); +}; + +} // namespace yaml + +} // namespace llvm + +LLVM_YAML_DECLARE_ENUM_TRAITS(llvm::minidump::ProcessorArchitecture) +LLVM_YAML_DECLARE_ENUM_TRAITS(llvm::minidump::OSPlatform) +LLVM_YAML_DECLARE_ENUM_TRAITS(llvm::minidump::StreamType) + +LLVM_YAML_DECLARE_MAPPING_TRAITS(llvm::minidump::CPUInfo::ArmInfo) +LLVM_YAML_DECLARE_MAPPING_TRAITS(llvm::minidump::CPUInfo::OtherInfo) +LLVM_YAML_DECLARE_MAPPING_TRAITS(llvm::minidump::CPUInfo::X86Info) +LLVM_YAML_DECLARE_MAPPING_TRAITS(llvm::minidump::VSFixedFileInfo) + +LLVM_YAML_DECLARE_MAPPING_TRAITS( + llvm::MinidumpYAML::MemoryListStream::entry_type) +LLVM_YAML_DECLARE_MAPPING_TRAITS( + llvm::MinidumpYAML::ModuleListStream::entry_type) +LLVM_YAML_DECLARE_MAPPING_TRAITS( + llvm::MinidumpYAML::ThreadListStream::entry_type) + +LLVM_YAML_IS_SEQUENCE_VECTOR(std::unique_ptr) +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::MinidumpYAML::MemoryListStream::entry_type) +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::MinidumpYAML::ModuleListStream::entry_type) +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::MinidumpYAML::ThreadListStream::entry_type) + +LLVM_YAML_DECLARE_MAPPING_TRAITS(llvm::MinidumpYAML::Object) + +#endif // LLVM_OBJECTYAML_MINIDUMPYAML_H diff --git a/include/llvm/ObjectYAML/ObjectYAML.h b/include/llvm/ObjectYAML/ObjectYAML.h index 00ce86430fca..0015fd3dc501 100644 --- a/include/llvm/ObjectYAML/ObjectYAML.h +++ b/include/llvm/ObjectYAML/ObjectYAML.h @@ -1,9 +1,8 @@ //===- ObjectYAML.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -13,6 +12,7 @@ #include "llvm/ObjectYAML/COFFYAML.h" #include "llvm/ObjectYAML/ELFYAML.h" #include "llvm/ObjectYAML/MachOYAML.h" +#include "llvm/ObjectYAML/MinidumpYAML.h" #include "llvm/ObjectYAML/WasmYAML.h" #include "llvm/Support/YAMLTraits.h" #include @@ -27,6 +27,7 @@ struct YamlObjectFile { std::unique_ptr Coff; std::unique_ptr MachO; std::unique_ptr FatMachO; + std::unique_ptr Minidump; std::unique_ptr Wasm; }; diff --git a/include/llvm/ObjectYAML/WasmYAML.h b/include/llvm/ObjectYAML/WasmYAML.h index 406dd7cb515f..2411dc7ac17d 100644 --- a/include/llvm/ObjectYAML/WasmYAML.h +++ b/include/llvm/ObjectYAML/WasmYAML.h @@ -1,9 +1,8 @@ //===- WasmYAML.h - Wasm YAMLIO implementation ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -39,6 +38,7 @@ LLVM_YAML_STRONG_TYPEDEF(uint32_t, SymbolKind) LLVM_YAML_STRONG_TYPEDEF(uint32_t, SegmentFlags) LLVM_YAML_STRONG_TYPEDEF(uint32_t, LimitFlags) LLVM_YAML_STRONG_TYPEDEF(uint32_t, ComdatKind) +LLVM_YAML_STRONG_TYPEDEF(uint32_t, FeaturePolicyPrefix) struct FileHeader { yaml::Hex32 Version; @@ -112,8 +112,9 @@ struct Relocation { }; struct DataSegment { - uint32_t MemoryIndex; uint32_t SectionOffset; + uint32_t InitFlags; + uint32_t MemoryIndex; wasm::WasmInitExpr Offset; yaml::BinaryRef Content; }; @@ -123,6 +124,16 @@ struct NameEntry { StringRef Name; }; +struct ProducerEntry { + std::string Name; + std::string Version; +}; + +struct FeatureEntry { + FeaturePolicyPrefix Prefix; + std::string Name; +}; + struct SegmentInfo { uint32_t Index; StringRef Name; @@ -224,6 +235,30 @@ struct LinkingSection : CustomSection { std::vector Comdats; }; +struct ProducersSection : CustomSection { + ProducersSection() : CustomSection("producers") {} + + static bool classof(const Section *S) { + auto C = dyn_cast(S); + return C && C->Name == "producers"; + } + + std::vector Languages; + std::vector Tools; + std::vector SDKs; +}; + +struct TargetFeaturesSection : CustomSection { + TargetFeaturesSection() : CustomSection("target_features") {} + + static bool classof(const Section *S) { + auto C = dyn_cast(S); + return C && C->Name == "target_features"; + } + + std::vector Features; +}; + struct TypeSection : Section { TypeSection() : Section(wasm::WASM_SEC_TYPE) {} @@ -344,6 +379,16 @@ struct DataSection : Section { std::vector Segments; }; +struct DataCountSection : Section { + DataCountSection() : Section(wasm::WASM_SEC_DATACOUNT) {} + + static bool classof(const Section *S) { + return S->Type == wasm::WASM_SEC_DATACOUNT; + } + + uint32_t Count; +}; + struct Object { FileHeader Header; std::vector> Sections; @@ -366,6 +411,8 @@ LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::Function) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::LocalDecl) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::Relocation) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::NameEntry) +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::ProducerEntry) +LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::FeatureEntry) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::SegmentInfo) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::SymbolInfo) LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::WasmYAML::InitFunction) @@ -444,6 +491,18 @@ template <> struct MappingTraits { static void mapping(IO &IO, WasmYAML::NameEntry &NameEntry); }; +template <> struct MappingTraits { + static void mapping(IO &IO, WasmYAML::ProducerEntry &ProducerEntry); +}; + +template <> struct ScalarEnumerationTraits { + static void enumeration(IO &IO, WasmYAML::FeaturePolicyPrefix &Prefix); +}; + +template <> struct MappingTraits { + static void mapping(IO &IO, WasmYAML::FeatureEntry &FeatureEntry); +}; + template <> struct MappingTraits { static void mapping(IO &IO, WasmYAML::SegmentInfo &SegmentInfo); }; diff --git a/include/llvm/ObjectYAML/XCOFFYAML.h b/include/llvm/ObjectYAML/XCOFFYAML.h new file mode 100644 index 000000000000..f99004e69762 --- /dev/null +++ b/include/llvm/ObjectYAML/XCOFFYAML.h @@ -0,0 +1,71 @@ +//===----- XCOFFYAML.h - XCOFF YAMLIO implementation ------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file declares classes for handling the YAML representation of XCOFF. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_OBJECTYAML_XCOFFYAML_H +#define LLVM_OBJECTYAML_XCOFFYAML_H + +#include "llvm/BinaryFormat/XCOFF.h" +#include "llvm/ObjectYAML/YAML.h" +#include + +namespace llvm { +namespace XCOFFYAML { + +struct FileHeader { + llvm::yaml::Hex16 Magic; + uint16_t NumberOfSections; + int32_t TimeStamp; + llvm::yaml::Hex32 SymbolTableOffset; // File offset to symbol table. + int32_t NumberOfSymTableEntries; + uint16_t AuxHeaderSize; + llvm::yaml::Hex16 Flags; +}; + +struct Symbol { + StringRef SymbolName; + llvm::yaml::Hex32 Value; // Symbol value; storage class-dependent. + StringRef SectionName; + llvm::yaml::Hex16 Type; + XCOFF::StorageClass StorageClass; + uint8_t NumberOfAuxEntries; // Number of auxiliary entries +}; + +struct Object { + FileHeader Header; + std::vector Symbols; + Object(); +}; +} // namespace XCOFFYAML +} // namespace llvm +LLVM_YAML_IS_SEQUENCE_VECTOR(XCOFFYAML::Symbol) +namespace llvm { +namespace yaml { + +template <> struct ScalarEnumerationTraits { + static void enumeration(IO &IO, XCOFF::StorageClass &Value); +}; + +template <> struct MappingTraits { + static void mapping(IO &IO, XCOFFYAML::FileHeader &H); +}; + +template <> struct MappingTraits { + static void mapping(IO &IO, XCOFFYAML::Object &Obj); +}; + +template <> struct MappingTraits { + static void mapping(IO &IO, XCOFFYAML::Symbol &S); +}; + +} // namespace yaml +} // namespace llvm + +#endif // LLVM_OBJECTYAML_XCOFFYAML_H diff --git a/include/llvm/ObjectYAML/YAML.h b/include/llvm/ObjectYAML/YAML.h index 163cd8dfcf08..37014109a615 100644 --- a/include/llvm/ObjectYAML/YAML.h +++ b/include/llvm/ObjectYAML/YAML.h @@ -1,9 +1,8 @@ //===- YAML.h ---------------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -74,8 +73,7 @@ class BinaryRef { public: BinaryRef() = default; BinaryRef(ArrayRef Data) : Data(Data), DataIsHexString(false) {} - BinaryRef(StringRef Data) - : Data(reinterpret_cast(Data.data()), Data.size()) {} + BinaryRef(StringRef Data) : Data(arrayRefFromStringRef(Data)) {} /// The number of bytes that are represented by this BinaryRef. /// This is the number of bytes that writeAsBinary() will write. diff --git a/include/llvm/Option/Arg.h b/include/llvm/Option/Arg.h index d0086bb6d611..22e2bcf06a6e 100644 --- a/include/llvm/Option/Arg.h +++ b/include/llvm/Option/Arg.h @@ -1,9 +1,8 @@ //===- Arg.h - Parsed Argument Classes --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -59,6 +58,11 @@ private: /// The argument values, as C strings. SmallVector Values; + /// If this arg was created through an alias, this is the original alias arg. + /// For example, *this might be "-finput-charset=utf-8" and Alias might + /// point to an arg representing "/source-charset:utf-8". + std::unique_ptr Alias; + public: Arg(const Option Opt, StringRef Spelling, unsigned Index, const Arg *BaseArg = nullptr); @@ -71,7 +75,15 @@ public: ~Arg(); const Option &getOption() const { return Opt; } + + /// Returns the used prefix and name of the option: + /// For `--foo=bar`, returns `--foo=`. + /// This is often the wrong function to call: + /// * Use `getValue()` to get `bar`. + /// * Use `getAsString()` to get a string suitable for printing an Arg in + /// a diagnostic. StringRef getSpelling() const { return Spelling; } + unsigned getIndex() const { return Index; } /// Return the base argument which generated this arg. @@ -83,6 +95,11 @@ public: } void setBaseArg(const Arg *BaseArg) { this->BaseArg = BaseArg; } + /// Args are converted to their unaliased form. For args that originally + /// came from an alias, this returns the alias the arg was produced from. + const Arg* getAlias() const { return Alias.get(); } + void setAlias(std::unique_ptr Alias) { this->Alias = std::move(Alias); } + bool getOwnsValues() const { return OwnsValues; } void setOwnsValues(bool Value) const { OwnsValues = Value; } @@ -120,8 +137,10 @@ public: void print(raw_ostream &O) const; void dump() const; - /// Return a formatted version of the argument and - /// its values, for debugging and diagnostics. + /// Return a formatted version of the argument and its values, for + /// diagnostics. Since this is for diagnostics, if this Arg was produced + /// through an alias, this returns the string representation of the alias + /// that the user wrote. std::string getAsString(const ArgList &Args) const; }; diff --git a/include/llvm/Option/ArgList.h b/include/llvm/Option/ArgList.h index 687c8cbb02f9..74bfadcba726 100644 --- a/include/llvm/Option/ArgList.h +++ b/include/llvm/Option/ArgList.h @@ -1,9 +1,8 @@ //===- ArgList.h - Argument List Management ---------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -302,10 +301,12 @@ public: bool hasFlag(OptSpecifier Pos, OptSpecifier PosAlias, OptSpecifier Neg, bool Default = true) const; - /// AddLastArg - Render only the last argument match \p Id0, if present. - void AddLastArg(ArgStringList &Output, OptSpecifier Id0) const; - void AddLastArg(ArgStringList &Output, OptSpecifier Id0, - OptSpecifier Id1) const; + /// Render only the last argument match \p Id0, if present. + template + void AddLastArg(ArgStringList &Output, OptSpecifiers ...Ids) const { + if (Arg *A = getLastArg(Ids...)) // Calls claim() on all Ids's Args. + A->render(*this, Output); + } /// AddAllArgsExcept - Render all arguments matching any of the given ids /// and not matching any of the excluded ids. diff --git a/include/llvm/Option/OptParser.td b/include/llvm/Option/OptParser.td index 9c373741770b..a68f17a8b10b 100644 --- a/include/llvm/Option/OptParser.td +++ b/include/llvm/Option/OptParser.td @@ -1,9 +1,8 @@ //===--- OptParser.td - Common Option Parsing Interfaces ------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Option/OptSpecifier.h b/include/llvm/Option/OptSpecifier.h index 84c3cf8ad534..7a5fcfb18b38 100644 --- a/include/llvm/Option/OptSpecifier.h +++ b/include/llvm/Option/OptSpecifier.h @@ -1,9 +1,8 @@ //===- OptSpecifier.h - Option Specifiers -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Option/OptTable.h b/include/llvm/Option/OptTable.h index fdb05d8a15af..5db30436069d 100644 --- a/include/llvm/Option/OptTable.h +++ b/include/llvm/Option/OptTable.h @@ -1,9 +1,8 @@ //===- OptTable.h - Option Table --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Option/Option.h b/include/llvm/Option/Option.h index b09f6043b7a9..33813d28d274 100644 --- a/include/llvm/Option/Option.h +++ b/include/llvm/Option/Option.h @@ -1,9 +1,8 @@ //===- Option.h - Abstract Driver Options -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -207,6 +206,11 @@ public: /// start. Arg *accept(const ArgList &Args, unsigned &Index, unsigned ArgSize) const; +private: + Arg *acceptInternal(const ArgList &Args, unsigned &Index, + unsigned ArgSize) const; + +public: void print(raw_ostream &O) const; void dump() const; }; diff --git a/include/llvm/Pass.h b/include/llvm/Pass.h index 5935a0853d32..329f7eaba73d 100644 --- a/include/llvm/Pass.h +++ b/include/llvm/Pass.h @@ -1,9 +1,8 @@ //===- llvm/Pass.h - Base class for Passes ----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/PassAnalysisSupport.h b/include/llvm/PassAnalysisSupport.h index a075eb557472..1228534deb95 100644 --- a/include/llvm/PassAnalysisSupport.h +++ b/include/llvm/PassAnalysisSupport.h @@ -1,9 +1,8 @@ //===- llvm/PassAnalysisSupport.h - Analysis Pass Support code --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/PassInfo.h b/include/llvm/PassInfo.h index 2f1ab4d43377..686fc044ebcb 100644 --- a/include/llvm/PassInfo.h +++ b/include/llvm/PassInfo.h @@ -1,9 +1,8 @@ //===- llvm/PassInfo.h - Pass Info class ------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/PassRegistry.h b/include/llvm/PassRegistry.h index 57462138c5ae..b9a015430c10 100644 --- a/include/llvm/PassRegistry.h +++ b/include/llvm/PassRegistry.h @@ -1,9 +1,8 @@ //===- llvm/PassRegistry.h - Pass Information Registry ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/PassSupport.h b/include/llvm/PassSupport.h index 1bf23dcba50b..ab90217ce4a8 100644 --- a/include/llvm/PassSupport.h +++ b/include/llvm/PassSupport.h @@ -1,9 +1,8 @@ //===- llvm/PassSupport.h - Pass Support code -------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Passes/PassBuilder.h b/include/llvm/Passes/PassBuilder.h index fa59345a02cf..5e6660599f93 100644 --- a/include/llvm/Passes/PassBuilder.h +++ b/include/llvm/Passes/PassBuilder.h @@ -1,9 +1,8 @@ //===- Parsing, selection, and construction of pass pipelines --*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -32,36 +31,85 @@ class ModuleSummaryIndex; /// A struct capturing PGO tunables. struct PGOOptions { - PGOOptions(std::string ProfileGenFile = "", std::string ProfileUseFile = "", - std::string SampleProfileFile = "", - std::string ProfileRemappingFile = "", - bool RunProfileGen = false, bool SamplePGOSupport = false) - : ProfileGenFile(ProfileGenFile), ProfileUseFile(ProfileUseFile), - SampleProfileFile(SampleProfileFile), - ProfileRemappingFile(ProfileRemappingFile), - RunProfileGen(RunProfileGen), - SamplePGOSupport(SamplePGOSupport || !SampleProfileFile.empty()) { - assert((RunProfileGen || - !SampleProfileFile.empty() || - !ProfileUseFile.empty() || - SamplePGOSupport) && "Illegal PGOOptions."); + enum PGOAction { NoAction, IRInstr, IRUse, SampleUse }; + enum CSPGOAction { NoCSAction, CSIRInstr, CSIRUse }; + PGOOptions(std::string ProfileFile = "", std::string CSProfileGenFile = "", + std::string ProfileRemappingFile = "", PGOAction Action = NoAction, + CSPGOAction CSAction = NoCSAction, bool SamplePGOSupport = false) + : ProfileFile(ProfileFile), CSProfileGenFile(CSProfileGenFile), + ProfileRemappingFile(ProfileRemappingFile), Action(Action), + CSAction(CSAction), + SamplePGOSupport(SamplePGOSupport || Action == SampleUse) { + // Note, we do allow ProfileFile.empty() for Action=IRUse LTO can + // callback with IRUse action without ProfileFile. + + // If there is a CSAction, PGOAction cannot be IRInstr or SampleUse. + assert(this->CSAction == NoCSAction || + (this->Action != IRInstr && this->Action != SampleUse)); + + // For CSIRInstr, CSProfileGenFile also needs to be nonempty. + assert(this->CSAction != CSIRInstr || !this->CSProfileGenFile.empty()); + + // If CSAction is CSIRUse, PGOAction needs to be IRUse as they share + // a profile. + assert(this->CSAction != CSIRUse || this->Action == IRUse); + + // If neither Action nor CSAction, SamplePGOSupport needs to be true. + assert(this->Action != NoAction || this->CSAction != NoCSAction || + this->SamplePGOSupport); } - std::string ProfileGenFile; - std::string ProfileUseFile; - std::string SampleProfileFile; + std::string ProfileFile; + std::string CSProfileGenFile; std::string ProfileRemappingFile; - bool RunProfileGen; + PGOAction Action; + CSPGOAction CSAction; bool SamplePGOSupport; }; +/// Tunable parameters for passes in the default pipelines. +class PipelineTuningOptions { +public: + /// Constructor sets pipeline tuning defaults based on cl::opts. Each option + /// can be set in the PassBuilder when using a LLVM as a library. + PipelineTuningOptions(); + + /// Tuning option to set loop interleaving on/off. Its default value is that + /// of the flag: `-interleave-loops`. + bool LoopInterleaving; + + /// Tuning option to enable/disable loop vectorization. Its default value is + /// that of the flag: `-vectorize-loops`. + bool LoopVectorization; + + /// Tuning option to enable/disable slp loop vectorization. Its default value + /// is that of the flag: `vectorize-slp`. + bool SLPVectorization; + + /// Tuning option to enable/disable loop unrolling. Its default value is true. + bool LoopUnrolling; + + /// Tuning option to forget all SCEV loops in LoopUnroll. Its default value + /// is that of the flag: `-forget-scev-loop-unroll`. + bool ForgetAllSCEVInLoopUnroll; + + /// Tuning option to cap the number of calls to retrive clobbering accesses in + /// MemorySSA, in LICM. + unsigned LicmMssaOptCap; + + /// Tuning option to disable promotion to scalars in LICM with MemorySSA, if + /// the number of access is too large. + unsigned LicmMssaNoAccForPromotionCap; +}; + /// This class provides access to building LLVM's passes. /// -/// It's members provide the baseline state available to passes during their +/// Its members provide the baseline state available to passes during their /// construction. The \c PassRegistry.def file specifies how to construct all /// of the built-in passes, and those may reference these members during /// construction. class PassBuilder { TargetMachine *TM; + PipelineTuningOptions PTO; Optional PGOOpt; PassInstrumentationCallbacks *PIC; @@ -85,9 +133,9 @@ public: enum class ThinLTOPhase { /// No ThinLTO behavior needed. None, - // ThinLTO prelink (summary) phase. + /// ThinLTO prelink (summary) phase. PreLink, - // ThinLTO postlink (backend compile) phase. + /// ThinLTO postlink (backend compile) phase. PostLink }; @@ -178,14 +226,15 @@ public: }; explicit PassBuilder(TargetMachine *TM = nullptr, + PipelineTuningOptions PTO = PipelineTuningOptions(), Optional PGOOpt = None, PassInstrumentationCallbacks *PIC = nullptr) - : TM(TM), PGOOpt(PGOOpt), PIC(PIC) {} + : TM(TM), PTO(PTO), PGOOpt(PGOOpt), PIC(PIC) {} /// Cross register the analysis managers through their proxies. /// /// This is an interface that can be used to cross register each - // AnalysisManager with all the others analysis managers. + /// AnalysisManager with all the others analysis managers. void crossRegisterProxies(LoopAnalysisManager &LAM, FunctionAnalysisManager &FAM, CGSCCAnalysisManager &CGAM, @@ -275,7 +324,8 @@ public: /// require some transformations for semantic reasons, they should explicitly /// build them. ModulePassManager buildModuleOptimizationPipeline(OptimizationLevel Level, - bool DebugLogging = false); + bool DebugLogging = false, + bool LTOPreLink = false); /// Build a per-module default optimization pipeline. /// @@ -289,7 +339,8 @@ public: /// require some transformations for semantic reasons, they should explicitly /// build them. ModulePassManager buildPerModuleDefaultPipeline(OptimizationLevel Level, - bool DebugLogging = false); + bool DebugLogging = false, + bool LTOPreLink = false); /// Build a pre-link, ThinLTO-targeting default optimization pipeline to /// a pass manager. @@ -392,7 +443,7 @@ public: /// {{@ Parse a textual pass pipeline description into a specific PassManager /// /// Automatic deduction of an appropriate pass manager stack is not supported. - /// For example, to insert a loop pass 'lpass' into a FunctinoPassManager, + /// For example, to insert a loop pass 'lpass' into a FunctionPassManager, /// this is the valid pipeline text: /// /// function(lpass) @@ -606,9 +657,8 @@ private: bool VerifyEachPass, bool DebugLogging); void addPGOInstrPasses(ModulePassManager &MPM, bool DebugLogging, - OptimizationLevel Level, bool RunProfileGen, - std::string ProfileGenFile, - std::string ProfileUseFile, + OptimizationLevel Level, bool RunProfileGen, bool IsCS, + std::string ProfileFile, std::string ProfileRemappingFile); void invokePeepholeEPCallbacks(FunctionPassManager &, OptimizationLevel); diff --git a/include/llvm/Passes/PassPlugin.h b/include/llvm/Passes/PassPlugin.h index af8f11a7a352..013b7a827c47 100644 --- a/include/llvm/Passes/PassPlugin.h +++ b/include/llvm/Passes/PassPlugin.h @@ -1,9 +1,8 @@ //===- llvm/Passes/PassPlugin.h - Public Plugin API -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Passes/StandardInstrumentations.h b/include/llvm/Passes/StandardInstrumentations.h index 8c6f5e1e22f7..3d3002eecce9 100644 --- a/include/llvm/Passes/StandardInstrumentations.h +++ b/include/llvm/Passes/StandardInstrumentations.h @@ -1,9 +1,8 @@ //===- StandardInstrumentations.h ------------------------------*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -64,6 +63,8 @@ public: StandardInstrumentations() = default; void registerCallbacks(PassInstrumentationCallbacks &PIC); + + TimePassesHandler &getTimePasses() { return TimePasses; } }; } // namespace llvm diff --git a/include/llvm/ProfileData/Coverage/CoverageMapping.h b/include/llvm/ProfileData/Coverage/CoverageMapping.h index beaa36553287..11758ac4cf2f 100644 --- a/include/llvm/ProfileData/Coverage/CoverageMapping.h +++ b/include/llvm/ProfileData/Coverage/CoverageMapping.h @@ -1,9 +1,8 @@ //===- CoverageMapping.h - Code coverage mapping support --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ProfileData/Coverage/CoverageMappingReader.h b/include/llvm/ProfileData/Coverage/CoverageMappingReader.h index c88c71a6d6f4..57a2aaefd660 100644 --- a/include/llvm/ProfileData/Coverage/CoverageMappingReader.h +++ b/include/llvm/ProfileData/Coverage/CoverageMappingReader.h @@ -1,9 +1,8 @@ //===- CoverageMappingReader.h - Code coverage mapping reader ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -204,9 +203,15 @@ public: BinaryCoverageReader(const BinaryCoverageReader &) = delete; BinaryCoverageReader &operator=(const BinaryCoverageReader &) = delete; + static Expected>> + create(MemoryBufferRef ObjectBuffer, StringRef Arch, + SmallVectorImpl> &ObjectFileBuffers); + static Expected> - create(std::unique_ptr &ObjectBuffer, - StringRef Arch); + createCoverageReaderFromBuffer(StringRef Coverage, + InstrProfSymtab &&ProfileNames, + uint8_t BytesInAddress, + support::endianness Endian); Error readNextRecord(CoverageMappingRecord &Record) override; }; diff --git a/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h b/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h index 86fb1bdf1773..5f88cacdfcbb 100644 --- a/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h +++ b/include/llvm/ProfileData/Coverage/CoverageMappingWriter.h @@ -1,9 +1,8 @@ //===- CoverageMappingWriter.h - Code coverage mapping writer ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ProfileData/GCOV.h b/include/llvm/ProfileData/GCOV.h index a088f63a6915..004ff3f4a2e2 100644 --- a/include/llvm/ProfileData/GCOV.h +++ b/include/llvm/ProfileData/GCOV.h @@ -1,9 +1,8 @@ //===- GCOV.h - LLVM coverage tool ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -45,9 +44,10 @@ enum GCOVVersion { V402, V404, V704 }; /// A struct for passing gcov options between functions. struct Options { - Options(bool A, bool B, bool C, bool F, bool P, bool U, bool L, bool N) + Options(bool A, bool B, bool C, bool F, bool P, bool U, bool L, bool N, bool X) : AllBlocks(A), BranchInfo(B), BranchCount(C), FuncCoverage(F), - PreservePaths(P), UncondBranch(U), LongFileNames(L), NoOutput(N) {} + PreservePaths(P), UncondBranch(U), LongFileNames(L), NoOutput(N), + HashFilenames(X) {} bool AllBlocks; bool BranchInfo; @@ -57,6 +57,7 @@ struct Options { bool UncondBranch; bool LongFileNames; bool NoOutput; + bool HashFilenames; }; } // end namespace GCOV @@ -317,12 +318,6 @@ class GCOVBlock { uint64_t Count = 0; }; - struct SortDstEdgesFunctor { - bool operator()(const GCOVEdge *E1, const GCOVEdge *E2) { - return E1->Dst.Number < E2->Dst.Number; - } - }; - public: using EdgeIterator = SmallVectorImpl::const_iterator; using BlockVector = SmallVector; diff --git a/include/llvm/ProfileData/InstrProf.h b/include/llvm/ProfileData/InstrProf.h index dc45021fc47d..c7d764ade30d 100644 --- a/include/llvm/ProfileData/InstrProf.h +++ b/include/llvm/ProfileData/InstrProf.h @@ -1,9 +1,8 @@ //===- InstrProf.h - Instrumented profiling format support ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -235,7 +234,7 @@ bool isIRPGOFlagSet(const Module *M); bool canRenameComdatFunc(const Function &F, bool CheckAddressTaken = false); enum InstrProfValueKind : uint32_t { -#define VALUE_PROF_KIND(Enumerator, Value) Enumerator = Value, +#define VALUE_PROF_KIND(Enumerator, Value, Descr) Enumerator = Value, #include "llvm/ProfileData/InstrProfData.inc" }; @@ -591,6 +590,70 @@ StringRef InstrProfSymtab::getOrigFuncName(uint64_t FuncMD5Hash) { return PGOName.drop_front(S + 1); } +// To store the sums of profile count values, or the percentage of +// the sums of the total count values. +struct CountSumOrPercent { + uint64_t NumEntries; + double CountSum; + double ValueCounts[IPVK_Last - IPVK_First + 1]; + CountSumOrPercent() : NumEntries(0), CountSum(0.0f), ValueCounts() {} + void reset() { + NumEntries = 0; + CountSum = 0.0f; + for (unsigned I = 0; I < IPVK_Last - IPVK_First + 1; I++) + ValueCounts[I] = 0.0f; + } +}; + +// Function level or program level overlap information. +struct OverlapStats { + enum OverlapStatsLevel { ProgramLevel, FunctionLevel }; + // Sum of the total count values for the base profile. + CountSumOrPercent Base; + // Sum of the total count values for the test profile. + CountSumOrPercent Test; + // Overlap lap score. Should be in range of [0.0f to 1.0f]. + CountSumOrPercent Overlap; + CountSumOrPercent Mismatch; + CountSumOrPercent Unique; + OverlapStatsLevel Level; + const std::string *BaseFilename; + const std::string *TestFilename; + StringRef FuncName; + uint64_t FuncHash; + bool Valid; + + OverlapStats(OverlapStatsLevel L = ProgramLevel) + : Level(L), BaseFilename(nullptr), TestFilename(nullptr), FuncHash(0), + Valid(false) {} + + void dump(raw_fd_ostream &OS) const; + + void setFuncInfo(StringRef Name, uint64_t Hash) { + FuncName = Name; + FuncHash = Hash; + } + + Error accumuateCounts(const std::string &BaseFilename, + const std::string &TestFilename, bool IsCS); + void addOneMismatch(const CountSumOrPercent &MismatchFunc); + void addOneUnique(const CountSumOrPercent &UniqueFunc); + + static inline double score(uint64_t Val1, uint64_t Val2, double Sum1, + double Sum2) { + if (Sum1 < 1.0f || Sum2 < 1.0f) + return 0.0f; + return std::min(Val1 / Sum1, Val2 / Sum2); + } +}; + +// This is used to filter the functions whose overlap information +// to be output. +struct OverlapFuncFilters { + uint64_t ValueCutoff; + const std::string NameFilter; +}; + struct InstrProfValueSiteRecord { /// Value profiling data pairs at a given value site. std::list ValueData; @@ -616,6 +679,10 @@ struct InstrProfValueSiteRecord { function_ref Warn); /// Scale up value profile data counts. void scale(uint64_t Weight, function_ref Warn); + + /// Compute the overlap b/w this record and Input record. + void overlap(InstrProfValueSiteRecord &Input, uint32_t ValueKind, + OverlapStats &Overlap, OverlapStats &FuncLevelOverlap); }; /// Profiling information for a single function. @@ -704,6 +771,18 @@ struct InstrProfRecord { /// Clear value data entries void clearValueData() { ValueData = nullptr; } + /// Compute the sums of all counts and store in Sum. + void accumuateCounts(CountSumOrPercent &Sum) const; + + /// Compute the overlap b/w this IntrprofRecord and Other. + void overlap(InstrProfRecord &Other, OverlapStats &Overlap, + OverlapStats &FuncLevelOverlap, uint64_t ValueCutoff); + + /// Compute the overlap of value profile counts. + void overlapValueProfData(uint32_t ValueKind, InstrProfRecord &Src, + OverlapStats &Overlap, + OverlapStats &FuncLevelOverlap); + private: struct ValueProfData { std::vector IndirectCallSites; @@ -768,10 +847,20 @@ struct NamedInstrProfRecord : InstrProfRecord { StringRef Name; uint64_t Hash; + // We reserve this bit as the flag for context sensitive profile record. + static const int CS_FLAG_IN_FUNC_HASH = 60; + NamedInstrProfRecord() = default; NamedInstrProfRecord(StringRef Name, uint64_t Hash, std::vector Counts) : InstrProfRecord(std::move(Counts)), Name(Name), Hash(Hash) {} + + static bool hasCSFlagInHash(uint64_t FuncHash) { + return ((FuncHash >> CS_FLAG_IN_FUNC_HASH) & 1); + } + static void setCSFlagInHash(uint64_t &FuncHash) { + FuncHash |= ((uint64_t)1 << CS_FLAG_IN_FUNC_HASH); + } }; uint32_t InstrProfRecord::getNumValueKinds() const { @@ -1005,6 +1094,8 @@ namespace RawInstrProf { // from control data struct is changed from raw pointer to Name's MD5 value. // Version 4: ValueDataBegin and ValueDataSizes fields are removed from the // raw header. +// Version 5: Bit 60 of FuncHash is reserved for the flag for the context +// sensitive records. const uint64_t Version = INSTR_PROF_RAW_VERSION; template inline uint64_t getMagic(); @@ -1041,6 +1132,12 @@ struct Header { void getMemOPSizeRangeFromOption(StringRef Str, int64_t &RangeStart, int64_t &RangeLast); -} // end namespace llvm +// Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime +// aware this is an ir_level profile so it can set the version flag. +void createIRLevelProfileFlagVar(Module &M, bool IsCS); +// Create the variable for the profile file name. +void createProfileFileNameVar(Module &M, StringRef InstrProfileOutput); + +} // end namespace llvm #endif // LLVM_PROFILEDATA_INSTRPROF_H diff --git a/include/llvm/ProfileData/InstrProfData.inc b/include/llvm/ProfileData/InstrProfData.inc index 454620ed997a..749781b9ac2d 100644 --- a/include/llvm/ProfileData/InstrProfData.inc +++ b/include/llvm/ProfileData/InstrProfData.inc @@ -1,9 +1,8 @@ /*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ /* @@ -170,7 +169,7 @@ VALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx)) /* VALUE_PROF_KIND start */ #ifndef VALUE_PROF_KIND -#define VALUE_PROF_KIND(Enumerator, Value) +#define VALUE_PROF_KIND(Enumerator, Value, Descr) #else #define INSTR_PROF_DATA_DEFINED #endif @@ -183,16 +182,16 @@ VALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx)) * For this remapping the ProfData is used. ProfData contains both the function * name hash and the function address. */ -VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0) +VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") /* For memory intrinsic functions size profiling. */ -VALUE_PROF_KIND(IPVK_MemOPSize, 1) +VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") /* These two kinds must be the last to be * declared. This is to make sure the string * array created with the template can be * indexed with the kind value. */ -VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget) -VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize) +VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first") +VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last") #undef VALUE_PROF_KIND /* VALUE_PROF_KIND end */ @@ -250,22 +249,25 @@ COVMAP_HEADER(uint32_t, Int32Ty, Version, \ #define INSTR_PROF_DATA_DEFINED INSTR_PROF_SECT_ENTRY(IPSK_data, \ INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_DATA_COFF), "__DATA,") + INSTR_PROF_DATA_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COFF), "__DATA,") + INSTR_PROF_CNTS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_name, \ INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_NAME_COFF), "__DATA,") + INSTR_PROF_NAME_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_vals, \ INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_VALS_COFF), "__DATA,") + INSTR_PROF_VALS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COFF), "__DATA,") + INSTR_PROF_VNODES_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_covmap, \ INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COFF), "__LLVM_COV,") + INSTR_PROF_COVMAP_COFF, "__LLVM_COV,") +INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") #undef INSTR_PROF_SECT_ENTRY #endif @@ -636,10 +638,12 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, * version for other variants of profile. We set the lowest bit of the upper 8 * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton * generated profile, and 0 if this is a Clang FE generated profile. + * 1 in bit 57 indicates there are context-sensitive records in the profile. */ #define VARIANT_MASKS_ALL 0xff00000000000000ULL #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) #define VARIANT_MASK_IR_PROF (0x1ULL << 56) +#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime @@ -655,13 +659,17 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VALS_COMMON __llvm_prf_vals #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds #define INSTR_PROF_COVMAP_COMMON __llvm_covmap -/* Win32 */ -#define INSTR_PROF_DATA_COFF .lprfd -#define INSTR_PROF_NAME_COFF .lprfn -#define INSTR_PROF_CNTS_COFF .lprfc -#define INSTR_PROF_VALS_COFF .lprfv -#define INSTR_PROF_VNODES_COFF .lprfnd -#define INSTR_PROF_COVMAP_COFF .lcovmap +#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile +/* Windows section names. Because these section names contain dollar characters, + * they must be quoted. + */ +#define INSTR_PROF_DATA_COFF ".lprfd$M" +#define INSTR_PROF_NAME_COFF ".lprfn$M" +#define INSTR_PROF_CNTS_COFF ".lprfc$M" +#define INSTR_PROF_VALS_COFF ".lprfv$M" +#define INSTR_PROF_VNODES_COFF ".lprfnd$M" +#define INSTR_PROF_COVMAP_COFF ".lcovmap$M" +#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" #ifdef _WIN32 /* Runtime section names and name strings. */ @@ -675,32 +683,30 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, /* Value profile nodes section. */ #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF +#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF #else /* Runtime section names and name strings. */ -#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COMMON -#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COMMON -#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COMMON +#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) +#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) +#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ -#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COMMON +#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON) /* Value profile nodes section. */ -#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COMMON -#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COMMON +#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) +#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) +/* Order file instrumentation. */ +#define INSTR_PROF_ORDERFILE_SECT_NAME \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) #endif -#define INSTR_PROF_DATA_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_DATA_SECT_NAME) -#define INSTR_PROF_NAME_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_NAME_SECT_NAME) -#define INSTR_PROF_CNTS_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_CNTS_SECT_NAME) -#define INSTR_PROF_COVMAP_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_SECT_NAME) -#define INSTR_PROF_VALS_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_VALS_SECT_NAME) -#define INSTR_PROF_VNODES_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_VNODES_SECT_NAME) +#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer +#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME) +#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx +#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME) /* Macros to define start/stop section symbol for a given * section on Linux. For instance @@ -735,6 +741,12 @@ typedef struct InstrProfValueData { #endif /* INSTR_PROF_DATA_INC */ +#ifndef INSTR_ORDER_FILE_INC +// The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). +#define INSTR_ORDER_FILE_BUFFER_SIZE 131072 +#define INSTR_ORDER_FILE_BUFFER_BITS 17 +#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff +#endif /* INSTR_ORDER_FILE_INC */ #else #undef INSTR_PROF_DATA_DEFINED #endif diff --git a/include/llvm/ProfileData/InstrProfReader.h b/include/llvm/ProfileData/InstrProfReader.h index 08d782276117..73751faab88e 100644 --- a/include/llvm/ProfileData/InstrProfReader.h +++ b/include/llvm/ProfileData/InstrProfReader.h @@ -1,9 +1,8 @@ //===- InstrProfReader.h - Instrumented profiling readers -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -78,6 +77,8 @@ public: virtual bool isIRLevelProfile() const = 0; + virtual bool hasCSIRLevelProfile() const = 0; + /// Return the PGO symtab. There are three different readers: /// Raw, Text, and Indexed profile readers. The first two types /// of readers are used only by llvm-profdata tool, while the indexed @@ -90,6 +91,9 @@ public: /// compiler. virtual InstrProfSymtab &getSymtab() = 0; + /// Compute the sum of counts and return in Sum. + void accumuateCounts(CountSumOrPercent &Sum, bool IsCS); + protected: std::unique_ptr Symtab; @@ -143,6 +147,7 @@ private: /// Iterator over the profile data. line_iterator Line; bool IsIRLevelProfile = false; + bool HasCSIRLevelProfile = false; Error readValueProfileData(InstrProfRecord &Record); @@ -157,6 +162,8 @@ public: bool isIRLevelProfile() const override { return IsIRLevelProfile; } + bool hasCSIRLevelProfile() const override { return HasCSIRLevelProfile; } + /// Read the header. Error readHeader() override; @@ -213,6 +220,10 @@ public: return (Version & VARIANT_MASK_IR_PROF) != 0; } + bool hasCSIRLevelProfile() const override { + return (Version & VARIANT_MASK_CSIR_PROF) != 0; + } + InstrProfSymtab &getSymtab() override { assert(Symtab.get()); return *Symtab.get(); @@ -342,6 +353,7 @@ struct InstrProfReaderIndexBase { virtual void setValueProfDataEndianness(support::endianness Endianness) = 0; virtual uint64_t getVersion() const = 0; virtual bool isIRLevelProfile() const = 0; + virtual bool hasCSIRLevelProfile() const = 0; virtual Error populateSymtab(InstrProfSymtab &) = 0; }; @@ -386,6 +398,10 @@ public: return (FormatVersion & VARIANT_MASK_IR_PROF) != 0; } + bool hasCSIRLevelProfile() const override { + return (FormatVersion & VARIANT_MASK_CSIR_PROF) != 0; + } + Error populateSymtab(InstrProfSymtab &Symtab) override { return Symtab.create(HashTable->keys()); } @@ -413,13 +429,16 @@ private: std::unique_ptr Remapper; /// Profile summary data. std::unique_ptr Summary; + /// Context sensitive profile summary data. + std::unique_ptr CS_Summary; // Index to the current record in the record array. unsigned RecordIndex; // Read the profile summary. Return a pointer pointing to one byte past the // end of the summary data if it exists or the input \c Cur. + // \c UseCS indicates whether to use the context-sensitive profile summary. const unsigned char *readSummary(IndexedInstrProf::ProfVersion Version, - const unsigned char *Cur); + const unsigned char *Cur, bool UseCS); public: IndexedInstrProfReader( @@ -433,6 +452,9 @@ public: /// Return the profile version. uint64_t getVersion() const { return Index->getVersion(); } bool isIRLevelProfile() const override { return Index->isIRLevelProfile(); } + bool hasCSIRLevelProfile() const override { + return Index->hasCSIRLevelProfile(); + } /// Return true if the given buffer is in an indexed instrprof format. static bool hasFormat(const MemoryBuffer &DataBuffer); @@ -451,7 +473,16 @@ public: std::vector &Counts); /// Return the maximum of all known function counts. - uint64_t getMaximumFunctionCount() { return Summary->getMaxFunctionCount(); } + /// \c UseCS indicates whether to use the context-sensitive count. + uint64_t getMaximumFunctionCount(bool UseCS) { + if (UseCS) { + assert(CS_Summary && "No context sensitive profile summary"); + return CS_Summary->getMaxFunctionCount(); + } else { + assert(Summary && "No profile summary"); + return Summary->getMaxFunctionCount(); + } + } /// Factory method to create an indexed reader. static Expected> @@ -470,7 +501,18 @@ public: // to be used by llvm-profdata (for dumping). Avoid using this when // the client is the compiler. InstrProfSymtab &getSymtab() override; - ProfileSummary &getSummary() { return *(Summary.get()); } + + /// Return the profile summary. + /// \c UseCS indicates whether to use the context-sensitive summary. + ProfileSummary &getSummary(bool UseCS) { + if (UseCS) { + assert(CS_Summary && "No context sensitive summary"); + return *(CS_Summary.get()); + } else { + assert(Summary && "No profile summary"); + return *(Summary.get()); + } + } }; } // end namespace llvm diff --git a/include/llvm/ProfileData/InstrProfWriter.h b/include/llvm/ProfileData/InstrProfWriter.h index 8107ab386fe2..5882fa2781e2 100644 --- a/include/llvm/ProfileData/InstrProfWriter.h +++ b/include/llvm/ProfileData/InstrProfWriter.h @@ -1,9 +1,8 @@ //===- InstrProfWriter.h - Instrumented profiling writer --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -34,7 +33,8 @@ class raw_fd_ostream; class InstrProfWriter { public: using ProfilingData = SmallDenseMap; - enum ProfKind { PF_Unknown = 0, PF_FE, PF_IRLevel }; + // PF_IRLevelWithCS is the profile from context sensitive IR instrumentation. + enum ProfKind { PF_Unknown = 0, PF_FE, PF_IRLevel, PF_IRLevelWithCS }; private: bool Sparse; @@ -75,20 +75,36 @@ public: std::unique_ptr writeBuffer(); /// Set the ProfileKind. Report error if mixing FE and IR level profiles. - Error setIsIRLevelProfile(bool IsIRLevel) { + /// \c WithCS indicates if this is for contenxt sensitive instrumentation. + Error setIsIRLevelProfile(bool IsIRLevel, bool WithCS) { if (ProfileKind == PF_Unknown) { - ProfileKind = IsIRLevel ? PF_IRLevel: PF_FE; + if (IsIRLevel) + ProfileKind = WithCS ? PF_IRLevelWithCS : PF_IRLevel; + else + ProfileKind = PF_FE; return Error::success(); } - return (IsIRLevel == (ProfileKind == PF_IRLevel)) - ? Error::success() - : make_error( - instrprof_error::unsupported_version); + + if (((ProfileKind != PF_FE) && !IsIRLevel) || + ((ProfileKind == PF_FE) && IsIRLevel)) + return make_error(instrprof_error::unsupported_version); + + // When merging a context-sensitive profile (WithCS == true) with an IRLevel + // profile, set the kind to PF_IRLevelWithCS. + if (ProfileKind == PF_IRLevel && WithCS) + ProfileKind = PF_IRLevelWithCS; + + return Error::success(); } // Internal interface for testing purpose only. void setValueProfDataEndianness(support::endianness Endianness); void setOutputSparse(bool Sparse); + // Compute the overlap b/w this object and Other. Program level result is + // stored in Overlap and function level result is stored in FuncLevelOverlap. + void overlapRecord(NamedInstrProfRecord &&Other, OverlapStats &Overlap, + OverlapStats &FuncLevelOverlap, + const OverlapFuncFilters &FuncFilter); private: void addRecord(StringRef Name, uint64_t Hash, InstrProfRecord &&I, diff --git a/include/llvm/ProfileData/ProfileCommon.h b/include/llvm/ProfileData/ProfileCommon.h index 087588f06340..f98a34387fdf 100644 --- a/include/llvm/ProfileData/ProfileCommon.h +++ b/include/llvm/ProfileData/ProfileCommon.h @@ -1,9 +1,8 @@ //===- ProfileCommon.h - Common profiling APIs. -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -84,7 +83,8 @@ public: SampleProfileSummaryBuilder(std::vector Cutoffs) : ProfileSummaryBuilder(std::move(Cutoffs)) {} - void addRecord(const sampleprof::FunctionSamples &FS); + void addRecord(const sampleprof::FunctionSamples &FS, + bool isCallsiteSample = false); std::unique_ptr getSummary(); }; diff --git a/include/llvm/ProfileData/SampleProf.h b/include/llvm/ProfileData/SampleProf.h index 927dfd246878..7fbc857b7230 100644 --- a/include/llvm/ProfileData/SampleProf.h +++ b/include/llvm/ProfileData/SampleProf.h @@ -1,9 +1,8 @@ //===- SampleProf.h - Sampling profiling format support ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -411,6 +410,34 @@ public: return getNameInModule(Name, M); } + /// Return the canonical name for a function, taking into account + /// suffix elision policy attributes. + static StringRef getCanonicalFnName(const Function &F) { + static const char *knownSuffixes[] = { ".llvm.", ".part." }; + auto AttrName = "sample-profile-suffix-elision-policy"; + auto Attr = F.getFnAttribute(AttrName).getValueAsString(); + if (Attr == "" || Attr == "all") { + return F.getName().split('.').first; + } else if (Attr == "selected") { + StringRef Cand(F.getName()); + for (const auto &Suf : knownSuffixes) { + StringRef Suffix(Suf); + auto It = Cand.rfind(Suffix); + if (It == StringRef::npos) + return Cand; + auto Dit = Cand.rfind('.'); + if (Dit == It + Suffix.size() - 1) + Cand = Cand.substr(0, It); + } + return Cand; + } else if (Attr == "none") { + return F.getName(); + } else { + assert(false && "internal error: unknown suffix elision policy"); + } + return F.getName(); + } + /// Translate \p Name into its original name in Module. /// When the Format is not SPF_Compact_Binary, \p Name needs no translation. /// When the Format is SPF_Compact_Binary, \p Name in current FunctionSamples @@ -466,11 +493,9 @@ public: /// built in post-thin-link phase and var promotion has been done, /// we need to add the substring of function name without the suffix /// into the GUIDToFuncNameMap. - auto pos = OrigName.find('.'); - if (pos != StringRef::npos) { - StringRef NewName = OrigName.substr(0, pos); - GUIDToFuncNameMap.insert({Function::getGUID(NewName), NewName}); - } + StringRef CanonName = getCanonicalFnName(F); + if (CanonName != OrigName) + GUIDToFuncNameMap.insert({Function::getGUID(CanonName), CanonName}); } CurrentModule = &M; } @@ -547,10 +572,9 @@ public: SampleSorter(const std::map &Samples) { for (const auto &I : Samples) V.push_back(&I); - std::stable_sort(V.begin(), V.end(), - [](const SamplesWithLoc *A, const SamplesWithLoc *B) { - return A->first < B->first; - }); + llvm::stable_sort(V, [](const SamplesWithLoc *A, const SamplesWithLoc *B) { + return A->first < B->first; + }); } const SamplesWithLocList &get() const { return V; } diff --git a/include/llvm/ProfileData/SampleProfReader.h b/include/llvm/ProfileData/SampleProfReader.h index 5cc729e42cc8..969cdea859c9 100644 --- a/include/llvm/ProfileData/SampleProfReader.h +++ b/include/llvm/ProfileData/SampleProfReader.h @@ -1,9 +1,8 @@ //===- SampleProfReader.h - Read LLVM sample profile data -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -287,10 +286,11 @@ public: /// Return the samples collected for function \p F. FunctionSamples *getSamplesFor(const Function &F) { - // The function name may have been updated by adding suffix. In sample - // profile, the function names are all stripped, so we need to strip - // the function name suffix before matching with profile. - return getSamplesFor(F.getName().split('.').first); + // The function name may have been updated by adding suffix. Call + // a helper to (optionally) strip off suffixes so that we can + // match against the original function name in the profile. + StringRef CanonName = FunctionSamples::getCanonicalFnName(F); + return getSamplesFor(CanonName); } /// Return the samples collected for function \p F. diff --git a/include/llvm/ProfileData/SampleProfWriter.h b/include/llvm/ProfileData/SampleProfWriter.h index d5ac6e53e4f7..81e6e3ab0b4a 100644 --- a/include/llvm/ProfileData/SampleProfWriter.h +++ b/include/llvm/ProfileData/SampleProfWriter.h @@ -1,9 +1,8 @@ //===- SampleProfWriter.h - Write LLVM sample profile data ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Remarks/Remark.h b/include/llvm/Remarks/Remark.h new file mode 100644 index 000000000000..05d0ea60accd --- /dev/null +++ b/include/llvm/Remarks/Remark.h @@ -0,0 +1,113 @@ +//===-- llvm/Remarks/Remark.h - The remark type -----------------*- C++/-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines an abstraction for handling remarks. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_REMARKS_REMARK_H +#define LLVM_REMARKS_REMARK_H + +#include "llvm-c/Remarks.h" +#include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/Optional.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/CBindingWrapping.h" +#include + +namespace llvm { +namespace remarks { + +constexpr uint64_t Version = 0; + +/// The debug location used to track a remark back to the source file. +struct RemarkLocation { + /// Absolute path of the source file corresponding to this remark. + StringRef SourceFilePath; + unsigned SourceLine; + unsigned SourceColumn; +}; + +// Create wrappers for C Binding types (see CBindingWrapping.h). +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(RemarkLocation, LLVMRemarkDebugLocRef) + +/// A key-value pair with a debug location that is used to display the remarks +/// at the right place in the source. +struct Argument { + StringRef Key; + // FIXME: We might want to be able to store other types than strings here. + StringRef Val; + // If set, the debug location corresponding to the value. + Optional Loc; +}; + +// Create wrappers for C Binding types (see CBindingWrapping.h). +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(Argument, LLVMRemarkArgRef) + +/// The type of the remark. +enum class Type { + Unknown, + Passed, + Missed, + Analysis, + AnalysisFPCommute, + AnalysisAliasing, + Failure, + LastTypeValue = Failure +}; + +/// A remark type used for both emission and parsing. +struct Remark { + /// The type of the remark. + Type RemarkType = Type::Unknown; + + /// Name of the pass that triggers the emission of this remark. + StringRef PassName; + + /// Textual identifier for the remark (single-word, camel-case). Can be used + /// by external tools reading the output file for remarks to identify the + /// remark. + StringRef RemarkName; + + /// Mangled name of the function that triggers the emssion of this remark. + StringRef FunctionName; + + /// The location in the source file of the remark. + Optional Loc; + + /// If profile information is available, this is the number of times the + /// corresponding code was executed in a profile instrumentation run. + Optional Hotness; + + /// Arguments collected via the streaming interface. + SmallVector Args; + + Remark() = default; + Remark(Remark &&) = default; + Remark &operator=(Remark &&) = default; + + /// Return a message composed from the arguments as a string. + std::string getArgsAsMsg() const; + + /// Clone this remark to explicitly ask for a copy. + Remark clone() const { return *this; } + +private: + /// In order to avoid unwanted copies, "delete" the copy constructor. + /// If a copy is needed, it should be done through `Remark::clone()`. + Remark(const Remark &) = default; + Remark& operator=(const Remark &) = default; +}; + +// Create wrappers for C Binding types (see CBindingWrapping.h). +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(Remark, LLVMRemarkEntryRef) + +} // end namespace remarks +} // end namespace llvm + +#endif /* LLVM_REMARKS_REMARK_H */ diff --git a/include/llvm/Remarks/RemarkFormat.h b/include/llvm/Remarks/RemarkFormat.h new file mode 100644 index 000000000000..e167d99d2517 --- /dev/null +++ b/include/llvm/Remarks/RemarkFormat.h @@ -0,0 +1,33 @@ +//===-- llvm/Remarks/RemarkFormat.h - The format of remarks -----*- C++/-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines utilities to deal with the format of remarks. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_REMARKS_REMARK_FORMAT_H +#define LLVM_REMARKS_REMARK_FORMAT_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Error.h" + +namespace llvm { +namespace remarks { + +constexpr StringRef Magic("REMARKS", 7); + +/// The format used for serializing/deserializing remarks. +enum class Format { Unknown, YAML }; + +/// Parse and validate a string for the remark format. +Expected parseFormat(StringRef FormatStr); + +} // end namespace remarks +} // end namespace llvm + +#endif /* LLVM_REMARKS_REMARK_FORMAT_H */ diff --git a/include/llvm/Remarks/RemarkParser.h b/include/llvm/Remarks/RemarkParser.h new file mode 100644 index 000000000000..671e1abe5ec7 --- /dev/null +++ b/include/llvm/Remarks/RemarkParser.h @@ -0,0 +1,77 @@ +//===-- llvm/Remarks/Remark.h - The remark type -----------------*- C++/-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides an interface for parsing remarks in LLVM. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_REMARKS_REMARK_PARSER_H +#define LLVM_REMARKS_REMARK_PARSER_H + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Remarks/Remark.h" +#include "llvm/Remarks/RemarkFormat.h" +#include "llvm/Support/Error.h" +#include + +namespace llvm { +namespace remarks { + +struct ParserImpl; +struct ParsedStringTable; + +class EndOfFileError : public ErrorInfo { +public: + static char ID; + + EndOfFileError() {} + + void log(raw_ostream &OS) const override { OS << "End of file reached."; } + std::error_code convertToErrorCode() const override { + return inconvertibleErrorCode(); + } +}; + +/// Parser used to parse a raw buffer to remarks::Remark objects. +struct Parser { + /// The format of the parser. + Format ParserFormat; + + Parser(Format ParserFormat) : ParserFormat(ParserFormat) {} + + /// If no error occurs, this returns a valid Remark object. + /// If an error of type EndOfFileError occurs, it is safe to recover from it + /// by stopping the parsing. + /// If any other error occurs, it should be propagated to the user. + /// The pointer should never be null. + virtual Expected> next() = 0; + + virtual ~Parser() = default; +}; + +/// In-memory representation of the string table parsed from a buffer (e.g. the +/// remarks section). +struct ParsedStringTable { + /// The buffer mapped from the section contents. + StringRef Buffer; + /// Collection of offsets in the buffer for each string entry. + SmallVector Offsets; + + Expected operator[](size_t Index) const; + ParsedStringTable(StringRef Buffer); +}; + +Expected> +createRemarkParser(Format ParserFormat, StringRef Buf, + Optional StrTab = None); + +} // end namespace remarks +} // end namespace llvm + +#endif /* LLVM_REMARKS_REMARK_PARSER_H */ diff --git a/include/llvm/Remarks/RemarkSerializer.h b/include/llvm/Remarks/RemarkSerializer.h new file mode 100644 index 000000000000..def5c2e16620 --- /dev/null +++ b/include/llvm/Remarks/RemarkSerializer.h @@ -0,0 +1,68 @@ +//===-- RemarkSerializer.h - Remark serialization interface -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides an interface for serializing remarks to different formats. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_REMARKS_REMARK_SERIALIZER_H +#define LLVM_REMARKS_REMARK_SERIALIZER_H + +#include "llvm/Remarks/Remark.h" +#include "llvm/Remarks/RemarkStringTable.h" +#include "llvm/Support/YAMLTraits.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { +namespace remarks { + +/// This is the base class for a remark serializer. +/// It includes support for using a string table while emitting. +struct Serializer { + /// The open raw_ostream that the remark diagnostics are emitted to. + raw_ostream &OS; + /// The string table containing all the unique strings used in the output. + /// The table can be serialized to be consumed after the compilation. + Optional StrTab; + + Serializer(raw_ostream &OS) : OS(OS), StrTab() {} + + /// This is just an interface. + virtual ~Serializer() = default; + virtual void emit(const Remark &Remark) = 0; +}; + +/// Wether the serializer should use a string table while emitting. +enum class UseStringTable { No, Yes }; + +/// Serialize the remarks to YAML. One remark entry looks like this: +/// --- ! +/// Pass: +/// Name: +/// DebugLoc: { File: , Line: , +/// Column: } +/// Function: +/// Args: +/// - : +/// DebugLoc: { File: , Line: , Column: } +/// ... +struct YAMLSerializer : public Serializer { + /// The YAML streamer. + yaml::Output YAMLOutput; + + YAMLSerializer(raw_ostream &OS, + UseStringTable UseStringTable = remarks::UseStringTable::No); + + /// Emit a remark to the stream. + void emit(const Remark &Remark) override; +}; + +} // end namespace remarks +} // end namespace llvm + +#endif /* LLVM_REMARKS_REMARK_SERIALIZER_H */ diff --git a/include/llvm/Remarks/RemarkStringTable.h b/include/llvm/Remarks/RemarkStringTable.h new file mode 100644 index 000000000000..f9b4fdbbfb8d --- /dev/null +++ b/include/llvm/Remarks/RemarkStringTable.h @@ -0,0 +1,59 @@ +//===-- RemarkStringTable.h - Serializing string table ----------*- C++/-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This class is used to deduplicate and serialize a string table used for +// generating remarks. +// +// For parsing a string table, use ParsedStringTable in RemarkParser.h +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_REMARKS_REMARK_STRING_TABLE_H +#define LLVM_REMARKS_REMARK_STRING_TABLE_H + +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Allocator.h" +#include + +namespace llvm { + +class raw_ostream; + +namespace remarks { + +/// The string table used for serializing remarks. +/// This table can be for example serialized in a section to be consumed after +/// the compilation. +struct StringTable { + /// Allocator holding all the memory used by the map. + BumpPtrAllocator Allocator; + /// The string table containing all the unique strings used in the output. + /// It maps a string to an unique ID. + StringMap StrTab; + /// Total size of the string table when serialized. + size_t SerializedSize = 0; + + StringTable() : Allocator(), StrTab(Allocator) {} + /// Add a string to the table. It returns an unique ID of the string. + std::pair add(StringRef Str); + /// Serialize the string table to a stream. It is serialized as a little + /// endian uint64 (the size of the table in bytes) followed by a sequence of + /// NULL-terminated strings, where the N-th string is the string with the ID N + /// in the StrTab map. + void serialize(raw_ostream &OS) const; + /// Serialize the string table to a vector. This allows users to do the actual + /// writing to file/memory/other. + /// The string with the ID == N should be the N-th element in the vector. + std::vector serialize() const; +}; + +} // end namespace remarks +} // end namespace llvm + +#endif /* LLVM_REMARKS_REMARK_STRING_TABLE_H */ diff --git a/include/llvm/Support/AArch64TargetParser.def b/include/llvm/Support/AArch64TargetParser.def index e03297b7c3c3..e152f383b3ec 100644 --- a/include/llvm/Support/AArch64TargetParser.def +++ b/include/llvm/Support/AArch64TargetParser.def @@ -1,9 +1,8 @@ //===- AARCH64TargetParser.def - AARCH64 target parsing defines ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -51,78 +50,92 @@ AARCH64_ARCH("armv8.5-a", ARMV8_5A, "8.5-A", "v8.5a", #define AARCH64_ARCH_EXT_NAME(NAME, ID, FEATURE, NEGFEATURE) #endif // FIXME: This would be nicer were it tablegen -AARCH64_ARCH_EXT_NAME("invalid", AArch64::AEK_INVALID, nullptr, nullptr) -AARCH64_ARCH_EXT_NAME("none", AArch64::AEK_NONE, nullptr, nullptr) -AARCH64_ARCH_EXT_NAME("crc", AArch64::AEK_CRC, "+crc", "-crc") -AARCH64_ARCH_EXT_NAME("lse", AArch64::AEK_LSE, "+lse", "-lse") -AARCH64_ARCH_EXT_NAME("rdm", AArch64::AEK_RDM, "+rdm", "-rdm") -AARCH64_ARCH_EXT_NAME("crypto", AArch64::AEK_CRYPTO, "+crypto","-crypto") -AARCH64_ARCH_EXT_NAME("sm4", AArch64::AEK_SM4, "+sm4", "-sm4") -AARCH64_ARCH_EXT_NAME("sha3", AArch64::AEK_SHA3, "+sha3", "-sha3") -AARCH64_ARCH_EXT_NAME("sha2", AArch64::AEK_SHA2, "+sha2", "-sha2") -AARCH64_ARCH_EXT_NAME("aes", AArch64::AEK_AES, "+aes", "-aes") -AARCH64_ARCH_EXT_NAME("dotprod", AArch64::AEK_DOTPROD, "+dotprod","-dotprod") -AARCH64_ARCH_EXT_NAME("fp", AArch64::AEK_FP, "+fp-armv8", "-fp-armv8") -AARCH64_ARCH_EXT_NAME("simd", AArch64::AEK_SIMD, "+neon", "-neon") -AARCH64_ARCH_EXT_NAME("fp16", AArch64::AEK_FP16, "+fullfp16", "-fullfp16") -AARCH64_ARCH_EXT_NAME("fp16fml", AArch64::AEK_FP16FML, "+fp16fml", "-fp16fml") -AARCH64_ARCH_EXT_NAME("profile", AArch64::AEK_PROFILE, "+spe", "-spe") -AARCH64_ARCH_EXT_NAME("ras", AArch64::AEK_RAS, "+ras", "-ras") -AARCH64_ARCH_EXT_NAME("sve", AArch64::AEK_SVE, "+sve", "-sve") -AARCH64_ARCH_EXT_NAME("rcpc", AArch64::AEK_RCPC, "+rcpc", "-rcpc") -AARCH64_ARCH_EXT_NAME("rng", AArch64::AEK_RAND, "+rand", "-rand") -AARCH64_ARCH_EXT_NAME("memtag", AArch64::AEK_MTE, "+mte", "-mte") -AARCH64_ARCH_EXT_NAME("ssbs", AArch64::AEK_SSBS, "+ssbs", "-ssbs") -AARCH64_ARCH_EXT_NAME("sb", AArch64::AEK_SB, "+sb", "-sb") -AARCH64_ARCH_EXT_NAME("predres", AArch64::AEK_PREDRES, "+predres", "-predres") +AARCH64_ARCH_EXT_NAME("invalid", AArch64::AEK_INVALID, nullptr, nullptr) +AARCH64_ARCH_EXT_NAME("none", AArch64::AEK_NONE, nullptr, nullptr) +AARCH64_ARCH_EXT_NAME("crc", AArch64::AEK_CRC, "+crc", "-crc") +AARCH64_ARCH_EXT_NAME("lse", AArch64::AEK_LSE, "+lse", "-lse") +AARCH64_ARCH_EXT_NAME("rdm", AArch64::AEK_RDM, "+rdm", "-rdm") +AARCH64_ARCH_EXT_NAME("crypto", AArch64::AEK_CRYPTO, "+crypto","-crypto") +AARCH64_ARCH_EXT_NAME("sm4", AArch64::AEK_SM4, "+sm4", "-sm4") +AARCH64_ARCH_EXT_NAME("sha3", AArch64::AEK_SHA3, "+sha3", "-sha3") +AARCH64_ARCH_EXT_NAME("sha2", AArch64::AEK_SHA2, "+sha2", "-sha2") +AARCH64_ARCH_EXT_NAME("aes", AArch64::AEK_AES, "+aes", "-aes") +AARCH64_ARCH_EXT_NAME("dotprod", AArch64::AEK_DOTPROD, "+dotprod","-dotprod") +AARCH64_ARCH_EXT_NAME("fp", AArch64::AEK_FP, "+fp-armv8", "-fp-armv8") +AARCH64_ARCH_EXT_NAME("simd", AArch64::AEK_SIMD, "+neon", "-neon") +AARCH64_ARCH_EXT_NAME("fp16", AArch64::AEK_FP16, "+fullfp16", "-fullfp16") +AARCH64_ARCH_EXT_NAME("fp16fml", AArch64::AEK_FP16FML, "+fp16fml", "-fp16fml") +AARCH64_ARCH_EXT_NAME("profile", AArch64::AEK_PROFILE, "+spe", "-spe") +AARCH64_ARCH_EXT_NAME("ras", AArch64::AEK_RAS, "+ras", "-ras") +AARCH64_ARCH_EXT_NAME("sve", AArch64::AEK_SVE, "+sve", "-sve") +AARCH64_ARCH_EXT_NAME("sve2", AArch64::AEK_SVE2, "+sve2", "-sve2") +AARCH64_ARCH_EXT_NAME("sve2-aes", AArch64::AEK_SVE2AES, "+sve2-aes", "-sve2-aes") +AARCH64_ARCH_EXT_NAME("sve2-sm4", AArch64::AEK_SVE2SM4, "+sve2-sm4", "-sve2-sm4") +AARCH64_ARCH_EXT_NAME("sve2-sha3", AArch64::AEK_SVE2SHA3, "+sve2-sha3", "-sve2-sha3") +AARCH64_ARCH_EXT_NAME("bitperm", AArch64::AEK_BITPERM, "+bitperm", "-bitperm") +AARCH64_ARCH_EXT_NAME("rcpc", AArch64::AEK_RCPC, "+rcpc", "-rcpc") +AARCH64_ARCH_EXT_NAME("rng", AArch64::AEK_RAND, "+rand", "-rand") +AARCH64_ARCH_EXT_NAME("memtag", AArch64::AEK_MTE, "+mte", "-mte") +AARCH64_ARCH_EXT_NAME("ssbs", AArch64::AEK_SSBS, "+ssbs", "-ssbs") +AARCH64_ARCH_EXT_NAME("sb", AArch64::AEK_SB, "+sb", "-sb") +AARCH64_ARCH_EXT_NAME("predres", AArch64::AEK_PREDRES, "+predres", "-predres") #undef AARCH64_ARCH_EXT_NAME #ifndef AARCH64_CPU_NAME #define AARCH64_CPU_NAME(NAME, ID, DEFAULT_FPU, IS_DEFAULT, DEFAULT_EXT) #endif AARCH64_CPU_NAME("cortex-a35", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_CRC)) + (AArch64::AEK_CRC)) AARCH64_CPU_NAME("cortex-a53", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, true, - (AArch64::AEK_CRC)) + (AArch64::AEK_CRC)) AARCH64_CPU_NAME("cortex-a55", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC)) AARCH64_CPU_NAME("cortex-a57", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_CRC)) + (AArch64::AEK_CRC)) AARCH64_CPU_NAME("cortex-a72", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_CRC)) + (AArch64::AEK_CRC)) AARCH64_CPU_NAME("cortex-a73", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_CRC)) + (AArch64::AEK_CRC)) AARCH64_CPU_NAME("cortex-a75", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC)) +AARCH64_CPU_NAME("cortex-a76", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, + (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC | + AArch64::AEK_SSBS)) +AARCH64_CPU_NAME("cortex-a76ae", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, + (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD | AArch64::AEK_RCPC | + AArch64::AEK_SSBS)) AARCH64_CPU_NAME("cyclone", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_NONE)) + (AArch64::AEK_NONE)) AARCH64_CPU_NAME("exynos-m1", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_CRC)) + (AArch64::AEK_CRC)) AARCH64_CPU_NAME("exynos-m2", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_CRC)) + (AArch64::AEK_CRC)) AARCH64_CPU_NAME("exynos-m3", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_CRC)) + (AArch64::AEK_CRC)) AARCH64_CPU_NAME("exynos-m4", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_FP16 | AArch64::AEK_DOTPROD)) + (AArch64::AEK_DOTPROD | AArch64::AEK_FP16)) +AARCH64_CPU_NAME("exynos-m5", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, + (AArch64::AEK_DOTPROD | AArch64::AEK_FP16)) AARCH64_CPU_NAME("falkor", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_CRC | AArch64::AEK_RDM)) + (AArch64::AEK_CRC | AArch64::AEK_RDM)) AARCH64_CPU_NAME("saphira", ARMV8_3A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_PROFILE)) + (AArch64::AEK_PROFILE)) AARCH64_CPU_NAME("kryo", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_CRC)) + (AArch64::AEK_CRC)) AARCH64_CPU_NAME("thunderx2t99", ARMV8_1A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_NONE)) + (AArch64::AEK_NONE)) AARCH64_CPU_NAME("thunderx", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_CRC | AArch64::AEK_PROFILE)) + (AArch64::AEK_CRC | AArch64::AEK_PROFILE)) AARCH64_CPU_NAME("thunderxt88", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_CRC | AArch64::AEK_PROFILE)) + (AArch64::AEK_CRC | AArch64::AEK_PROFILE)) AARCH64_CPU_NAME("thunderxt81", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_CRC | AArch64::AEK_PROFILE)) + (AArch64::AEK_CRC | AArch64::AEK_PROFILE)) AARCH64_CPU_NAME("thunderxt83", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_CRC | AArch64::AEK_PROFILE)) + (AArch64::AEK_CRC | AArch64::AEK_PROFILE)) AARCH64_CPU_NAME("tsv110", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, - (AArch64::AEK_PROFILE | AArch64::AEK_FP16 | AArch64::AEK_FP16FML | - AArch64::AEK_DOTPROD)) + (AArch64::AEK_DOTPROD | + AArch64::AEK_FP16 | AArch64::AEK_FP16FML | + AArch64::AEK_PROFILE)) // Invalid CPU AARCH64_CPU_NAME("invalid", INVALID, FK_INVALID, true, AArch64::AEK_INVALID) #undef AARCH64_CPU_NAME diff --git a/include/llvm/Support/AArch64TargetParser.h b/include/llvm/Support/AArch64TargetParser.h index 76b77d474428..965d38535e74 100644 --- a/include/llvm/Support/AArch64TargetParser.h +++ b/include/llvm/Support/AArch64TargetParser.h @@ -1,9 +1,8 @@ //===-- AArch64TargetParser - Parser for AArch64 features -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -50,6 +49,11 @@ enum ArchExtKind : unsigned { AEK_SSBS = 1 << 20, AEK_SB = 1 << 21, AEK_PREDRES = 1 << 22, + AEK_SVE2 = 1 << 23, + AEK_SVE2AES = 1 << 24, + AEK_SVE2SM4 = 1 << 25, + AEK_SVE2SHA3 = 1 << 26, + AEK_BITPERM = 1 << 27, }; enum class ArchKind { diff --git a/include/llvm/Support/AMDGPUMetadata.h b/include/llvm/Support/AMDGPUMetadata.h index 84851c07499d..f7f1ec40dde9 100644 --- a/include/llvm/Support/AMDGPUMetadata.h +++ b/include/llvm/Support/AMDGPUMetadata.h @@ -1,9 +1,8 @@ //===--- AMDGPUMetadata.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -75,6 +74,7 @@ enum class ValueKind : uint8_t { HiddenPrintfBuffer = 11, HiddenDefaultQueue = 12, HiddenCompletionAction = 13, + HiddenMultiGridSyncArg = 14, Unknown = 0xff }; @@ -157,6 +157,8 @@ constexpr char Name[] = "Name"; constexpr char TypeName[] = "TypeName"; /// Key for Kernel::Arg::Metadata::mSize. constexpr char Size[] = "Size"; +/// Key for Kernel::Arg::Metadata::mOffset. +constexpr char Offset[] = "Offset"; /// Key for Kernel::Arg::Metadata::mAlign. constexpr char Align[] = "Align"; /// Key for Kernel::Arg::Metadata::mValueKind. @@ -189,6 +191,8 @@ struct Metadata final { std::string mTypeName = std::string(); /// Size in bytes. Required. uint32_t mSize = 0; + /// Offset in bytes. Required for code object v3, unused for code object v2. + uint32_t mOffset = 0; /// Alignment in bytes. Required. uint32_t mAlign = 0; /// Value kind. Required. @@ -453,11 +457,30 @@ constexpr char AssemblerDirectiveEnd[] = ".end_amdgpu_metadata"; //===----------------------------------------------------------------------===// namespace PALMD { -/// PAL metadata assembler directive. +/// PAL metadata (old linear format) assembler directive. constexpr char AssemblerDirective[] = ".amd_amdgpu_pal_metadata"; +/// PAL metadata (new MsgPack format) beginning assembler directive. +constexpr char AssemblerDirectiveBegin[] = ".amdgpu_pal_metadata"; + +/// PAL metadata (new MsgPack format) ending assembler directive. +constexpr char AssemblerDirectiveEnd[] = ".end_amdgpu_pal_metadata"; + /// PAL metadata keys. enum Key : uint32_t { + R_2E12_COMPUTE_PGM_RSRC1 = 0x2e12, + R_2D4A_SPI_SHADER_PGM_RSRC1_LS = 0x2d4a, + R_2D0A_SPI_SHADER_PGM_RSRC1_HS = 0x2d0a, + R_2CCA_SPI_SHADER_PGM_RSRC1_ES = 0x2cca, + R_2C8A_SPI_SHADER_PGM_RSRC1_GS = 0x2c8a, + R_2C4A_SPI_SHADER_PGM_RSRC1_VS = 0x2c4a, + R_2C0A_SPI_SHADER_PGM_RSRC1_PS = 0x2c0a, + R_2E00_COMPUTE_DISPATCH_INITIATOR = 0x2e00, + R_A1B3_SPI_PS_INPUT_ENA = 0xa1b3, + R_A1B4_SPI_PS_INPUT_ADDR = 0xa1b4, + R_A1B6_SPI_PS_IN_CONTROL = 0xa1b6, + R_A2D5_VGT_SHADER_STAGES_EN = 0xa2d5, + LS_NUM_USED_VGPRS = 0x10000021, HS_NUM_USED_VGPRS = 0x10000022, ES_NUM_USED_VGPRS = 0x10000023, @@ -483,12 +506,6 @@ enum Key : uint32_t { CS_SCRATCH_SIZE = 0x1000004a }; -/// PAL metadata represented as a vector. -typedef std::vector Metadata; - -/// Converts \p PALMetadata to \p String. -std::error_code toString(const Metadata &PALMetadata, std::string &String); - } // end namespace PALMD } // end namespace AMDGPU } // end namespace llvm diff --git a/include/llvm/Support/AMDHSAKernelDescriptor.h b/include/llvm/Support/AMDHSAKernelDescriptor.h index 751699e3a19f..d1c2147536a7 100644 --- a/include/llvm/Support/AMDHSAKernelDescriptor.h +++ b/include/llvm/Support/AMDHSAKernelDescriptor.h @@ -1,9 +1,8 @@ //===--- AMDHSAKernelDescriptor.h -----------------------------*- C++ -*---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -89,8 +88,11 @@ enum : int32_t { COMPUTE_PGM_RSRC1(ENABLE_IEEE_MODE, 23, 1), COMPUTE_PGM_RSRC1(BULKY, 24, 1), COMPUTE_PGM_RSRC1(CDBG_USER, 25, 1), - COMPUTE_PGM_RSRC1(FP16_OVFL, 26, 1), // GFX9+ - COMPUTE_PGM_RSRC1(RESERVED0, 27, 5), + COMPUTE_PGM_RSRC1(FP16_OVFL, 26, 1), // GFX9+ + COMPUTE_PGM_RSRC1(RESERVED0, 27, 2), + COMPUTE_PGM_RSRC1(WGP_MODE, 29, 1), // GFX10+ + COMPUTE_PGM_RSRC1(MEM_ORDERED, 30, 1), // GFX10+ + COMPUTE_PGM_RSRC1(FWD_PROGRESS, 31, 1), // GFX10+ }; #undef COMPUTE_PGM_RSRC1 @@ -120,6 +122,15 @@ enum : int32_t { }; #undef COMPUTE_PGM_RSRC2 +// Compute program resource register 3. Must match hardware definition. +#define COMPUTE_PGM_RSRC3(NAME, SHIFT, WIDTH) \ + AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC3_ ## NAME, SHIFT, WIDTH) +enum : int32_t { + COMPUTE_PGM_RSRC3(SHARED_VGPR_COUNT, 0, 4), // GFX10+ + COMPUTE_PGM_RSRC3(RESERVED0, 4, 28), +}; +#undef COMPUTE_PGM_RSRC3 + // Kernel code properties. Must be kept backwards compatible. #define KERNEL_CODE_PROPERTY(NAME, SHIFT, WIDTH) \ AMDHSA_BITS_ENUM_ENTRY(KERNEL_CODE_PROPERTY_ ## NAME, SHIFT, WIDTH) @@ -131,7 +142,9 @@ enum : int32_t { KERNEL_CODE_PROPERTY(ENABLE_SGPR_DISPATCH_ID, 4, 1), KERNEL_CODE_PROPERTY(ENABLE_SGPR_FLAT_SCRATCH_INIT, 5, 1), KERNEL_CODE_PROPERTY(ENABLE_SGPR_PRIVATE_SEGMENT_SIZE, 6, 1), - KERNEL_CODE_PROPERTY(RESERVED0, 7, 9), + KERNEL_CODE_PROPERTY(RESERVED0, 7, 3), + KERNEL_CODE_PROPERTY(ENABLE_WAVEFRONT_SIZE32, 10, 1), // GFX10+ + KERNEL_CODE_PROPERTY(RESERVED1, 11, 5), }; #undef KERNEL_CODE_PROPERTY @@ -141,7 +154,8 @@ struct kernel_descriptor_t { uint32_t private_segment_fixed_size; uint8_t reserved0[8]; int64_t kernel_code_entry_byte_offset; - uint8_t reserved1[24]; + uint8_t reserved1[20]; + uint32_t compute_pgm_rsrc3; // GFX10+ uint32_t compute_pgm_rsrc1; uint32_t compute_pgm_rsrc2; uint16_t kernel_code_properties; @@ -166,6 +180,9 @@ static_assert( static_assert( offsetof(kernel_descriptor_t, reserved1) == 24, "invalid offset for reserved1"); +static_assert( + offsetof(kernel_descriptor_t, compute_pgm_rsrc3) == 44, + "invalid offset for compute_pgm_rsrc3"); static_assert( offsetof(kernel_descriptor_t, compute_pgm_rsrc1) == 48, "invalid offset for compute_pgm_rsrc1"); diff --git a/include/llvm/Support/ARMAttributeParser.h b/include/llvm/Support/ARMAttributeParser.h index 919f39721f86..f6c39abb4f21 100644 --- a/include/llvm/Support/ARMAttributeParser.h +++ b/include/llvm/Support/ARMAttributeParser.h @@ -1,9 +1,8 @@ //===--- ARMAttributeParser.h - ARM Attribute Information Printer ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -54,6 +53,8 @@ class ARMAttributeParser { uint32_t &Offset); void Advanced_SIMD_arch(ARMBuildAttrs::AttrType Tag, const uint8_t *Data, uint32_t &Offset); + void MVE_arch(ARMBuildAttrs::AttrType Tag, const uint8_t *Data, + uint32_t &Offset); void PCS_config(ARMBuildAttrs::AttrType Tag, const uint8_t *Data, uint32_t &Offset); void ABI_PCS_R9_use(ARMBuildAttrs::AttrType Tag, const uint8_t *Data, diff --git a/include/llvm/Support/ARMBuildAttributes.h b/include/llvm/Support/ARMBuildAttributes.h index b8a03765a7c0..90481eaa1677 100644 --- a/include/llvm/Support/ARMBuildAttributes.h +++ b/include/llvm/Support/ARMBuildAttributes.h @@ -1,9 +1,8 @@ //===-- ARMBuildAttributes.h - ARM Build Attributes -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -68,6 +67,7 @@ enum AttrType { MPextension_use = 42, // recoded from 70 (ABI r2.08) DIV_use = 44, DSP_extension = 46, + MVE_arch = 48, also_compatible_with = 65, conformance = 67, Virtualization_use = 68, @@ -111,6 +111,7 @@ enum CPUArch { v8_R = 15, // e.g. Cortex R52 v8_M_Base= 16, // v8_M_Base AArch32 v8_M_Main= 17, // v8_M_Main AArch32 + v8_1_M_Main=21, // v8_1_M_Main AArch32 }; enum CPUArchProfile { // (=7), uleb128 @@ -152,6 +153,10 @@ enum { AllowNeonARMv8 = 3, // ARM v8-A SIMD was permitted AllowNeonARMv8_1a = 4,// ARM v8.1-A SIMD was permitted (RDMA) + // Tag_MVE_arch, (=48), uleb128 + AllowMVEInteger = 1, // integer-only MVE was permitted + AllowMVEIntegerAndFloat = 2, // both integer and floating point MVE were permitted + // Tag_ABI_PCS_R9_use, (=14), uleb128 R9IsGPR = 0, // R9 used as v6 (just another callee-saved register) R9IsSB = 1, // R9 used as a global static base rgister diff --git a/include/llvm/Support/ARMEHABI.h b/include/llvm/Support/ARMEHABI.h index 9b052df0a908..3fbb56d65eb8 100644 --- a/include/llvm/Support/ARMEHABI.h +++ b/include/llvm/Support/ARMEHABI.h @@ -1,9 +1,8 @@ //===--- ARMEHABI.h - ARM Exception Handling ABI ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/ARMTargetParser.def b/include/llvm/Support/ARMTargetParser.def index 9e844e2b464d..f466b3252748 100644 --- a/include/llvm/Support/ARMTargetParser.def +++ b/include/llvm/Support/ARMTargetParser.def @@ -1,9 +1,8 @@ //===- ARMTargetParser.def - ARM target parsing defines ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -32,6 +31,8 @@ ARM_FPU("fpv4-sp-d16", FK_FPV4_SP_D16, FPUVersion::VFPV4, NeonSupportLevel::None ARM_FPU("fpv5-d16", FK_FPV5_D16, FPUVersion::VFPV5, NeonSupportLevel::None, FPURestriction::D16) ARM_FPU("fpv5-sp-d16", FK_FPV5_SP_D16, FPUVersion::VFPV5, NeonSupportLevel::None, FPURestriction::SP_D16) ARM_FPU("fp-armv8", FK_FP_ARMV8, FPUVersion::VFPV5, NeonSupportLevel::None, FPURestriction::None) +ARM_FPU("fp-armv8-fullfp16-d16", FK_FP_ARMV8_FULLFP16_D16, FPUVersion::VFPV5_FULLFP16, NeonSupportLevel::None, FPURestriction::D16) +ARM_FPU("fp-armv8-fullfp16-sp-d16", FK_FP_ARMV8_FULLFP16_SP_D16, FPUVersion::VFPV5_FULLFP16, NeonSupportLevel::None, FPURestriction::SP_D16) ARM_FPU("neon", FK_NEON, FPUVersion::VFPV3, NeonSupportLevel::Neon, FPURestriction::None) ARM_FPU("neon-fp16", FK_NEON_FP16, FPUVersion::VFPV3_FP16, NeonSupportLevel::Neon, FPURestriction::None) ARM_FPU("neon-vfpv4", FK_NEON_VFPV4, FPUVersion::VFPV4, NeonSupportLevel::Neon, FPURestriction::None) @@ -119,6 +120,8 @@ ARM_ARCH("armv8-m.base", ARMV8MBaseline, "8-M.Baseline", "v8m.base", ARMBuildAttrs::CPUArch::v8_M_Base, FK_NONE, ARM::AEK_HWDIVTHUMB) ARM_ARCH("armv8-m.main", ARMV8MMainline, "8-M.Mainline", "v8m.main", ARMBuildAttrs::CPUArch::v8_M_Main, FK_FPV5_D16, ARM::AEK_HWDIVTHUMB) +ARM_ARCH("armv8.1-m.main", ARMV8_1MMainline, "8.1-M.Mainline", "v8.1m.main", + ARMBuildAttrs::CPUArch::v8_1_M_Main, FK_FP_ARMV8_FULLFP16_SP_D16, ARM::AEK_HWDIVTHUMB | ARM::AEK_RAS | ARM::AEK_LOB) // Non-standard Arch names. ARM_ARCH("iwmmxt", IWMMXT, "iwmmxt", "", ARMBuildAttrs::CPUArch::v5TE, FK_NONE, ARM::AEK_NONE) @@ -145,6 +148,9 @@ ARM_ARCH_EXT_NAME("aes", ARM::AEK_AES, "+aes", "-aes") ARM_ARCH_EXT_NAME("dotprod", ARM::AEK_DOTPROD, "+dotprod","-dotprod") ARM_ARCH_EXT_NAME("dsp", ARM::AEK_DSP, "+dsp", "-dsp") ARM_ARCH_EXT_NAME("fp", ARM::AEK_FP, nullptr, nullptr) +ARM_ARCH_EXT_NAME("fp.dp", ARM::AEK_FP_DP, nullptr, nullptr) +ARM_ARCH_EXT_NAME("mve", (ARM::AEK_DSP | ARM::AEK_SIMD), "+mve", "-mve") +ARM_ARCH_EXT_NAME("mve.fp", (ARM::AEK_DSP | ARM::AEK_SIMD | ARM::AEK_FP), "+mve.fp", "-mve.fp") ARM_ARCH_EXT_NAME("idiv", (ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB), nullptr, nullptr) ARM_ARCH_EXT_NAME("mp", ARM::AEK_MP, nullptr, nullptr) ARM_ARCH_EXT_NAME("simd", ARM::AEK_SIMD, nullptr, nullptr) @@ -159,6 +165,7 @@ ARM_ARCH_EXT_NAME("maverick", ARM::AEK_MAVERICK, nullptr, nullptr) ARM_ARCH_EXT_NAME("xscale", ARM::AEK_XSCALE, nullptr, nullptr) ARM_ARCH_EXT_NAME("fp16fml", ARM::AEK_FP16FML, "+fp16fml", "-fp16fml") ARM_ARCH_EXT_NAME("sb", ARM::AEK_SB, "+sb", "-sb") +ARM_ARCH_EXT_NAME("lob", ARM::AEK_LOB, "+lob", "-lob") #undef ARM_ARCH_EXT_NAME #ifndef ARM_HW_DIV_NAME @@ -252,6 +259,7 @@ ARM_CPU_NAME("cortex-m4", ARMV7EM, FK_FPV4_SP_D16, true, ARM::AEK_NONE) ARM_CPU_NAME("cortex-m7", ARMV7EM, FK_FPV5_D16, false, ARM::AEK_NONE) ARM_CPU_NAME("cortex-m23", ARMV8MBaseline, FK_NONE, false, ARM::AEK_NONE) ARM_CPU_NAME("cortex-m33", ARMV8MMainline, FK_FPV5_SP_D16, false, ARM::AEK_DSP) +ARM_CPU_NAME("cortex-m35p", ARMV8MMainline, FK_FPV5_SP_D16, false, ARM::AEK_DSP) ARM_CPU_NAME("cortex-a32", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) ARM_CPU_NAME("cortex-a35", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) ARM_CPU_NAME("cortex-a53", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) @@ -262,12 +270,18 @@ ARM_CPU_NAME("cortex-a72", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) ARM_CPU_NAME("cortex-a73", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) ARM_CPU_NAME("cortex-a75", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, (ARM::AEK_FP16 | ARM::AEK_DOTPROD)) +ARM_CPU_NAME("cortex-a76", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, + (ARM::AEK_FP16 | ARM::AEK_DOTPROD)) +ARM_CPU_NAME("cortex-a76ae", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, + (ARM::AEK_FP16 | ARM::AEK_DOTPROD)) ARM_CPU_NAME("cyclone", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) ARM_CPU_NAME("exynos-m1", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) ARM_CPU_NAME("exynos-m2", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) ARM_CPU_NAME("exynos-m3", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) ARM_CPU_NAME("exynos-m4", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, (ARM::AEK_FP16 | ARM::AEK_DOTPROD)) +ARM_CPU_NAME("exynos-m5", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, + (ARM::AEK_FP16 | ARM::AEK_DOTPROD)) ARM_CPU_NAME("kryo", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) // Non-standard Arch names. ARM_CPU_NAME("iwmmxt", IWMMXT, FK_NONE, true, ARM::AEK_NONE) diff --git a/include/llvm/Support/ARMTargetParser.h b/include/llvm/Support/ARMTargetParser.h index 71acc0dc72d0..4b9070dea596 100644 --- a/include/llvm/Support/ARMTargetParser.h +++ b/include/llvm/Support/ARMTargetParser.h @@ -1,9 +1,8 @@ //===-- ARMTargetParser - Parser for ARM target features --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -46,6 +45,13 @@ enum ArchExtKind : unsigned { AEK_AES = 1 << 16, AEK_FP16FML = 1 << 17, AEK_SB = 1 << 18, + AEK_SVE2 = 1 << 19, + AEK_SVE2AES = 1 << 20, + AEK_SVE2SM4 = 1 << 21, + AEK_SVE2SHA3 = 1 << 22, + AEK_BITPERM = 1 << 23, + AEK_FP_DP = 1 << 24, + AEK_LOB = 1 << 25, // Unsupported extensions. AEK_OS = 0x8000000, AEK_IWMMXT = 0x10000000, @@ -127,7 +133,8 @@ enum class FPUVersion { VFPV3, VFPV3_FP16, VFPV4, - VFPV5 + VFPV5, + VFPV5_FULLFP16, }; // An FPU name restricts the FPU in one of three ways: @@ -234,6 +241,8 @@ StringRef getCPUAttr(ArchKind AK); StringRef getSubArch(ArchKind AK); StringRef getArchExtName(unsigned ArchExtKind); StringRef getArchExtFeature(StringRef ArchExt); +bool appendArchExtFeatures(StringRef CPU, ARM::ArchKind AK, StringRef ArchExt, + std::vector &Features); StringRef getHWDivName(unsigned HWDivKind); // Information by Name diff --git a/include/llvm/Support/ARMWinEH.h b/include/llvm/Support/ARMWinEH.h index 60174503ad49..857a0d3814a8 100644 --- a/include/llvm/Support/ARMWinEH.h +++ b/include/llvm/Support/ARMWinEH.h @@ -1,9 +1,8 @@ -//===-- llvm/Support/WinARMEH.h - Windows on ARM EH Constants ---*- C++ -*-===// +//===-- llvm/Support/ARMWinEH.h - Windows on ARM EH Constants ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -383,7 +382,7 @@ struct ExceptionDataRecord { return ((Data[0] & 0x00400000) >> 22); } - uint8_t EpilogueCount() const { + uint16_t EpilogueCount() const { if (HeaderWords(*this) == 1) { if (isAArch64) return (Data[0] & 0x07C00000) >> 22; diff --git a/include/llvm/Support/AlignOf.h b/include/llvm/Support/AlignOf.h index 9e7a62b85e34..d12401f0eb49 100644 --- a/include/llvm/Support/AlignOf.h +++ b/include/llvm/Support/AlignOf.h @@ -1,9 +1,8 @@ //===--- AlignOf.h - Portable calculation of type alignment -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/Allocator.h b/include/llvm/Support/Allocator.h index 42d08378a677..09e967b98abc 100644 --- a/include/llvm/Support/Allocator.h +++ b/include/llvm/Support/Allocator.h @@ -1,9 +1,8 @@ //===- Allocator.h - Simple memory allocation abstraction -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/Support/ArrayRecycler.h b/include/llvm/Support/ArrayRecycler.h index 68696be6bf3d..5256ce80c028 100644 --- a/include/llvm/Support/ArrayRecycler.h +++ b/include/llvm/Support/ArrayRecycler.h @@ -1,9 +1,8 @@ //==- llvm/Support/ArrayRecycler.h - Recycling of Arrays ---------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/Atomic.h b/include/llvm/Support/Atomic.h index 552313f0c241..a8445fddc1a8 100644 --- a/include/llvm/Support/Atomic.h +++ b/include/llvm/Support/Atomic.h @@ -1,9 +1,8 @@ //===- llvm/Support/Atomic.h - Atomic Operations -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/AtomicOrdering.h b/include/llvm/Support/AtomicOrdering.h index a679ab30243e..763bc3ea7b28 100644 --- a/include/llvm/Support/AtomicOrdering.h +++ b/include/llvm/Support/AtomicOrdering.h @@ -1,9 +1,8 @@ //===-- llvm/Support/AtomicOrdering.h ---Atomic Ordering---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/include/llvm/Support/BinaryByteStream.h b/include/llvm/Support/BinaryByteStream.h index 9808d3b72157..7acce9a03888 100644 --- a/include/llvm/Support/BinaryByteStream.h +++ b/include/llvm/Support/BinaryByteStream.h @@ -1,9 +1,8 @@ //===- BinaryByteStream.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception //===----------------------------------------------------------------------===// // A BinaryStream which stores data in a single continguous memory buffer. //===----------------------------------------------------------------------===// diff --git a/include/llvm/Support/BinaryItemStream.h b/include/llvm/Support/BinaryItemStream.h index 278723ddf8da..4cd66adcc01a 100644 --- a/include/llvm/Support/BinaryItemStream.h +++ b/include/llvm/Support/BinaryItemStream.h @@ -1,9 +1,8 @@ //===- BinaryItemStream.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Support/BinaryStream.h b/include/llvm/Support/BinaryStream.h index 7677214e48ee..fcf4398550ee 100644 --- a/include/llvm/Support/BinaryStream.h +++ b/include/llvm/Support/BinaryStream.h @@ -1,9 +1,8 @@ //===- BinaryStream.h - Base interface for a stream of data -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Support/BinaryStreamArray.h b/include/llvm/Support/BinaryStreamArray.h index 7c110fcb6a4b..96d09db69ae5 100644 --- a/include/llvm/Support/BinaryStreamArray.h +++ b/include/llvm/Support/BinaryStreamArray.h @@ -1,9 +1,8 @@ //===- BinaryStreamArray.h - Array backed by an arbitrary stream *- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Support/BinaryStreamError.h b/include/llvm/Support/BinaryStreamError.h index 7d9699d53639..cf6e034ffd2c 100644 --- a/include/llvm/Support/BinaryStreamError.h +++ b/include/llvm/Support/BinaryStreamError.h @@ -1,9 +1,8 @@ //===- BinaryStreamError.h - Error extensions for Binary Streams *- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Support/BinaryStreamReader.h b/include/llvm/Support/BinaryStreamReader.h index 392958de30d5..d8fddde66bfa 100644 --- a/include/llvm/Support/BinaryStreamReader.h +++ b/include/llvm/Support/BinaryStreamReader.h @@ -1,9 +1,8 @@ //===- BinaryStreamReader.h - Reads objects from a binary stream *- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -97,6 +96,18 @@ public: return Error::success(); } + /// Read an unsigned LEB128 encoded value. + /// + /// \returns a success error code if the data was successfully read, otherwise + /// returns an appropriate error code. + Error readULEB128(uint64_t &Dest); + + /// Read a signed LEB128 encoded value. + /// + /// \returns a success error code if the data was successfully read, otherwise + /// returns an appropriate error code. + Error readSLEB128(int64_t &Dest); + /// Read a null terminated string from \p Dest. Whether a copy occurs depends /// on the implementation of the underlying stream. Updates the stream's /// offset to point after the newly read data. diff --git a/include/llvm/Support/BinaryStreamRef.h b/include/llvm/Support/BinaryStreamRef.h index d8dc1392c01c..7427b8da5b43 100644 --- a/include/llvm/Support/BinaryStreamRef.h +++ b/include/llvm/Support/BinaryStreamRef.h @@ -1,9 +1,8 @@ //===- BinaryStreamRef.h - A copyable reference to a stream -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Support/BinaryStreamWriter.h b/include/llvm/Support/BinaryStreamWriter.h index 6e8a68a30474..86d2389d9182 100644 --- a/include/llvm/Support/BinaryStreamWriter.h +++ b/include/llvm/Support/BinaryStreamWriter.h @@ -1,9 +1,8 @@ //===- BinaryStreamWriter.h - Writes objects to a BinaryStream ---*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -80,6 +79,20 @@ public: return writeInteger(static_cast(Num)); } + /// Write the unsigned integer Value to the underlying stream using ULEB128 + /// encoding. + /// + /// \returns a success error code if the data was successfully written, + /// otherwise returns an appropriate error code. + Error writeULEB128(uint64_t Value); + + /// Write the unsigned integer Value to the underlying stream using ULEB128 + /// encoding. + /// + /// \returns a success error code if the data was successfully written, + /// otherwise returns an appropriate error code. + Error writeSLEB128(int64_t Value); + /// Write the string \p Str to the underlying stream followed by a null /// terminator. On success, updates the offset so that subsequent writes /// occur at the next unwritten position. \p Str need not be null terminated diff --git a/include/llvm/Support/BlockFrequency.h b/include/llvm/Support/BlockFrequency.h index 4b468f7acb32..18fb60e1904b 100644 --- a/include/llvm/Support/BlockFrequency.h +++ b/include/llvm/Support/BlockFrequency.h @@ -1,9 +1,8 @@ //===-------- BlockFrequency.h - Block Frequency Wrapper --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/BranchProbability.h b/include/llvm/Support/BranchProbability.h index 3a88e71c2480..cd9d369b4f4e 100644 --- a/include/llvm/Support/BranchProbability.h +++ b/include/llvm/Support/BranchProbability.h @@ -1,9 +1,8 @@ //===- BranchProbability.h - Branch Probability Wrapper ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -119,6 +118,13 @@ public: return *this; } + BranchProbability &operator/=(BranchProbability RHS) { + assert(N != UnknownN && RHS.N != UnknownN && + "Unknown probability cannot participate in arithmetics."); + N = (static_cast(N) * D + RHS.N / 2) / RHS.N; + return *this; + } + BranchProbability &operator/=(uint32_t RHS) { assert(N != UnknownN && "Unknown probability cannot participate in arithmetics."); @@ -129,27 +135,38 @@ public: BranchProbability operator+(BranchProbability RHS) const { BranchProbability Prob(*this); - return Prob += RHS; + Prob += RHS; + return Prob; } BranchProbability operator-(BranchProbability RHS) const { BranchProbability Prob(*this); - return Prob -= RHS; + Prob -= RHS; + return Prob; } BranchProbability operator*(BranchProbability RHS) const { BranchProbability Prob(*this); - return Prob *= RHS; + Prob *= RHS; + return Prob; } BranchProbability operator*(uint32_t RHS) const { BranchProbability Prob(*this); - return Prob *= RHS; + Prob *= RHS; + return Prob; + } + + BranchProbability operator/(BranchProbability RHS) const { + BranchProbability Prob(*this); + Prob /= RHS; + return Prob; } BranchProbability operator/(uint32_t RHS) const { BranchProbability Prob(*this); - return Prob /= RHS; + Prob /= RHS; + return Prob; } bool operator==(BranchProbability RHS) const { return N == RHS.N; } diff --git a/include/llvm/Support/BuryPointer.h b/include/llvm/Support/BuryPointer.h index 53f1f395b922..276a5b7089c3 100644 --- a/include/llvm/Support/BuryPointer.h +++ b/include/llvm/Support/BuryPointer.h @@ -1,9 +1,8 @@ //===- llvm/Support/BuryPointer.h - Memory Manipulation/Leak ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Support/CBindingWrapping.h b/include/llvm/Support/CBindingWrapping.h index f60f99d376ad..46d6b4e3fa7d 100644 --- a/include/llvm/Support/CBindingWrapping.h +++ b/include/llvm/Support/CBindingWrapping.h @@ -1,9 +1,8 @@ -//===- llvm/Support/CBindingWrapph.h - C Interface Wrapping -----*- C++ -*-===// +//===- llvm/Support/CBindingWrapping.h - C Interface Wrapping ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/CFGUpdate.h b/include/llvm/Support/CFGUpdate.h index 63c24a3d2a20..eeaf5d0a21ac 100644 --- a/include/llvm/Support/CFGUpdate.h +++ b/include/llvm/Support/CFGUpdate.h @@ -1,9 +1,8 @@ //===- CFGUpdate.h - Encode a CFG Edge Update. ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/COM.h b/include/llvm/Support/COM.h index a2d5a7a68ba9..d59966f849b4 100644 --- a/include/llvm/Support/COM.h +++ b/include/llvm/Support/COM.h @@ -1,9 +1,8 @@ //===- llvm/Support/COM.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/Support/CRC.h b/include/llvm/Support/CRC.h new file mode 100644 index 000000000000..6ea8e3edcea4 --- /dev/null +++ b/include/llvm/Support/CRC.h @@ -0,0 +1,25 @@ +//===-- llvm/Support/CRC.h - Cyclic Redundancy Check-------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains basic functions for calculating Cyclic Redundancy Check +// or CRC. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_CRC_H +#define LLVM_SUPPORT_CRC_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/DataTypes.h" + +namespace llvm { +/// zlib independent CRC32 calculation. +uint32_t crc32(uint32_t CRC, StringRef S); +} // end namespace llvm + +#endif diff --git a/include/llvm/Support/CachePruning.h b/include/llvm/Support/CachePruning.h index cf3f8ec67a52..a72a86439f6a 100644 --- a/include/llvm/Support/CachePruning.h +++ b/include/llvm/Support/CachePruning.h @@ -1,9 +1,8 @@ //=- CachePruning.h - Helper to manage the pruning of a cache dir -*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/Capacity.h b/include/llvm/Support/Capacity.h index 7460f9825bd3..6b99e0aaa488 100644 --- a/include/llvm/Support/Capacity.h +++ b/include/llvm/Support/Capacity.h @@ -1,9 +1,8 @@ //===--- Capacity.h - Generic computation of ADT memory use -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/Casting.h b/include/llvm/Support/Casting.h index 3f21e0f9ebc3..46bdedb04cfe 100644 --- a/include/llvm/Support/Casting.h +++ b/include/llvm/Support/Casting.h @@ -1,9 +1,8 @@ //===- llvm/Support/Casting.h - Allow flexible, checked, casts --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -144,6 +143,16 @@ template LLVM_NODISCARD inline bool isa(const Y &Val) { typename simplify_type::SimpleType>::doit(Val); } +// isa_and_nonnull - Functionally identical to isa, except that a null value +// is accepted. +// +template +LLVM_NODISCARD inline bool isa_and_nonnull(const Y &Val) { + if (!Val) + return false; + return isa(Val); +} + //===----------------------------------------------------------------------===// // cast Support Templates //===----------------------------------------------------------------------===// diff --git a/include/llvm/Support/CheckedArithmetic.h b/include/llvm/Support/CheckedArithmetic.h index 039c374136ff..8a50e3d5ddf6 100644 --- a/include/llvm/Support/CheckedArithmetic.h +++ b/include/llvm/Support/CheckedArithmetic.h @@ -1,9 +1,8 @@ //==-- llvm/Support/CheckedArithmetic.h - Safe arithmetical operations *- C++ // // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -50,6 +49,15 @@ checkedAdd(T LHS, T RHS) { return checkedOp(LHS, RHS, &llvm::APInt::sadd_ov); } +/// Subtract two signed integers \p LHS and \p RHS. +/// \return Optional of sum if no signed overflow occurred, +/// \c None otherwise. +template +typename std::enable_if::value, llvm::Optional>::type +checkedSub(T LHS, T RHS) { + return checkedOp(LHS, RHS, &llvm::APInt::ssub_ov); +} + /// Multiply two signed integers \p LHS and \p RHS. /// \return Optional of product if no signed overflow occurred, /// \c None otherwise. diff --git a/include/llvm/Support/Chrono.h b/include/llvm/Support/Chrono.h index 57677e8d5cf1..334ab60835a4 100644 --- a/include/llvm/Support/Chrono.h +++ b/include/llvm/Support/Chrono.h @@ -1,9 +1,8 @@ //===- llvm/Support/Chrono.h - Utilities for Timing Manipulation-*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -34,21 +33,21 @@ template using TimePoint = std::chrono::time_point; /// Convert a TimePoint to std::time_t -LLVM_ATTRIBUTE_ALWAYS_INLINE inline std::time_t toTimeT(TimePoint<> TP) { +inline std::time_t toTimeT(TimePoint<> TP) { using namespace std::chrono; return system_clock::to_time_t( time_point_cast(TP)); } /// Convert a std::time_t to a TimePoint -LLVM_ATTRIBUTE_ALWAYS_INLINE inline TimePoint +inline TimePoint toTimePoint(std::time_t T) { using namespace std::chrono; return time_point_cast(system_clock::from_time_t(T)); } /// Convert a std::time_t + nanoseconds to a TimePoint -LLVM_ATTRIBUTE_ALWAYS_INLINE inline TimePoint<> +inline TimePoint<> toTimePoint(std::time_t T, uint32_t nsec) { using namespace std::chrono; return time_point_cast(system_clock::from_time_t(T)) diff --git a/include/llvm/Support/CodeGen.h b/include/llvm/Support/CodeGen.h index 22e74167266c..a3f423e558cf 100644 --- a/include/llvm/Support/CodeGen.h +++ b/include/llvm/Support/CodeGen.h @@ -1,9 +1,8 @@ //===-- llvm/Support/CodeGen.h - CodeGen Concepts ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,13 +18,14 @@ namespace llvm { // Relocation model types. namespace Reloc { - enum Model { Static, PIC_, DynamicNoPIC, ROPI, RWPI, ROPI_RWPI }; + // Cannot be named PIC due to collision with -DPIC + enum Model { Static, PIC_, DynamicNoPIC, ROPI, RWPI, ROPI_RWPI }; } // Code model types. namespace CodeModel { // Sync changes with CodeGenCWrappers.h. - enum Model { Tiny, Small, Kernel, Medium, Large }; + enum Model { Tiny, Small, Kernel, Medium, Large }; } namespace PICLevel { @@ -50,10 +50,10 @@ namespace llvm { // Code generation optimization level. namespace CodeGenOpt { enum Level { - None, // -O0 - Less, // -O1 - Default, // -O2, -Os - Aggressive // -O3 + None = 0, // -O0 + Less = 1, // -O1 + Default = 2, // -O2, -Os + Aggressive = 3 // -O3 }; } diff --git a/include/llvm/Support/CodeGenCoverage.h b/include/llvm/Support/CodeGenCoverage.h index c863be35b822..0b1af779ffb0 100644 --- a/include/llvm/Support/CodeGenCoverage.h +++ b/include/llvm/Support/CodeGenCoverage.h @@ -1,9 +1,8 @@ //== llvm/Support/CodeGenCoverage.h ------------------------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file This file provides rule coverage tracking for tablegen-erated CodeGen. diff --git a/include/llvm/Support/CommandLine.h b/include/llvm/Support/CommandLine.h index a8ad89384d17..3cc2c3c0121b 100644 --- a/include/llvm/Support/CommandLine.h +++ b/include/llvm/Support/CommandLine.h @@ -1,9 +1,8 @@ //===- llvm/Support/CommandLine.h - Command line handler --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -67,7 +66,8 @@ namespace cl { bool ParseCommandLineOptions(int argc, const char *const *argv, StringRef Overview = "", raw_ostream *Errs = nullptr, - const char *EnvVar = nullptr); + const char *EnvVar = nullptr, + bool LongOptionsUseDoubleDash = false); //===----------------------------------------------------------------------===// // ParseEnvironmentOptions - Environment variable option processing alternate @@ -159,23 +159,27 @@ enum OptionHidden { // Control whether -help shows this option // AlwaysPrefix - Only allow the behavior enabled by the Prefix flag and reject // the Option=Value form. // -// Grouping - With this option enabled, multiple letter options are allowed to -// bunch together with only a single hyphen for the whole group. This allows -// emulation of the behavior that ls uses for example: ls -la === ls -l -a -// enum FormattingFlags { NormalFormatting = 0x00, // Nothing special Positional = 0x01, // Is a positional argument, no '-' required Prefix = 0x02, // Can this option directly prefix its value? - AlwaysPrefix = 0x03, // Can this option only directly prefix its value? - Grouping = 0x04 // Can this option group with other options? + AlwaysPrefix = 0x03 // Can this option only directly prefix its value? }; enum MiscFlags { // Miscellaneous flags to adjust argument CommaSeparated = 0x01, // Should this cl::list split between commas? PositionalEatsArgs = 0x02, // Should this positional cl::list eat -args? - Sink = 0x04 // Should this cl::list eat all unknown options? + Sink = 0x04, // Should this cl::list eat all unknown options? + + // Grouping - Can this option group with other options? + // If this is enabled, multiple letter options are allowed to bunch together + // with only a single hyphen for the whole group. This allows emulation + // of the behavior that ls uses for example: ls -la === ls -l -a + Grouping = 0x08, + + // Default option + DefaultOption = 0x10 }; //===----------------------------------------------------------------------===// @@ -261,26 +265,27 @@ class Option { // Out of line virtual function to provide home for the class. virtual void anchor(); - int NumOccurrences = 0; // The number of times specified + uint16_t NumOccurrences; // The number of times specified // Occurrences, HiddenFlag, and Formatting are all enum types but to avoid // problems with signed enums in bitfields. - unsigned Occurrences : 3; // enum NumOccurrencesFlag + uint16_t Occurrences : 3; // enum NumOccurrencesFlag // not using the enum type for 'Value' because zero is an implementation // detail representing the non-value - unsigned Value : 2; - unsigned HiddenFlag : 2; // enum OptionHidden - unsigned Formatting : 3; // enum FormattingFlags - unsigned Misc : 3; - unsigned Position = 0; // Position of last occurrence of the option - unsigned AdditionalVals = 0; // Greater than 0 for multi-valued option. + uint16_t Value : 2; + uint16_t HiddenFlag : 2; // enum OptionHidden + uint16_t Formatting : 2; // enum FormattingFlags + uint16_t Misc : 5; + uint16_t FullyInitialized : 1; // Has addArgument been called? + uint16_t Position; // Position of last occurrence of the option + uint16_t AdditionalVals; // Greater than 0 for multi-valued option. public: StringRef ArgStr; // The argument string itself (ex: "help", "o") StringRef HelpStr; // The descriptive text message for -help StringRef ValueStr; // String describing what the value of this option is - OptionCategory *Category; // The Category this option belongs to - SmallPtrSet Subs; // The subcommands this option belongs to. - bool FullyInitialized = false; // Has addArgument been called? + SmallVector + Categories; // The Categories this option belongs to + SmallPtrSet Subs; // The subcommands this option belongs to. inline enum NumOccurrencesFlag getNumOccurrencesFlag() const { return (enum NumOccurrencesFlag)Occurrences; @@ -306,6 +311,7 @@ public: bool hasArgStr() const { return !ArgStr.empty(); } bool isPositional() const { return getFormattingFlag() == cl::Positional; } bool isSink() const { return getMiscFlags() & cl::Sink; } + bool isDefaultOption() const { return getMiscFlags() & cl::DefaultOption; } bool isConsumeAfter() const { return getNumOccurrencesFlag() == cl::ConsumeAfter; @@ -329,14 +335,17 @@ public: void setFormattingFlag(enum FormattingFlags V) { Formatting = V; } void setMiscFlag(enum MiscFlags M) { Misc |= M; } void setPosition(unsigned pos) { Position = pos; } - void setCategory(OptionCategory &C) { Category = &C; } + void addCategory(OptionCategory &C); void addSubCommand(SubCommand &S) { Subs.insert(&S); } protected: explicit Option(enum NumOccurrencesFlag OccurrencesFlag, enum OptionHidden Hidden) - : Occurrences(OccurrencesFlag), Value(0), HiddenFlag(Hidden), - Formatting(NormalFormatting), Misc(0), Category(&GeneralCategory) {} + : NumOccurrences(0), Occurrences(OccurrencesFlag), Value(0), + HiddenFlag(Hidden), Formatting(NormalFormatting), Misc(0), + FullyInitialized(false), Position(0), AdditionalVals(0) { + Categories.push_back(&GeneralCategory); + } inline void setNumAdditionalVals(unsigned n) { AdditionalVals = n; } @@ -382,7 +391,7 @@ public: } inline int getNumOccurrences() const { return NumOccurrences; } - inline void reset() { NumOccurrences = 0; } + void reset(); }; //===----------------------------------------------------------------------===// @@ -447,7 +456,7 @@ struct cat { cat(OptionCategory &c) : Category(c) {} - template void apply(Opt &O) const { O.setCategory(Category); } + template void apply(Opt &O) const { O.addCategory(Category); } }; // sub - Specify the subcommand that this option belongs to. @@ -823,6 +832,8 @@ class basic_parser_impl { // non-template implementation of basic_parser public: basic_parser_impl(Option &) {} + virtual ~basic_parser_impl() {} + enum ValueExpected getValueExpectedFlagDefault() const { return ValueRequired; } @@ -850,8 +861,6 @@ public: virtual void anchor(); protected: - ~basic_parser_impl() = default; - // A helper for basic_parser::printOptionDiff. void printOptionName(const Option &O, size_t GlobalWidth) const; }; @@ -865,15 +874,12 @@ public: using OptVal = OptionValue; basic_parser(Option &O) : basic_parser_impl(O) {} - -protected: - ~basic_parser() = default; }; //-------------------------------------------------- // parser // -template <> class parser final : public basic_parser { +template <> class parser : public basic_parser { public: parser(Option &O) : basic_parser(O) {} @@ -900,8 +906,7 @@ extern template class basic_parser; //-------------------------------------------------- // parser -template <> -class parser final : public basic_parser { +template <> class parser : public basic_parser { public: parser(Option &O) : basic_parser(O) {} @@ -927,7 +932,7 @@ extern template class basic_parser; //-------------------------------------------------- // parser // -template <> class parser final : public basic_parser { +template <> class parser : public basic_parser { public: parser(Option &O) : basic_parser(O) {} @@ -949,7 +954,7 @@ extern template class basic_parser; //-------------------------------------------------- // parser // -template <> class parser final : public basic_parser { +template <> class parser : public basic_parser { public: parser(Option &O) : basic_parser(O) {} @@ -968,12 +973,34 @@ public: extern template class basic_parser; +//-------------------------------------------------- +// parser +// +template <> +class parser final : public basic_parser { +public: + parser(Option &O) : basic_parser(O) {} + + // parse - Return true on error. + bool parse(Option &O, StringRef ArgName, StringRef Arg, unsigned long &Val); + + // getValueName - Overload in subclass to provide a better default value. + StringRef getValueName() const override { return "ulong"; } + + void printOptionDiff(const Option &O, unsigned long V, OptVal Default, + size_t GlobalWidth) const; + + // An out-of-line virtual method to provide a 'home' for this class. + void anchor() override; +}; + +extern template class basic_parser; + //-------------------------------------------------- // parser // template <> -class parser final - : public basic_parser { +class parser : public basic_parser { public: parser(Option &O) : basic_parser(O) {} @@ -982,7 +1009,7 @@ public: unsigned long long &Val); // getValueName - Overload in subclass to provide a better default value. - StringRef getValueName() const override { return "uint"; } + StringRef getValueName() const override { return "ulong"; } void printOptionDiff(const Option &O, unsigned long long V, OptVal Default, size_t GlobalWidth) const; @@ -996,7 +1023,7 @@ extern template class basic_parser; //-------------------------------------------------- // parser // -template <> class parser final : public basic_parser { +template <> class parser : public basic_parser { public: parser(Option &O) : basic_parser(O) {} @@ -1018,7 +1045,7 @@ extern template class basic_parser; //-------------------------------------------------- // parser // -template <> class parser final : public basic_parser { +template <> class parser : public basic_parser { public: parser(Option &O) : basic_parser(O) {} @@ -1040,7 +1067,7 @@ extern template class basic_parser; //-------------------------------------------------- // parser // -template <> class parser final : public basic_parser { +template <> class parser : public basic_parser { public: parser(Option &O) : basic_parser(O) {} @@ -1065,7 +1092,7 @@ extern template class basic_parser; //-------------------------------------------------- // parser // -template <> class parser final : public basic_parser { +template <> class parser : public basic_parser { public: parser(Option &O) : basic_parser(O) {} @@ -1178,7 +1205,11 @@ template <> struct applicator { }; template <> struct applicator { - static void opt(MiscFlags MF, Option &O) { O.setMiscFlag(MF); } + static void opt(MiscFlags MF, Option &O) { + assert((MF != Grouping || O.ArgStr.size() == 1) && + "cl::Grouping can only apply to single charater Options."); + O.setMiscFlag(MF); + } }; // apply method - Apply modifiers to an option in a type safe way. @@ -1398,6 +1429,8 @@ template class list_storage { public: list_storage() = default; + void clear() {} + bool setLocation(Option &O, StorageClass &L) { if (Location) return O.error("cl::location(x) specified more than once!"); @@ -1449,6 +1482,10 @@ public: reference operator[](size_type pos) { return Storage[pos]; } const_reference operator[](size_type pos) const { return Storage[pos]; } + void clear() { + Storage.clear(); + } + iterator erase(const_iterator pos) { return Storage.erase(pos); } iterator erase(const_iterator first, const_iterator last) { return Storage.erase(first, last); @@ -1526,7 +1563,10 @@ class list : public Option, public list_storage { void printOptionValue(size_t /*GlobalWidth*/, bool /*Force*/) const override { } - void setDefault() override {} + void setDefault() override { + Positions.clear(); + list_storage::clear(); + } void done() { addArgument(); @@ -1732,7 +1772,10 @@ class alias : public Option { error("cl::alias must have argument name specified!"); if (!AliasFor) error("cl::alias must have an cl::aliasopt(option) specified!"); + if (!Subs.empty()) + error("cl::alias must not have cl::sub(), aliased option's cl::sub() will be used!"); Subs = AliasFor->Subs; + Categories = AliasFor->Categories; addArgument(); } diff --git a/include/llvm/Support/Compiler.h b/include/llvm/Support/Compiler.h index 14e4d6e97140..3f4f465f3960 100644 --- a/include/llvm/Support/Compiler.h +++ b/include/llvm/Support/Compiler.h @@ -1,9 +1,8 @@ //===-- llvm/Support/Compiler.h - Compiler abstraction support --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -255,6 +254,15 @@ #define LLVM_FALLTHROUGH #endif +/// LLVM_REQUIRE_CONSTANT_INITIALIZATION - Apply this to globals to ensure that +/// they are constant initialized. +#if __has_cpp_attribute(clang::require_constant_initialization) +#define LLVM_REQUIRE_CONSTANT_INITIALIZATION \ + [[clang::require_constant_initialization]] +#else +#define LLVM_REQUIRE_CONSTANT_INITIALIZATION +#endif + /// LLVM_EXTENSION - Support compilers where we have a keyword to suppress /// pedantic diagnostics. #ifdef __GNUC__ diff --git a/include/llvm/Support/Compression.h b/include/llvm/Support/Compression.h index f7258f4bf8f8..5bc0e56913fe 100644 --- a/include/llvm/Support/Compression.h +++ b/include/llvm/Support/Compression.h @@ -1,9 +1,8 @@ //===-- llvm/Support/Compression.h ---Compression----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/ConvertUTF.h b/include/llvm/Support/ConvertUTF.h index 6ae56c2470bb..1add185330fa 100644 --- a/include/llvm/Support/ConvertUTF.h +++ b/include/llvm/Support/ConvertUTF.h @@ -1,9 +1,8 @@ /*===--- ConvertUTF.h - Universal Character Names conversions ---------------=== * - * The LLVM Compiler Infrastructure - * - * This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * *==------------------------------------------------------------------------==*/ /* diff --git a/include/llvm/Support/CrashRecoveryContext.h b/include/llvm/Support/CrashRecoveryContext.h index 7b3fd4f882e4..feb449e2899c 100644 --- a/include/llvm/Support/CrashRecoveryContext.h +++ b/include/llvm/Support/CrashRecoveryContext.h @@ -1,9 +1,8 @@ //===--- CrashRecoveryContext.h - Crash Recovery ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Support/DJB.h b/include/llvm/Support/DJB.h index e03111473362..8a04a324a5dc 100644 --- a/include/llvm/Support/DJB.h +++ b/include/llvm/Support/DJB.h @@ -1,9 +1,8 @@ //===-- llvm/Support/DJB.h ---DJB Hash --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/DOTGraphTraits.h b/include/llvm/Support/DOTGraphTraits.h index 4381b5bf1633..ec01b7d9576a 100644 --- a/include/llvm/Support/DOTGraphTraits.h +++ b/include/llvm/Support/DOTGraphTraits.h @@ -1,9 +1,8 @@ -//===-- llvm/Support/DotGraphTraits.h - Customize .dot output ---*- C++ -*-===// +//===-- llvm/Support/DOTGraphTraits.h - Customize .dot output ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/DataExtractor.h b/include/llvm/Support/DataExtractor.h index 2b1639856e79..6b08a2a2a445 100644 --- a/include/llvm/Support/DataExtractor.h +++ b/include/llvm/Support/DataExtractor.h @@ -1,9 +1,8 @@ //===-- DataExtractor.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Support/DataTypes.h b/include/llvm/Support/DataTypes.h index ad60a5b3f300..a3fcc82531b7 100644 --- a/include/llvm/Support/DataTypes.h +++ b/include/llvm/Support/DataTypes.h @@ -1,9 +1,8 @@ //===-- llvm/Support/DataTypes.h - Define fixed size types ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/Debug.h b/include/llvm/Support/Debug.h index df86dbb82414..64b730951bda 100644 --- a/include/llvm/Support/Debug.h +++ b/include/llvm/Support/Debug.h @@ -1,9 +1,8 @@ //===- llvm/Support/Debug.h - Easy way to add debug output ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/DebugCounter.h b/include/llvm/Support/DebugCounter.h index 6eadd5c6aeff..e7d1fa68f21a 100644 --- a/include/llvm/Support/DebugCounter.h +++ b/include/llvm/Support/DebugCounter.h @@ -1,9 +1,8 @@ //===- llvm/Support/DebugCounter.h - Debug counter support ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/Support/DynamicLibrary.h b/include/llvm/Support/DynamicLibrary.h index 9563b483f6d5..95d5ba281e22 100644 --- a/include/llvm/Support/DynamicLibrary.h +++ b/include/llvm/Support/DynamicLibrary.h @@ -1,9 +1,8 @@ //===-- llvm/Support/DynamicLibrary.h - Portable Dynamic Library -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/Endian.h b/include/llvm/Support/Endian.h index a4d3f4ff793d..d8be94427d7e 100644 --- a/include/llvm/Support/Endian.h +++ b/include/llvm/Support/Endian.h @@ -1,9 +1,8 @@ //===- Endian.h - Utilities for IO with endian specific data ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -204,10 +203,14 @@ inline void writeAtBitAlignment(void *memory, value_type value, namespace detail { -template +template struct packed_endian_specific_integral { + using value_type = ValueType; + static constexpr endianness endian = Endian; + static constexpr std::size_t alignment = Alignment; + packed_endian_specific_integral() = default; explicit packed_endian_specific_integral(value_type val) { *this = val; } @@ -335,6 +338,17 @@ using unaligned_int32_t = using unaligned_int64_t = detail::packed_endian_specific_integral; +template +using little_t = detail::packed_endian_specific_integral; +template +using big_t = detail::packed_endian_specific_integral; + +template +using aligned_little_t = + detail::packed_endian_specific_integral; +template +using aligned_big_t = detail::packed_endian_specific_integral; + namespace endian { template inline T read(const void *P, endianness E) { diff --git a/include/llvm/Support/EndianStream.h b/include/llvm/Support/EndianStream.h index 9742e253ad3e..87898038d216 100644 --- a/include/llvm/Support/EndianStream.h +++ b/include/llvm/Support/EndianStream.h @@ -1,9 +1,8 @@ //===- EndianStream.h - Stream ops with endian specific data ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/Errc.h b/include/llvm/Support/Errc.h index dce42782a0d3..9be8e5705a54 100644 --- a/include/llvm/Support/Errc.h +++ b/include/llvm/Support/Errc.h @@ -1,9 +1,8 @@ //===- llvm/Support/Errc.h - Defines the llvm::errc enum --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/Errno.h b/include/llvm/Support/Errno.h index 8069c3639df3..aedb5fb292b8 100644 --- a/include/llvm/Support/Errno.h +++ b/include/llvm/Support/Errno.h @@ -1,9 +1,8 @@ //===- llvm/Support/Errno.h - Portable+convenient errno handling -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/Error.h b/include/llvm/Support/Error.h index ee2cbeec97a8..299fce7a1368 100644 --- a/include/llvm/Support/Error.h +++ b/include/llvm/Support/Error.h @@ -1,9 +1,8 @@ //===- llvm/Support/Error.h - Recoverable error handling --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -1161,8 +1160,8 @@ private: /// Create formatted StringError object. template -Error createStringError(std::error_code EC, char const *Fmt, - const Ts &... Vals) { +inline Error createStringError(std::error_code EC, char const *Fmt, + const Ts &... Vals) { std::string Buffer; raw_string_ostream Stream(Buffer); Stream << format(Fmt, Vals...); @@ -1171,18 +1170,27 @@ Error createStringError(std::error_code EC, char const *Fmt, Error createStringError(std::error_code EC, char const *Msg); +template +inline Error createStringError(std::errc EC, char const *Fmt, + const Ts &... Vals) { + return createStringError(std::make_error_code(EC), Fmt, Vals...); +} + /// This class wraps a filename and another Error. /// /// In some cases, an error needs to live along a 'source' name, in order to /// show more detailed information to the user. class FileError final : public ErrorInfo { - friend Error createFileError(std::string, Error); + friend Error createFileError(const Twine &, Error); + friend Error createFileError(const Twine &, size_t, Error); public: void log(raw_ostream &OS) const override { assert(Err && !FileName.empty() && "Trying to log after takeError()."); OS << "'" << FileName << "': "; + if (Line.hasValue()) + OS << "line " << Line.getValue() << ": "; Err->log(OS); } @@ -1194,29 +1202,51 @@ public: static char ID; private: - FileError(std::string F, std::unique_ptr E) { + FileError(const Twine &F, Optional LineNum, + std::unique_ptr E) { assert(E && "Cannot create FileError from Error success value."); - assert(!F.empty() && + assert(!F.isTriviallyEmpty() && "The file name provided to FileError must not be empty."); - FileName = F; + FileName = F.str(); Err = std::move(E); + Line = std::move(LineNum); } - static Error build(std::string F, Error E) { - return Error(std::unique_ptr(new FileError(F, E.takePayload()))); + static Error build(const Twine &F, Optional Line, Error E) { + return Error( + std::unique_ptr(new FileError(F, Line, E.takePayload()))); } std::string FileName; + Optional Line; std::unique_ptr Err; }; /// Concatenate a source file path and/or name with an Error. The resulting /// Error is unchecked. -inline Error createFileError(std::string F, Error E) { - return FileError::build(F, std::move(E)); +inline Error createFileError(const Twine &F, Error E) { + return FileError::build(F, Optional(), std::move(E)); +} + +/// Concatenate a source file path and/or name with line number and an Error. +/// The resulting Error is unchecked. +inline Error createFileError(const Twine &F, size_t Line, Error E) { + return FileError::build(F, Optional(Line), std::move(E)); +} + +/// Concatenate a source file path and/or name with a std::error_code +/// to form an Error object. +inline Error createFileError(const Twine &F, std::error_code EC) { + return createFileError(F, errorCodeToError(EC)); +} + +/// Concatenate a source file path and/or name with line number and +/// std::error_code to form an Error object. +inline Error createFileError(const Twine &F, size_t Line, std::error_code EC) { + return createFileError(F, Line, errorCodeToError(EC)); } -Error createFileError(std::string F, ErrorSuccess) = delete; +Error createFileError(const Twine &F, ErrorSuccess) = delete; /// Helper for check-and-exit error handling. /// diff --git a/include/llvm/Support/ErrorHandling.h b/include/llvm/Support/ErrorHandling.h index fec39e59a717..f75c2984a9ff 100644 --- a/include/llvm/Support/ErrorHandling.h +++ b/include/llvm/Support/ErrorHandling.h @@ -1,9 +1,8 @@ //===- llvm/Support/ErrorHandling.h - Fatal error handling ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/ErrorOr.h b/include/llvm/Support/ErrorOr.h index e6ce764ad822..8211f4d8a098 100644 --- a/include/llvm/Support/ErrorOr.h +++ b/include/llvm/Support/ErrorOr.h @@ -1,9 +1,8 @@ //===- llvm/Support/ErrorOr.h - Error Smart Pointer -------------*- C++ -*-===// // -// The LLVM Linker -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/include/llvm/Support/FileCheck.h b/include/llvm/Support/FileCheck.h index 4061a26e22c5..0cd25a71a3b3 100644 --- a/include/llvm/Support/FileCheck.h +++ b/include/llvm/Support/FileCheck.h @@ -1,9 +1,8 @@ //==-- llvm/Support/FileCheck.h ---------------------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -37,9 +36,218 @@ struct FileCheckRequest { bool VerboseVerbose = false; }; +//===----------------------------------------------------------------------===// +// Numeric substitution handling code. +//===----------------------------------------------------------------------===// + +/// Base class representing the AST of a given expression. +class FileCheckExpressionAST { +public: + virtual ~FileCheckExpressionAST() = default; + + /// Evaluates and \returns the value of the expression represented by this + /// AST or an error if evaluation fails. + virtual Expected eval() const = 0; +}; + +/// Class representing an unsigned literal in the AST of an expression. +class FileCheckExpressionLiteral : public FileCheckExpressionAST { +private: + /// Actual value of the literal. + uint64_t Value; + +public: + /// Constructs a literal with the specified value. + FileCheckExpressionLiteral(uint64_t Val) : Value(Val) {} + + /// \returns the literal's value. + Expected eval() const { return Value; } +}; + +/// Class to represent an undefined variable error, which quotes that +/// variable's name when printed. +class FileCheckUndefVarError : public ErrorInfo { +private: + StringRef VarName; + +public: + static char ID; + + FileCheckUndefVarError(StringRef VarName) : VarName(VarName) {} + + StringRef getVarName() const { return VarName; } + + std::error_code convertToErrorCode() const override { + return inconvertibleErrorCode(); + } + + /// Print name of variable associated with this error. + void log(raw_ostream &OS) const override { + OS << "\""; + OS.write_escaped(VarName) << "\""; + } +}; + +/// Class representing a numeric variable and its associated current value. +class FileCheckNumericVariable { +private: + /// Name of the numeric variable. + StringRef Name; + + /// Value of numeric variable, if defined, or None otherwise. + Optional Value; + + /// Line number where this variable is defined, or None if defined before + /// input is parsed. Used to determine whether a variable is defined on the + /// same line as a given use. + Optional DefLineNumber; + +public: + /// Constructor for a variable \p Name defined at line \p DefLineNumber or + /// defined before input is parsed if DefLineNumber is None. + FileCheckNumericVariable(StringRef Name, + Optional DefLineNumber = None) + : Name(Name), DefLineNumber(DefLineNumber) {} + + /// \returns name of this numeric variable. + StringRef getName() const { return Name; } + + /// \returns this variable's value. + Optional getValue() const { return Value; } + + /// Sets value of this numeric variable, if undefined. Triggers an assertion + /// failure if the variable is actually defined. + void setValue(uint64_t Value); + + /// Clears value of this numeric variable, regardless of whether it is + /// currently defined or not. + void clearValue(); + + /// \returns the line number where this variable is defined, if any, or None + /// if defined before input is parsed. + Optional getDefLineNumber() { return DefLineNumber; } +}; + +/// Class representing the use of a numeric variable in the AST of an +/// expression. +class FileCheckNumericVariableUse : public FileCheckExpressionAST { +private: + /// Name of the numeric variable. + StringRef Name; + + /// Pointer to the class instance for the variable this use is about. + FileCheckNumericVariable *NumericVariable; + +public: + FileCheckNumericVariableUse(StringRef Name, + FileCheckNumericVariable *NumericVariable) + : Name(Name), NumericVariable(NumericVariable) {} + + /// \returns the value of the variable referenced by this instance. + Expected eval() const; +}; + +/// Type of functions evaluating a given binary operation. +using binop_eval_t = uint64_t (*)(uint64_t, uint64_t); + +/// Class representing a single binary operation in the AST of an expression. +class FileCheckASTBinop : public FileCheckExpressionAST { +private: + /// Left operand. + std::unique_ptr LeftOperand; + + /// Right operand. + std::unique_ptr RightOperand; + + /// Pointer to function that can evaluate this binary operation. + binop_eval_t EvalBinop; + +public: + FileCheckASTBinop(binop_eval_t EvalBinop, + std::unique_ptr LeftOp, + std::unique_ptr RightOp) + : EvalBinop(EvalBinop) { + LeftOperand = std::move(LeftOp); + RightOperand = std::move(RightOp); + } + + /// Evaluates the value of the binary operation represented by this AST, + /// using EvalBinop on the result of recursively evaluating the operands. + /// \returns the expression value or an error if an undefined numeric + /// variable is used in one of the operands. + Expected eval() const; +}; + +class FileCheckPatternContext; + +/// Class representing a substitution to perform in the RegExStr string. +class FileCheckSubstitution { +protected: + /// Pointer to a class instance holding, among other things, the table with + /// the values of live string variables at the start of any given CHECK line. + /// Used for substituting string variables with the text they were defined + /// as. Expressions are linked to the numeric variables they use at + /// parse time and directly access the value of the numeric variable to + /// evaluate their value. + FileCheckPatternContext *Context; + + /// The string that needs to be substituted for something else. For a + /// string variable this is its name, otherwise this is the whole expression. + StringRef FromStr; + + // Index in RegExStr of where to do the substitution. + size_t InsertIdx; + +public: + FileCheckSubstitution(FileCheckPatternContext *Context, StringRef VarName, + size_t InsertIdx) + : Context(Context), FromStr(VarName), InsertIdx(InsertIdx) {} + + virtual ~FileCheckSubstitution() = default; + + /// \returns the string to be substituted for something else. + StringRef getFromString() const { return FromStr; } + + /// \returns the index where the substitution is to be performed in RegExStr. + size_t getIndex() const { return InsertIdx; } + + /// \returns a string containing the result of the substitution represented + /// by this class instance or an error if substitution failed. + virtual Expected getResult() const = 0; +}; + +class FileCheckStringSubstitution : public FileCheckSubstitution { +public: + FileCheckStringSubstitution(FileCheckPatternContext *Context, + StringRef VarName, size_t InsertIdx) + : FileCheckSubstitution(Context, VarName, InsertIdx) {} + + /// \returns the text that the string variable in this substitution matched + /// when defined, or an error if the variable is undefined. + Expected getResult() const override; +}; + +class FileCheckNumericSubstitution : public FileCheckSubstitution { +private: + /// Pointer to the class representing the expression whose value is to be + /// substituted. + std::unique_ptr ExpressionAST; + +public: + FileCheckNumericSubstitution(FileCheckPatternContext *Context, StringRef Expr, + std::unique_ptr ExprAST, + size_t InsertIdx) + : FileCheckSubstitution(Context, Expr, InsertIdx) { + ExpressionAST = std::move(ExprAST); + } + + /// \returns a string containing the result of evaluating the expression in + /// this substitution, or an error if evaluation failed. + Expected getResult() const override; +}; //===----------------------------------------------------------------------===// -// Pattern Handling Code. +// Pattern handling code. //===----------------------------------------------------------------------===// namespace Check { @@ -78,12 +286,133 @@ public: int getCount() const { return Count; } FileCheckType &setCount(int C); + // \returns a description of \p Prefix. std::string getDescription(StringRef Prefix) const; }; -} +} // namespace Check struct FileCheckDiag; +/// Class holding the FileCheckPattern global state, shared by all patterns: +/// tables holding values of variables and whether they are defined or not at +/// any given time in the matching process. +class FileCheckPatternContext { + friend class FileCheckPattern; + +private: + /// When matching a given pattern, this holds the value of all the string + /// variables defined in previous patterns. In a pattern, only the last + /// definition for a given variable is recorded in this table. + /// Back-references are used for uses after any the other definition. + StringMap GlobalVariableTable; + + /// Map of all string variables defined so far. Used at parse time to detect + /// a name conflict between a numeric variable and a string variable when + /// the former is defined on a later line than the latter. + StringMap DefinedVariableTable; + + /// When matching a given pattern, this holds the pointers to the classes + /// representing the numeric variables defined in previous patterns. When + /// matching a pattern all definitions for that pattern are recorded in the + /// NumericVariableDefs table in the FileCheckPattern instance of that + /// pattern. + StringMap GlobalNumericVariableTable; + + /// Pointer to the class instance representing the @LINE pseudo variable for + /// easily updating its value. + FileCheckNumericVariable *LineVariable = nullptr; + + /// Vector holding pointers to all parsed numeric variables. Used to + /// automatically free them once they are guaranteed to no longer be used. + std::vector> NumericVariables; + + /// Vector holding pointers to all substitutions. Used to automatically free + /// them once they are guaranteed to no longer be used. + std::vector> Substitutions; + +public: + /// \returns the value of string variable \p VarName or an error if no such + /// variable has been defined. + Expected getPatternVarValue(StringRef VarName); + + /// Defines string and numeric variables from definitions given on the + /// command line, passed as a vector of [#]VAR=VAL strings in + /// \p CmdlineDefines. \returns an error list containing diagnostics against + /// \p SM for all definition parsing failures, if any, or Success otherwise. + Error defineCmdlineVariables(std::vector &CmdlineDefines, + SourceMgr &SM); + + /// Create @LINE pseudo variable. Value is set when pattern are being + /// matched. + void createLineVariable(); + + /// Undefines local variables (variables whose name does not start with a '$' + /// sign), i.e. removes them from GlobalVariableTable and from + /// GlobalNumericVariableTable and also clears the value of numeric + /// variables. + void clearLocalVars(); + +private: + /// Makes a new numeric variable and registers it for destruction when the + /// context is destroyed. + template + FileCheckNumericVariable *makeNumericVariable(Types... args); + + /// Makes a new string substitution and registers it for destruction when the + /// context is destroyed. + FileCheckSubstitution *makeStringSubstitution(StringRef VarName, + size_t InsertIdx); + + /// Makes a new numeric substitution and registers it for destruction when + /// the context is destroyed. + FileCheckSubstitution * + makeNumericSubstitution(StringRef ExpressionStr, + std::unique_ptr ExpressionAST, + size_t InsertIdx); +}; + +/// Class to represent an error holding a diagnostic with location information +/// used when printing it. +class FileCheckErrorDiagnostic : public ErrorInfo { +private: + SMDiagnostic Diagnostic; + +public: + static char ID; + + FileCheckErrorDiagnostic(SMDiagnostic &&Diag) : Diagnostic(Diag) {} + + std::error_code convertToErrorCode() const override { + return inconvertibleErrorCode(); + } + + /// Print diagnostic associated with this error when printing the error. + void log(raw_ostream &OS) const override { Diagnostic.print(nullptr, OS); } + + static Error get(const SourceMgr &SM, SMLoc Loc, const Twine &ErrMsg) { + return make_error( + SM.GetMessage(Loc, SourceMgr::DK_Error, ErrMsg)); + } + + static Error get(const SourceMgr &SM, StringRef Buffer, const Twine &ErrMsg) { + return get(SM, SMLoc::getFromPointer(Buffer.data()), ErrMsg); + } +}; + +class FileCheckNotFoundError : public ErrorInfo { +public: + static char ID; + + std::error_code convertToErrorCode() const override { + return inconvertibleErrorCode(); + } + + /// Print diagnostic associated with this error when printing the error. + void log(raw_ostream &OS) const override { + OS << "String not found in input"; + } +}; + class FileCheckPattern { SMLoc PatternLoc; @@ -95,43 +424,143 @@ class FileCheckPattern { /// a fixed string to match. std::string RegExStr; - /// Entries in this vector map to uses of a variable in the pattern, e.g. - /// "foo[[bar]]baz". In this case, the RegExStr will contain "foobaz" and - /// we'll get an entry in this vector that tells us to insert the value of - /// bar at offset 3. - std::vector> VariableUses; + /// Entries in this vector represent a substitution of a string variable or + /// an expression in the RegExStr regex at match time. For example, in the + /// case of a CHECK directive with the pattern "foo[[bar]]baz[[#N+1]]", + /// RegExStr will contain "foobaz" and we'll get two entries in this vector + /// that tells us to insert the value of string variable "bar" at offset 3 + /// and the value of expression "N+1" at offset 6. + std::vector Substitutions; - /// Maps definitions of variables to their parenthesized capture numbers. + /// Maps names of string variables defined in a pattern to the number of + /// their parenthesis group in RegExStr capturing their last definition. + /// + /// E.g. for the pattern "foo[[bar:.*]]baz([[bar]][[QUUX]][[bar:.*]])", + /// RegExStr will be "foo(.*)baz(\1(.*))" where is + /// the value captured for QUUX on the earlier line where it was defined, and + /// VariableDefs will map "bar" to the third parenthesis group which captures + /// the second definition of "bar". /// - /// E.g. for the pattern "foo[[bar:.*]]baz", VariableDefs will map "bar" to - /// 1. + /// Note: uses std::map rather than StringMap to be able to get the key when + /// iterating over values. std::map VariableDefs; + /// Structure representing the definition of a numeric variable in a pattern. + /// It holds the pointer to the class representing the numeric variable whose + /// value is being defined and the number of the parenthesis group in + /// RegExStr to capture that value. + struct FileCheckNumericVariableMatch { + /// Pointer to class representing the numeric variable whose value is being + /// defined. + FileCheckNumericVariable *DefinedNumericVariable; + + /// Number of the parenthesis group in RegExStr that captures the value of + /// this numeric variable definition. + unsigned CaptureParenGroup; + }; + + /// Holds the number of the parenthesis group in RegExStr and pointer to the + /// corresponding FileCheckNumericVariable class instance of all numeric + /// variable definitions. Used to set the matched value of all those + /// variables. + StringMap NumericVariableDefs; + + /// Pointer to a class instance holding the global state shared by all + /// patterns: + /// - separate tables with the values of live string and numeric variables + /// respectively at the start of any given CHECK line; + /// - table holding whether a string variable has been defined at any given + /// point during the parsing phase. + FileCheckPatternContext *Context; + Check::FileCheckType CheckTy; - /// Contains the number of line this pattern is in. - unsigned LineNumber; + /// Line number for this CHECK pattern or None if it is an implicit pattern. + /// Used to determine whether a variable definition is made on an earlier + /// line to the one with this CHECK. + Optional LineNumber; public: - explicit FileCheckPattern(Check::FileCheckType Ty) - : CheckTy(Ty) {} + FileCheckPattern(Check::FileCheckType Ty, FileCheckPatternContext *Context, + Optional Line = None) + : Context(Context), CheckTy(Ty), LineNumber(Line) {} - /// Returns the location in source code. + /// \returns the location in source code. SMLoc getLoc() const { return PatternLoc; } - bool ParsePattern(StringRef PatternStr, StringRef Prefix, SourceMgr &SM, - unsigned LineNumber, const FileCheckRequest &Req); - size_t Match(StringRef Buffer, size_t &MatchLen, - StringMap &VariableTable) const; - void PrintVariableUses(const SourceMgr &SM, StringRef Buffer, - const StringMap &VariableTable, - SMRange MatchRange = None) const; - void PrintFuzzyMatch(const SourceMgr &SM, StringRef Buffer, - const StringMap &VariableTable, + /// \returns the pointer to the global state for all patterns in this + /// FileCheck instance. + FileCheckPatternContext *getContext() const { return Context; } + + /// \returns whether \p C is a valid first character for a variable name. + static bool isValidVarNameStart(char C); + + /// Parsing information about a variable. + struct VariableProperties { + StringRef Name; + bool IsPseudo; + }; + + /// Parses the string at the start of \p Str for a variable name. \returns + /// a VariableProperties structure holding the variable name and whether it + /// is the name of a pseudo variable, or an error holding a diagnostic + /// against \p SM if parsing fail. If parsing was successful, also strips + /// \p Str from the variable name. + static Expected parseVariable(StringRef &Str, + const SourceMgr &SM); + /// Parses \p Expr for the name of a numeric variable to be defined at line + /// \p LineNumber or before input is parsed if \p LineNumber is None. + /// \returns a pointer to the class instance representing that variable, + /// creating it if needed, or an error holding a diagnostic against \p SM + /// should defining such a variable be invalid. + static Expected parseNumericVariableDefinition( + StringRef &Expr, FileCheckPatternContext *Context, + Optional LineNumber, const SourceMgr &SM); + /// Parses \p Expr for a numeric substitution block. Parameter + /// \p IsLegacyLineExpr indicates whether \p Expr should be a legacy @LINE + /// expression. \returns a pointer to the class instance representing the AST + /// of the expression whose value must be substituted, or an error holding a + /// diagnostic against \p SM if parsing fails. If substitution was + /// successful, sets \p DefinedNumericVariable to point to the class + /// representing the numeric variable being defined in this numeric + /// substitution block, or None if this block does not define any variable. + Expected> + parseNumericSubstitutionBlock( + StringRef Expr, + Optional &DefinedNumericVariable, + bool IsLegacyLineExpr, const SourceMgr &SM) const; + /// Parses the pattern in \p PatternStr and initializes this FileCheckPattern + /// instance accordingly. + /// + /// \p Prefix provides which prefix is being matched, \p Req describes the + /// global options that influence the parsing such as whitespace + /// canonicalization, \p SM provides the SourceMgr used for error reports. + /// \returns true in case of an error, false otherwise. + bool parsePattern(StringRef PatternStr, StringRef Prefix, SourceMgr &SM, + const FileCheckRequest &Req); + /// Matches the pattern string against the input buffer \p Buffer + /// + /// \returns the position that is matched or an error indicating why matching + /// failed. If there is a match, updates \p MatchLen with the size of the + /// matched string. + /// + /// The GlobalVariableTable StringMap in the FileCheckPatternContext class + /// instance provides the current values of FileCheck string variables and + /// is updated if this match defines new values. Likewise, the + /// GlobalNumericVariableTable StringMap in the same class provides the + /// current values of FileCheck numeric variables and is updated if this + /// match defines new numeric values. + Expected match(StringRef Buffer, size_t &MatchLen, + const SourceMgr &SM) const; + /// Prints the value of successful substitutions or the name of the undefined + /// string or numeric variables preventing a successful substitution. + void printSubstitutions(const SourceMgr &SM, StringRef Buffer, + SMRange MatchRange = None) const; + void printFuzzyMatch(const SourceMgr &SM, StringRef Buffer, std::vector *Diags) const; bool hasVariable() const { - return !(VariableUses.empty() && VariableDefs.empty()); + return !(Substitutions.empty() && VariableDefs.empty()); } Check::FileCheckType getCheckTy() const { return CheckTy; } @@ -141,11 +570,40 @@ public: private: bool AddRegExToRegEx(StringRef RS, unsigned &CurParen, SourceMgr &SM); void AddBackrefToRegEx(unsigned BackrefNum); - unsigned - ComputeMatchDistance(StringRef Buffer, - const StringMap &VariableTable) const; - bool EvaluateExpression(StringRef Expr, std::string &Value) const; + /// Computes an arbitrary estimate for the quality of matching this pattern + /// at the start of \p Buffer; a distance of zero should correspond to a + /// perfect match. + unsigned computeMatchDistance(StringRef Buffer) const; + /// Finds the closing sequence of a regex variable usage or definition. + /// + /// \p Str has to point in the beginning of the definition (right after the + /// opening sequence). \p SM holds the SourceMgr used for error repporting. + /// \returns the offset of the closing sequence within Str, or npos if it + /// was not found. size_t FindRegexVarEnd(StringRef Str, SourceMgr &SM); + + /// Parses \p Name as a (pseudo if \p IsPseudo is true) numeric variable use. + /// \returns the pointer to the class instance representing that variable if + /// successful, or an error holding a diagnostic against \p SM otherwise. + Expected> + parseNumericVariableUse(StringRef Name, bool IsPseudo, + const SourceMgr &SM) const; + enum class AllowedOperand { LineVar, Literal, Any }; + /// Parses \p Expr for use of a numeric operand. Accepts both literal values + /// and numeric variables, depending on the value of \p AO. \returns the + /// class representing that operand in the AST of the expression or an error + /// holding a diagnostic against \p SM otherwise. + Expected> + parseNumericOperand(StringRef &Expr, AllowedOperand AO, + const SourceMgr &SM) const; + /// Parses \p Expr for a binary operation. The left operand of this binary + /// operation is given in \p LeftOp and \p IsLegacyLineExpr indicates whether + /// we are parsing a legacy @LINE expression. \returns the class representing + /// the binary operation in the AST of the expression, or an error holding a + /// diagnostic against \p SM otherwise. + Expected> + parseBinop(StringRef &Expr, std::unique_ptr LeftOp, + bool IsLegacyLineExpr, const SourceMgr &SM) const; }; //===----------------------------------------------------------------------===// @@ -223,20 +681,27 @@ struct FileCheckString { FileCheckString(const FileCheckPattern &P, StringRef S, SMLoc L) : Pat(P), Prefix(S), Loc(L) {} + /// Matches check string and its "not strings" and/or "dag strings". size_t Check(const SourceMgr &SM, StringRef Buffer, bool IsLabelScanMode, - size_t &MatchLen, StringMap &VariableTable, - FileCheckRequest &Req, std::vector *Diags) const; + size_t &MatchLen, FileCheckRequest &Req, + std::vector *Diags) const; + /// Verifies that there is a single line in the given \p Buffer. Errors are + /// reported against \p SM. bool CheckNext(const SourceMgr &SM, StringRef Buffer) const; + /// Verifies that there is no newline in the given \p Buffer. Errors are + /// reported against \p SM. bool CheckSame(const SourceMgr &SM, StringRef Buffer) const; + /// Verifies that none of the strings in \p NotStrings are found in the given + /// \p Buffer. Errors are reported against \p SM and diagnostics recorded in + /// \p Diags according to the verbosity level set in \p Req. bool CheckNot(const SourceMgr &SM, StringRef Buffer, const std::vector &NotStrings, - StringMap &VariableTable, const FileCheckRequest &Req, std::vector *Diags) const; + /// Matches "dag strings" and their mixed "not strings". size_t CheckDag(const SourceMgr &SM, StringRef Buffer, std::vector &NotStrings, - StringMap &VariableTable, const FileCheckRequest &Req, std::vector *Diags) const; }; @@ -245,6 +710,7 @@ struct FileCheckString { /// use information from the request. class FileCheck { FileCheckRequest Req; + FileCheckPatternContext PatternContext; public: FileCheck(FileCheckRequest Req) : Req(Req) {} @@ -256,24 +722,27 @@ public: // library. Regex buildCheckPrefixRegex(); - /// Read the check file, which specifies the sequence of expected strings. + /// Reads the check file from \p Buffer and records the expected strings it + /// contains in the \p CheckStrings vector. Errors are reported against + /// \p SM. /// - /// The strings are added to the CheckStrings vector. Returns true in case of - /// an error, false otherwise. + /// Only expected strings whose prefix is one of those listed in \p PrefixRE + /// are recorded. \returns true in case of an error, false otherwise. bool ReadCheckFile(SourceMgr &SM, StringRef Buffer, Regex &PrefixRE, std::vector &CheckStrings); bool ValidateCheckPrefixes(); - /// Canonicalize whitespaces in the file. Line endings are replaced with + /// Canonicalizes whitespaces in the file. Line endings are replaced with /// UNIX-style '\n'. StringRef CanonicalizeFile(MemoryBuffer &MB, SmallVectorImpl &OutputBuffer); - /// Check the input to FileCheck provided in the \p Buffer against the \p - /// CheckStrings read from the check file. + /// Checks the input to FileCheck provided in the \p Buffer against the + /// \p CheckStrings read from the check file and record diagnostics emitted + /// in \p Diags. Errors are recorded against \p SM. /// - /// Returns false if the input fails to satisfy the checks. + /// \returns false if the input fails to satisfy the checks. bool CheckInput(SourceMgr &SM, StringRef Buffer, ArrayRef CheckStrings, std::vector *Diags = nullptr); diff --git a/include/llvm/Support/FileOutputBuffer.h b/include/llvm/Support/FileOutputBuffer.h index 68226ca55502..999f551ebf2d 100644 --- a/include/llvm/Support/FileOutputBuffer.h +++ b/include/llvm/Support/FileOutputBuffer.h @@ -1,9 +1,8 @@ //=== FileOutputBuffer.h - File Output Buffer -------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -33,11 +32,6 @@ public: enum { /// set the 'x' bit on the resulting file F_executable = 1, - - /// the contents of the new file are initialized from the file that exists - /// at the location (if present). This allows in-place modification of an - /// existing file. - F_modify = 2 }; /// Factory method to create an OutputBuffer object which manages a read/write diff --git a/include/llvm/Support/FileSystem.h b/include/llvm/Support/FileSystem.h index d2042f51d8c1..1bec27bddad9 100644 --- a/include/llvm/Support/FileSystem.h +++ b/include/llvm/Support/FileSystem.h @@ -1,9 +1,8 @@ //===- llvm/Support/FileSystem.h - File System OS Concept -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -649,6 +648,19 @@ std::error_code status(const Twine &path, file_status &result, /// A version for when a file descriptor is already available. std::error_code status(int FD, file_status &Result); +#ifdef _WIN32 +/// A version for when a file descriptor is already available. +std::error_code status(file_t FD, file_status &Result); +#endif + +/// Get file creation mode mask of the process. +/// +/// @returns Mask reported by umask(2) +/// @note There is no umask on Windows. This function returns 0 always +/// on Windows. This function does not return an error_code because +/// umask(2) never fails. It is not thread safe. +unsigned getUmask(); + /// Set file permissions. /// /// @param Path File to set permissions on. @@ -660,6 +672,11 @@ std::error_code status(int FD, file_status &Result); /// Otherwise, the file will be marked as read-only. std::error_code setPermissions(const Twine &Path, perms Permissions); +/// Vesion of setPermissions accepting a file descriptor. +/// TODO Delete the path based overload once we implement the FD based overload +/// on Windows. +std::error_code setPermissions(int FD, perms Permissions); + /// Get file permissions. /// /// @param Path File to get permissions from. @@ -765,11 +782,32 @@ enum OpenFlags : unsigned { OF_UpdateAtime = 16, }; +/// Create a potentially unique file name but does not create it. +/// +/// Generates a unique path suitable for a temporary file but does not +/// open or create the file. The name is based on \a Model with '%' +/// replaced by a random char in [0-9a-f]. If \a MakeAbsolute is true +/// then the system's temp directory is prepended first. If \a MakeAbsolute +/// is false the current directory will be used instead. +/// +/// This function does not check if the file exists. If you want to be sure +/// that the file does not yet exist, you should use use enough '%' characters +/// in your model to ensure this. Each '%' gives 4-bits of entropy so you can +/// use 32 of them to get 128 bits of entropy. +/// +/// Example: clang-%%-%%-%%-%%-%%.s => clang-a0-b1-c2-d3-e4.s +/// +/// @param Model Name to base unique path off of. +/// @param ResultPath Set to the file's path. +/// @param MakeAbsolute Whether to use the system temp directory. +void createUniquePath(const Twine &Model, SmallVectorImpl &ResultPath, + bool MakeAbsolute); + /// Create a uniquely named file. /// /// Generates a unique path suitable for a temporary file and then opens it as a -/// file. The name is based on \a model with '%' replaced by a random char in -/// [0-9a-f]. If \a model is not an absolute path, the temporary file will be +/// file. The name is based on \a Model with '%' replaced by a random char in +/// [0-9a-f]. If \a Model is not an absolute path, the temporary file will be /// created in the current directory. /// /// Example: clang-%%-%%-%%-%%-%%.s => clang-a0-b1-c2-d3-e4.s @@ -932,6 +970,51 @@ Expected openNativeFile(const Twine &Name, CreationDisposition Disp, FileAccess Access, OpenFlags Flags, unsigned Mode = 0666); +/// Converts from a Posix file descriptor number to a native file handle. +/// On Windows, this retreives the underlying handle. On non-Windows, this is a +/// no-op. +file_t convertFDToNativeFile(int FD); + +#ifndef _WIN32 +inline file_t convertFDToNativeFile(int FD) { return FD; } +#endif + +/// Return an open handle to standard in. On Unix, this is typically FD 0. +/// Returns kInvalidFile when the stream is closed. +file_t getStdinHandle(); + +/// Return an open handle to standard out. On Unix, this is typically FD 1. +/// Returns kInvalidFile when the stream is closed. +file_t getStdoutHandle(); + +/// Return an open handle to standard error. On Unix, this is typically FD 2. +/// Returns kInvalidFile when the stream is closed. +file_t getStderrHandle(); + +/// Reads \p Buf.size() bytes from \p FileHandle into \p Buf. The number of +/// bytes actually read is returned in \p BytesRead. On Unix, this is equivalent +/// to `*BytesRead = ::read(FD, Buf.data(), Buf.size())`, with error reporting. +/// BytesRead will contain zero when reaching EOF. +/// +/// @param FileHandle File to read from. +/// @param Buf Buffer to read into. +/// @param BytesRead Output parameter of the number of bytes read. +/// @returns The error, if any, or errc::success. +std::error_code readNativeFile(file_t FileHandle, MutableArrayRef Buf, + size_t *BytesRead); + +/// Reads \p Buf.size() bytes from \p FileHandle at offset \p Offset into \p +/// Buf. If 'pread' is available, this will use that, otherwise it will use +/// 'lseek'. Bytes requested beyond the end of the file will be zero +/// initialized. +/// +/// @param FileHandle File to read from. +/// @param Buf Buffer to read into. +/// @param Offset Offset into the file at which the read should occur. +/// @returns The error, if any, or errc::success. +std::error_code readNativeFileSlice(file_t FileHandle, + MutableArrayRef Buf, size_t Offset); + /// @brief Opens the file with the given name in a write-only or read-write /// mode, returning its open file descriptor. If the file does not exist, it /// is created. @@ -1051,11 +1134,15 @@ openNativeFileForRead(const Twine &Name, OpenFlags Flags = OF_None, SmallVectorImpl *RealPath = nullptr); /// @brief Close the file object. This should be used instead of ::close for -/// portability. +/// portability. On error, the caller should assume the file is closed, as is +/// the case for Process::SafelyCloseFileDescriptor /// /// @param F On input, this is the file to close. On output, the file is /// set to kInvalidFile. -void closeFile(file_t &F); +/// +/// @returns An error code if closing the file failed. Typically, an error here +/// means that the filesystem may have failed to perform some buffered writes. +std::error_code closeFile(file_t &F); std::error_code getUniqueID(const Twine Path, UniqueID &Result); @@ -1085,21 +1172,19 @@ private: size_t Size; void *Mapping; #ifdef _WIN32 - void *FileHandle; + sys::fs::file_t FileHandle; #endif mapmode Mode; - std::error_code init(int FD, uint64_t Offset, mapmode Mode); + std::error_code init(sys::fs::file_t FD, uint64_t Offset, mapmode Mode); public: mapped_file_region() = delete; mapped_file_region(mapped_file_region&) = delete; mapped_file_region &operator =(mapped_file_region&) = delete; - /// \param fd An open file descriptor to map. mapped_file_region takes - /// ownership if closefd is true. It must have been opended in the correct - /// mode. - mapped_file_region(int fd, mapmode mode, size_t length, uint64_t offset, + /// \param fd An open file descriptor to map. Does not take ownership of fd. + mapped_file_region(sys::fs::file_t fd, mapmode mode, size_t length, uint64_t offset, std::error_code &ec); ~mapped_file_region(); diff --git a/include/llvm/Support/FileUtilities.h b/include/llvm/Support/FileUtilities.h index 2ee2c60b9964..16b2206924c3 100644 --- a/include/llvm/Support/FileUtilities.h +++ b/include/llvm/Support/FileUtilities.h @@ -1,9 +1,8 @@ //===- llvm/Support/FileUtilities.h - File System Utilities -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/Format.h b/include/llvm/Support/Format.h index bcbd2bec5722..77dcbaebf1a3 100644 --- a/include/llvm/Support/Format.h +++ b/include/llvm/Support/Format.h @@ -1,9 +1,8 @@ //===- Format.h - Efficient printf-style formatting for streams -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/FormatAdapters.h b/include/llvm/Support/FormatAdapters.h index 8320eaad39a9..a0e8cc439191 100644 --- a/include/llvm/Support/FormatAdapters.h +++ b/include/llvm/Support/FormatAdapters.h @@ -1,9 +1,8 @@ //===- FormatAdapters.h - Formatters for common LLVM types -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Support/FormatCommon.h b/include/llvm/Support/FormatCommon.h index 36fbad296c3f..3c119d12529a 100644 --- a/include/llvm/Support/FormatCommon.h +++ b/include/llvm/Support/FormatCommon.h @@ -1,9 +1,8 @@ -//===- FormatAdapters.h - Formatters for common LLVM types -----*- C++ -*-===// +//===- FormatCommon.h - Formatters for common LLVM types --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Support/FormatProviders.h b/include/llvm/Support/FormatProviders.h index 4e57034ff98e..629a4845716a 100644 --- a/include/llvm/Support/FormatProviders.h +++ b/include/llvm/Support/FormatProviders.h @@ -1,9 +1,8 @@ //===- FormatProviders.h - Formatters for common LLVM types -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/FormatVariadic.h b/include/llvm/Support/FormatVariadic.h index b0f582513e07..5bbda9dd626e 100644 --- a/include/llvm/Support/FormatVariadic.h +++ b/include/llvm/Support/FormatVariadic.h @@ -1,9 +1,8 @@ //===- FormatVariadic.h - Efficient type-safe string formatting --*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/FormatVariadicDetails.h b/include/llvm/Support/FormatVariadicDetails.h index e8bd90f50941..e3c185134daa 100644 --- a/include/llvm/Support/FormatVariadicDetails.h +++ b/include/llvm/Support/FormatVariadicDetails.h @@ -1,9 +1,8 @@ //===- FormatVariadicDetails.h - Helpers for FormatVariadic.h ----*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Support/FormattedStream.h b/include/llvm/Support/FormattedStream.h index 4a135cd23174..b49c8d86531d 100644 --- a/include/llvm/Support/FormattedStream.h +++ b/include/llvm/Support/FormattedStream.h @@ -1,9 +1,8 @@ //===-- llvm/Support/FormattedStream.h - Formatted streams ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/GenericDomTree.h b/include/llvm/Support/GenericDomTree.h index b3018bac310a..99620802505b 100644 --- a/include/llvm/Support/GenericDomTree.h +++ b/include/llvm/Support/GenericDomTree.h @@ -1,9 +1,8 @@ //===- GenericDomTree.h - Generic dominator trees for graphs ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -670,14 +669,12 @@ protected: // The postdom tree can have a null root if there are no returns. if (getRootNode()) PrintDomTree(getRootNode(), O, 1); - if (IsPostDominator) { - O << "Roots: "; - for (const NodePtr Block : Roots) { - Block->printAsOperand(O, false); - O << " "; - } - O << "\n"; + O << "Roots: "; + for (const NodePtr Block : Roots) { + Block->printAsOperand(O, false); + O << " "; } + O << "\n"; } public: diff --git a/include/llvm/Support/GenericDomTreeConstruction.h b/include/llvm/Support/GenericDomTreeConstruction.h index 971e8305a112..ccceba881718 100644 --- a/include/llvm/Support/GenericDomTreeConstruction.h +++ b/include/llvm/Support/GenericDomTreeConstruction.h @@ -1,9 +1,8 @@ //===- GenericDomTreeConstruction.h - Dominator Calculation ------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -16,9 +15,12 @@ /// Loukas Georgiadis, Princeton University, November 2005, pp. 21-23: /// ftp://ftp.cs.princeton.edu/reports/2005/737.pdf /// -/// This implements the O(n*log(n)) versions of EVAL and LINK, because it turns -/// out that the theoretically slower O(n*log(n)) implementation is actually -/// faster than the almost-linear O(n*alpha(n)) version, even for large CFGs. +/// Semi-NCA algorithm runs in O(n^2) worst-case time but usually slightly +/// faster than Simple Lengauer-Tarjan in practice. +/// +/// O(n^2) worst cases happen when the computation of nearest common ancestors +/// requires O(n) average time, which is very unlikely in real world. If this +/// ever turns out to be an issue, consider implementing a hybrid algorithm. /// /// The file uses the Depth Based Search algorithm to perform incremental /// updates (insertion and deletions). The implemented algorithm is based on @@ -255,42 +257,47 @@ struct SemiNCAInfo { return LastNum; } - NodePtr eval(NodePtr VIn, unsigned LastLinked) { - auto &VInInfo = NodeToInfo[VIn]; - if (VInInfo.DFSNum < LastLinked) - return VIn; - - SmallVector Work; - SmallPtrSet Visited; - - if (VInInfo.Parent >= LastLinked) - Work.push_back(VIn); - - while (!Work.empty()) { - NodePtr V = Work.back(); - auto &VInfo = NodeToInfo[V]; - NodePtr VAncestor = NumToNode[VInfo.Parent]; - - // Process Ancestor first - if (Visited.insert(VAncestor).second && VInfo.Parent >= LastLinked) { - Work.push_back(VAncestor); - continue; - } - Work.pop_back(); - - // Update VInfo based on Ancestor info - if (VInfo.Parent < LastLinked) - continue; - - auto &VAInfo = NodeToInfo[VAncestor]; - NodePtr VAncestorLabel = VAInfo.Label; - NodePtr VLabel = VInfo.Label; - if (NodeToInfo[VAncestorLabel].Semi < NodeToInfo[VLabel].Semi) - VInfo.Label = VAncestorLabel; - VInfo.Parent = VAInfo.Parent; - } - - return VInInfo.Label; + // V is a predecessor of W. eval() returns V if V < W, otherwise the minimum + // of sdom(U), where U > W and there is a virtual forest path from U to V. The + // virtual forest consists of linked edges of processed vertices. + // + // We can follow Parent pointers (virtual forest edges) to determine the + // ancestor U with minimum sdom(U). But it is slow and thus we employ the path + // compression technique to speed up to O(m*log(n)). Theoretically the virtual + // forest can be organized as balanced trees to achieve almost linear + // O(m*alpha(m,n)) running time. But it requires two auxiliary arrays (Size + // and Child) and is unlikely to be faster than the simple implementation. + // + // For each vertex V, its Label points to the vertex with the minimal sdom(U) + // (Semi) in its path from V (included) to NodeToInfo[V].Parent (excluded). + NodePtr eval(NodePtr V, unsigned LastLinked, + SmallVectorImpl &Stack) { + InfoRec *VInfo = &NodeToInfo[V]; + if (VInfo->Parent < LastLinked) + return VInfo->Label; + + // Store ancestors except the last (root of a virtual tree) into a stack. + assert(Stack.empty()); + do { + Stack.push_back(VInfo); + VInfo = &NodeToInfo[NumToNode[VInfo->Parent]]; + } while (VInfo->Parent >= LastLinked); + + // Path compression. Point each vertex's Parent to the root and update its + // Label if any of its ancestors (PInfo->Label) has a smaller Semi. + const InfoRec *PInfo = VInfo; + const InfoRec *PLabelInfo = &NodeToInfo[PInfo->Label]; + do { + VInfo = Stack.pop_back_val(); + VInfo->Parent = PInfo->Parent; + const InfoRec *VLabelInfo = &NodeToInfo[VInfo->Label]; + if (PLabelInfo->Semi < VLabelInfo->Semi) + VInfo->Label = PInfo->Label; + else + PLabelInfo = VLabelInfo; + PInfo = VInfo; + } while (!Stack.empty()); + return VInfo->Label; } // This function requires DFS to be run before calling it. @@ -304,6 +311,7 @@ struct SemiNCAInfo { } // Step #1: Calculate the semidominators of all vertices. + SmallVector EvalStack; for (unsigned i = NextDFSNum - 1; i >= 2; --i) { NodePtr W = NumToNode[i]; auto &WInfo = NodeToInfo[W]; @@ -319,7 +327,7 @@ struct SemiNCAInfo { if (TN && TN->getLevel() < MinLevel) continue; - unsigned SemiU = NodeToInfo[eval(N, i + 1)].Semi; + unsigned SemiU = NodeToInfo[eval(N, i + 1, EvalStack)].Semi; if (SemiU < WInfo.Semi) WInfo.Semi = SemiU; } } @@ -620,21 +628,22 @@ struct SemiNCAInfo { // Helper struct used during edge insertions. struct InsertionInfo { - using BucketElementTy = std::pair; - struct DecreasingLevel { - bool operator()(const BucketElementTy &First, - const BucketElementTy &Second) const { - return First.first > Second.first; + struct Compare { + bool operator()(TreeNodePtr LHS, TreeNodePtr RHS) const { + return LHS->getLevel() < RHS->getLevel(); } }; - std::priority_queue, - DecreasingLevel> - Bucket; // Queue of tree nodes sorted by level in descending order. - SmallDenseSet Affected; - SmallDenseMap Visited; - SmallVector AffectedQueue; - SmallVector VisitedNotAffectedQueue; + // Bucket queue of tree nodes ordered by descending level. For simplicity, + // we use a priority_queue here. + std::priority_queue, + Compare> + Bucket; + SmallDenseSet Visited; + SmallVector Affected; +#ifndef NDEBUG + SmallVector VisitedUnaffected; +#endif }; static void InsertEdge(DomTreeT &DT, const BatchUpdatePtr BUI, @@ -689,6 +698,17 @@ struct SemiNCAInfo { return true; } + static bool isPermutation(const SmallVectorImpl &A, + const SmallVectorImpl &B) { + if (A.size() != B.size()) + return false; + SmallPtrSet Set(A.begin(), A.end()); + for (NodePtr N : B) + if (Set.count(N) == 0) + return false; + return true; + } + // Updates the set of roots after insertion or deletion. This ensures that // roots are the same when after a series of updates and when the tree would // be built from scratch. @@ -702,9 +722,8 @@ struct SemiNCAInfo { return; // Recalculate the set of roots. - auto Roots = FindRoots(DT, BUI); - if (DT.Roots.size() != Roots.size() || - !std::is_permutation(DT.Roots.begin(), DT.Roots.end(), Roots.begin())) { + RootsT Roots = FindRoots(DT, BUI); + if (!isPermutation(DT.Roots, Roots)) { // The roots chosen in the CFG have changed. This is because the // incremental algorithm does not really know or use the set of roots and // can make a different (implicit) decision about which node within an @@ -715,7 +734,6 @@ struct SemiNCAInfo { // It may be possible to update the tree without recalculating it, but // we do not know yet how to do it, and it happens rarely in practise. CalculateFromScratch(DT, BUI); - return; } } @@ -737,128 +755,113 @@ struct SemiNCAInfo { assert(NCD); LLVM_DEBUG(dbgs() << "\t\tNCA == " << BlockNamePrinter(NCD) << "\n"); - const TreeNodePtr ToIDom = To->getIDom(); + const unsigned NCDLevel = NCD->getLevel(); - // Nothing affected -- NCA property holds. - // (Based on the lemma 2.5 from the second paper.) - if (NCD == To || NCD == ToIDom) return; + // Based on Lemma 2.5 from the second paper, after insertion of (From,To), v + // is affected iff depth(NCD)+1 < depth(v) && a path P from To to v exists + // where every w on P s.t. depth(v) <= depth(w) + // + // This reduces to a widest path problem (maximizing the depth of the + // minimum vertex in the path) which can be solved by a modified version of + // Dijkstra with a bucket queue (named depth-based search in the paper). + + // To is in the path, so depth(NCD)+1 < depth(v) <= depth(To). Nothing + // affected if this does not hold. + if (NCDLevel + 1 >= To->getLevel()) + return; - // Identify and collect affected nodes. InsertionInfo II; - LLVM_DEBUG(dbgs() << "Marking " << BlockNamePrinter(To) - << " as affected\n"); - II.Affected.insert(To); - const unsigned ToLevel = To->getLevel(); - LLVM_DEBUG(dbgs() << "Putting " << BlockNamePrinter(To) - << " into a Bucket\n"); - II.Bucket.push({ToLevel, To}); + SmallVector UnaffectedOnCurrentLevel; + II.Bucket.push(To); + II.Visited.insert(To); while (!II.Bucket.empty()) { - const TreeNodePtr CurrentNode = II.Bucket.top().second; - const unsigned CurrentLevel = CurrentNode->getLevel(); + TreeNodePtr TN = II.Bucket.top(); II.Bucket.pop(); - LLVM_DEBUG(dbgs() << "\tAdding to Visited and AffectedQueue: " - << BlockNamePrinter(CurrentNode) << "\n"); - - II.Visited.insert({CurrentNode, CurrentLevel}); - II.AffectedQueue.push_back(CurrentNode); + II.Affected.push_back(TN); + + const unsigned CurrentLevel = TN->getLevel(); + LLVM_DEBUG(dbgs() << "Mark " << BlockNamePrinter(TN) << + "as affected, CurrentLevel " << CurrentLevel << "\n"); + + assert(TN->getBlock() && II.Visited.count(TN) && "Preconditions!"); + + while (true) { + // Unlike regular Dijkstra, we have an inner loop to expand more + // vertices. The first iteration is for the (affected) vertex popped + // from II.Bucket and the rest are for vertices in + // UnaffectedOnCurrentLevel, which may eventually expand to affected + // vertices. + // + // Invariant: there is an optimal path from `To` to TN with the minimum + // depth being CurrentLevel. + for (const NodePtr Succ : + ChildrenGetter::Get(TN->getBlock(), BUI)) { + const TreeNodePtr SuccTN = DT.getNode(Succ); + assert(SuccTN && + "Unreachable successor found at reachable insertion"); + const unsigned SuccLevel = SuccTN->getLevel(); + + LLVM_DEBUG(dbgs() << "\tSuccessor " << BlockNamePrinter(Succ) + << ", level = " << SuccLevel << "\n"); + + // There is an optimal path from `To` to Succ with the minimum depth + // being min(CurrentLevel, SuccLevel). + // + // If depth(NCD)+1 < depth(Succ) is not satisfied, Succ is unaffected + // and no affected vertex may be reached by a path passing through it. + // Stop here. Also, Succ may be visited by other predecessors but the + // first visit has the optimal path. Stop if Succ has been visited. + if (SuccLevel <= NCDLevel + 1 || !II.Visited.insert(SuccTN).second) + continue; + + if (SuccLevel > CurrentLevel) { + // Succ is unaffected but it may (transitively) expand to affected + // vertices. Store it in UnaffectedOnCurrentLevel. + LLVM_DEBUG(dbgs() << "\t\tMarking visited not affected " + << BlockNamePrinter(Succ) << "\n"); + UnaffectedOnCurrentLevel.push_back(SuccTN); +#ifndef NDEBUG + II.VisitedUnaffected.push_back(SuccTN); +#endif + } else { + // The condition is satisfied (Succ is affected). Add Succ to the + // bucket queue. + LLVM_DEBUG(dbgs() << "\t\tAdd " << BlockNamePrinter(Succ) + << " to a Bucket\n"); + II.Bucket.push(SuccTN); + } + } - // Discover and collect affected successors of the current node. - VisitInsertion(DT, BUI, CurrentNode, CurrentLevel, NCD, II); + if (UnaffectedOnCurrentLevel.empty()) + break; + TN = UnaffectedOnCurrentLevel.pop_back_val(); + LLVM_DEBUG(dbgs() << " Next: " << BlockNamePrinter(TN) << "\n"); + } } // Finish by updating immediate dominators and levels. UpdateInsertion(DT, BUI, NCD, II); } - // Visits an affected node and collect its affected successors. - static void VisitInsertion(DomTreeT &DT, const BatchUpdatePtr BUI, - const TreeNodePtr TN, const unsigned RootLevel, - const TreeNodePtr NCD, InsertionInfo &II) { - const unsigned NCDLevel = NCD->getLevel(); - LLVM_DEBUG(dbgs() << "Visiting " << BlockNamePrinter(TN) << ", RootLevel " - << RootLevel << "\n"); - - SmallVector Stack = {TN}; - assert(TN->getBlock() && II.Visited.count(TN) && "Preconditions!"); - - SmallPtrSet Processed; - - do { - TreeNodePtr Next = Stack.pop_back_val(); - LLVM_DEBUG(dbgs() << " Next: " << BlockNamePrinter(Next) << "\n"); - - for (const NodePtr Succ : - ChildrenGetter::Get(Next->getBlock(), BUI)) { - const TreeNodePtr SuccTN = DT.getNode(Succ); - assert(SuccTN && "Unreachable successor found at reachable insertion"); - const unsigned SuccLevel = SuccTN->getLevel(); - - LLVM_DEBUG(dbgs() << "\tSuccessor " << BlockNamePrinter(Succ) - << ", level = " << SuccLevel << "\n"); - - // Do not process the same node multiple times. - if (Processed.count(Next) > 0) - continue; - - // Succ dominated by subtree From -- not affected. - // (Based on the lemma 2.5 from the second paper.) - if (SuccLevel > RootLevel) { - LLVM_DEBUG(dbgs() << "\t\tDominated by subtree From\n"); - if (II.Visited.count(SuccTN) != 0) { - LLVM_DEBUG(dbgs() << "\t\t\talready visited at level " - << II.Visited[SuccTN] << "\n\t\t\tcurrent level " - << RootLevel << ")\n"); - - // A node can be necessary to visit again if we see it again at - // a lower level than before. - if (II.Visited[SuccTN] >= RootLevel) - continue; - } - - LLVM_DEBUG(dbgs() << "\t\tMarking visited not affected " - << BlockNamePrinter(Succ) << "\n"); - II.Visited.insert({SuccTN, RootLevel}); - II.VisitedNotAffectedQueue.push_back(SuccTN); - Stack.push_back(SuccTN); - } else if ((SuccLevel > NCDLevel + 1) && - II.Affected.count(SuccTN) == 0) { - LLVM_DEBUG(dbgs() << "\t\tMarking affected and adding " - << BlockNamePrinter(Succ) << " to a Bucket\n"); - II.Affected.insert(SuccTN); - II.Bucket.push({SuccLevel, SuccTN}); - } - } - - Processed.insert(Next); - } while (!Stack.empty()); - } - // Updates immediate dominators and levels after insertion. static void UpdateInsertion(DomTreeT &DT, const BatchUpdatePtr BUI, const TreeNodePtr NCD, InsertionInfo &II) { LLVM_DEBUG(dbgs() << "Updating NCD = " << BlockNamePrinter(NCD) << "\n"); - for (const TreeNodePtr TN : II.AffectedQueue) { + for (const TreeNodePtr TN : II.Affected) { LLVM_DEBUG(dbgs() << "\tIDom(" << BlockNamePrinter(TN) << ") = " << BlockNamePrinter(NCD) << "\n"); TN->setIDom(NCD); } - UpdateLevelsAfterInsertion(II); - if (IsPostDom) UpdateRootsAfterUpdate(DT, BUI); - } - - static void UpdateLevelsAfterInsertion(InsertionInfo &II) { - LLVM_DEBUG( - dbgs() << "Updating levels for visited but not affected nodes\n"); +#ifndef NDEBUG + for (const TreeNodePtr TN : II.VisitedUnaffected) + assert(TN->getLevel() == TN->getIDom()->getLevel() + 1 && + "TN should have been updated by an affected ancestor"); +#endif - for (const TreeNodePtr TN : II.VisitedNotAffectedQueue) { - LLVM_DEBUG(dbgs() << "\tlevel(" << BlockNamePrinter(TN) << ") = (" - << BlockNamePrinter(TN->getIDom()) << ") " - << TN->getIDom()->getLevel() << " + 1\n"); - TN->UpdateLevel(); - } + if (IsPostDom) UpdateRootsAfterUpdate(DT, BUI); } // Handles insertion to previously unreachable nodes. @@ -1182,6 +1185,10 @@ struct SemiNCAInfo { BUI.FuturePredecessors[U.getTo()].push_back({U.getFrom(), U.getKind()}); } +#if 0 + // FIXME: The LLVM_DEBUG macro only plays well with a modular + // build of LLVM when the header is marked as textual, but doing + // so causes redefinition errors. LLVM_DEBUG(dbgs() << "About to apply " << NumLegalized << " updates\n"); LLVM_DEBUG(if (NumLegalized < 32) for (const auto &U : reverse(BUI.Updates)) { @@ -1190,6 +1197,7 @@ struct SemiNCAInfo { dbgs() << "\n"; }); LLVM_DEBUG(dbgs() << "\n"); +#endif // Recalculate the DominatorTree when the number of updates // exceeds a threshold, which usually makes direct updating slower than @@ -1215,8 +1223,13 @@ struct SemiNCAInfo { static void ApplyNextUpdate(DomTreeT &DT, BatchUpdateInfo &BUI) { assert(!BUI.Updates.empty() && "No updates to apply!"); UpdateT CurrentUpdate = BUI.Updates.pop_back_val(); +#if 0 + // FIXME: The LLVM_DEBUG macro only plays well with a modular + // build of LLVM when the header is marked as textual, but doing + // so causes redefinition errors. LLVM_DEBUG(dbgs() << "Applying update: "); LLVM_DEBUG(CurrentUpdate.dump(); dbgs() << "\n"); +#endif // Move to the next snapshot of the CFG by removing the reverse-applied // current update. Since updates are performed in the same order they are @@ -1270,9 +1283,7 @@ struct SemiNCAInfo { } RootsT ComputedRoots = FindRoots(DT, nullptr); - if (DT.Roots.size() != ComputedRoots.size() || - !std::is_permutation(DT.Roots.begin(), DT.Roots.end(), - ComputedRoots.begin())) { + if (!isPermutation(DT.Roots, ComputedRoots)) { errs() << "Tree has different roots than freshly computed ones!\n"; errs() << "\tPDT roots: "; for (const NodePtr N : DT.Roots) errs() << BlockNamePrinter(N) << ", "; diff --git a/include/llvm/Support/GenericIteratedDominanceFrontier.h b/include/llvm/Support/GenericIteratedDominanceFrontier.h new file mode 100644 index 000000000000..25eb7cd7b6d5 --- /dev/null +++ b/include/llvm/Support/GenericIteratedDominanceFrontier.h @@ -0,0 +1,209 @@ +//===- IteratedDominanceFrontier.h - Calculate IDF --------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// Compute iterated dominance frontiers using a linear time algorithm. +/// +/// The algorithm used here is based on: +/// +/// Sreedhar and Gao. A linear time algorithm for placing phi-nodes. +/// In Proceedings of the 22nd ACM SIGPLAN-SIGACT Symposium on Principles of +/// Programming Languages +/// POPL '95. ACM, New York, NY, 62-73. +/// +/// It has been modified to not explicitly use the DJ graph data structure and +/// to directly compute pruned SSA using per-variable liveness information. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_GENERIC_IDF_H +#define LLVM_SUPPORT_GENERIC_IDF_H + +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/Support/GenericDomTree.h" +#include + +namespace llvm { + +namespace IDFCalculatorDetail { + +/// Generic utility class used for getting the children of a basic block. +/// May be specialized if, for example, one wouldn't like to return nullpointer +/// successors. +template struct ChildrenGetterTy { + using NodeRef = typename GraphTraits::NodeRef; + using ChildrenTy = SmallVector; + + ChildrenTy get(const NodeRef &N); +}; + +} // end of namespace IDFCalculatorDetail + +/// Determine the iterated dominance frontier, given a set of defining +/// blocks, and optionally, a set of live-in blocks. +/// +/// In turn, the results can be used to place phi nodes. +/// +/// This algorithm is a linear time computation of Iterated Dominance Frontiers, +/// pruned using the live-in set. +/// By default, liveness is not used to prune the IDF computation. +/// The template parameters should be of a CFG block type. +template class IDFCalculatorBase { +public: + using OrderedNodeTy = + typename std::conditional, NodeTy *>::type; + using ChildrenGetterTy = + IDFCalculatorDetail::ChildrenGetterTy; + + IDFCalculatorBase(DominatorTreeBase &DT) : DT(DT) {} + + IDFCalculatorBase(DominatorTreeBase &DT, + const ChildrenGetterTy &C) + : DT(DT), ChildrenGetter(C) {} + + /// Give the IDF calculator the set of blocks in which the value is + /// defined. This is equivalent to the set of starting blocks it should be + /// calculating the IDF for (though later gets pruned based on liveness). + /// + /// Note: This set *must* live for the entire lifetime of the IDF calculator. + void setDefiningBlocks(const SmallPtrSetImpl &Blocks) { + DefBlocks = &Blocks; + } + + /// Give the IDF calculator the set of blocks in which the value is + /// live on entry to the block. This is used to prune the IDF calculation to + /// not include blocks where any phi insertion would be dead. + /// + /// Note: This set *must* live for the entire lifetime of the IDF calculator. + void setLiveInBlocks(const SmallPtrSetImpl &Blocks) { + LiveInBlocks = &Blocks; + useLiveIn = true; + } + + /// Reset the live-in block set to be empty, and tell the IDF + /// calculator to not use liveness anymore. + void resetLiveInBlocks() { + LiveInBlocks = nullptr; + useLiveIn = false; + } + + /// Calculate iterated dominance frontiers + /// + /// This uses the linear-time phi algorithm based on DJ-graphs mentioned in + /// the file-level comment. It performs DF->IDF pruning using the live-in + /// set, to avoid computing the IDF for blocks where an inserted PHI node + /// would be dead. + void calculate(SmallVectorImpl &IDFBlocks); + +private: + DominatorTreeBase &DT; + ChildrenGetterTy ChildrenGetter; + bool useLiveIn = false; + const SmallPtrSetImpl *LiveInBlocks; + const SmallPtrSetImpl *DefBlocks; +}; + +//===----------------------------------------------------------------------===// +// Implementation. +//===----------------------------------------------------------------------===// + +namespace IDFCalculatorDetail { + +template +typename ChildrenGetterTy::ChildrenTy +ChildrenGetterTy::get(const NodeRef &N) { + using OrderedNodeTy = + typename IDFCalculatorBase::OrderedNodeTy; + + auto Children = children(N); + return {Children.begin(), Children.end()}; +} + +} // end of namespace IDFCalculatorDetail + +template +void IDFCalculatorBase::calculate( + SmallVectorImpl &PHIBlocks) { + // Use a priority queue keyed on dominator tree level so that inserted nodes + // are handled from the bottom of the dominator tree upwards. We also augment + // the level with a DFS number to ensure that the blocks are ordered in a + // deterministic way. + using DomTreeNodePair = + std::pair *, std::pair>; + using IDFPriorityQueue = + std::priority_queue, + less_second>; + + IDFPriorityQueue PQ; + + DT.updateDFSNumbers(); + + for (NodeTy *BB : *DefBlocks) { + if (DomTreeNodeBase *Node = DT.getNode(BB)) + PQ.push({Node, std::make_pair(Node->getLevel(), Node->getDFSNumIn())}); + } + + SmallVector *, 32> Worklist; + SmallPtrSet *, 32> VisitedPQ; + SmallPtrSet *, 32> VisitedWorklist; + + while (!PQ.empty()) { + DomTreeNodePair RootPair = PQ.top(); + PQ.pop(); + DomTreeNodeBase *Root = RootPair.first; + unsigned RootLevel = RootPair.second.first; + + // Walk all dominator tree children of Root, inspecting their CFG edges with + // targets elsewhere on the dominator tree. Only targets whose level is at + // most Root's level are added to the iterated dominance frontier of the + // definition set. + + Worklist.clear(); + Worklist.push_back(Root); + VisitedWorklist.insert(Root); + + while (!Worklist.empty()) { + DomTreeNodeBase *Node = Worklist.pop_back_val(); + NodeTy *BB = Node->getBlock(); + // Succ is the successor in the direction we are calculating IDF, so it is + // successor for IDF, and predecessor for Reverse IDF. + auto DoWork = [&](NodeTy *Succ) { + DomTreeNodeBase *SuccNode = DT.getNode(Succ); + + const unsigned SuccLevel = SuccNode->getLevel(); + if (SuccLevel > RootLevel) + return; + + if (!VisitedPQ.insert(SuccNode).second) + return; + + NodeTy *SuccBB = SuccNode->getBlock(); + if (useLiveIn && !LiveInBlocks->count(SuccBB)) + return; + + PHIBlocks.emplace_back(SuccBB); + if (!DefBlocks->count(SuccBB)) + PQ.push(std::make_pair( + SuccNode, std::make_pair(SuccLevel, SuccNode->getDFSNumIn()))); + }; + + for (auto Succ : ChildrenGetter.get(BB)) + DoWork(Succ); + + for (auto DomChild : *Node) { + if (VisitedWorklist.insert(DomChild).second) + Worklist.push_back(DomChild); + } + } + } +} + +} // end of namespace llvm + +#endif diff --git a/include/llvm/Support/GlobPattern.h b/include/llvm/Support/GlobPattern.h index c9436a13c1a3..66a4cd94c12a 100644 --- a/include/llvm/Support/GlobPattern.h +++ b/include/llvm/Support/GlobPattern.h @@ -1,9 +1,8 @@ //===-- GlobPattern.h - glob pattern matcher implementation -*- C++ -*-----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/GraphWriter.h b/include/llvm/Support/GraphWriter.h index 02d98bec16e2..466a0449e257 100644 --- a/include/llvm/Support/GraphWriter.h +++ b/include/llvm/Support/GraphWriter.h @@ -1,9 +1,8 @@ //===- llvm/Support/GraphWriter.h - Write graph to a .dot file --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/Host.h b/include/llvm/Support/Host.h index 57c79c0b9fdf..b37cc514c92e 100644 --- a/include/llvm/Support/Host.h +++ b/include/llvm/Support/Host.h @@ -1,9 +1,8 @@ //===- llvm/Support/Host.h - Host machine characteristics --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/InitLLVM.h b/include/llvm/Support/InitLLVM.h index 0f629c9ac92d..8069859a3e0b 100644 --- a/include/llvm/Support/InitLLVM.h +++ b/include/llvm/Support/InitLLVM.h @@ -1,9 +1,8 @@ //===- InitLLVM.h -----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -20,7 +19,10 @@ // 1. Setting up a signal handler so that pretty stack trace is printed out // if a process crashes. // -// 2. If running on Windows, obtain command line arguments using a +// 2. Set up the global new-handler which is called when a memory allocation +// attempt fails. +// +// 3. If running on Windows, obtain command line arguments using a // multibyte character-aware API and convert arguments into UTF-8 // encoding, so that you can assume that command line arguments are // always encoded in UTF-8 on any platform. diff --git a/include/llvm/Support/ItaniumManglingCanonicalizer.h b/include/llvm/Support/ItaniumManglingCanonicalizer.h index 34eb9f7deaaf..6920000340d4 100644 --- a/include/llvm/Support/ItaniumManglingCanonicalizer.h +++ b/include/llvm/Support/ItaniumManglingCanonicalizer.h @@ -1,9 +1,8 @@ //===--- ItaniumManglingCanonicalizer.h -------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/JSON.h b/include/llvm/Support/JSON.h index 7a04fd52bc50..0ca41097dddd 100644 --- a/include/llvm/Support/JSON.h +++ b/include/llvm/Support/JSON.h @@ -1,9 +1,8 @@ //===--- JSON.h - JSON values, parsing and serialization -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===---------------------------------------------------------------------===// /// @@ -22,6 +21,9 @@ /// - a convention and helpers for mapping between json::Value and user-defined /// types. See fromJSON(), ObjectMapper, and the class comment on Value. /// +/// - an output API json::OStream which can emit JSON without materializing +/// all structures as json::Value. +/// /// Typically, JSON data would be read from an external source, parsed into /// a Value, and then converted into some native data structure before doing /// real work on it. (And vice versa when writing). @@ -37,7 +39,7 @@ /// /// - LLVM bitstream is a space- and CPU- efficient binary format. Typically it /// encodes LLVM IR ("bitcode"), but it can be a container for other data. -/// Low-level reader/writer libraries are in Bitcode/Bitstream*.h +/// Low-level reader/writer libraries are in Bitstream/Bitstream*.h /// //===---------------------------------------------------------------------===// @@ -96,7 +98,7 @@ public: using iterator = Storage::iterator; using const_iterator = Storage::const_iterator; - explicit Object() = default; + Object() = default; // KV is a trivial key-value struct for list-initialization. // (using std::pair forces extra copies). struct KV; @@ -157,7 +159,7 @@ public: using iterator = std::vector::iterator; using const_iterator = std::vector::const_iterator; - explicit Array() = default; + Array() = default; explicit Array(std::initializer_list Elements); template explicit Array(const Collection &C) { for (const auto &V : C) @@ -180,6 +182,7 @@ public: bool empty() const { return V.empty(); } size_t size() const { return V.size(); } + void reserve(size_t S) { V.reserve(S); } void clear() { V.clear(); } void push_back(const Value &E) { V.push_back(E); } @@ -310,8 +313,8 @@ public: create(std::move(V)); } Value(const llvm::SmallVectorImpl &V) - : Value(std::string(V.begin(), V.end())){}; - Value(const llvm::formatv_object_base &V) : Value(V.str()){}; + : Value(std::string(V.begin(), V.end())) {} + Value(const llvm::formatv_object_base &V) : Value(V.str()) {} // Strings: types with reference semantics. Must be valid UTF-8. Value(StringRef V) : Type(T_StringRef) { create(V); @@ -437,11 +440,6 @@ public: return LLVM_LIKELY(Type == T_Array) ? &as() : nullptr; } - /// Serializes this Value to JSON, writing it to the provided stream. - /// The formatting is compact (no extra whitespace) and deterministic. - /// For pretty-printing, use the formatv() format_provider below. - friend llvm::raw_ostream &operator<<(llvm::raw_ostream &, const Value &); - private: void destroy(); void copyFrom(const Value &M); @@ -462,9 +460,7 @@ private: return *static_cast(Storage); } - template - void print(llvm::raw_ostream &, const Indenter &) const; - friend struct llvm::format_provider; + friend class OStream; enum ValueType : char { T_Null, @@ -481,11 +477,11 @@ private: mutable llvm::AlignedCharArrayUnion Union; + friend bool operator==(const Value &, const Value &); }; bool operator==(const Value &, const Value &); inline bool operator!=(const Value &L, const Value &R) { return !(L == R); } -llvm::raw_ostream &operator<<(llvm::raw_ostream &, const Value &); /// ObjectKey is a used to capture keys in Object. Like Value but: /// - only strings are allowed @@ -698,6 +694,154 @@ public: return llvm::inconvertibleErrorCode(); } }; + +/// json::OStream allows writing well-formed JSON without materializing +/// all structures as json::Value ahead of time. +/// It's faster, lower-level, and less safe than OS << json::Value. +/// +/// Only one "top-level" object can be written to a stream. +/// Simplest usage involves passing lambdas (Blocks) to fill in containers: +/// +/// json::OStream J(OS); +/// J.array([&]{ +/// for (const Event &E : Events) +/// J.object([&] { +/// J.attribute("timestamp", int64_t(E.Time)); +/// J.attributeArray("participants", [&] { +/// for (const Participant &P : E.Participants) +/// J.string(P.toString()); +/// }); +/// }); +/// }); +/// +/// This would produce JSON like: +/// +/// [ +/// { +/// "timestamp": 19287398741, +/// "participants": [ +/// "King Kong", +/// "Miley Cyrus", +/// "Cleopatra" +/// ] +/// }, +/// ... +/// ] +/// +/// The lower level begin/end methods (arrayBegin()) are more flexible but +/// care must be taken to pair them correctly: +/// +/// json::OStream J(OS); +// J.arrayBegin(); +/// for (const Event &E : Events) { +/// J.objectBegin(); +/// J.attribute("timestamp", int64_t(E.Time)); +/// J.attributeBegin("participants"); +/// for (const Participant &P : E.Participants) +/// J.value(P.toString()); +/// J.attributeEnd(); +/// J.objectEnd(); +/// } +/// J.arrayEnd(); +/// +/// If the call sequence isn't valid JSON, asserts will fire in debug mode. +/// This can be mismatched begin()/end() pairs, trying to emit attributes inside +/// an array, and so on. +/// With asserts disabled, this is undefined behavior. +class OStream { + public: + using Block = llvm::function_ref; + // If IndentSize is nonzero, output is pretty-printed. + explicit OStream(llvm::raw_ostream &OS, unsigned IndentSize = 0) + : OS(OS), IndentSize(IndentSize) { + Stack.emplace_back(); + } + ~OStream() { + assert(Stack.size() == 1 && "Unmatched begin()/end()"); + assert(Stack.back().Ctx == Singleton); + assert(Stack.back().HasValue && "Did not write top-level value"); + } + + /// Flushes the underlying ostream. OStream does not buffer internally. + void flush() { OS.flush(); } + + // High level functions to output a value. + // Valid at top-level (exactly once), in an attribute value (exactly once), + // or in an array (any number of times). + + /// Emit a self-contained value (number, string, vector etc). + void value(const Value &V); + /// Emit an array whose elements are emitted in the provided Block. + void array(Block Contents) { + arrayBegin(); + Contents(); + arrayEnd(); + } + /// Emit an object whose elements are emitted in the provided Block. + void object(Block Contents) { + objectBegin(); + Contents(); + objectEnd(); + } + + // High level functions to output object attributes. + // Valid only within an object (any number of times). + + /// Emit an attribute whose value is self-contained (number, vector etc). + void attribute(llvm::StringRef Key, const Value& Contents) { + attributeImpl(Key, [&] { value(Contents); }); + } + /// Emit an attribute whose value is an array with elements from the Block. + void attributeArray(llvm::StringRef Key, Block Contents) { + attributeImpl(Key, [&] { array(Contents); }); + } + /// Emit an attribute whose value is an object with attributes from the Block. + void attributeObject(llvm::StringRef Key, Block Contents) { + attributeImpl(Key, [&] { object(Contents); }); + } + + // Low-level begin/end functions to output arrays, objects, and attributes. + // Must be correctly paired. Allowed contexts are as above. + + void arrayBegin(); + void arrayEnd(); + void objectBegin(); + void objectEnd(); + void attributeBegin(llvm::StringRef Key); + void attributeEnd(); + + private: + void attributeImpl(llvm::StringRef Key, Block Contents) { + attributeBegin(Key); + Contents(); + attributeEnd(); + } + + void valueBegin(); + void newline(); + + enum Context { + Singleton, // Top level, or object attribute. + Array, + Object, + }; + struct State { + Context Ctx = Singleton; + bool HasValue = false; + }; + llvm::SmallVector Stack; // Never empty. + llvm::raw_ostream &OS; + unsigned IndentSize; + unsigned Indent = 0; +}; + +/// Serializes this Value to JSON, writing it to the provided stream. +/// The formatting is compact (no extra whitespace) and deterministic. +/// For pretty-printing, use the formatv() format_provider below. +inline llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Value &V) { + OStream(OS).value(V); + return OS; +} } // namespace json /// Allow printing json::Value with formatv(). diff --git a/include/llvm/Support/JamCRC.h b/include/llvm/Support/JamCRC.h index 846d6cea9828..b6fc4e7b9b03 100644 --- a/include/llvm/Support/JamCRC.h +++ b/include/llvm/Support/JamCRC.h @@ -1,9 +1,8 @@ //===-- llvm/Support/JamCRC.h - Cyclic Redundancy Check ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/KnownBits.h b/include/llvm/Support/KnownBits.h index 259df9546c57..07fd94e29a1f 100644 --- a/include/llvm/Support/KnownBits.h +++ b/include/llvm/Support/KnownBits.h @@ -1,9 +1,8 @@ //===- llvm/Support/KnownBits.h - Stores known zeros/ones -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -110,25 +109,36 @@ public: /// Truncate the underlying known Zero and One bits. This is equivalent /// to truncating the value we're tracking. - KnownBits trunc(unsigned BitWidth) { + KnownBits trunc(unsigned BitWidth) const { return KnownBits(Zero.trunc(BitWidth), One.trunc(BitWidth)); } - /// Zero extends the underlying known Zero and One bits. This is equivalent - /// to zero extending the value we're tracking. - KnownBits zext(unsigned BitWidth) { - return KnownBits(Zero.zext(BitWidth), One.zext(BitWidth)); + /// Extends the underlying known Zero and One bits. + /// By setting ExtendedBitsAreKnownZero=true this will be equivalent to + /// zero extending the value we're tracking. + /// With ExtendedBitsAreKnownZero=false the extended bits are set to unknown. + KnownBits zext(unsigned BitWidth, bool ExtendedBitsAreKnownZero) const { + unsigned OldBitWidth = getBitWidth(); + APInt NewZero = Zero.zext(BitWidth); + if (ExtendedBitsAreKnownZero) + NewZero.setBitsFrom(OldBitWidth); + return KnownBits(NewZero, One.zext(BitWidth)); } /// Sign extends the underlying known Zero and One bits. This is equivalent /// to sign extending the value we're tracking. - KnownBits sext(unsigned BitWidth) { + KnownBits sext(unsigned BitWidth) const { return KnownBits(Zero.sext(BitWidth), One.sext(BitWidth)); } - /// Zero extends or truncates the underlying known Zero and One bits. This is - /// equivalent to zero extending or truncating the value we're tracking. - KnownBits zextOrTrunc(unsigned BitWidth) { + /// Extends or truncates the underlying known Zero and One bits. When + /// extending the extended bits can either be set as known zero (if + /// ExtendedBitsAreKnownZero=true) or as unknown (if + /// ExtendedBitsAreKnownZero=false). + KnownBits zextOrTrunc(unsigned BitWidth, + bool ExtendedBitsAreKnownZero) const { + if (BitWidth > getBitWidth()) + return zext(BitWidth, ExtendedBitsAreKnownZero); return KnownBits(Zero.zextOrTrunc(BitWidth), One.zextOrTrunc(BitWidth)); } @@ -192,6 +202,10 @@ public: return getBitWidth() - Zero.countPopulation(); } + /// Compute known bits resulting from adding LHS, RHS and a 1-bit Carry. + static KnownBits computeForAddCarry( + const KnownBits &LHS, const KnownBits &RHS, const KnownBits &Carry); + /// Compute known bits resulting from adding LHS and RHS. static KnownBits computeForAddSub(bool Add, bool NSW, const KnownBits &LHS, KnownBits RHS); diff --git a/include/llvm/Support/LEB128.h b/include/llvm/Support/LEB128.h index 9feb07229225..a02b83ca9597 100644 --- a/include/llvm/Support/LEB128.h +++ b/include/llvm/Support/LEB128.h @@ -1,9 +1,8 @@ //===- llvm/Support/LEB128.h - [SU]LEB128 utility functions -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -166,6 +165,8 @@ inline int64_t decodeSLEB128(const uint8_t *p, unsigned *n = nullptr, int64_t Value = 0; unsigned Shift = 0; uint8_t Byte; + if (error) + *error = nullptr; do { if (end && p == end) { if (error) @@ -175,11 +176,11 @@ inline int64_t decodeSLEB128(const uint8_t *p, unsigned *n = nullptr, return 0; } Byte = *p++; - Value |= (int64_t(Byte & 0x7f) << Shift); + Value |= (uint64_t(Byte & 0x7f) << Shift); Shift += 7; } while (Byte >= 128); - // Sign extend negative numbers. - if (Byte & 0x40) + // Sign extend negative numbers if needed. + if (Shift < 64 && (Byte & 0x40)) Value |= (-1ULL) << Shift; if (n) *n = (unsigned)(p - orig_p); diff --git a/include/llvm/Support/LineIterator.h b/include/llvm/Support/LineIterator.h index 892d289976cb..c9f10ca975ae 100644 --- a/include/llvm/Support/LineIterator.h +++ b/include/llvm/Support/LineIterator.h @@ -1,9 +1,8 @@ //===- LineIterator.h - Iterator to read a text buffer's lines --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Support/LockFileManager.h b/include/llvm/Support/LockFileManager.h index 86db0b2b1020..57e4fbd84cd9 100644 --- a/include/llvm/Support/LockFileManager.h +++ b/include/llvm/Support/LockFileManager.h @@ -1,9 +1,8 @@ //===--- LockFileManager.h - File-level locking utility ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef LLVM_SUPPORT_LOCKFILEMANAGER_H diff --git a/include/llvm/Support/LowLevelTypeImpl.h b/include/llvm/Support/LowLevelTypeImpl.h index 2a1075c9a48d..0e02b6e7d750 100644 --- a/include/llvm/Support/LowLevelTypeImpl.h +++ b/include/llvm/Support/LowLevelTypeImpl.h @@ -1,9 +1,8 @@ //== llvm/Support/LowLevelTypeImpl.h --------------------------- -*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -46,8 +45,8 @@ public: SizeInBits, /*AddressSpace=*/0}; } - /// Get a low-level pointer in the given address space (defaulting to 0). - static LLT pointer(uint16_t AddressSpace, unsigned SizeInBits) { + /// Get a low-level pointer in the given address space. + static LLT pointer(unsigned AddressSpace, unsigned SizeInBits) { assert(SizeInBits > 0 && "invalid pointer size"); return LLT{/*isPointer=*/true, /*isVector=*/false, /*NumElements=*/0, SizeInBits, AddressSpace}; @@ -71,6 +70,14 @@ public: ScalarTy.isPointer() ? ScalarTy.getAddressSpace() : 0}; } + static LLT scalarOrVector(uint16_t NumElements, LLT ScalarTy) { + return NumElements == 1 ? ScalarTy : LLT::vector(NumElements, ScalarTy); + } + + static LLT scalarOrVector(uint16_t NumElements, unsigned ScalarSize) { + return scalarOrVector(NumElements, LLT::scalar(ScalarSize)); + } + explicit LLT(bool isPointer, bool isVector, uint16_t NumElements, unsigned SizeInBits, unsigned AddressSpace) { init(isPointer, isVector, NumElements, SizeInBits, AddressSpace); @@ -104,6 +111,32 @@ public: return getScalarSizeInBits() * getNumElements(); } + /// Returns the total size of the type in bytes, i.e. number of whole bytes + /// needed to represent the size in bits. Must only be called on sized types. + unsigned getSizeInBytes() const { + return (getSizeInBits() + 7) / 8; + } + + LLT getScalarType() const { + return isVector() ? getElementType() : *this; + } + + /// If this type is a vector, return a vector with the same number of elements + /// but the new element type. Otherwise, return the new element type. + LLT changeElementType(LLT NewEltTy) const { + return isVector() ? LLT::vector(getNumElements(), NewEltTy) : NewEltTy; + } + + /// If this type is a vector, return a vector with the same number of elements + /// but the new element size. Otherwise, return the new element type. Invalid + /// for pointer types. For pointer types, use changeElementType. + LLT changeElementSize(unsigned NewEltSize) const { + assert(!getScalarType().isPointer() && + "invalid to directly change element size for pointers"); + return isVector() ? LLT::vector(getNumElements(), NewEltSize) + : LLT::scalar(NewEltSize); + } + unsigned getScalarSizeInBits() const { assert(RawData != 0 && "Invalid Type"); if (!IsVector) { @@ -170,10 +203,10 @@ private: static const constexpr BitFieldInfo ScalarSizeFieldInfo{32, 0}; /// * Pointer (isPointer == 1 && isVector == 0): /// SizeInBits: 16; - /// AddressSpace: 23; + /// AddressSpace: 24; static const constexpr BitFieldInfo PointerSizeFieldInfo{16, 0}; static const constexpr BitFieldInfo PointerAddressSpaceFieldInfo{ - 23, PointerSizeFieldInfo[0] + PointerSizeFieldInfo[1]}; + 24, PointerSizeFieldInfo[0] + PointerSizeFieldInfo[1]}; /// * Vector-of-non-pointer (isPointer == 0 && isVector == 1): /// NumElements: 16; /// SizeOfElement: 32; @@ -183,13 +216,13 @@ private: /// * Vector-of-pointer (isPointer == 1 && isVector == 1): /// NumElements: 16; /// SizeOfElement: 16; - /// AddressSpace: 23; + /// AddressSpace: 24; static const constexpr BitFieldInfo PointerVectorElementsFieldInfo{16, 0}; static const constexpr BitFieldInfo PointerVectorSizeFieldInfo{ 16, PointerVectorElementsFieldInfo[1] + PointerVectorElementsFieldInfo[0]}; static const constexpr BitFieldInfo PointerVectorAddressSpaceFieldInfo{ - 23, PointerVectorSizeFieldInfo[1] + PointerVectorSizeFieldInfo[0]}; + 24, PointerVectorSizeFieldInfo[1] + PointerVectorSizeFieldInfo[0]}; uint64_t IsPointer : 1; uint64_t IsVector : 1; diff --git a/include/llvm/Support/MSVCErrorWorkarounds.h b/include/llvm/Support/MSVCErrorWorkarounds.h index 053ecf64d1e9..30e8febae20b 100644 --- a/include/llvm/Support/MSVCErrorWorkarounds.h +++ b/include/llvm/Support/MSVCErrorWorkarounds.h @@ -1,9 +1,8 @@ //===--- MSVCErrorWorkarounds.h - Enable future in MSVC --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/MachineValueType.h b/include/llvm/Support/MachineValueType.h index 552dea05029c..b94d2c4836cc 100644 --- a/include/llvm/Support/MachineValueType.h +++ b/include/llvm/Support/MachineValueType.h @@ -1,9 +1,8 @@ //===- Support/MachineValueType.h - Machine-Level types ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -87,58 +86,65 @@ namespace llvm { v64i16 = 39, // 64 x i16 v128i16 = 40, //128 x i16 - v1i32 = 41, // 1 x i32 - v2i32 = 42, // 2 x i32 - v4i32 = 43, // 4 x i32 - v8i32 = 44, // 8 x i32 - v16i32 = 45, // 16 x i32 - v32i32 = 46, // 32 x i32 - v64i32 = 47, // 64 x i32 - - v1i64 = 48, // 1 x i64 - v2i64 = 49, // 2 x i64 - v4i64 = 50, // 4 x i64 - v8i64 = 51, // 8 x i64 - v16i64 = 52, // 16 x i64 - v32i64 = 53, // 32 x i64 - - v1i128 = 54, // 1 x i128 + v1i32 = 41, // 1 x i32 + v2i32 = 42, // 2 x i32 + v3i32 = 43, // 3 x i32 + v4i32 = 44, // 4 x i32 + v5i32 = 45, // 5 x i32 + v8i32 = 46, // 8 x i32 + v16i32 = 47, // 16 x i32 + v32i32 = 48, // 32 x i32 + v64i32 = 49, // 64 x i32 + v128i32 = 50, // 128 x i32 + v256i32 = 51, // 256 x i32 + v512i32 = 52, // 512 x i32 + v1024i32 = 53, // 1024 x i32 + v2048i32 = 54, // 2048 x i32 + + v1i64 = 55, // 1 x i64 + v2i64 = 56, // 2 x i64 + v4i64 = 57, // 4 x i64 + v8i64 = 58, // 8 x i64 + v16i64 = 59, // 16 x i64 + v32i64 = 60, // 32 x i64 + + v1i128 = 61, // 1 x i128 // Scalable integer types - nxv1i1 = 55, // n x 1 x i1 - nxv2i1 = 56, // n x 2 x i1 - nxv4i1 = 57, // n x 4 x i1 - nxv8i1 = 58, // n x 8 x i1 - nxv16i1 = 59, // n x 16 x i1 - nxv32i1 = 60, // n x 32 x i1 - - nxv1i8 = 61, // n x 1 x i8 - nxv2i8 = 62, // n x 2 x i8 - nxv4i8 = 63, // n x 4 x i8 - nxv8i8 = 64, // n x 8 x i8 - nxv16i8 = 65, // n x 16 x i8 - nxv32i8 = 66, // n x 32 x i8 - - nxv1i16 = 67, // n x 1 x i16 - nxv2i16 = 68, // n x 2 x i16 - nxv4i16 = 69, // n x 4 x i16 - nxv8i16 = 70, // n x 8 x i16 - nxv16i16 = 71, // n x 16 x i16 - nxv32i16 = 72, // n x 32 x i16 - - nxv1i32 = 73, // n x 1 x i32 - nxv2i32 = 74, // n x 2 x i32 - nxv4i32 = 75, // n x 4 x i32 - nxv8i32 = 76, // n x 8 x i32 - nxv16i32 = 77, // n x 16 x i32 - nxv32i32 = 78, // n x 32 x i32 - - nxv1i64 = 79, // n x 1 x i64 - nxv2i64 = 80, // n x 2 x i64 - nxv4i64 = 81, // n x 4 x i64 - nxv8i64 = 82, // n x 8 x i64 - nxv16i64 = 83, // n x 16 x i64 - nxv32i64 = 84, // n x 32 x i64 + nxv1i1 = 62, // n x 1 x i1 + nxv2i1 = 63, // n x 2 x i1 + nxv4i1 = 64, // n x 4 x i1 + nxv8i1 = 65, // n x 8 x i1 + nxv16i1 = 66, // n x 16 x i1 + nxv32i1 = 67, // n x 32 x i1 + + nxv1i8 = 68, // n x 1 x i8 + nxv2i8 = 69, // n x 2 x i8 + nxv4i8 = 70, // n x 4 x i8 + nxv8i8 = 71, // n x 8 x i8 + nxv16i8 = 72, // n x 16 x i8 + nxv32i8 = 73, // n x 32 x i8 + + nxv1i16 = 74, // n x 1 x i16 + nxv2i16 = 75, // n x 2 x i16 + nxv4i16 = 76, // n x 4 x i16 + nxv8i16 = 77, // n x 8 x i16 + nxv16i16 = 78, // n x 16 x i16 + nxv32i16 = 79, // n x 32 x i16 + + nxv1i32 = 80, // n x 1 x i32 + nxv2i32 = 81, // n x 2 x i32 + nxv4i32 = 82, // n x 4 x i32 + nxv8i32 = 83, // n x 8 x i32 + nxv16i32 = 84, // n x 16 x i32 + nxv32i32 = 85, // n x 32 x i32 + + nxv1i64 = 86, // n x 1 x i64 + nxv2i64 = 87, // n x 2 x i64 + nxv4i64 = 88, // n x 4 x i64 + nxv8i64 = 89, // n x 8 x i64 + nxv16i64 = 90, // n x 16 x i64 + nxv32i64 = 91, // n x 32 x i64 FIRST_INTEGER_VECTOR_VALUETYPE = v1i1, LAST_INTEGER_VECTOR_VALUETYPE = nxv32i64, @@ -146,31 +152,40 @@ namespace llvm { FIRST_INTEGER_SCALABLE_VALUETYPE = nxv1i1, LAST_INTEGER_SCALABLE_VALUETYPE = nxv32i64, - v2f16 = 85, // 2 x f16 - v4f16 = 86, // 4 x f16 - v8f16 = 87, // 8 x f16 - v1f32 = 88, // 1 x f32 - v2f32 = 89, // 2 x f32 - v4f32 = 90, // 4 x f32 - v8f32 = 91, // 8 x f32 - v16f32 = 92, // 16 x f32 - v1f64 = 93, // 1 x f64 - v2f64 = 94, // 2 x f64 - v4f64 = 95, // 4 x f64 - v8f64 = 96, // 8 x f64 - - nxv2f16 = 97, // n x 2 x f16 - nxv4f16 = 98, // n x 4 x f16 - nxv8f16 = 99, // n x 8 x f16 - nxv1f32 = 100, // n x 1 x f32 - nxv2f32 = 101, // n x 2 x f32 - nxv4f32 = 102, // n x 4 x f32 - nxv8f32 = 103, // n x 8 x f32 - nxv16f32 = 104, // n x 16 x f32 - nxv1f64 = 105, // n x 1 x f64 - nxv2f64 = 106, // n x 2 x f64 - nxv4f64 = 107, // n x 4 x f64 - nxv8f64 = 108, // n x 8 x f64 + v2f16 = 92, // 2 x f16 + v4f16 = 93, // 4 x f16 + v8f16 = 94, // 8 x f16 + v1f32 = 95, // 1 x f32 + v2f32 = 96, // 2 x f32 + v3f32 = 97, // 3 x f32 + v4f32 = 98, // 4 x f32 + v5f32 = 99, // 5 x f32 + v8f32 = 100, // 8 x f32 + v16f32 = 101, // 16 x f32 + v32f32 = 102, // 32 x f32 + v64f32 = 103, // 64 x f32 + v128f32 = 104, // 128 x f32 + v256f32 = 105, // 256 x f32 + v512f32 = 106, // 512 x f32 + v1024f32 = 107, // 1024 x f32 + v2048f32 = 108, // 2048 x f32 + v1f64 = 109, // 1 x f64 + v2f64 = 110, // 2 x f64 + v4f64 = 111, // 4 x f64 + v8f64 = 112, // 8 x f64 + + nxv2f16 = 113, // n x 2 x f16 + nxv4f16 = 114, // n x 4 x f16 + nxv8f16 = 115, // n x 8 x f16 + nxv1f32 = 116, // n x 1 x f32 + nxv2f32 = 117, // n x 2 x f32 + nxv4f32 = 118, // n x 4 x f32 + nxv8f32 = 119, // n x 8 x f32 + nxv16f32 = 120, // n x 16 x f32 + nxv1f64 = 121, // n x 1 x f64 + nxv2f64 = 122, // n x 2 x f64 + nxv4f64 = 123, // n x 4 x f64 + nxv8f64 = 124, // n x 8 x f64 FIRST_FP_VECTOR_VALUETYPE = v2f16, LAST_FP_VECTOR_VALUETYPE = nxv8f64, @@ -181,25 +196,25 @@ namespace llvm { FIRST_VECTOR_VALUETYPE = v1i1, LAST_VECTOR_VALUETYPE = nxv8f64, - x86mmx = 109, // This is an X86 MMX value + x86mmx = 125, // This is an X86 MMX value - Glue = 110, // This glues nodes together during pre-RA sched + Glue = 126, // This glues nodes together during pre-RA sched - isVoid = 111, // This has no value + isVoid = 127, // This has no value - Untyped = 112, // This value takes a register, but has + Untyped = 128, // This value takes a register, but has // unspecified type. The register class // will be determined by the opcode. - ExceptRef = 113, // WebAssembly's except_ref type + exnref = 129, // WebAssembly's exnref type FIRST_VALUETYPE = 1, // This is always the beginning of the list. - LAST_VALUETYPE = 114, // This always remains at the end of the list. + LAST_VALUETYPE = 130, // This always remains at the end of the list. // This is the current maximum for LAST_VALUETYPE. // MVT::MAX_ALLOWED_VALUETYPE is used for asserts and to size bit vectors // This value must be a multiple of 32. - MAX_ALLOWED_VALUETYPE = 128, + MAX_ALLOWED_VALUETYPE = 160, // A value of type llvm::TokenTy token = 248, @@ -464,11 +479,18 @@ namespace llvm { case nxv32i16: return i16; case v1i32: case v2i32: + case v3i32: case v4i32: + case v5i32: case v8i32: case v16i32: case v32i32: case v64i32: + case v128i32: + case v256i32: + case v512i32: + case v1024i32: + case v2048i32: case nxv1i32: case nxv2i32: case nxv4i32: @@ -496,9 +518,18 @@ namespace llvm { case nxv8f16: return f16; case v1f32: case v2f32: + case v3f32: case v4f32: + case v5f32: case v8f32: case v16f32: + case v32f32: + case v64f32: + case v128f32: + case v256f32: + case v512f32: + case v1024f32: + case v2048f32: case nxv1f32: case nxv2f32: case nxv4f32: @@ -519,21 +550,33 @@ namespace llvm { switch (SimpleTy) { default: llvm_unreachable("Not a vector MVT!"); - case v1024i1: return 1024; - case v512i1: return 512; - case v256i8: return 256; + case v2048i32: + case v2048f32: return 2048; + case v1024i1: + case v1024i32: + case v1024f32: return 1024; + case v512i1: + case v512i32: + case v512f32: return 512; + case v256i8: + case v256i32: + case v256f32: return 256; case v128i1: case v128i8: - case v128i16: return 128; + case v128i16: + case v128i32: + case v128f32: return 128; case v64i1: case v64i8: case v64i16: - case v64i32: return 64; + case v64i32: + case v64f32: return 64; case v32i1: case v32i8: case v32i16: case v32i32: case v32i64: + case v32f32: case nxv32i1: case nxv32i8: case nxv32i16: @@ -567,6 +610,8 @@ namespace llvm { case nxv8f16: case nxv8f32: case nxv8f64: return 8; + case v5i32: + case v5f32: return 5; case v4i1: case v4i8: case v4i16: @@ -583,6 +628,8 @@ namespace llvm { case nxv4f16: case nxv4f32: case nxv4f64: return 4; + case v3i32: + case v3f32: return 3; case v2i1: case v2i8: case v2i16: @@ -693,6 +740,8 @@ namespace llvm { case nxv2f32: case nxv1f64: return 64; case f80 : return 80; + case v3i32: + case v3f32: return 96; case f128: case ppcf128: case i128: @@ -712,6 +761,8 @@ namespace llvm { case nxv8f16: case nxv4f32: case nxv2f64: return 128; + case v5i32: + case v5f32: return 160; case v32i8: case v16i16: case v8i32: @@ -741,14 +792,26 @@ namespace llvm { case v64i16: case v32i32: case v16i64: + case v32f32: case nxv32i32: case nxv16i64: return 1024; case v256i8: case v128i16: case v64i32: case v32i64: + case v64f32: case nxv32i64: return 2048; - case ExceptRef: return 0; // opaque type + case v128i32: + case v128f32: return 4096; + case v256i32: + case v256f32: return 8192; + case v512i32: + case v512f32: return 16384; + case v1024i32: + case v1024f32: return 32768; + case v2048i32: + case v2048f32: return 65536; + case exnref: return 0; // opaque type } } @@ -862,13 +925,20 @@ namespace llvm { if (NumElements == 128) return MVT::v128i16; break; case MVT::i32: - if (NumElements == 1) return MVT::v1i32; - if (NumElements == 2) return MVT::v2i32; - if (NumElements == 4) return MVT::v4i32; - if (NumElements == 8) return MVT::v8i32; - if (NumElements == 16) return MVT::v16i32; - if (NumElements == 32) return MVT::v32i32; - if (NumElements == 64) return MVT::v64i32; + if (NumElements == 1) return MVT::v1i32; + if (NumElements == 2) return MVT::v2i32; + if (NumElements == 3) return MVT::v3i32; + if (NumElements == 4) return MVT::v4i32; + if (NumElements == 5) return MVT::v5i32; + if (NumElements == 8) return MVT::v8i32; + if (NumElements == 16) return MVT::v16i32; + if (NumElements == 32) return MVT::v32i32; + if (NumElements == 64) return MVT::v64i32; + if (NumElements == 128) return MVT::v128i32; + if (NumElements == 256) return MVT::v256i32; + if (NumElements == 512) return MVT::v512i32; + if (NumElements == 1024) return MVT::v1024i32; + if (NumElements == 2048) return MVT::v2048i32; break; case MVT::i64: if (NumElements == 1) return MVT::v1i64; @@ -887,11 +957,20 @@ namespace llvm { if (NumElements == 8) return MVT::v8f16; break; case MVT::f32: - if (NumElements == 1) return MVT::v1f32; - if (NumElements == 2) return MVT::v2f32; - if (NumElements == 4) return MVT::v4f32; - if (NumElements == 8) return MVT::v8f32; - if (NumElements == 16) return MVT::v16f32; + if (NumElements == 1) return MVT::v1f32; + if (NumElements == 2) return MVT::v2f32; + if (NumElements == 3) return MVT::v3f32; + if (NumElements == 4) return MVT::v4f32; + if (NumElements == 5) return MVT::v5f32; + if (NumElements == 8) return MVT::v8f32; + if (NumElements == 16) return MVT::v16f32; + if (NumElements == 32) return MVT::v32f32; + if (NumElements == 64) return MVT::v64f32; + if (NumElements == 128) return MVT::v128f32; + if (NumElements == 256) return MVT::v256f32; + if (NumElements == 512) return MVT::v512f32; + if (NumElements == 1024) return MVT::v1024f32; + if (NumElements == 2048) return MVT::v2048f32; break; case MVT::f64: if (NumElements == 1) return MVT::v1f64; diff --git a/include/llvm/Support/ManagedStatic.h b/include/llvm/Support/ManagedStatic.h index b4bf3210cc73..e65bb051f181 100644 --- a/include/llvm/Support/ManagedStatic.h +++ b/include/llvm/Support/ManagedStatic.h @@ -1,9 +1,8 @@ //===-- llvm/Support/ManagedStatic.h - Static Global wrapper ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -33,18 +32,41 @@ template struct object_deleter { static void call(void *Ptr) { delete[](T *)Ptr; } }; +// ManagedStatic must be initialized to zero, and it must *not* have a dynamic +// initializer because managed statics are often created while running other +// dynamic initializers. In standard C++11, the best way to accomplish this is +// with a constexpr default constructor. However, different versions of the +// Visual C++ compiler have had bugs where, even though the constructor may be +// constexpr, a dynamic initializer may be emitted depending on optimization +// settings. For the affected versions of MSVC, use the old linker +// initialization pattern of not providing a constructor and leaving the fields +// uninitialized. +#if !defined(_MSC_VER) || defined(__clang__) +#define LLVM_USE_CONSTEXPR_CTOR +#endif + /// ManagedStaticBase - Common base class for ManagedStatic instances. class ManagedStaticBase { protected: +#ifdef LLVM_USE_CONSTEXPR_CTOR + mutable std::atomic Ptr{}; + mutable void (*DeleterFn)(void *) = nullptr; + mutable const ManagedStaticBase *Next = nullptr; +#else // This should only be used as a static variable, which guarantees that this // will be zero initialized. mutable std::atomic Ptr; - mutable void (*DeleterFn)(void*); + mutable void (*DeleterFn)(void *); mutable const ManagedStaticBase *Next; +#endif void RegisterManagedStatic(void *(*creator)(), void (*deleter)(void*)) const; public: +#ifdef LLVM_USE_CONSTEXPR_CTOR + constexpr ManagedStaticBase() = default; +#endif + /// isConstructed - Return true if this object has not been created yet. bool isConstructed() const { return Ptr != nullptr; } diff --git a/include/llvm/Support/MathExtras.h b/include/llvm/Support/MathExtras.h index b59f21b4998e..249139e824b5 100644 --- a/include/llvm/Support/MathExtras.h +++ b/include/llvm/Support/MathExtras.h @@ -1,9 +1,8 @@ //===-- llvm/Support/MathExtras.h - Useful math functions -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -52,14 +51,14 @@ enum ZeroBehavior { namespace detail { template struct TrailingZerosCounter { - static std::size_t count(T Val, ZeroBehavior) { + static unsigned count(T Val, ZeroBehavior) { if (!Val) return std::numeric_limits::digits; if (Val & 0x1) return 0; // Bisection method. - std::size_t ZeroBits = 0; + unsigned ZeroBits = 0; T Shift = std::numeric_limits::digits >> 1; T Mask = std::numeric_limits::max() >> Shift; while (Shift) { @@ -76,7 +75,7 @@ template struct TrailingZerosCounter { #if __GNUC__ >= 4 || defined(_MSC_VER) template struct TrailingZerosCounter { - static std::size_t count(T Val, ZeroBehavior ZB) { + static unsigned count(T Val, ZeroBehavior ZB) { if (ZB != ZB_Undefined && Val == 0) return 32; @@ -92,7 +91,7 @@ template struct TrailingZerosCounter { #if !defined(_MSC_VER) || defined(_M_X64) template struct TrailingZerosCounter { - static std::size_t count(T Val, ZeroBehavior ZB) { + static unsigned count(T Val, ZeroBehavior ZB) { if (ZB != ZB_Undefined && Val == 0) return 64; @@ -117,7 +116,7 @@ template struct TrailingZerosCounter { /// \param ZB the behavior on an input of 0. Only ZB_Width and ZB_Undefined are /// valid arguments. template -std::size_t countTrailingZeros(T Val, ZeroBehavior ZB = ZB_Width) { +unsigned countTrailingZeros(T Val, ZeroBehavior ZB = ZB_Width) { static_assert(std::numeric_limits::is_integer && !std::numeric_limits::is_signed, "Only unsigned integral types are allowed."); @@ -126,12 +125,12 @@ std::size_t countTrailingZeros(T Val, ZeroBehavior ZB = ZB_Width) { namespace detail { template struct LeadingZerosCounter { - static std::size_t count(T Val, ZeroBehavior) { + static unsigned count(T Val, ZeroBehavior) { if (!Val) return std::numeric_limits::digits; // Bisection method. - std::size_t ZeroBits = 0; + unsigned ZeroBits = 0; for (T Shift = std::numeric_limits::digits >> 1; Shift; Shift >>= 1) { T Tmp = Val >> Shift; if (Tmp) @@ -145,7 +144,7 @@ template struct LeadingZerosCounter { #if __GNUC__ >= 4 || defined(_MSC_VER) template struct LeadingZerosCounter { - static std::size_t count(T Val, ZeroBehavior ZB) { + static unsigned count(T Val, ZeroBehavior ZB) { if (ZB != ZB_Undefined && Val == 0) return 32; @@ -161,7 +160,7 @@ template struct LeadingZerosCounter { #if !defined(_MSC_VER) || defined(_M_X64) template struct LeadingZerosCounter { - static std::size_t count(T Val, ZeroBehavior ZB) { + static unsigned count(T Val, ZeroBehavior ZB) { if (ZB != ZB_Undefined && Val == 0) return 64; @@ -186,7 +185,7 @@ template struct LeadingZerosCounter { /// \param ZB the behavior on an input of 0. Only ZB_Width and ZB_Undefined are /// valid arguments. template -std::size_t countLeadingZeros(T Val, ZeroBehavior ZB = ZB_Width) { +unsigned countLeadingZeros(T Val, ZeroBehavior ZB = ZB_Width) { static_assert(std::numeric_limits::is_integer && !std::numeric_limits::is_signed, "Only unsigned integral types are allowed."); @@ -459,7 +458,7 @@ inline uint64_t ByteSwap_64(uint64_t Value) { /// \param ZB the behavior on an input of all ones. Only ZB_Width and /// ZB_Undefined are valid arguments. template -std::size_t countLeadingOnes(T Value, ZeroBehavior ZB = ZB_Width) { +unsigned countLeadingOnes(T Value, ZeroBehavior ZB = ZB_Width) { static_assert(std::numeric_limits::is_integer && !std::numeric_limits::is_signed, "Only unsigned integral types are allowed."); @@ -475,7 +474,7 @@ std::size_t countLeadingOnes(T Value, ZeroBehavior ZB = ZB_Width) { /// \param ZB the behavior on an input of all ones. Only ZB_Width and /// ZB_Undefined are valid arguments. template -std::size_t countTrailingOnes(T Value, ZeroBehavior ZB = ZB_Width) { +unsigned countTrailingOnes(T Value, ZeroBehavior ZB = ZB_Width) { static_assert(std::numeric_limits::is_integer && !std::numeric_limits::is_signed, "Only unsigned integral types are allowed."); @@ -560,15 +559,20 @@ inline unsigned Log2_64_Ceil(uint64_t Value) { } /// Return the greatest common divisor of the values using Euclid's algorithm. -inline uint64_t GreatestCommonDivisor64(uint64_t A, uint64_t B) { +template +inline T greatestCommonDivisor(T A, T B) { while (B) { - uint64_t T = B; + T Tmp = B; B = A % B; - A = T; + A = Tmp; } return A; } +inline uint64_t GreatestCommonDivisor64(uint64_t A, uint64_t B) { + return greatestCommonDivisor(A, B); +} + /// This function takes a 64-bit integer and returns the bit equivalent double. inline double BitsToDouble(uint64_t Bits) { double D; diff --git a/include/llvm/Support/MemAlloc.h b/include/llvm/Support/MemAlloc.h index d06c659cfba6..0e5869141fd3 100644 --- a/include/llvm/Support/MemAlloc.h +++ b/include/llvm/Support/MemAlloc.h @@ -1,9 +1,8 @@ //===- MemAlloc.h - Memory allocation functions -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -25,23 +24,41 @@ namespace llvm { LLVM_ATTRIBUTE_RETURNS_NONNULL inline void *safe_malloc(size_t Sz) { void *Result = std::malloc(Sz); - if (Result == nullptr) + if (Result == nullptr) { + // It is implementation-defined whether allocation occurs if the space + // requested is zero (ISO/IEC 9899:2018 7.22.3). Retry, requesting + // non-zero, if the space requested was zero. + if (Sz == 0) + return safe_malloc(1); report_bad_alloc_error("Allocation failed"); + } return Result; } LLVM_ATTRIBUTE_RETURNS_NONNULL inline void *safe_calloc(size_t Count, size_t Sz) { void *Result = std::calloc(Count, Sz); - if (Result == nullptr) + if (Result == nullptr) { + // It is implementation-defined whether allocation occurs if the space + // requested is zero (ISO/IEC 9899:2018 7.22.3). Retry, requesting + // non-zero, if the space requested was zero. + if (Count == 0 || Sz == 0) + return safe_malloc(1); report_bad_alloc_error("Allocation failed"); + } return Result; } LLVM_ATTRIBUTE_RETURNS_NONNULL inline void *safe_realloc(void *Ptr, size_t Sz) { void *Result = std::realloc(Ptr, Sz); - if (Result == nullptr) + if (Result == nullptr) { + // It is implementation-defined whether allocation occurs if the space + // requested is zero (ISO/IEC 9899:2018 7.22.3). Retry, requesting + // non-zero, if the space requested was zero. + if (Sz == 0) + return safe_malloc(1); report_bad_alloc_error("Allocation failed"); + } return Result; } diff --git a/include/llvm/Support/Memory.h b/include/llvm/Support/Memory.h index fa026d49a61b..6f22dd7080cd 100644 --- a/include/llvm/Support/Memory.h +++ b/include/llvm/Support/Memory.h @@ -1,9 +1,8 @@ //===- llvm/Support/Memory.h - Memory Support -------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,6 +18,10 @@ #include namespace llvm { + +// Forward declare raw_ostream: it is used for debug dumping below. +class raw_ostream; + namespace sys { /// This class encapsulates the notion of a memory block which has an address @@ -28,14 +31,18 @@ namespace sys { /// Memory block abstraction. class MemoryBlock { public: - MemoryBlock() : Address(nullptr), Size(0) { } - MemoryBlock(void *addr, size_t size) : Address(addr), Size(size) { } + MemoryBlock() : Address(nullptr), AllocatedSize(0) {} + MemoryBlock(void *addr, size_t allocatedSize) + : Address(addr), AllocatedSize(allocatedSize) {} void *base() const { return Address; } - size_t size() const { return Size; } - + /// The size as it was allocated. This is always greater or equal to the + /// size that was originally requested. + size_t allocatedSize() const { return AllocatedSize; } + private: void *Address; ///< Address of first byte of memory area - size_t Size; ///< Size, in bytes of the memory area + size_t AllocatedSize; ///< Size, in bytes of the memory area + unsigned Flags = 0; friend class Memory; }; @@ -46,9 +53,11 @@ namespace sys { class Memory { public: enum ProtectionFlags { - MF_READ = 0x1000000, + MF_READ = 0x1000000, MF_WRITE = 0x2000000, - MF_EXEC = 0x4000000 + MF_EXEC = 0x4000000, + MF_RWE_MASK = 0x7000000, + MF_HUGE_HINT = 0x0000001 }; /// This method allocates a block of memory that is suitable for loading @@ -133,13 +142,22 @@ namespace sys { Memory::releaseMappedMemory(M); } void *base() const { return M.base(); } - size_t size() const { return M.size(); } + /// The size as it was allocated. This is always greater or equal to the + /// size that was originally requested. + size_t allocatedSize() const { return M.allocatedSize(); } MemoryBlock getMemoryBlock() const { return M; } private: MemoryBlock M; }; -} -} +#ifndef NDEBUG + /// Debugging output for Memory::ProtectionFlags. + raw_ostream &operator<<(raw_ostream &OS, const Memory::ProtectionFlags &PF); + + /// Debugging output for MemoryBlock. + raw_ostream &operator<<(raw_ostream &OS, const MemoryBlock &MB); +#endif // ifndef NDEBUG + } // end namespace sys + } // end namespace llvm #endif diff --git a/include/llvm/Support/MemoryBuffer.h b/include/llvm/Support/MemoryBuffer.h index 8933295d4ea4..b5196cd84cb4 100644 --- a/include/llvm/Support/MemoryBuffer.h +++ b/include/llvm/Support/MemoryBuffer.h @@ -1,9 +1,8 @@ //===--- MemoryBuffer.h - Memory Buffer Interface ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -91,7 +90,7 @@ public: /// MemoryBuffer. The slice is specified by an \p Offset and \p MapSize. /// Since this is in the middle of a file, the buffer is not null terminated. static ErrorOr> - getOpenFileSlice(int FD, const Twine &Filename, uint64_t MapSize, + getOpenFileSlice(sys::fs::file_t FD, const Twine &Filename, uint64_t MapSize, int64_t Offset, bool IsVolatile = false); /// Given an already-open file descriptor, read the file and return a @@ -101,7 +100,7 @@ public: /// can change outside the user's control, e.g. when libclang tries to parse /// while the user is editing/updating the file or if the file is on an NFS. static ErrorOr> - getOpenFile(int FD, const Twine &Filename, uint64_t FileSize, + getOpenFile(sys::fs::file_t FD, const Twine &Filename, uint64_t FileSize, bool RequiresNullTerminator = true, bool IsVolatile = false); /// Open the specified memory range as a MemoryBuffer. Note that InputData @@ -265,7 +264,7 @@ class MemoryBufferRef { public: MemoryBufferRef() = default; - MemoryBufferRef(MemoryBuffer& Buffer) + MemoryBufferRef(const MemoryBuffer& Buffer) : Buffer(Buffer.getBuffer()), Identifier(Buffer.getBufferIdentifier()) {} MemoryBufferRef(StringRef Buffer, StringRef Identifier) : Buffer(Buffer), Identifier(Identifier) {} diff --git a/include/llvm/Support/MipsABIFlags.h b/include/llvm/Support/MipsABIFlags.h index 12c350015b21..d3233f645fb9 100644 --- a/include/llvm/Support/MipsABIFlags.h +++ b/include/llvm/Support/MipsABIFlags.h @@ -1,9 +1,8 @@ //===--- MipsABIFlags.h - MIPS ABI flags ----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/Mutex.h b/include/llvm/Support/Mutex.h index 680d94b24ef5..c3abfc7a7806 100644 --- a/include/llvm/Support/Mutex.h +++ b/include/llvm/Support/Mutex.h @@ -1,9 +1,8 @@ //===- llvm/Support/Mutex.h - Mutex Operating System Concept -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/MutexGuard.h b/include/llvm/Support/MutexGuard.h index 641d64d94988..d86ced145816 100644 --- a/include/llvm/Support/MutexGuard.h +++ b/include/llvm/Support/MutexGuard.h @@ -1,9 +1,8 @@ //===-- Support/MutexGuard.h - Acquire/Release Mutex In Scope ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/NativeFormatting.h b/include/llvm/Support/NativeFormatting.h index 6d1dd7b422fe..825a44c77c00 100644 --- a/include/llvm/Support/NativeFormatting.h +++ b/include/llvm/Support/NativeFormatting.h @@ -1,9 +1,8 @@ //===- NativeFormatting.h - Low level formatting helpers ---------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Support/OnDiskHashTable.h b/include/llvm/Support/OnDiskHashTable.h index 912e2700d1a0..d84da92aab9b 100644 --- a/include/llvm/Support/OnDiskHashTable.h +++ b/include/llvm/Support/OnDiskHashTable.h @@ -1,9 +1,8 @@ //===--- OnDiskHashTable.h - On-Disk Hash Table Implementation --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/include/llvm/Support/Options.h b/include/llvm/Support/Options.h index dd321c6a1984..d02ef85a75bf 100644 --- a/include/llvm/Support/Options.h +++ b/include/llvm/Support/Options.h @@ -1,9 +1,8 @@ //===- llvm/Support/Options.h - Debug options support -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/Support/Parallel.h b/include/llvm/Support/Parallel.h index 1462265343be..eab9b492c4a5 100644 --- a/include/llvm/Support/Parallel.h +++ b/include/llvm/Support/Parallel.h @@ -1,9 +1,8 @@ //===- llvm/Support/Parallel.h - Parallel algorithms ----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -74,8 +73,12 @@ public: class TaskGroup { Latch L; + bool Parallel; public: + TaskGroup(); + ~TaskGroup(); + void spawn(std::function f); void sync() const { L.sync(); } diff --git a/include/llvm/Support/Path.h b/include/llvm/Support/Path.h index 76de887b7cb4..5c0bee58f188 100644 --- a/include/llvm/Support/Path.h +++ b/include/llvm/Support/Path.h @@ -1,9 +1,8 @@ //===- llvm/Support/Path.h - Path Operating System Concept ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/PluginLoader.h b/include/llvm/Support/PluginLoader.h index bdbb134b28eb..c0c516bdae03 100644 --- a/include/llvm/Support/PluginLoader.h +++ b/include/llvm/Support/PluginLoader.h @@ -1,9 +1,8 @@ //===-- llvm/Support/PluginLoader.h - Plugin Loader for Tools ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/PointerLikeTypeTraits.h b/include/llvm/Support/PointerLikeTypeTraits.h index 1710b57131d1..1e7e5b53ca65 100644 --- a/include/llvm/Support/PointerLikeTypeTraits.h +++ b/include/llvm/Support/PointerLikeTypeTraits.h @@ -1,9 +1,8 @@ //===- llvm/Support/PointerLikeTypeTraits.h - Pointer Traits ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/PrettyStackTrace.h b/include/llvm/Support/PrettyStackTrace.h index 4d64fe4ef727..6eb070b2297e 100644 --- a/include/llvm/Support/PrettyStackTrace.h +++ b/include/llvm/Support/PrettyStackTrace.h @@ -1,9 +1,8 @@ //===- llvm/Support/PrettyStackTrace.h - Pretty Crash Handling --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -22,8 +21,22 @@ namespace llvm { class raw_ostream; + /// Enables dumping a "pretty" stack trace when the program crashes. + /// + /// \see PrettyStackTraceEntry void EnablePrettyStackTrace(); + /// Enables (or disables) dumping a "pretty" stack trace when the user sends + /// SIGINFO or SIGUSR1 to the current process. + /// + /// This is a per-thread decision so that a program can choose to print stack + /// traces only on a primary thread, or on all threads that use + /// PrettyStackTraceEntry. + /// + /// \see EnablePrettyStackTrace + /// \see PrettyStackTraceEntry + void EnablePrettyStackTraceOnSigInfoForThisThread(bool ShouldEnable = true); + /// PrettyStackTraceEntry - This class is used to represent a frame of the /// "pretty" stack trace that is dumped when a program crashes. You can define /// subclasses of this and declare them on the program stack: when they are diff --git a/include/llvm/Support/Printable.h b/include/llvm/Support/Printable.h index cb55d41316e3..0f8670d0419c 100644 --- a/include/llvm/Support/Printable.h +++ b/include/llvm/Support/Printable.h @@ -1,9 +1,8 @@ //===--- Printable.h - Print function helpers -------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/Process.h b/include/llvm/Support/Process.h index f9f1cac86278..67e37912519b 100644 --- a/include/llvm/Support/Process.h +++ b/include/llvm/Support/Process.h @@ -1,9 +1,8 @@ //===- llvm/Support/Process.h -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -29,6 +28,7 @@ #include "llvm/Support/Allocator.h" #include "llvm/Support/Chrono.h" #include "llvm/Support/DataTypes.h" +#include "llvm/Support/Error.h" #include namespace llvm { @@ -42,7 +42,25 @@ namespace sys { /// current executing process. class Process { public: - static unsigned getPageSize(); + /// Get the process's page size. + /// This may fail if the underlying syscall returns an error. In most cases, + /// page size information is used for optimization, and this error can be + /// safely discarded by calling consumeError, and an estimated page size + /// substituted instead. + static Expected getPageSize(); + + /// Get the process's estimated page size. + /// This function always succeeds, but if the underlying syscall to determine + /// the page size fails then this will silently return an estimated page size. + /// The estimated page size is guaranteed to be a power of 2. + static unsigned getPageSizeEstimate() { + if (auto PageSize = getPageSize()) + return *PageSize; + else { + consumeError(PageSize.takeError()); + return 4096; + } + } /// Return process memory usage. /// This static function will return the total amount of memory allocated diff --git a/include/llvm/Support/Program.h b/include/llvm/Support/Program.h index 1f4dbdce3323..6b2315c5da8d 100644 --- a/include/llvm/Support/Program.h +++ b/include/llvm/Support/Program.h @@ -1,9 +1,8 @@ //===- llvm/Support/Program.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/RWMutex.h b/include/llvm/Support/RWMutex.h index 5ac3e558999b..9cd57cbd65a1 100644 --- a/include/llvm/Support/RWMutex.h +++ b/include/llvm/Support/RWMutex.h @@ -1,9 +1,8 @@ //===- RWMutex.h - Reader/Writer Mutual Exclusion Lock ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/RandomNumberGenerator.h b/include/llvm/Support/RandomNumberGenerator.h index 1399dab815f8..55d6876cc5e4 100644 --- a/include/llvm/Support/RandomNumberGenerator.h +++ b/include/llvm/Support/RandomNumberGenerator.h @@ -1,9 +1,8 @@ //==- llvm/Support/RandomNumberGenerator.h - RNG for diversity ---*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/Recycler.h b/include/llvm/Support/Recycler.h index 53db2e86d12d..bbd9ae321ae3 100644 --- a/include/llvm/Support/Recycler.h +++ b/include/llvm/Support/Recycler.h @@ -1,9 +1,8 @@ //==- llvm/Support/Recycler.h - Recycling Allocator --------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/RecyclingAllocator.h b/include/llvm/Support/RecyclingAllocator.h index 32b033b17946..2c29dacfe212 100644 --- a/include/llvm/Support/RecyclingAllocator.h +++ b/include/llvm/Support/RecyclingAllocator.h @@ -1,9 +1,8 @@ //==- llvm/Support/RecyclingAllocator.h - Recycling Allocator ----*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/Regex.h b/include/llvm/Support/Regex.h index d901eb1e3ffb..2d19b10fd890 100644 --- a/include/llvm/Support/Regex.h +++ b/include/llvm/Support/Regex.h @@ -1,9 +1,8 @@ //===-- Regex.h - Regular Expression matcher implementation -*- C++ -*-----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/Registry.h b/include/llvm/Support/Registry.h index 02fd5b9354a1..4d8aa5f1470d 100644 --- a/include/llvm/Support/Registry.h +++ b/include/llvm/Support/Registry.h @@ -1,9 +1,8 @@ //=== Registry.h - Linker-supported plugin registries -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -81,17 +80,17 @@ namespace llvm { /// Iterators for registry entries. /// - class iterator { + class iterator + : public llvm::iterator_facade_base { const node *Cur; public: explicit iterator(const node *N) : Cur(N) {} bool operator==(const iterator &That) const { return Cur == That.Cur; } - bool operator!=(const iterator &That) const { return Cur != That.Cur; } iterator &operator++() { Cur = Cur->Next; return *this; } const entry &operator*() const { return Cur->Val; } - const entry *operator->() const { return &Cur->Val; } }; // begin is not defined here in order to avoid usage of an undefined static diff --git a/include/llvm/Support/SHA1.h b/include/llvm/Support/SHA1.h index 1fc60a878f94..87fe94bbd5cd 100644 --- a/include/llvm/Support/SHA1.h +++ b/include/llvm/Support/SHA1.h @@ -1,9 +1,8 @@ //==- SHA1.h - SHA1 implementation for LLVM --*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This code is taken from public domain diff --git a/include/llvm/Support/SMLoc.h b/include/llvm/Support/SMLoc.h index c74feff378d6..d8607034ee86 100644 --- a/include/llvm/Support/SMLoc.h +++ b/include/llvm/Support/SMLoc.h @@ -1,9 +1,8 @@ //===- SMLoc.h - Source location for use with diagnostics -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/SMTAPI.h b/include/llvm/Support/SMTAPI.h new file mode 100644 index 000000000000..24dcd124593e --- /dev/null +++ b/include/llvm/Support/SMTAPI.h @@ -0,0 +1,447 @@ +//===- SMTAPI.h -------------------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file defines a SMT generic Solver API, which will be the base class +// for every SMT solver specific class. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_SMTAPI_H +#define LLVM_SUPPORT_SMTAPI_H + +#include "llvm/ADT/APFloat.h" +#include "llvm/ADT/APSInt.h" +#include "llvm/ADT/FoldingSet.h" +#include "llvm/Support/raw_ostream.h" +#include + +namespace llvm { + +/// Generic base class for SMT sorts +class SMTSort { +public: + SMTSort() = default; + virtual ~SMTSort() = default; + + /// Returns true if the sort is a bitvector, calls isBitvectorSortImpl(). + virtual bool isBitvectorSort() const { return isBitvectorSortImpl(); } + + /// Returns true if the sort is a floating-point, calls isFloatSortImpl(). + virtual bool isFloatSort() const { return isFloatSortImpl(); } + + /// Returns true if the sort is a boolean, calls isBooleanSortImpl(). + virtual bool isBooleanSort() const { return isBooleanSortImpl(); } + + /// Returns the bitvector size, fails if the sort is not a bitvector + /// Calls getBitvectorSortSizeImpl(). + virtual unsigned getBitvectorSortSize() const { + assert(isBitvectorSort() && "Not a bitvector sort!"); + unsigned Size = getBitvectorSortSizeImpl(); + assert(Size && "Size is zero!"); + return Size; + }; + + /// Returns the floating-point size, fails if the sort is not a floating-point + /// Calls getFloatSortSizeImpl(). + virtual unsigned getFloatSortSize() const { + assert(isFloatSort() && "Not a floating-point sort!"); + unsigned Size = getFloatSortSizeImpl(); + assert(Size && "Size is zero!"); + return Size; + }; + + virtual void Profile(llvm::FoldingSetNodeID &ID) const = 0; + + bool operator<(const SMTSort &Other) const { + llvm::FoldingSetNodeID ID1, ID2; + Profile(ID1); + Other.Profile(ID2); + return ID1 < ID2; + } + + friend bool operator==(SMTSort const &LHS, SMTSort const &RHS) { + return LHS.equal_to(RHS); + } + + virtual void print(raw_ostream &OS) const = 0; + + LLVM_DUMP_METHOD void dump() const; + +protected: + /// Query the SMT solver and returns true if two sorts are equal (same kind + /// and bit width). This does not check if the two sorts are the same objects. + virtual bool equal_to(SMTSort const &other) const = 0; + + /// Query the SMT solver and checks if a sort is bitvector. + virtual bool isBitvectorSortImpl() const = 0; + + /// Query the SMT solver and checks if a sort is floating-point. + virtual bool isFloatSortImpl() const = 0; + + /// Query the SMT solver and checks if a sort is boolean. + virtual bool isBooleanSortImpl() const = 0; + + /// Query the SMT solver and returns the sort bit width. + virtual unsigned getBitvectorSortSizeImpl() const = 0; + + /// Query the SMT solver and returns the sort bit width. + virtual unsigned getFloatSortSizeImpl() const = 0; +}; + +/// Shared pointer for SMTSorts, used by SMTSolver API. +using SMTSortRef = const SMTSort *; + +/// Generic base class for SMT exprs +class SMTExpr { +public: + SMTExpr() = default; + virtual ~SMTExpr() = default; + + bool operator<(const SMTExpr &Other) const { + llvm::FoldingSetNodeID ID1, ID2; + Profile(ID1); + Other.Profile(ID2); + return ID1 < ID2; + } + + virtual void Profile(llvm::FoldingSetNodeID &ID) const = 0; + + friend bool operator==(SMTExpr const &LHS, SMTExpr const &RHS) { + return LHS.equal_to(RHS); + } + + virtual void print(raw_ostream &OS) const = 0; + + LLVM_DUMP_METHOD void dump() const; + +protected: + /// Query the SMT solver and returns true if two sorts are equal (same kind + /// and bit width). This does not check if the two sorts are the same objects. + virtual bool equal_to(SMTExpr const &other) const = 0; +}; + +/// Shared pointer for SMTExprs, used by SMTSolver API. +using SMTExprRef = const SMTExpr *; + +/// Generic base class for SMT Solvers +/// +/// This class is responsible for wrapping all sorts and expression generation, +/// through the mk* methods. It also provides methods to create SMT expressions +/// straight from clang's AST, through the from* methods. +class SMTSolver { +public: + SMTSolver() = default; + virtual ~SMTSolver() = default; + + LLVM_DUMP_METHOD void dump() const; + + // Returns an appropriate floating-point sort for the given bitwidth. + SMTSortRef getFloatSort(unsigned BitWidth) { + switch (BitWidth) { + case 16: + return getFloat16Sort(); + case 32: + return getFloat32Sort(); + case 64: + return getFloat64Sort(); + case 128: + return getFloat128Sort(); + default:; + } + llvm_unreachable("Unsupported floating-point bitwidth!"); + } + + // Returns a boolean sort. + virtual SMTSortRef getBoolSort() = 0; + + // Returns an appropriate bitvector sort for the given bitwidth. + virtual SMTSortRef getBitvectorSort(const unsigned BitWidth) = 0; + + // Returns a floating-point sort of width 16 + virtual SMTSortRef getFloat16Sort() = 0; + + // Returns a floating-point sort of width 32 + virtual SMTSortRef getFloat32Sort() = 0; + + // Returns a floating-point sort of width 64 + virtual SMTSortRef getFloat64Sort() = 0; + + // Returns a floating-point sort of width 128 + virtual SMTSortRef getFloat128Sort() = 0; + + // Returns an appropriate sort for the given AST. + virtual SMTSortRef getSort(const SMTExprRef &AST) = 0; + + /// Given a constraint, adds it to the solver + virtual void addConstraint(const SMTExprRef &Exp) const = 0; + + /// Creates a bitvector addition operation + virtual SMTExprRef mkBVAdd(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a bitvector subtraction operation + virtual SMTExprRef mkBVSub(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a bitvector multiplication operation + virtual SMTExprRef mkBVMul(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a bitvector signed modulus operation + virtual SMTExprRef mkBVSRem(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a bitvector unsigned modulus operation + virtual SMTExprRef mkBVURem(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a bitvector signed division operation + virtual SMTExprRef mkBVSDiv(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a bitvector unsigned division operation + virtual SMTExprRef mkBVUDiv(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a bitvector logical shift left operation + virtual SMTExprRef mkBVShl(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a bitvector arithmetic shift right operation + virtual SMTExprRef mkBVAshr(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a bitvector logical shift right operation + virtual SMTExprRef mkBVLshr(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a bitvector negation operation + virtual SMTExprRef mkBVNeg(const SMTExprRef &Exp) = 0; + + /// Creates a bitvector not operation + virtual SMTExprRef mkBVNot(const SMTExprRef &Exp) = 0; + + /// Creates a bitvector xor operation + virtual SMTExprRef mkBVXor(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a bitvector or operation + virtual SMTExprRef mkBVOr(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a bitvector and operation + virtual SMTExprRef mkBVAnd(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a bitvector unsigned less-than operation + virtual SMTExprRef mkBVUlt(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a bitvector signed less-than operation + virtual SMTExprRef mkBVSlt(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a bitvector unsigned greater-than operation + virtual SMTExprRef mkBVUgt(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a bitvector signed greater-than operation + virtual SMTExprRef mkBVSgt(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a bitvector unsigned less-equal-than operation + virtual SMTExprRef mkBVUle(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a bitvector signed less-equal-than operation + virtual SMTExprRef mkBVSle(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a bitvector unsigned greater-equal-than operation + virtual SMTExprRef mkBVUge(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a bitvector signed greater-equal-than operation + virtual SMTExprRef mkBVSge(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a boolean not operation + virtual SMTExprRef mkNot(const SMTExprRef &Exp) = 0; + + /// Creates a boolean equality operation + virtual SMTExprRef mkEqual(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a boolean and operation + virtual SMTExprRef mkAnd(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a boolean or operation + virtual SMTExprRef mkOr(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a boolean ite operation + virtual SMTExprRef mkIte(const SMTExprRef &Cond, const SMTExprRef &T, + const SMTExprRef &F) = 0; + + /// Creates a bitvector sign extension operation + virtual SMTExprRef mkBVSignExt(unsigned i, const SMTExprRef &Exp) = 0; + + /// Creates a bitvector zero extension operation + virtual SMTExprRef mkBVZeroExt(unsigned i, const SMTExprRef &Exp) = 0; + + /// Creates a bitvector extract operation + virtual SMTExprRef mkBVExtract(unsigned High, unsigned Low, + const SMTExprRef &Exp) = 0; + + /// Creates a bitvector concat operation + virtual SMTExprRef mkBVConcat(const SMTExprRef &LHS, + const SMTExprRef &RHS) = 0; + + /// Creates a predicate that checks for overflow in a bitvector addition + /// operation + virtual SMTExprRef mkBVAddNoOverflow(const SMTExprRef &LHS, + const SMTExprRef &RHS, + bool isSigned) = 0; + + /// Creates a predicate that checks for underflow in a signed bitvector + /// addition operation + virtual SMTExprRef mkBVAddNoUnderflow(const SMTExprRef &LHS, + const SMTExprRef &RHS) = 0; + + /// Creates a predicate that checks for overflow in a signed bitvector + /// subtraction operation + virtual SMTExprRef mkBVSubNoOverflow(const SMTExprRef &LHS, + const SMTExprRef &RHS) = 0; + + /// Creates a predicate that checks for underflow in a bitvector subtraction + /// operation + virtual SMTExprRef mkBVSubNoUnderflow(const SMTExprRef &LHS, + const SMTExprRef &RHS, + bool isSigned) = 0; + + /// Creates a predicate that checks for overflow in a signed bitvector + /// division/modulus operation + virtual SMTExprRef mkBVSDivNoOverflow(const SMTExprRef &LHS, + const SMTExprRef &RHS) = 0; + + /// Creates a predicate that checks for overflow in a bitvector negation + /// operation + virtual SMTExprRef mkBVNegNoOverflow(const SMTExprRef &Exp) = 0; + + /// Creates a predicate that checks for overflow in a bitvector multiplication + /// operation + virtual SMTExprRef mkBVMulNoOverflow(const SMTExprRef &LHS, + const SMTExprRef &RHS, + bool isSigned) = 0; + + /// Creates a predicate that checks for underflow in a signed bitvector + /// multiplication operation + virtual SMTExprRef mkBVMulNoUnderflow(const SMTExprRef &LHS, + const SMTExprRef &RHS) = 0; + + /// Creates a floating-point negation operation + virtual SMTExprRef mkFPNeg(const SMTExprRef &Exp) = 0; + + /// Creates a floating-point isInfinite operation + virtual SMTExprRef mkFPIsInfinite(const SMTExprRef &Exp) = 0; + + /// Creates a floating-point isNaN operation + virtual SMTExprRef mkFPIsNaN(const SMTExprRef &Exp) = 0; + + /// Creates a floating-point isNormal operation + virtual SMTExprRef mkFPIsNormal(const SMTExprRef &Exp) = 0; + + /// Creates a floating-point isZero operation + virtual SMTExprRef mkFPIsZero(const SMTExprRef &Exp) = 0; + + /// Creates a floating-point multiplication operation + virtual SMTExprRef mkFPMul(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a floating-point division operation + virtual SMTExprRef mkFPDiv(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a floating-point remainder operation + virtual SMTExprRef mkFPRem(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a floating-point addition operation + virtual SMTExprRef mkFPAdd(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a floating-point subtraction operation + virtual SMTExprRef mkFPSub(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a floating-point less-than operation + virtual SMTExprRef mkFPLt(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a floating-point greater-than operation + virtual SMTExprRef mkFPGt(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a floating-point less-than-or-equal operation + virtual SMTExprRef mkFPLe(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a floating-point greater-than-or-equal operation + virtual SMTExprRef mkFPGe(const SMTExprRef &LHS, const SMTExprRef &RHS) = 0; + + /// Creates a floating-point equality operation + virtual SMTExprRef mkFPEqual(const SMTExprRef &LHS, + const SMTExprRef &RHS) = 0; + + /// Creates a floating-point conversion from floatint-point to floating-point + /// operation + virtual SMTExprRef mkFPtoFP(const SMTExprRef &From, const SMTSortRef &To) = 0; + + /// Creates a floating-point conversion from signed bitvector to + /// floatint-point operation + virtual SMTExprRef mkSBVtoFP(const SMTExprRef &From, + const SMTSortRef &To) = 0; + + /// Creates a floating-point conversion from unsigned bitvector to + /// floatint-point operation + virtual SMTExprRef mkUBVtoFP(const SMTExprRef &From, + const SMTSortRef &To) = 0; + + /// Creates a floating-point conversion from floatint-point to signed + /// bitvector operation + virtual SMTExprRef mkFPtoSBV(const SMTExprRef &From, unsigned ToWidth) = 0; + + /// Creates a floating-point conversion from floatint-point to unsigned + /// bitvector operation + virtual SMTExprRef mkFPtoUBV(const SMTExprRef &From, unsigned ToWidth) = 0; + + /// Creates a new symbol, given a name and a sort + virtual SMTExprRef mkSymbol(const char *Name, SMTSortRef Sort) = 0; + + // Returns an appropriate floating-point rounding mode. + virtual SMTExprRef getFloatRoundingMode() = 0; + + // If the a model is available, returns the value of a given bitvector symbol + virtual llvm::APSInt getBitvector(const SMTExprRef &Exp, unsigned BitWidth, + bool isUnsigned) = 0; + + // If the a model is available, returns the value of a given boolean symbol + virtual bool getBoolean(const SMTExprRef &Exp) = 0; + + /// Constructs an SMTExprRef from a boolean. + virtual SMTExprRef mkBoolean(const bool b) = 0; + + /// Constructs an SMTExprRef from a finite APFloat. + virtual SMTExprRef mkFloat(const llvm::APFloat Float) = 0; + + /// Constructs an SMTExprRef from an APSInt and its bit width + virtual SMTExprRef mkBitvector(const llvm::APSInt Int, unsigned BitWidth) = 0; + + /// Given an expression, extract the value of this operand in the model. + virtual bool getInterpretation(const SMTExprRef &Exp, llvm::APSInt &Int) = 0; + + /// Given an expression extract the value of this operand in the model. + virtual bool getInterpretation(const SMTExprRef &Exp, + llvm::APFloat &Float) = 0; + + /// Check if the constraints are satisfiable + virtual Optional check() const = 0; + + /// Push the current solver state + virtual void push() = 0; + + /// Pop the previous solver state + virtual void pop(unsigned NumStates = 1) = 0; + + /// Reset the solver and remove all constraints. + virtual void reset() = 0; + + /// Checks if the solver supports floating-points. + virtual bool isFPSupported() = 0; + + virtual void print(raw_ostream &OS) const = 0; +}; + +/// Shared pointer for SMTSolvers. +using SMTSolverRef = std::shared_ptr; + +/// Convenience method to create and Z3Solver object +SMTSolverRef CreateZ3Solver(); + +} // namespace llvm + +#endif diff --git a/include/llvm/Support/SaveAndRestore.h b/include/llvm/Support/SaveAndRestore.h index 8e11789907ad..3c0333b7119a 100644 --- a/include/llvm/Support/SaveAndRestore.h +++ b/include/llvm/Support/SaveAndRestore.h @@ -1,9 +1,8 @@ //===-- SaveAndRestore.h - Utility -------------------------------*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/include/llvm/Support/ScalableSize.h b/include/llvm/Support/ScalableSize.h new file mode 100644 index 000000000000..96bf043773a0 --- /dev/null +++ b/include/llvm/Support/ScalableSize.h @@ -0,0 +1,43 @@ +//===- ScalableSize.h - Scalable vector size info ---------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file provides a struct that can be used to query the size of IR types +// which may be scalable vectors. It provides convenience operators so that +// it can be used in much the same way as a single scalar value. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_SCALABLESIZE_H +#define LLVM_SUPPORT_SCALABLESIZE_H + +namespace llvm { + +class ElementCount { +public: + unsigned Min; // Minimum number of vector elements. + bool Scalable; // If true, NumElements is a multiple of 'Min' determined + // at runtime rather than compile time. + + ElementCount(unsigned Min, bool Scalable) + : Min(Min), Scalable(Scalable) {} + + ElementCount operator*(unsigned RHS) { + return { Min * RHS, Scalable }; + } + ElementCount operator/(unsigned RHS) { + return { Min / RHS, Scalable }; + } + + bool operator==(const ElementCount& RHS) const { + return Min == RHS.Min && Scalable == RHS.Scalable; + } +}; + +} // end namespace llvm + +#endif // LLVM_SUPPORT_SCALABLESIZE_H diff --git a/include/llvm/Support/ScaledNumber.h b/include/llvm/Support/ScaledNumber.h index 3bd3ccedc42c..552da34f357b 100644 --- a/include/llvm/Support/ScaledNumber.h +++ b/include/llvm/Support/ScaledNumber.h @@ -1,9 +1,8 @@ //===- llvm/Support/ScaledNumber.h - Support for scaled numbers -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -887,10 +886,6 @@ template void ScaledNumber::shiftRight(int32_t Shift) { Digits >>= Shift; } -template struct isPodLike; -template struct isPodLike> { - static const bool value = true; -}; } // end namespace llvm diff --git a/include/llvm/Support/ScopedPrinter.h b/include/llvm/Support/ScopedPrinter.h index 34c1a287ee10..88daedc8713b 100644 --- a/include/llvm/Support/ScopedPrinter.h +++ b/include/llvm/Support/ScopedPrinter.h @@ -1,9 +1,8 @@ -//===-- ScopedPrinter.h ---------------------------------------------------===// +//===-- ScopedPrinter.h ----------------------------------------*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Support/Signals.h b/include/llvm/Support/Signals.h index f25a04969904..a6b215a24311 100644 --- a/include/llvm/Support/Signals.h +++ b/include/llvm/Support/Signals.h @@ -1,9 +1,8 @@ //===- llvm/Support/Signals.h - Signal Handling support ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -66,13 +65,25 @@ namespace sys { /// This function registers a function to be called when the user "interrupts" /// the program (typically by pressing ctrl-c). When the user interrupts the /// program, the specified interrupt function is called instead of the program - /// being killed, and the interrupt function automatically disabled. Note - /// that interrupt functions are not allowed to call any non-reentrant + /// being killed, and the interrupt function automatically disabled. + /// + /// Note that interrupt functions are not allowed to call any non-reentrant /// functions. An null interrupt function pointer disables the current /// installed function. Note also that the handler may be executed on a /// different thread on some platforms. - /// Register a function to be called when ctrl-c is pressed. void SetInterruptFunction(void (*IF)()); + + /// Registers a function to be called when an "info" signal is delivered to + /// the process. + /// + /// On POSIX systems, this will be SIGUSR1; on systems that have it, SIGINFO + /// will also be used (typically ctrl-t). + /// + /// Note that signal handlers are not allowed to call any non-reentrant + /// functions. An null function pointer disables the current installed + /// function. Note also that the handler may be executed on a different + /// thread on some platforms. + void SetInfoSignalFunction(void (*Handler)()); } // End sys namespace } // End llvm namespace diff --git a/include/llvm/Support/Signposts.h b/include/llvm/Support/Signposts.h new file mode 100644 index 000000000000..b5a8c3d61e3e --- /dev/null +++ b/include/llvm/Support/Signposts.h @@ -0,0 +1,43 @@ +//===-- llvm/Support/Signposts.h - Interval debug annotations ---*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +/// \file Some OS's provide profilers that allow applications to provide custom +/// annotations to the profiler. For example, on Xcode 10 and later 'signposts' +/// can be emitted by the application and these will be rendered to the Points +/// of Interest track on the instruments timeline. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_SIGNPOSTS_H +#define LLVM_SUPPORT_SIGNPOSTS_H + +namespace llvm { +class SignpostEmitterImpl; +class Timer; + +/// Manages the emission of signposts into the recording method supported by +/// the OS. +class SignpostEmitter { + SignpostEmitterImpl *Impl; + +public: + SignpostEmitter(); + ~SignpostEmitter(); + + bool isEnabled() const; + + /// Begin a signposted interval for the given timer. + void startTimerInterval(Timer *T); + /// End a signposted interval for the given timer. + void endTimerInterval(Timer *T); +}; + +} // end namespace llvm + +#endif // ifndef LLVM_SUPPORT_SIGNPOSTS_H diff --git a/include/llvm/Support/SmallVectorMemoryBuffer.h b/include/llvm/Support/SmallVectorMemoryBuffer.h index c4a600e7f37d..b63b58e3a8ba 100644 --- a/include/llvm/Support/SmallVectorMemoryBuffer.h +++ b/include/llvm/Support/SmallVectorMemoryBuffer.h @@ -1,9 +1,8 @@ //===- SmallVectorMemoryBuffer.h --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/Solaris/sys/regset.h b/include/llvm/Support/Solaris/sys/regset.h index 6a69ebe718a1..6bd98fa826a6 100644 --- a/include/llvm/Support/Solaris/sys/regset.h +++ b/include/llvm/Support/Solaris/sys/regset.h @@ -1,9 +1,8 @@ /*===- llvm/Support/Solaris/sys/regset.h ------------------------*- C++ -*-===* * - * The LLVM Compiler Infrastructure - * - * This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * *===----------------------------------------------------------------------===* * diff --git a/include/llvm/Support/SourceMgr.h b/include/llvm/Support/SourceMgr.h index 63ac893239d1..aa6026c23d07 100644 --- a/include/llvm/Support/SourceMgr.h +++ b/include/llvm/Support/SourceMgr.h @@ -1,9 +1,8 @@ //===- SourceMgr.h - Manager for Source Buffers & Diagnostics ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -107,6 +106,8 @@ public: SourceMgr() = default; SourceMgr(const SourceMgr &) = delete; SourceMgr &operator=(const SourceMgr &) = delete; + SourceMgr(SourceMgr &&) = default; + SourceMgr &operator=(SourceMgr &&) = default; ~SourceMgr() = default; void setIncludeDirs(const std::vector &Dirs) { diff --git a/include/llvm/Support/SpecialCaseList.h b/include/llvm/Support/SpecialCaseList.h index fd62fc48047b..b7400266f4df 100644 --- a/include/llvm/Support/SpecialCaseList.h +++ b/include/llvm/Support/SpecialCaseList.h @@ -1,9 +1,8 @@ //===-- SpecialCaseList.h - special case list for sanitizers ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception //===----------------------------------------------------------------------===// // // This is a utility class used to parse user-provided text files with diff --git a/include/llvm/Support/StringPool.h b/include/llvm/Support/StringPool.h index bb5fd07f0d00..a4f45916f53d 100644 --- a/include/llvm/Support/StringPool.h +++ b/include/llvm/Support/StringPool.h @@ -1,9 +1,8 @@ //===- StringPool.h - Interned string pool ----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/StringSaver.h b/include/llvm/Support/StringSaver.h index 6b77d487333b..c54044e3986c 100644 --- a/include/llvm/Support/StringSaver.h +++ b/include/llvm/Support/StringSaver.h @@ -1,9 +1,8 @@ //===- llvm/Support/StringSaver.h -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Support/SwapByteOrder.h b/include/llvm/Support/SwapByteOrder.h index 71d3724950ab..06a447a27c2a 100644 --- a/include/llvm/Support/SwapByteOrder.h +++ b/include/llvm/Support/SwapByteOrder.h @@ -1,9 +1,8 @@ //===- SwapByteOrder.h - Generic and optimized byte swaps -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,6 +17,7 @@ #include "llvm/Support/Compiler.h" #include "llvm/Support/DataTypes.h" #include +#include #if defined(_MSC_VER) && !defined(_DEBUG) #include #endif @@ -116,6 +116,13 @@ inline double getSwappedBytes(double C) { return out.d; } +template +inline typename std::enable_if::value, T>::type +getSwappedBytes(T C) { + return static_cast( + getSwappedBytes(static_cast::type>(C))); +} + template inline void swapByteOrder(T &Value) { Value = getSwappedBytes(Value); diff --git a/include/llvm/Support/SymbolRemappingReader.h b/include/llvm/Support/SymbolRemappingReader.h index b457b9e817e4..2b9ab570eb8b 100644 --- a/include/llvm/Support/SymbolRemappingReader.h +++ b/include/llvm/Support/SymbolRemappingReader.h @@ -1,9 +1,8 @@ //===- SymbolRemappingReader.h - Read symbol remapping file -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/SystemUtils.h b/include/llvm/Support/SystemUtils.h index bd60793d1554..77deddb9ee1c 100644 --- a/include/llvm/Support/SystemUtils.h +++ b/include/llvm/Support/SystemUtils.h @@ -1,9 +1,8 @@ //===- SystemUtils.h - Utilities to do low-level system stuff ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/TarWriter.h b/include/llvm/Support/TarWriter.h index 639f61b53892..71164e2ef961 100644 --- a/include/llvm/Support/TarWriter.h +++ b/include/llvm/Support/TarWriter.h @@ -1,9 +1,8 @@ //===-- llvm/Support/TarWriter.h - Tar archive file creator -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Support/TargetOpcodes.def b/include/llvm/Support/TargetOpcodes.def index 3e8193a5cdcf..598c1064efd0 100644 --- a/include/llvm/Support/TargetOpcodes.def +++ b/include/llvm/Support/TargetOpcodes.def @@ -1,9 +1,8 @@ //===-- llvm/Support/TargetOpcodes.def - Target Indep Opcodes ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -29,6 +28,7 @@ /// HANDLE_TARGET_OPCODE(PHI) HANDLE_TARGET_OPCODE(INLINEASM) +HANDLE_TARGET_OPCODE(INLINEASM_BR) HANDLE_TARGET_OPCODE(CFI_INSTRUCTION) HANDLE_TARGET_OPCODE(EH_LABEL) HANDLE_TARGET_OPCODE(GC_LABEL) @@ -316,6 +316,9 @@ HANDLE_TARGET_OPCODE(G_ATOMICRMW_MIN) HANDLE_TARGET_OPCODE(G_ATOMICRMW_UMAX) HANDLE_TARGET_OPCODE(G_ATOMICRMW_UMIN) +// Generic atomic fence +HANDLE_TARGET_OPCODE(G_FENCE) + /// Generic conditional branch instruction. HANDLE_TARGET_OPCODE(G_BRCOND) @@ -481,6 +484,27 @@ HANDLE_TARGET_OPCODE(G_UITOFP) /// Generic FP absolute value. HANDLE_TARGET_OPCODE(G_FABS) +/// FCOPYSIGN(X, Y) - Return the value of X with the sign of Y. NOTE: This does +/// not require that X and Y have the same type, just that they are both +/// floating point. X and the result must have the same type. FCOPYSIGN(f32, +/// f64) is allowed. +HANDLE_TARGET_OPCODE(G_FCOPYSIGN) + +/// Generic FP canonicalize value. +HANDLE_TARGET_OPCODE(G_FCANONICALIZE) + +/// FP min/max matching libm's fmin/fmax +HANDLE_TARGET_OPCODE(G_FMINNUM) +HANDLE_TARGET_OPCODE(G_FMAXNUM) + +/// FP min/max matching IEEE-754 2008's minnum/maxnum semantics. +HANDLE_TARGET_OPCODE(G_FMINNUM_IEEE) +HANDLE_TARGET_OPCODE(G_FMAXNUM_IEEE) + +/// FP min/max matching IEEE-754 2018 draft semantics. +HANDLE_TARGET_OPCODE(G_FMINIMUM) +HANDLE_TARGET_OPCODE(G_FMAXIMUM) + /// Generic pointer offset HANDLE_TARGET_OPCODE(G_GEP) @@ -488,9 +512,24 @@ HANDLE_TARGET_OPCODE(G_GEP) /// *down* to the given alignment. HANDLE_TARGET_OPCODE(G_PTR_MASK) +/// Generic signed integer minimum. +HANDLE_TARGET_OPCODE(G_SMIN) + +/// Generic signed integer maximum. +HANDLE_TARGET_OPCODE(G_SMAX) + +/// Generic unsigned integer maximum. +HANDLE_TARGET_OPCODE(G_UMIN) + +/// Generic unsigned integer maximum. +HANDLE_TARGET_OPCODE(G_UMAX) + /// Generic BRANCH instruction. This is an unconditional branch. HANDLE_TARGET_OPCODE(G_BR) +/// Generic branch to jump table entry. +HANDLE_TARGET_OPCODE(G_BRJT) + /// Generic insertelement. HANDLE_TARGET_OPCODE(G_INSERT_VECTOR_ELT) @@ -521,18 +560,39 @@ HANDLE_TARGET_OPCODE(G_BSWAP) /// Floating point ceil. HANDLE_TARGET_OPCODE(G_FCEIL) +/// Floating point cosine. +HANDLE_TARGET_OPCODE(G_FCOS) + +/// Floating point sine. +HANDLE_TARGET_OPCODE(G_FSIN) + +/// Floating point square root. +HANDLE_TARGET_OPCODE(G_FSQRT) + +/// Floating point floor. +HANDLE_TARGET_OPCODE(G_FFLOOR) + +/// Floating point round to next integer. +HANDLE_TARGET_OPCODE(G_FRINT) + +/// Floating point round to nearest integer. +HANDLE_TARGET_OPCODE(G_FNEARBYINT) + /// Generic AddressSpaceCast. HANDLE_TARGET_OPCODE(G_ADDRSPACE_CAST) /// Generic block address HANDLE_TARGET_OPCODE(G_BLOCK_ADDR) +/// Generic jump table address +HANDLE_TARGET_OPCODE(G_JUMP_TABLE) + // TODO: Add more generic opcodes as we move along. /// Marker for the end of the generic opcode. /// This is used to check if an opcode is in the range of the /// generic opcodes. -HANDLE_TARGET_OPCODE_MARKER(PRE_ISEL_GENERIC_OPCODE_END, G_BLOCK_ADDR) +HANDLE_TARGET_OPCODE_MARKER(PRE_ISEL_GENERIC_OPCODE_END, G_JUMP_TABLE) /// BUILTIN_OP_END - This must be the last enum value in this list. /// The target-specific post-isel opcode values start here. diff --git a/include/llvm/Support/TargetParser.h b/include/llvm/Support/TargetParser.h index ace11ed410a3..a7e1a752d081 100644 --- a/include/llvm/Support/TargetParser.h +++ b/include/llvm/Support/TargetParser.h @@ -1,9 +1,8 @@ //===-- TargetParser - Parser for target features ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -122,10 +121,15 @@ enum GPUKind : uint32_t { GK_GFX902 = 61, GK_GFX904 = 62, GK_GFX906 = 63, + GK_GFX908 = 64, GK_GFX909 = 65, + GK_GFX1010 = 71, + GK_GFX1011 = 72, + GK_GFX1012 = 73, + GK_AMDGCN_FIRST = GK_GFX600, - GK_AMDGCN_LAST = GK_GFX909, + GK_AMDGCN_LAST = GK_GFX1012, }; /// Instruction set architecture version. diff --git a/include/llvm/Support/TargetRegistry.h b/include/llvm/Support/TargetRegistry.h index 1bafc4e687da..bf75650760d0 100644 --- a/include/llvm/Support/TargetRegistry.h +++ b/include/llvm/Support/TargetRegistry.h @@ -1,9 +1,8 @@ //===- Support/TargetRegistry.h - Target Registration -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -101,6 +100,11 @@ MCStreamer *createWasmStreamer(MCContext &Ctx, std::unique_ptr &&OW, std::unique_ptr &&CE, bool RelaxAll); +MCStreamer *createXCOFFStreamer(MCContext &Ctx, + std::unique_ptr &&TAB, + std::unique_ptr &&OW, + std::unique_ptr &&CE, + bool RelaxAll); MCRelocationInfo *createMCRelocationInfo(const Triple &TT, MCContext &Ctx); @@ -471,7 +475,7 @@ public: bool DWARFMustBeAtTheEnd) const { MCStreamer *S; switch (T.getObjectFormat()) { - default: + case Triple::UnknownObjectFormat: llvm_unreachable("Unknown object format"); case Triple::COFF: assert(T.isOSWindows() && "only Windows COFF is supported"); @@ -505,6 +509,10 @@ public: S = createWasmStreamer(Ctx, std::move(TAB), std::move(OW), std::move(Emitter), RelaxAll); break; + case Triple::XCOFF: + S = createXCOFFStreamer(Ctx, std::move(TAB), std::move(OW), + std::move(Emitter), RelaxAll); + break; } if (ObjectTargetStreamerCtorFn) ObjectTargetStreamerCtorFn(*S, STI); diff --git a/include/llvm/Support/TargetSelect.h b/include/llvm/Support/TargetSelect.h index 582785cb69a5..9ffb84c4a570 100644 --- a/include/llvm/Support/TargetSelect.h +++ b/include/llvm/Support/TargetSelect.h @@ -1,9 +1,8 @@ //===- TargetSelect.h - Target Selection & Registration ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/TaskQueue.h b/include/llvm/Support/TaskQueue.h index 49981adb763d..df2ffdee2cc2 100644 --- a/include/llvm/Support/TaskQueue.h +++ b/include/llvm/Support/TaskQueue.h @@ -1,9 +1,8 @@ //===-- llvm/Support/TaskQueue.h - A TaskQueue implementation ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/ThreadLocal.h b/include/llvm/Support/ThreadLocal.h index 885bd18e8356..d6838c15fc34 100644 --- a/include/llvm/Support/ThreadLocal.h +++ b/include/llvm/Support/ThreadLocal.h @@ -1,9 +1,8 @@ //===- llvm/Support/ThreadLocal.h - Thread Local Data ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/ThreadPool.h b/include/llvm/Support/ThreadPool.h index 4fdbd528b212..4bcbaa3142fd 100644 --- a/include/llvm/Support/ThreadPool.h +++ b/include/llvm/Support/ThreadPool.h @@ -1,9 +1,8 @@ //===-- llvm/Support/ThreadPool.h - A ThreadPool implementation -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/Threading.h b/include/llvm/Support/Threading.h index ba7ece5e72ba..46d413dc487b 100644 --- a/include/llvm/Support/Threading.h +++ b/include/llvm/Support/Threading.h @@ -1,9 +1,8 @@ //===-- llvm/Support/Threading.h - Control multithreading mode --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -33,6 +32,9 @@ // implementations like libstdc++ are known to have problems on NetBSD, // OpenBSD and PowerPC. #define LLVM_THREADING_USE_STD_CALL_ONCE 1 +#elif defined(LLVM_ON_UNIX) && \ + ((defined(__ppc__) || defined(__PPC__)) && defined(__LITTLE_ENDIAN__)) +#define LLVM_THREADING_USE_STD_CALL_ONCE 1 #else #define LLVM_THREADING_USE_STD_CALL_ONCE 0 #endif @@ -165,6 +167,19 @@ void llvm_execute_on_thread(void (*UserFn)(void *), void *UserData, /// purposes, and as with setting a thread's name no indication of whether /// the operation succeeded or failed is returned. void get_thread_name(SmallVectorImpl &Name); + + enum class ThreadPriority { + Background = 0, + Default = 1, + }; + /// If priority is Background tries to lower current threads priority such + /// that it does not affect foreground tasks significantly. Can be used for + /// long-running, latency-insensitive tasks to make sure cpu is not hogged by + /// this task. + /// If the priority is default tries to restore current threads priority to + /// default scheduling priority. + enum class SetThreadPriorityResult { FAILURE, SUCCESS }; + SetThreadPriorityResult set_thread_priority(ThreadPriority Priority); } #endif diff --git a/include/llvm/Support/TimeProfiler.h b/include/llvm/Support/TimeProfiler.h new file mode 100644 index 000000000000..72b6f7180bde --- /dev/null +++ b/include/llvm/Support/TimeProfiler.h @@ -0,0 +1,76 @@ +//===- llvm/Support/TimeProfiler.h - Hierarchical Time Profiler -*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_TIME_PROFILER_H +#define LLVM_SUPPORT_TIME_PROFILER_H + +#include "llvm/Support/raw_ostream.h" + +namespace llvm { + +struct TimeTraceProfiler; +extern TimeTraceProfiler *TimeTraceProfilerInstance; + +/// Initialize the time trace profiler. +/// This sets up the global \p TimeTraceProfilerInstance +/// variable to be the profiler instance. +void timeTraceProfilerInitialize(); + +/// Cleanup the time trace profiler, if it was initialized. +void timeTraceProfilerCleanup(); + +/// Is the time trace profiler enabled, i.e. initialized? +inline bool timeTraceProfilerEnabled() { + return TimeTraceProfilerInstance != nullptr; +} + +/// Write profiling data to output file. +/// Data produced is JSON, in Chrome "Trace Event" format, see +/// https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview +void timeTraceProfilerWrite(raw_pwrite_stream &OS); + +/// Manually begin a time section, with the given \p Name and \p Detail. +/// Profiler copies the string data, so the pointers can be given into +/// temporaries. Time sections can be hierarchical; every Begin must have a +/// matching End pair but they can nest. +void timeTraceProfilerBegin(StringRef Name, StringRef Detail); +void timeTraceProfilerBegin(StringRef Name, + llvm::function_ref Detail); + +/// Manually end the last time section. +void timeTraceProfilerEnd(); + +/// The TimeTraceScope is a helper class to call the begin and end functions +/// of the time trace profiler. When the object is constructed, it begins +/// the section; and when it is destroyed, it stops it. If the time profiler +/// is not initialized, the overhead is a single branch. +struct TimeTraceScope { + + TimeTraceScope() = delete; + TimeTraceScope(const TimeTraceScope &) = delete; + TimeTraceScope &operator=(const TimeTraceScope &) = delete; + TimeTraceScope(TimeTraceScope &&) = delete; + TimeTraceScope &operator=(TimeTraceScope &&) = delete; + + TimeTraceScope(StringRef Name, StringRef Detail) { + if (TimeTraceProfilerInstance != nullptr) + timeTraceProfilerBegin(Name, Detail); + } + TimeTraceScope(StringRef Name, llvm::function_ref Detail) { + if (TimeTraceProfilerInstance != nullptr) + timeTraceProfilerBegin(Name, Detail); + } + ~TimeTraceScope() { + if (TimeTraceProfilerInstance != nullptr) + timeTraceProfilerEnd(); + } +}; + +} // end namespace llvm + +#endif diff --git a/include/llvm/Support/Timer.h b/include/llvm/Support/Timer.h index a11c3ce3ff22..76c9bc7b6863 100644 --- a/include/llvm/Support/Timer.h +++ b/include/llvm/Support/Timer.h @@ -1,9 +1,8 @@ //===-- llvm/Support/Timer.h - Interval Timing Support ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -206,8 +205,9 @@ public: Description.assign(NewDescription.begin(), NewDescription.end()); } - /// Print any started timers in this group. - void print(raw_ostream &OS); + /// Print any started timers in this group, optionally resetting timers after + /// printing them. + void print(raw_ostream &OS, bool ResetAfterPrint = false); /// Clear all timers in this group. void clear(); @@ -234,7 +234,7 @@ private: friend void PrintStatisticsJSON(raw_ostream &OS); void addTimer(Timer &T); void removeTimer(Timer &T); - void prepareToPrintList(); + void prepareToPrintList(bool reset_time = false); void PrintQueuedTimers(raw_ostream &OS); void printJSONValue(raw_ostream &OS, const PrintRecord &R, const char *suffix, double Value); diff --git a/include/llvm/Support/ToolOutputFile.h b/include/llvm/Support/ToolOutputFile.h index cf3bc2fb0171..a99e327f8db7 100644 --- a/include/llvm/Support/ToolOutputFile.h +++ b/include/llvm/Support/ToolOutputFile.h @@ -1,9 +1,8 @@ //===- ToolOutputFile.h - Output files for compiler-like tools -----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/TrailingObjects.h b/include/llvm/Support/TrailingObjects.h index 490bd94f4cd5..8cf4f7aed7f8 100644 --- a/include/llvm/Support/TrailingObjects.h +++ b/include/llvm/Support/TrailingObjects.h @@ -1,9 +1,8 @@ //===--- TrailingObjects.h - Variable-length classes ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/include/llvm/Support/TrigramIndex.h b/include/llvm/Support/TrigramIndex.h index da0b6daf47ed..9351c2db169a 100644 --- a/include/llvm/Support/TrigramIndex.h +++ b/include/llvm/Support/TrigramIndex.h @@ -1,9 +1,8 @@ //===-- TrigramIndex.h - a heuristic for SpecialCaseList --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception //===----------------------------------------------------------------------===// // // TrigramIndex implements a heuristic for SpecialCaseList that allows to diff --git a/include/llvm/Support/TypeName.h b/include/llvm/Support/TypeName.h index 0eb7ead98b21..236490a25011 100644 --- a/include/llvm/Support/TypeName.h +++ b/include/llvm/Support/TypeName.h @@ -1,9 +1,8 @@ //===- TypeName.h -----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Support/Unicode.h b/include/llvm/Support/Unicode.h index 983acaf03635..ca17bba2fbb4 100644 --- a/include/llvm/Support/Unicode.h +++ b/include/llvm/Support/Unicode.h @@ -1,9 +1,8 @@ //===- llvm/Support/Unicode.h - Unicode character properties -*- C++ -*-=====// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/UnicodeCharRanges.h b/include/llvm/Support/UnicodeCharRanges.h index 3cf4a6d96602..4b59f8a92b76 100644 --- a/include/llvm/Support/UnicodeCharRanges.h +++ b/include/llvm/Support/UnicodeCharRanges.h @@ -1,9 +1,8 @@ //===--- UnicodeCharRanges.h - Types and functions for character ranges ---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef LLVM_SUPPORT_UNICODECHARRANGES_H diff --git a/include/llvm/Support/UniqueLock.h b/include/llvm/Support/UniqueLock.h index 91dc911036d5..0a887ad5965d 100644 --- a/include/llvm/Support/UniqueLock.h +++ b/include/llvm/Support/UniqueLock.h @@ -1,9 +1,8 @@ //===- Support/UniqueLock.h - Acquire/Release Mutex In Scope ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/Valgrind.h b/include/llvm/Support/Valgrind.h index 084b901b326c..1e14dfec9a61 100644 --- a/include/llvm/Support/Valgrind.h +++ b/include/llvm/Support/Valgrind.h @@ -1,9 +1,8 @@ //===- llvm/Support/Valgrind.h - Communication with Valgrind ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/VersionTuple.h b/include/llvm/Support/VersionTuple.h index e85a188e54b4..14736d6b28f0 100644 --- a/include/llvm/Support/VersionTuple.h +++ b/include/llvm/Support/VersionTuple.h @@ -1,9 +1,8 @@ //===- VersionTuple.h - Version Number Handling -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/include/llvm/Support/VirtualFileSystem.h b/include/llvm/Support/VirtualFileSystem.h index 61c3d2f46e9c..31c9e851daed 100644 --- a/include/llvm/Support/VirtualFileSystem.h +++ b/include/llvm/Support/VirtualFileSystem.h @@ -1,9 +1,8 @@ //===- VirtualFileSystem.h - Virtual File System Layer ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -59,15 +58,15 @@ public: Status() = default; Status(const llvm::sys::fs::file_status &Status); - Status(StringRef Name, llvm::sys::fs::UniqueID UID, + Status(const Twine &Name, llvm::sys::fs::UniqueID UID, llvm::sys::TimePoint<> MTime, uint32_t User, uint32_t Group, uint64_t Size, llvm::sys::fs::file_type Type, llvm::sys::fs::perms Perms); /// Get a copy of a Status with a different name. - static Status copyWithNewName(const Status &In, StringRef NewName); + static Status copyWithNewName(const Status &In, const Twine &NewName); static Status copyWithNewName(const llvm::sys::fs::file_status &In, - StringRef NewName); + const Twine &NewName); /// Returns the name that should be used for this file or directory. StringRef getName() const { return Name; } @@ -299,8 +298,16 @@ public: /// Gets an \p vfs::FileSystem for the 'real' file system, as seen by /// the operating system. +/// The working directory is linked to the process's working directory. +/// (This is usually thread-hostile). IntrusiveRefCntPtr getRealFileSystem(); +/// Create an \p vfs::FileSystem for the 'real' file system, as seen by +/// the operating system. +/// It has its own working directory, independent of (but initially equal to) +/// that of the process. +std::unique_ptr createPhysicalFileSystem(); + /// A file system that allows overlaying one \p AbstractFileSystem on top /// of another. /// @@ -336,15 +343,24 @@ public: using iterator = FileSystemList::reverse_iterator; using const_iterator = FileSystemList::const_reverse_iterator; + using reverse_iterator = FileSystemList::iterator; + using const_reverse_iterator = FileSystemList::const_iterator; /// Get an iterator pointing to the most recently added file system. iterator overlays_begin() { return FSList.rbegin(); } const_iterator overlays_begin() const { return FSList.rbegin(); } - /// Get an iterator pointing one-past the least recently added file - /// system. + /// Get an iterator pointing one-past the least recently added file system. iterator overlays_end() { return FSList.rend(); } const_iterator overlays_end() const { return FSList.rend(); } + + /// Get an iterator pointing to the least recently added file system. + reverse_iterator overlays_rbegin() { return FSList.begin(); } + const_reverse_iterator overlays_rbegin() const { return FSList.begin(); } + + /// Get an iterator pointing one-past the most recently added file system. + reverse_iterator overlays_rend() { return FSList.end(); } + const_reverse_iterator overlays_rend() const { return FSList.end(); } }; /// By default, this delegates all calls to the underlying file system. This diff --git a/include/llvm/Support/Watchdog.h b/include/llvm/Support/Watchdog.h index 01e1d926eb95..281595e8f272 100644 --- a/include/llvm/Support/Watchdog.h +++ b/include/llvm/Support/Watchdog.h @@ -1,9 +1,8 @@ //===--- Watchdog.h - Watchdog timer ----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/Win64EH.h b/include/llvm/Support/Win64EH.h index e27bf1b3a1a5..bdd23b41594e 100644 --- a/include/llvm/Support/Win64EH.h +++ b/include/llvm/Support/Win64EH.h @@ -1,9 +1,8 @@ //===-- llvm/Support/Win64EH.h ---Win64 EH Constants-------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/WindowsError.h b/include/llvm/Support/WindowsError.h index 63bfe5976546..195405224124 100644 --- a/include/llvm/Support/WindowsError.h +++ b/include/llvm/Support/WindowsError.h @@ -1,9 +1,8 @@ //===-- WindowsError.h - Support for mapping windows errors to posix-------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Support/WithColor.h b/include/llvm/Support/WithColor.h index 76842d1c3dc8..f4e107581179 100644 --- a/include/llvm/Support/WithColor.h +++ b/include/llvm/Support/WithColor.h @@ -1,9 +1,8 @@ //===- WithColor.h ----------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Support/X86DisassemblerDecoderCommon.h b/include/llvm/Support/X86DisassemblerDecoderCommon.h index 466dd309909a..baf842b12a27 100644 --- a/include/llvm/Support/X86DisassemblerDecoderCommon.h +++ b/include/llvm/Support/X86DisassemblerDecoderCommon.h @@ -1,9 +1,8 @@ //===-- X86DisassemblerDecoderCommon.h - Disassembler decoder ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -47,29 +46,23 @@ namespace X86Disassembler { // Attributes of an instruction that must be known before the opcode can be // processed correctly. Most of these indicate the presence of particular // prefixes, but ATTR_64BIT is simply an attribute of the decoding context. -#define ATTRIBUTE_BITS \ - ENUM_ENTRY(ATTR_NONE, 0x00) \ - ENUM_ENTRY(ATTR_64BIT, (0x1 << 0)) \ - ENUM_ENTRY(ATTR_XS, (0x1 << 1)) \ - ENUM_ENTRY(ATTR_XD, (0x1 << 2)) \ - ENUM_ENTRY(ATTR_REXW, (0x1 << 3)) \ - ENUM_ENTRY(ATTR_OPSIZE, (0x1 << 4)) \ - ENUM_ENTRY(ATTR_ADSIZE, (0x1 << 5)) \ - ENUM_ENTRY(ATTR_VEX, (0x1 << 6)) \ - ENUM_ENTRY(ATTR_VEXL, (0x1 << 7)) \ - ENUM_ENTRY(ATTR_EVEX, (0x1 << 8)) \ - ENUM_ENTRY(ATTR_EVEXL, (0x1 << 9)) \ - ENUM_ENTRY(ATTR_EVEXL2, (0x1 << 10)) \ - ENUM_ENTRY(ATTR_EVEXK, (0x1 << 11)) \ - ENUM_ENTRY(ATTR_EVEXKZ, (0x1 << 12)) \ - ENUM_ENTRY(ATTR_EVEXB, (0x1 << 13)) - -#define ENUM_ENTRY(n, v) n = v, enum attributeBits { - ATTRIBUTE_BITS - ATTR_max + ATTR_NONE = 0x00, + ATTR_64BIT = 0x1 << 0, + ATTR_XS = 0x1 << 1, + ATTR_XD = 0x1 << 2, + ATTR_REXW = 0x1 << 3, + ATTR_OPSIZE = 0x1 << 4, + ATTR_ADSIZE = 0x1 << 5, + ATTR_VEX = 0x1 << 6, + ATTR_VEXL = 0x1 << 7, + ATTR_EVEX = 0x1 << 8, + ATTR_EVEXL2 = 0x1 << 9, + ATTR_EVEXK = 0x1 << 10, + ATTR_EVEXKZ = 0x1 << 11, + ATTR_EVEXB = 0x1 << 12, + ATTR_max = 0x1 << 13, }; -#undef ENUM_ENTRY // Combinations of the above attributes that are relevant to instruction // decode. Although other combinations are possible, they can be reduced to @@ -394,6 +387,7 @@ enum ModRMDecisionType { ENUM_ENTRY(ENCODING_IRC, "Immediate for static rounding control") \ ENUM_ENTRY(ENCODING_Rv, "Register code of operand size added to the " \ "opcode byte") \ + ENUM_ENTRY(ENCODING_CC, "Condition code encoded in opcode") \ ENUM_ENTRY(ENCODING_DUP, "Duplicate of another operand; ID is encoded " \ "in type") \ ENUM_ENTRY(ENCODING_SI, "Source index; encoded in OpSize/Adsize prefix") \ @@ -415,9 +409,6 @@ enum OperandEncoding { ENUM_ENTRY(TYPE_R32, "4-byte") \ ENUM_ENTRY(TYPE_R64, "8-byte") \ ENUM_ENTRY(TYPE_IMM, "immediate operand") \ - ENUM_ENTRY(TYPE_IMM3, "1-byte immediate operand between 0 and 7") \ - ENUM_ENTRY(TYPE_IMM5, "1-byte immediate operand between 0 and 31") \ - ENUM_ENTRY(TYPE_AVX512ICC, "1-byte immediate operand for AVX512 icmp") \ ENUM_ENTRY(TYPE_UIMM8, "1-byte unsigned immediate operand") \ ENUM_ENTRY(TYPE_M, "Memory operand") \ ENUM_ENTRY(TYPE_MVSIBX, "Memory operand using XMM index") \ @@ -432,6 +423,7 @@ enum OperandEncoding { ENUM_ENTRY(TYPE_YMM, "32-byte") \ ENUM_ENTRY(TYPE_ZMM, "64-byte") \ ENUM_ENTRY(TYPE_VK, "mask register") \ + ENUM_ENTRY(TYPE_VK_PAIR, "mask register pair") \ ENUM_ENTRY(TYPE_SEGMENTREG, "Segment register operand") \ ENUM_ENTRY(TYPE_DEBUGREG, "Debug register operand") \ ENUM_ENTRY(TYPE_CONTROLREG, "Control register operand") \ diff --git a/include/llvm/Support/X86TargetParser.def b/include/llvm/Support/X86TargetParser.def index e9bede545d3f..1749be3b3ae2 100644 --- a/include/llvm/Support/X86TargetParser.def +++ b/include/llvm/Support/X86TargetParser.def @@ -1,9 +1,8 @@ //===- X86TargetParser.def - X86 target parsing defines ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -34,6 +33,8 @@ X86_VENDOR(VENDOR_AMD, "amd") #ifndef X86_CPU_TYPE #define X86_CPU_TYPE(ARCHNAME, ENUM) #endif +// The first part of this list must match what is implemented in libgcc and +// compilert-rt. Clang uses this to know how to implement __builtin_cpu_is. X86_CPU_TYPE_COMPAT_WITH_ALIAS("bonnell", INTEL_BONNELL, "bonnell", "atom") X86_CPU_TYPE_COMPAT ("core2", INTEL_CORE2, "core2") X86_CPU_TYPE_COMPAT ("nehalem", INTEL_COREI7, "corei7") @@ -80,6 +81,8 @@ X86_CPU_TYPE ("k8-sse3", AMD_K8SSE3) #define X86_CPU_SUBTYPE(ARCHNAME, ENUM) #endif +// The first part of this list must match what is implemented in libgcc and +// compilert-rt. Clang uses this to know how to implement __builtin_cpu_is. X86_CPU_SUBTYPE_COMPAT("nehalem", INTEL_COREI7_NEHALEM, "nehalem") X86_CPU_SUBTYPE_COMPAT("westmere", INTEL_COREI7_WESTMERE, "westmere") X86_CPU_SUBTYPE_COMPAT("sandybridge", INTEL_COREI7_SANDYBRIDGE, "sandybridge") @@ -99,14 +102,16 @@ X86_CPU_SUBTYPE_COMPAT("skylake-avx512", INTEL_COREI7_SKYLAKE_AVX512, "skylake-a X86_CPU_SUBTYPE_COMPAT("cannonlake", INTEL_COREI7_CANNONLAKE, "cannonlake") X86_CPU_SUBTYPE_COMPAT("icelake-client", INTEL_COREI7_ICELAKE_CLIENT, "icelake-client") X86_CPU_SUBTYPE_COMPAT("icelake-server", INTEL_COREI7_ICELAKE_SERVER, "icelake-server") +X86_CPU_SUBTYPE_COMPAT("znver2", AMDFAM17H_ZNVER2, "znver2") +X86_CPU_SUBTYPE_COMPAT("cascadelake", INTEL_COREI7_CASCADELAKE, "cascadelake") // Entries below this are not in libgcc/compiler-rt. X86_CPU_SUBTYPE ("core2", INTEL_CORE2_65) X86_CPU_SUBTYPE ("penryn", INTEL_CORE2_45) -X86_CPU_SUBTYPE ("cascadelake", INTEL_COREI7_CASCADELAKE) X86_CPU_SUBTYPE ("k6", AMDPENTIUM_K6) X86_CPU_SUBTYPE ("k6-2", AMDPENTIUM_K62) X86_CPU_SUBTYPE ("k6-3", AMDPENTIUM_K63) X86_CPU_SUBTYPE ("geode", AMDPENTIUM_GEODE) +X86_CPU_SUBTYPE ("cooperlake", INTEL_COREI7_COOPERLAKE) #undef X86_CPU_SUBTYPE_COMPAT #undef X86_CPU_SUBTYPE @@ -161,5 +166,6 @@ X86_FEATURE (65, FEATURE_ADX) X86_FEATURE (66, FEATURE_EM64T) X86_FEATURE (67, FEATURE_CLFLUSHOPT) X86_FEATURE (68, FEATURE_SHA) +X86_FEATURE (69, FEATURE_AVX512BF16) #undef X86_FEATURE_COMPAT #undef X86_FEATURE diff --git a/include/llvm/Support/YAMLParser.h b/include/llvm/Support/YAMLParser.h index 5b031a9a4270..3570119a3bfd 100644 --- a/include/llvm/Support/YAMLParser.h +++ b/include/llvm/Support/YAMLParser.h @@ -1,9 +1,8 @@ //===- YAMLParser.h - Simple YAML parser ------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/YAMLTraits.h b/include/llvm/Support/YAMLTraits.h index 3d790e96fff7..5181dc56d81d 100644 --- a/include/llvm/Support/YAMLTraits.h +++ b/include/llvm/Support/YAMLTraits.h @@ -1,9 +1,8 @@ //===- llvm/Support/YAMLTraits.h --------------------------------*- C++ -*-===// // -// The LLVM Linker -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -102,8 +101,7 @@ template struct MappingContextTraits { /// io.enumCase(value, "green", cGreen); /// } /// }; -template -struct ScalarEnumerationTraits { +template struct ScalarEnumerationTraits { // Must provide: // static void enumeration(IO &io, T &value); }; @@ -119,8 +117,7 @@ struct ScalarEnumerationTraits { /// io.bitSetCase(value, "round", flagRound); /// } /// }; -template -struct ScalarBitSetTraits { +template struct ScalarBitSetTraits { // Must provide: // static void bitset(IO &io, T &value); }; @@ -146,8 +143,7 @@ enum class QuotingType { None, Single, Double }; /// } /// static QuotingType mustQuote(StringRef) { return QuotingType::Single; } /// }; -template -struct ScalarTraits { +template struct ScalarTraits { // Must provide: // // Function to write the value as a string: @@ -864,8 +860,8 @@ public: mapOptionalWithContext(Key, Val, Ctx); } - template - void mapOptional(const char *Key, T &Val, const T &Default) { + template + void mapOptional(const char *Key, T &Val, const DefaultT &Default) { EmptyContext Ctx; mapOptionalWithContext(Key, Val, Default, Ctx); } @@ -891,10 +887,13 @@ public: this->processKey(Key, Val, false, Ctx); } - template - void mapOptionalWithContext(const char *Key, T &Val, const T &Default, + template + void mapOptionalWithContext(const char *Key, T &Val, const DefaultT &Default, Context &Ctx) { - this->processKeyWithDefault(Key, Val, Default, false, Ctx); + static_assert(std::is_convertible::value, + "Default type must be implicitly convertible to value type!"); + this->processKeyWithDefault(Key, Val, static_cast(Default), + false, Ctx); } private: @@ -978,7 +977,7 @@ yamlize(IO &io, T &Val, bool, EmptyContext &Ctx) { bool DoClear; if ( io.beginBitSetScalar(DoClear) ) { if ( DoClear ) - Val = static_cast(0); + Val = T(); ScalarBitSetTraits::bitset(io, Val); io.endBitSetScalar(); } @@ -1243,12 +1242,14 @@ struct ScalarTraits { static QuotingType mustQuote(StringRef) { return QuotingType::None; } }; -// For endian types, we just use the existing ScalarTraits for the underlying -// type. This way endian aware types are supported whenever a ScalarTraits -// is defined for the underlying type. +// For endian types, we use existing scalar Traits class for the underlying +// type. This way endian aware types are supported whenever the traits are +// defined for the underlying type. template -struct ScalarTraits> { +struct ScalarTraits< + support::detail::packed_endian_specific_integral, + typename std::enable_if::value>::type> { using endian_type = support::detail::packed_endian_specific_integral; @@ -1269,6 +1270,38 @@ struct ScalarTraits +struct ScalarEnumerationTraits< + support::detail::packed_endian_specific_integral, + typename std::enable_if< + has_ScalarEnumerationTraits::value>::type> { + using endian_type = + support::detail::packed_endian_specific_integral; + + static void enumeration(IO &io, endian_type &E) { + value_type V = E; + ScalarEnumerationTraits::enumeration(io, V); + E = V; + } +}; + +template +struct ScalarBitSetTraits< + support::detail::packed_endian_specific_integral, + typename std::enable_if::value>::type> { + using endian_type = + support::detail::packed_endian_specific_integral; + static void bitset(IO &io, endian_type &E) { + value_type V = E; + ScalarBitSetTraits::bitset(io, V); + E = V; + } +}; + // Utility for use within MappingTraits<>::mapping() method // to [de]normalize an object for use with YAML conversion. template @@ -1587,8 +1620,9 @@ private: bool NeedBitValueComma = false; bool NeedFlowSequenceComma = false; bool EnumerationMatchFound = false; - bool NeedsNewLine = false; bool WriteDefaultValues = false; + StringRef Padding; + StringRef PaddingBeforeContainer; }; /// YAML I/O does conversion based on types. But often native data types @@ -1872,6 +1906,11 @@ struct SequenceTraits, typename std::enable_if::flow>::value>::type> : SequenceTraitsImpl, SequenceElementTraits::flow> {}; +template +struct SequenceTraits, + typename std::enable_if::flow>::value>::type> + : SequenceTraitsImpl, SequenceElementTraits::flow> {}; // Sequences of fundamental types use flow formatting. template diff --git a/include/llvm/Support/circular_raw_ostream.h b/include/llvm/Support/circular_raw_ostream.h index b46fd7f730c9..4ecdb17376f1 100644 --- a/include/llvm/Support/circular_raw_ostream.h +++ b/include/llvm/Support/circular_raw_ostream.h @@ -1,9 +1,8 @@ //===-- llvm/Support/circular_raw_ostream.h - Buffered streams --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/raw_os_ostream.h b/include/llvm/Support/raw_os_ostream.h index a983aeb90879..c51a94da3a28 100644 --- a/include/llvm/Support/raw_os_ostream.h +++ b/include/llvm/Support/raw_os_ostream.h @@ -1,9 +1,8 @@ //===- raw_os_ostream.h - std::ostream adaptor for raw_ostream --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/raw_ostream.h b/include/llvm/Support/raw_ostream.h index d062e716209d..48bb623b0638 100644 --- a/include/llvm/Support/raw_ostream.h +++ b/include/llvm/Support/raw_ostream.h @@ -1,9 +1,8 @@ //===--- raw_ostream.h - Raw output stream ----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -346,7 +345,7 @@ public: explicit raw_pwrite_stream(bool Unbuffered = false) : raw_ostream(Unbuffered) {} void pwrite(const char *Ptr, size_t Size, uint64_t Offset) { -#ifndef NDBEBUG +#ifndef NDEBUG uint64_t Pos = tell(); // /dev/null always reports a pos of 0, so we cannot perform this check // in that case. diff --git a/include/llvm/Support/raw_sha1_ostream.h b/include/llvm/Support/raw_sha1_ostream.h index bd55d98b7c1d..3991691796b5 100644 --- a/include/llvm/Support/raw_sha1_ostream.h +++ b/include/llvm/Support/raw_sha1_ostream.h @@ -1,9 +1,8 @@ //==- raw_sha1_ostream.h - raw_ostream that compute SHA1 --*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/thread.h b/include/llvm/Support/thread.h index 787a513d6017..084ed16166fe 100644 --- a/include/llvm/Support/thread.h +++ b/include/llvm/Support/thread.h @@ -1,9 +1,8 @@ //===-- llvm/Support/thread.h - Wrapper for ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Support/type_traits.h b/include/llvm/Support/type_traits.h index e7b8f2517b8a..c8c6a76a90f1 100644 --- a/include/llvm/Support/type_traits.h +++ b/include/llvm/Support/type_traits.h @@ -1,9 +1,8 @@ //===- llvm/Support/type_traits.h - Simplfied type traits -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,35 +24,6 @@ namespace llvm { -/// isPodLike - This is a type trait that is used to determine whether a given -/// type can be copied around with memcpy instead of running ctors etc. -template -struct isPodLike { - // std::is_trivially_copyable is available in libc++ with clang, libstdc++ - // that comes with GCC 5. MSVC 2015 and newer also have - // std::is_trivially_copyable. -#if (__has_feature(is_trivially_copyable) && defined(_LIBCPP_VERSION)) || \ - (defined(__GNUC__) && __GNUC__ >= 5) || defined(_MSC_VER) - // If the compiler supports the is_trivially_copyable trait use it, as it - // matches the definition of isPodLike closely. - static const bool value = std::is_trivially_copyable::value; -#elif __has_feature(is_trivially_copyable) - // Use the internal name if the compiler supports is_trivially_copyable but we - // don't know if the standard library does. This is the case for clang in - // conjunction with libstdc++ from GCC 4.x. - static const bool value = __is_trivially_copyable(T); -#else - // If we don't know anything else, we can (at least) assume that all non-class - // types are PODs. - static const bool value = !std::is_class::value; -#endif -}; - -// std::pair's are pod-like if their elements are. -template -struct isPodLike> { - static const bool value = isPodLike::value && isPodLike::value; -}; /// Metafunction that determines whether the given type is either an /// integral type or an enumeration type, including enum classes. @@ -120,6 +90,12 @@ template union move_construction_triviality_helper { move_construction_triviality_helper(move_construction_triviality_helper&&) = default; ~move_construction_triviality_helper() = default; }; + +template +union trivial_helper { + T t; +}; + } // end namespace detail /// An implementation of `std::is_trivially_copy_constructible` since we have @@ -144,6 +120,78 @@ struct is_trivially_move_constructible : std::true_type {}; template struct is_trivially_move_constructible : std::true_type {}; + +template +struct is_copy_assignable { + template + static auto get(F*) -> decltype(std::declval() = std::declval(), std::true_type{}); + static std::false_type get(...); + static constexpr bool value = decltype(get((T*)nullptr))::value; +}; + +template +struct is_move_assignable { + template + static auto get(F*) -> decltype(std::declval() = std::declval(), std::true_type{}); + static std::false_type get(...); + static constexpr bool value = decltype(get((T*)nullptr))::value; +}; + + +// An implementation of `std::is_trivially_copyable` since STL version +// is not equally supported by all compilers, especially GCC 4.9. +// Uniform implementation of this trait is important for ABI compatibility +// as it has an impact on SmallVector's ABI (among others). +template +class is_trivially_copyable { + + // copy constructors + static constexpr bool has_trivial_copy_constructor = + std::is_copy_constructible>::value; + static constexpr bool has_deleted_copy_constructor = + !std::is_copy_constructible::value; + + // move constructors + static constexpr bool has_trivial_move_constructor = + std::is_move_constructible>::value; + static constexpr bool has_deleted_move_constructor = + !std::is_move_constructible::value; + + // copy assign + static constexpr bool has_trivial_copy_assign = + is_copy_assignable>::value; + static constexpr bool has_deleted_copy_assign = + !is_copy_assignable::value; + + // move assign + static constexpr bool has_trivial_move_assign = + is_move_assignable>::value; + static constexpr bool has_deleted_move_assign = + !is_move_assignable::value; + + // destructor + static constexpr bool has_trivial_destructor = + std::is_destructible>::value; + + public: + + static constexpr bool value = + has_trivial_destructor && + (has_deleted_move_assign || has_trivial_move_assign) && + (has_deleted_move_constructor || has_trivial_move_constructor) && + (has_deleted_copy_assign || has_trivial_copy_assign) && + (has_deleted_copy_constructor || has_trivial_copy_constructor); + +#ifdef HAVE_STD_IS_TRIVIALLY_COPYABLE + static_assert(value == std::is_trivially_copyable::value, + "inconsistent behavior between llvm:: and std:: implementation of is_trivially_copyable"); +#endif +}; +template +class is_trivially_copyable : public std::true_type { +}; + + } // end namespace llvm // If the compiler supports detecting whether a class is final, define diff --git a/include/llvm/TableGen/Error.h b/include/llvm/TableGen/Error.h index de4d3bf54782..7c83b6298620 100644 --- a/include/llvm/TableGen/Error.h +++ b/include/llvm/TableGen/Error.h @@ -1,9 +1,8 @@ //===- llvm/TableGen/Error.h - tblgen error handling helpers ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/TableGen/Main.h b/include/llvm/TableGen/Main.h index 670572dc8103..e464cd4d4fb5 100644 --- a/include/llvm/TableGen/Main.h +++ b/include/llvm/TableGen/Main.h @@ -1,9 +1,8 @@ //===- llvm/TableGen/Main.h - tblgen entry point ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/TableGen/Record.h b/include/llvm/TableGen/Record.h index e022bc82b4e4..bf7f02208c28 100644 --- a/include/llvm/TableGen/Record.h +++ b/include/llvm/TableGen/Record.h @@ -1,9 +1,8 @@ //===- llvm/TableGen/Record.h - Classes for Table Records -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -316,6 +315,7 @@ protected: IK_TernOpInit, IK_UnOpInit, IK_LastOpInit, + IK_CondOpInit, IK_FoldOpInit, IK_IsAOpInit, IK_StringInit, @@ -623,10 +623,11 @@ public: class CodeInit : public TypedInit { StringRef Value; + SMLoc Loc; - explicit CodeInit(StringRef V) + explicit CodeInit(StringRef V, const SMLoc &Loc) : TypedInit(IK_CodeInit, static_cast(CodeRecTy::get())), - Value(V) {} + Value(V), Loc(Loc) {} public: CodeInit(const StringInit &) = delete; @@ -636,9 +637,10 @@ public: return I->getKind() == IK_CodeInit; } - static CodeInit *get(StringRef); + static CodeInit *get(StringRef, const SMLoc &Loc); StringRef getValue() const { return Value; } + const SMLoc &getLoc() const { return Loc; } Init *convertInitializerTo(RecTy *Ty) const override; @@ -798,8 +800,9 @@ public: /// !op (X, Y) - Combine two inits. class BinOpInit : public OpInit, public FoldingSetNode { public: - enum BinaryOp : uint8_t { ADD, AND, OR, SHL, SRA, SRL, LISTCONCAT, - STRCONCAT, CONCAT, EQ, NE, LE, LT, GE, GT }; + enum BinaryOp : uint8_t { ADD, MUL, AND, OR, SHL, SRA, SRL, LISTCONCAT, + LISTSPLAT, STRCONCAT, CONCAT, EQ, NE, LE, LT, GE, + GT }; private: Init *LHS, *RHS; @@ -818,6 +821,8 @@ public: static BinOpInit *get(BinaryOp opc, Init *lhs, Init *rhs, RecTy *Type); static Init *getStrConcat(Init *lhs, Init *rhs); + static Init *getListConcat(TypedInit *lhs, Init *rhs); + static Init *getListSplat(TypedInit *lhs, Init *rhs); void Profile(FoldingSetNodeID &ID) const; @@ -912,6 +917,83 @@ public: std::string getAsString() const override; }; +/// !cond(condition_1: value1, ... , condition_n: value) +/// Selects the first value for which condition is true. +/// Otherwise reports an error. +class CondOpInit final : public TypedInit, public FoldingSetNode, + public TrailingObjects { + unsigned NumConds; + RecTy *ValType; + + CondOpInit(unsigned NC, RecTy *Type) + : TypedInit(IK_CondOpInit, Type), + NumConds(NC), ValType(Type) {} + + size_t numTrailingObjects(OverloadToken) const { + return 2*NumConds; + } + +public: + CondOpInit(const CondOpInit &) = delete; + CondOpInit &operator=(const CondOpInit &) = delete; + + static bool classof(const Init *I) { + return I->getKind() == IK_CondOpInit; + } + + static CondOpInit *get(ArrayRef C, ArrayRef V, + RecTy *Type); + + void Profile(FoldingSetNodeID &ID) const; + + RecTy *getValType() const { return ValType; } + + unsigned getNumConds() const { return NumConds; } + + Init *getCond(unsigned Num) const { + assert(Num < NumConds && "Condition number out of range!"); + return getTrailingObjects()[Num]; + } + + Init *getVal(unsigned Num) const { + assert(Num < NumConds && "Val number out of range!"); + return getTrailingObjects()[Num+NumConds]; + } + + ArrayRef getConds() const { + return makeArrayRef(getTrailingObjects(), NumConds); + } + + ArrayRef getVals() const { + return makeArrayRef(getTrailingObjects()+NumConds, NumConds); + } + + Init *Fold(Record *CurRec) const; + + Init *resolveReferences(Resolver &R) const override; + + bool isConcrete() const override; + bool isComplete() const override; + std::string getAsString() const override; + + using const_case_iterator = SmallVectorImpl::const_iterator; + using const_val_iterator = SmallVectorImpl::const_iterator; + + inline const_case_iterator arg_begin() const { return getConds().begin(); } + inline const_case_iterator arg_end () const { return getConds().end(); } + + inline size_t case_size () const { return NumConds; } + inline bool case_empty() const { return NumConds == 0; } + + inline const_val_iterator name_begin() const { return getVals().begin();} + inline const_val_iterator name_end () const { return getVals().end(); } + + inline size_t val_size () const { return NumConds; } + inline bool val_empty() const { return NumConds == 0; } + + Init *getBit(unsigned Bit) const override; +}; + /// !foldl (a, b, expr, start, lst) - Fold over a list. class FoldOpInit : public TypedInit, public FoldingSetNode { private: diff --git a/include/llvm/TableGen/SearchableTable.td b/include/llvm/TableGen/SearchableTable.td index 1089d363eb6f..2680c71218ea 100644 --- a/include/llvm/TableGen/SearchableTable.td +++ b/include/llvm/TableGen/SearchableTable.td @@ -1,9 +1,8 @@ //===- SearchableTable.td ----------------------------------*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/TableGen/SetTheory.h b/include/llvm/TableGen/SetTheory.h index 4b32f9e3da8f..35156424b0d3 100644 --- a/include/llvm/TableGen/SetTheory.h +++ b/include/llvm/TableGen/SetTheory.h @@ -1,9 +1,8 @@ //===- SetTheory.h - Generate ordered sets from DAG expressions -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/TableGen/StringMatcher.h b/include/llvm/TableGen/StringMatcher.h index 3aa3540d616d..795b7a6d41dc 100644 --- a/include/llvm/TableGen/StringMatcher.h +++ b/include/llvm/TableGen/StringMatcher.h @@ -1,9 +1,8 @@ //===- StringMatcher.h - Generate a matcher for input strings ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/TableGen/StringToOffsetTable.h b/include/llvm/TableGen/StringToOffsetTable.h index 4b11e889ea6c..76ce51893907 100644 --- a/include/llvm/TableGen/StringToOffsetTable.h +++ b/include/llvm/TableGen/StringToOffsetTable.h @@ -1,9 +1,8 @@ //===- StringToOffsetTable.h - Emit a big concatenated string ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/TableGen/TableGenBackend.h b/include/llvm/TableGen/TableGenBackend.h index d226f1f1af7b..a426e4217578 100644 --- a/include/llvm/TableGen/TableGenBackend.h +++ b/include/llvm/TableGen/TableGenBackend.h @@ -1,9 +1,8 @@ //===- llvm/TableGen/TableGenBackend.h - Backend utilities ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -23,6 +22,8 @@ class raw_ostream; /// raw_ostream. void emitSourceFileHeader(StringRef Desc, raw_ostream &OS); +extern bool TimeRegions; + } // End llvm namespace #endif diff --git a/include/llvm/Target/CodeGenCWrappers.h b/include/llvm/Target/CodeGenCWrappers.h index 3ad77c5d5e00..a99546357053 100644 --- a/include/llvm/Target/CodeGenCWrappers.h +++ b/include/llvm/Target/CodeGenCWrappers.h @@ -1,9 +1,8 @@ //===- llvm/Target/CodeGenCWrappers.h - CodeGen C Wrappers ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Target/GenericOpcodes.td b/include/llvm/Target/GenericOpcodes.td index 045fe2520047..45718327b4a7 100644 --- a/include/llvm/Target/GenericOpcodes.td +++ b/include/llvm/Target/GenericOpcodes.td @@ -1,9 +1,8 @@ //===-- GenericOpcodes.td - Opcodes used with GlobalISel ---*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -93,12 +92,14 @@ def G_BITCAST : GenericInstruction { let hasSideEffects = 0; } +// Only supports scalar result types def G_CONSTANT : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins unknown:$imm); let hasSideEffects = 0; } +// Only supports scalar result types def G_FCONSTANT : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins unknown:$imm); @@ -122,31 +123,31 @@ def G_VAARG : GenericInstruction { def G_CTLZ : GenericInstruction { let OutOperandList = (outs type0:$dst); - let InOperandList = (ins type0:$src); + let InOperandList = (ins type1:$src); let hasSideEffects = 0; } def G_CTLZ_ZERO_UNDEF : GenericInstruction { let OutOperandList = (outs type0:$dst); - let InOperandList = (ins type0:$src); + let InOperandList = (ins type1:$src); let hasSideEffects = 0; } def G_CTTZ : GenericInstruction { let OutOperandList = (outs type0:$dst); - let InOperandList = (ins type0:$src); + let InOperandList = (ins type1:$src); let hasSideEffects = 0; } def G_CTTZ_ZERO_UNDEF : GenericInstruction { let OutOperandList = (outs type0:$dst); - let InOperandList = (ins type0:$src); + let InOperandList = (ins type1:$src); let hasSideEffects = 0; } def G_CTPOP : GenericInstruction { let OutOperandList = (outs type0:$dst); - let InOperandList = (ins type0:$src); + let InOperandList = (ins type1:$src); let hasSideEffects = 0; } @@ -168,6 +169,12 @@ def G_BLOCK_ADDR : GenericInstruction { let hasSideEffects = 0; } +def G_JUMP_TABLE : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins unknown:$jti); + let hasSideEffects = 0; +} + //------------------------------------------------------------------------------ // Binary ops. //------------------------------------------------------------------------------ @@ -255,21 +262,21 @@ def G_XOR : GenericInstruction { // Generic left-shift. def G_SHL : GenericInstruction { let OutOperandList = (outs type0:$dst); - let InOperandList = (ins type0:$src1, type0:$src2); + let InOperandList = (ins type0:$src1, type1:$src2); let hasSideEffects = 0; } // Generic logical right-shift. def G_LSHR : GenericInstruction { let OutOperandList = (outs type0:$dst); - let InOperandList = (ins type0:$src1, type0:$src2); + let InOperandList = (ins type0:$src1, type1:$src2); let hasSideEffects = 0; } // Generic arithmetic right-shift. def G_ASHR : GenericInstruction { let OutOperandList = (outs type0:$dst); - let InOperandList = (ins type0:$src1, type0:$src2); + let InOperandList = (ins type0:$src1, type1:$src2); let hasSideEffects = 0; } @@ -307,6 +314,38 @@ def G_PTR_MASK : GenericInstruction { let hasSideEffects = 0; } +// Generic signed integer minimum. +def G_SMIN : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src1, type0:$src2); + let hasSideEffects = 0; + let isCommutable = 1; +} + +// Generic signed integer maximum. +def G_SMAX : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src1, type0:$src2); + let hasSideEffects = 0; + let isCommutable = 1; +} + +// Generic unsigned integer minimum. +def G_UMIN : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src1, type0:$src2); + let hasSideEffects = 0; + let isCommutable = 1; +} + +// Generic unsigned integer maximum. +def G_UMAX : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src1, type0:$src2); + let hasSideEffects = 0; + let isCommutable = 1; +} + //------------------------------------------------------------------------------ // Overflow ops //------------------------------------------------------------------------------ @@ -454,6 +493,74 @@ def G_FABS : GenericInstruction { let hasSideEffects = 0; } +def G_FCOPYSIGN : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src0, type1:$src1); + let hasSideEffects = 0; +} + +def G_FCANONICALIZE : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src); + let hasSideEffects = 0; +} + +// FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two +// values. +// +// In the case where a single input is a NaN (either signaling or quiet), +// the non-NaN input is returned. +// +// The return value of (FMINNUM 0.0, -0.0) could be either 0.0 or -0.0. +def G_FMINNUM : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src1, type0:$src2); + let hasSideEffects = 0; + let isCommutable = 1; +} + +def G_FMAXNUM : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src1, type0:$src2); + let hasSideEffects = 0; + let isCommutable = 1; +} + +// FMINNUM_IEEE/FMAXNUM_IEEE - Perform floating-point minimum or maximum on +// two values, following the IEEE-754 2008 definition. This differs from +// FMINNUM/FMAXNUM in the handling of signaling NaNs. If one input is a +// signaling NaN, returns a quiet NaN. +def G_FMINNUM_IEEE : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src1, type0:$src2); + let hasSideEffects = 0; + let isCommutable = 1; +} + +def G_FMAXNUM_IEEE : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src1, type0:$src2); + let hasSideEffects = 0; + let isCommutable = 1; +} + +// FMINIMUM/FMAXIMUM - NaN-propagating minimum/maximum that also treat -0.0 +// as less than 0.0. While FMINNUM_IEEE/FMAXNUM_IEEE follow IEEE 754-2008 +// semantics, FMINIMUM/FMAXIMUM follow IEEE 754-2018 draft semantics. +def G_FMINIMUM : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src1, type0:$src2); + let hasSideEffects = 0; + let isCommutable = 1; +} + +def G_FMAXIMUM : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src1, type0:$src2); + let hasSideEffects = 0; + let isCommutable = 1; +} + //------------------------------------------------------------------------------ // Floating Point Binary ops. //------------------------------------------------------------------------------ @@ -554,6 +661,51 @@ def G_FCEIL : GenericInstruction { let hasSideEffects = 0; } +// Floating point cosine of a value. +def G_FCOS : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src1); + let hasSideEffects = 0; +} + +// Floating point sine of a value. +def G_FSIN : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src1); + let hasSideEffects = 0; +} + +// Floating point square root of a value. +// This returns NaN for negative nonzero values. +// NOTE: Unlike libm sqrt(), this never sets errno. In all other respects it's +// libm-conformant. +def G_FSQRT : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src1); + let hasSideEffects = 0; +} + +// Floating point floor of a value. +def G_FFLOOR : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src1); + let hasSideEffects = 0; +} + +// Floating point round to next integer. +def G_FRINT : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src1); + let hasSideEffects = 0; +} + +// Floating point round to the nearest integer. +def G_FNEARBYINT : GenericInstruction { + let OutOperandList = (outs type0:$dst); + let InOperandList = (ins type0:$src1); + let hasSideEffects = 0; +} + //------------------------------------------------------------------------------ // Opcodes for LLVM Intrinsics //------------------------------------------------------------------------------ @@ -647,6 +799,12 @@ def G_ATOMICRMW_MIN : G_ATOMICRMW_OP; def G_ATOMICRMW_UMAX : G_ATOMICRMW_OP; def G_ATOMICRMW_UMIN : G_ATOMICRMW_OP; +def G_FENCE : GenericInstruction { + let OutOperandList = (outs); + let InOperandList = (ins i32imm:$ordering, i32imm:$scope); + let hasSideEffects = 1; +} + //------------------------------------------------------------------------------ // Variadic ops //------------------------------------------------------------------------------ @@ -689,7 +847,9 @@ def G_MERGE_VALUES : GenericInstruction { let hasSideEffects = 0; } -/// Create a vector from multiple scalar registers. +/// Create a vector from multiple scalar registers. No implicit +/// conversion is performed (i.e. the result element type must be the +/// same as all source operands) def G_BUILD_VECTOR : GenericInstruction { let OutOperandList = (outs type0:$dst); let InOperandList = (ins type1:$src0, variable_ops); @@ -759,6 +919,15 @@ def G_BRINDIRECT : GenericInstruction { let isTerminator = 1; } +// Generic branch to jump table entry +def G_BRJT : GenericInstruction { + let OutOperandList = (outs); + let InOperandList = (ins ptype0:$tbl, unknown:$jti, type1:$idx); + let hasSideEffects = 0; + let isBranch = 1; + let isTerminator = 1; +} + //------------------------------------------------------------------------------ // Vector ops //------------------------------------------------------------------------------ diff --git a/include/llvm/Target/GlobalISel/RegisterBank.td b/include/llvm/Target/GlobalISel/RegisterBank.td index 4dfd139e9fb6..51578b66b160 100644 --- a/include/llvm/Target/GlobalISel/RegisterBank.td +++ b/include/llvm/Target/GlobalISel/RegisterBank.td @@ -1,9 +1,8 @@ //===- RegisterBank.td - Register bank definitions ---------*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/include/llvm/Target/GlobalISel/SelectionDAGCompat.td index 31d26361260d..6cc58d6521da 100644 --- a/include/llvm/Target/GlobalISel/SelectionDAGCompat.td +++ b/include/llvm/Target/GlobalISel/SelectionDAGCompat.td @@ -1,9 +1,8 @@ //===- TargetGlobalISel.td - Common code for GlobalISel ----*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -50,6 +49,8 @@ def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; +def : GINodeEquiv; +def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; @@ -77,6 +78,7 @@ def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; +def : GINodeEquiv; def : GINodeEquiv; // ISD::INTRINSIC_VOID can also be handled with G_INTRINSIC_W_SIDE_EFFECTS. def : GINodeEquiv; @@ -89,7 +91,19 @@ def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; +def : GINodeEquiv; def : GINodeEquiv; +def : GINodeEquiv; +def : GINodeEquiv; +def : GINodeEquiv; +def : GINodeEquiv; +def : GINodeEquiv; +def : GINodeEquiv; +def : GINodeEquiv; +def : GINodeEquiv; +def : GINodeEquiv; +def : GINodeEquiv; +def : GINodeEquiv; // Broadly speaking G_LOAD is equivalent to ISD::LOAD but there are some // complications that tablegen must take care of. For example, Predicates such @@ -124,6 +138,7 @@ def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; def : GINodeEquiv; +def : GINodeEquiv; // Specifies the GlobalISel equivalents for SelectionDAG's ComplexPattern. // Should be used on defs that subclass GIComplexOperandMatcher<>. diff --git a/include/llvm/Target/GlobalISel/Target.td b/include/llvm/Target/GlobalISel/Target.td index 6740f404a9d3..538ca65e1162 100644 --- a/include/llvm/Target/GlobalISel/Target.td +++ b/include/llvm/Target/GlobalISel/Target.td @@ -1,9 +1,8 @@ //===- Target.td - Define GlobalISel rules -----------------*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Target/Target.td b/include/llvm/Target/Target.td index e4b827babb92..d58662e128e0 100644 --- a/include/llvm/Target/Target.td +++ b/include/llvm/Target/Target.td @@ -1,9 +1,8 @@ //===- Target.td - Target Independent TableGen interface ---*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -122,6 +121,10 @@ class ComposedSubRegIndex // this register class when printing. class RegAltNameIndex { string Namespace = ""; + + // A set to be used if the name for a register is not defined in this set. + // This allows creating name sets with only a few alternative names. + RegAltNameIndex FallbackRegAltNameIndex = ?; } def NoRegAltName : RegAltNameIndex; @@ -395,11 +398,49 @@ include "llvm/Target/TargetSchedule.td" class Predicate; // Forward def +class InstructionEncoding { + // Size of encoded instruction. + int Size; + + // The "namespace" in which this instruction exists, on targets like ARM + // which multiple ISA namespaces exist. + string DecoderNamespace = ""; + + // List of predicates which will be turned into isel matching code. + list Predicates = []; + + string DecoderMethod = ""; + + // Is the instruction decoder method able to completely determine if the + // given instruction is valid or not. If the TableGen definition of the + // instruction specifies bitpattern A??B where A and B are static bits, the + // hasCompleteDecoder flag says whether the decoder method fully handles the + // ?? space, i.e. if it is a final arbiter for the instruction validity. + // If not then the decoder attempts to continue decoding when the decoder + // method fails. + // + // This allows to handle situations where the encoding is not fully + // orthogonal. Example: + // * InstA with bitpattern 0b0000????, + // * InstB with bitpattern 0b000000?? but the associated decoder method + // DecodeInstB() returns Fail when ?? is 0b00 or 0b11. + // + // The decoder tries to decode a bitpattern that matches both InstA and + // InstB bitpatterns first as InstB (because it is the most specific + // encoding). In the default case (hasCompleteDecoder = 1), when + // DecodeInstB() returns Fail the bitpattern gets rejected. By setting + // hasCompleteDecoder = 0 in InstB, the decoder is informed that + // DecodeInstB() is not able to determine if all possible values of ?? are + // valid or not. If DecodeInstB() returns Fail the decoder will attempt to + // decode the bitpattern as InstA too. + bit hasCompleteDecoder = 1; +} + //===----------------------------------------------------------------------===// // Instruction set description - These classes correspond to the C++ classes in // the Target/TargetInstrInfo.h file. // -class Instruction { +class Instruction : InstructionEncoding { string Namespace = ""; dag OutOperandList; // An dag containing the MI def operand list. @@ -424,10 +465,6 @@ class Instruction { // from the opcode. int Size = 0; - // DecoderNamespace - The "namespace" in which this instruction exists, on - // targets like ARM which multiple ISA namespaces exist. - string DecoderNamespace = ""; - // Code size, for instruction selection. // FIXME: What does this actually mean? int CodeSize = 0; @@ -453,11 +490,16 @@ class Instruction { bit canFoldAsLoad = 0; // Can this be folded as a simple memory operand? bit mayLoad = ?; // Is it possible for this inst to read memory? bit mayStore = ?; // Is it possible for this inst to write memory? + bit mayRaiseFPException = 0; // Can this raise a floating-point exception? bit isConvertibleToThreeAddress = 0; // Can this 2-addr instruction promote? bit isCommutable = 0; // Is this 3 operand instruction commutable? bit isTerminator = 0; // Is this part of the terminator for a basic block? bit isReMaterializable = 0; // Is this instruction re-materializable? - bit isPredicable = 0; // Is this instruction predicable? + bit isPredicable = 0; // 1 means this instruction is predicable + // even if it does not have any operand + // tablegen can identify as a predicate + bit isUnpredicable = 0; // 1 means this instruction is not predicable + // even if it _does_ have a predicate operand bit hasDelaySlot = 0; // Does this instruction have an delay slot? bit usesCustomInserter = 0; // Pseudo instr needing special help. bit hasPostISelHook = 0; // To be *adjusted* after isel by target hook. @@ -524,31 +566,6 @@ class Instruction { string DisableEncoding = ""; string PostEncoderMethod = ""; - string DecoderMethod = ""; - - // Is the instruction decoder method able to completely determine if the - // given instruction is valid or not. If the TableGen definition of the - // instruction specifies bitpattern A??B where A and B are static bits, the - // hasCompleteDecoder flag says whether the decoder method fully handles the - // ?? space, i.e. if it is a final arbiter for the instruction validity. - // If not then the decoder attempts to continue decoding when the decoder - // method fails. - // - // This allows to handle situations where the encoding is not fully - // orthogonal. Example: - // * InstA with bitpattern 0b0000????, - // * InstB with bitpattern 0b000000?? but the associated decoder method - // DecodeInstB() returns Fail when ?? is 0b00 or 0b11. - // - // The decoder tries to decode a bitpattern that matches both InstA and - // InstB bitpatterns first as InstB (because it is the most specific - // encoding). In the default case (hasCompleteDecoder = 1), when - // DecodeInstB() returns Fail the bitpattern gets rejected. By setting - // hasCompleteDecoder = 0 in InstB, the decoder is informed that - // DecodeInstB() is not able to determine if all possible values of ?? are - // valid or not. If DecodeInstB() returns Fail the decoder will attempt to - // decode the bitpattern as InstA too. - bit hasCompleteDecoder = 1; /// Target-specific flags. This becomes the TSFlags field in TargetInstrDesc. bits<64> TSFlags = 0; @@ -585,6 +602,13 @@ class Instruction { bit FastISelShouldIgnore = 0; } +/// Defines an additional encoding that disassembles to the given instruction +/// Like Instruction, the Inst and SoftFail fields are omitted to allow targets +// to specify their size. +class AdditionalEncoding : InstructionEncoding { + Instruction AliasOf = I; +} + /// PseudoInstExpansion - Expansion information for a pseudo-instruction. /// Which instruction it expands to and how the operands map from the /// pseudo. @@ -909,7 +933,7 @@ class InstrInfo { } // Standard Pseudo Instructions. -// This list must match TargetOpcodes.h and CodeGenTarget.cpp. +// This list must match TargetOpcodes.def. // Only these instructions are allowed in the TargetOpcode namespace. // Ensure mayLoad and mayStore have a default value, so as not to break // targets that set guessInstructionProperties=0. Any local definition of @@ -934,6 +958,15 @@ def INLINEASM : StandardPseudoInstruction { let AsmString = ""; let hasSideEffects = 0; // Note side effect is encoded in an operand. } +def INLINEASM_BR : StandardPseudoInstruction { + let OutOperandList = (outs); + let InOperandList = (ins variable_ops); + let AsmString = ""; + let hasSideEffects = 0; // Note side effect is encoded in an operand. + let isTerminator = 1; + let isBranch = 1; + let isIndirectBranch = 1; +} def CFI_INSTRUCTION : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins i32imm:$id); @@ -1037,7 +1070,7 @@ def BUNDLE : StandardPseudoInstruction { let OutOperandList = (outs); let InOperandList = (ins variable_ops); let AsmString = "BUNDLE"; - let hasSideEffects = 1; + let hasSideEffects = 0; } def LIFETIME_START : StandardPseudoInstruction { let OutOperandList = (outs); @@ -1174,7 +1207,7 @@ def FENTRY_CALL : StandardPseudoInstruction { let hasSideEffects = 1; } def ICALL_BRANCH_FUNNEL : StandardPseudoInstruction { - let OutOperandList = (outs unknown:$dst); + let OutOperandList = (outs); let InOperandList = (ins variable_ops); let AsmString = ""; let hasSideEffects = 1; diff --git a/include/llvm/Target/TargetCallingConv.td b/include/llvm/Target/TargetCallingConv.td index 95d2b4226294..1bc03cf8a49d 100644 --- a/include/llvm/Target/TargetCallingConv.td +++ b/include/llvm/Target/TargetCallingConv.td @@ -1,9 +1,8 @@ //===- TargetCallingConv.td - Target Calling Conventions ---*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -83,6 +82,15 @@ class CCIfVarArg : CCIf<"State.isVarArg()", A> {} /// CCIfNotVarArg - If the current function is not vararg - apply the action class CCIfNotVarArg : CCIf<"!State.isVarArg()", A> {} +/// CCIfPtrAddrSpace - If the top-level parent of the current argument has +/// pointer type in the specified address-space. +class CCIfPtrAddrSpace + : CCIf<"(ArgFlags.isPointer() && ArgFlags.getPointerAddrSpace() == " # AS # ")", A> {} + +/// CCIfPtr - If the top-level parent of the current argument had +/// pointer type in some address-space. +class CCIfPtr : CCIf<"ArgFlags.isPointer()", A> {} + /// CCAssignToReg - This action matches if there is a register in the specified /// list that is still available. If so, it assigns the value to the first /// available register and succeeds. @@ -160,6 +168,11 @@ class CCDelegateTo : CCAction { /// that the target supports. class CallingConv actions> { list Actions = actions; + + /// If true, this calling convention will be emitted as externally visible in + /// the llvm namespaces instead of as a static function. + bit Entry = 0; + bit Custom = 0; } diff --git a/include/llvm/Target/TargetInstrPredicate.td b/include/llvm/Target/TargetInstrPredicate.td index 4b2c57b34c2e..5623461c648d 100644 --- a/include/llvm/Target/TargetInstrPredicate.td +++ b/include/llvm/Target/TargetInstrPredicate.td @@ -1,9 +1,8 @@ //===- TargetInstrPredicate.td - ---------------------------*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Target/TargetIntrinsicInfo.h b/include/llvm/Target/TargetIntrinsicInfo.h index 6a92bdee747e..ef571b15153e 100644 --- a/include/llvm/Target/TargetIntrinsicInfo.h +++ b/include/llvm/Target/TargetIntrinsicInfo.h @@ -1,9 +1,8 @@ //===-- llvm/Target/TargetIntrinsicInfo.h - Instruction Info ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Target/TargetItinerary.td b/include/llvm/Target/TargetItinerary.td index 182054d8444e..b68ed045520c 100644 --- a/include/llvm/Target/TargetItinerary.td +++ b/include/llvm/Target/TargetItinerary.td @@ -1,9 +1,8 @@ //===- TargetItinerary.td - Target Itinierary Description --*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Target/TargetLoweringObjectFile.h b/include/llvm/Target/TargetLoweringObjectFile.h index e80f2bf82f26..3a2497bff11e 100644 --- a/include/llvm/Target/TargetLoweringObjectFile.h +++ b/include/llvm/Target/TargetLoweringObjectFile.h @@ -1,9 +1,8 @@ //===-- llvm/Target/TargetLoweringObjectFile.h - Object Info ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -52,6 +51,7 @@ protected: unsigned PersonalityEncoding = 0; unsigned LSDAEncoding = 0; unsigned TTypeEncoding = 0; + unsigned CallSiteEncoding = 0; /// This section contains the static constructor pointer list. MCSection *StaticCtorSection = nullptr; @@ -80,6 +80,9 @@ public: /// Emit the module-level metadata that the platform cares about. virtual void emitModuleMetadata(MCStreamer &Streamer, Module &M) const {} + /// Get the module-level metadata that the platform cares about. + virtual void getModuleMetadata(Module &M) {} + /// Given a constant with the SectionKind, return a section that it should be /// placed in. virtual MCSection *getSectionForConstant(const DataLayout &DL, @@ -145,6 +148,7 @@ public: unsigned getPersonalityEncoding() const { return PersonalityEncoding; } unsigned getLSDAEncoding() const { return LSDAEncoding; } unsigned getTTypeEncoding() const { return TTypeEncoding; } + unsigned getCallSiteEncoding() const { return CallSiteEncoding; } const MCExpr *getTTypeReference(const MCSymbolRefExpr *Sym, unsigned Encoding, MCStreamer &Streamer) const; diff --git a/include/llvm/Target/TargetMachine.h b/include/llvm/Target/TargetMachine.h index 3eafcc25583a..cdf9f8bfd5ea 100644 --- a/include/llvm/Target/TargetMachine.h +++ b/include/llvm/Target/TargetMachine.h @@ -1,9 +1,8 @@ //===-- llvm/Target/TargetMachine.h - Target Information --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -36,6 +35,9 @@ class MCSubtargetInfo; class MCSymbol; class raw_pwrite_stream; class PassManagerBuilder; +struct PerFunctionMIParsingState; +class SMDiagnostic; +class SMRange; class Target; class TargetIntrinsicInfo; class TargetIRAnalysis; @@ -50,6 +52,10 @@ class PassManagerBase; } using legacy::PassManagerBase; +namespace yaml { +struct MachineFunctionInfo; +} + //===----------------------------------------------------------------------===// /// /// Primary interface to the complete machine description for the target @@ -115,6 +121,27 @@ public: return nullptr; } + /// Allocate and return a default initialized instance of the YAML + /// representation for the MachineFunctionInfo. + virtual yaml::MachineFunctionInfo *createDefaultFuncInfoYAML() const { + return nullptr; + } + + /// Allocate and initialize an instance of the YAML representation of the + /// MachineFunctionInfo. + virtual yaml::MachineFunctionInfo * + convertFuncInfoToYAML(const MachineFunction &MF) const { + return nullptr; + } + + /// Parse out the target's MachineFunctionInfo from the YAML reprsentation. + virtual bool parseMachineFunctionInfo(const yaml::MachineFunctionInfo &, + PerFunctionMIParsingState &PFS, + SMDiagnostic &Error, + SMRange &SourceRange) const { + return false; + } + /// This method returns a pointer to the specified type of /// TargetSubtargetInfo. In debug builds, it verifies that the object being /// returned is of the correct type. @@ -363,9 +390,9 @@ inline CodeModel::Model getEffectiveCodeModel(Optional CM, if (CM) { // By default, targets do not support the tiny and kernel models. if (*CM == CodeModel::Tiny) - report_fatal_error("Target does not support the tiny CodeModel"); + report_fatal_error("Target does not support the tiny CodeModel", false); if (*CM == CodeModel::Kernel) - report_fatal_error("Target does not support the kernel CodeModel"); + report_fatal_error("Target does not support the kernel CodeModel", false); return *CM; } return Default; diff --git a/include/llvm/Target/TargetOptions.h b/include/llvm/Target/TargetOptions.h index b18101d92833..8cc2a6010879 100644 --- a/include/llvm/Target/TargetOptions.h +++ b/include/llvm/Target/TargetOptions.h @@ -1,9 +1,8 @@ //===-- llvm/Target/TargetOptions.h - Target Options ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -119,7 +118,8 @@ namespace llvm { NoTrapAfterNoreturn(false), EmulatedTLS(false), ExplicitEmulatedTLS(false), EnableIPRA(false), EmitStackSizeSection(false), EnableMachineOutliner(false), - SupportsDefaultOutlining(false), EmitAddrsig(false) {} + SupportsDefaultOutlining(false), EmitAddrsig(false), + EnableDebugEntryValues(false) {} /// PrintMachineCode - This flag is enabled when the -print-machineinstrs /// option is specified on the command line, and should enable debugging @@ -253,6 +253,9 @@ namespace llvm { /// Emit address-significance table. unsigned EmitAddrsig : 1; + /// Emit debug info about parameter's entry values. + unsigned EnableDebugEntryValues : 1; + /// FloatABIType - This setting is set by -float-abi=xxx option is specfied /// on the command line. This setting may either be Default, Soft, or Hard. /// Default selects the target's default behavior. Soft selects the ABI for diff --git a/include/llvm/Target/TargetPfmCounters.td b/include/llvm/Target/TargetPfmCounters.td index dac150f03445..e1d5013c1291 100644 --- a/include/llvm/Target/TargetPfmCounters.td +++ b/include/llvm/Target/TargetPfmCounters.td @@ -1,9 +1,8 @@ //===- TargetPfmCounters.td - Target Pfm Counters -*- tablegen ----------*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Target/TargetSchedule.td b/include/llvm/Target/TargetSchedule.td index 808e183f5a5f..a36d259df831 100644 --- a/include/llvm/Target/TargetSchedule.td +++ b/include/llvm/Target/TargetSchedule.td @@ -1,9 +1,8 @@ //===- TargetSchedule.td - Target Independent Scheduling ---*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -136,7 +135,7 @@ class ProcResourceKind; // // An optional Super resource may be given to model these resources as // a subset of the more general super resources. Using one of these -// resources implies using one of the super resoruces. +// resources implies using one of the super resources. // // ProcResourceUnits normally model a few buffered resources within an // out-of-order engine. Buffered resources may be held for multiple diff --git a/include/llvm/Target/TargetSelectionDAG.td b/include/llvm/Target/TargetSelectionDAG.td index eb5a14bd21b8..b913a054ac2c 100644 --- a/include/llvm/Target/TargetSelectionDAG.td +++ b/include/llvm/Target/TargetSelectionDAG.td @@ -1,9 +1,8 @@ //===- TargetSelectionDAG.td - Common code for DAG isels ---*- tablegen -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -125,7 +124,7 @@ def SDTIntSatNoShOp : SDTypeProfile<1, 2, [ // ssat with no shift def SDTIntBinHiLoOp : SDTypeProfile<2, 2, [ // mulhi, mullo, sdivrem, udivrem SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisSameAs<0, 3>,SDTCisInt<0> ]>; -def SDTIntScaledBinOp : SDTypeProfile<1, 3, [ // smulfix +def SDTIntScaledBinOp : SDTypeProfile<1, 3, [ // smulfix, umulfix SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisInt<0>, SDTCisInt<3> ]>; @@ -265,6 +264,11 @@ def SDTAtomic3 : SDTypeProfile<1, 3, [ def SDTAtomic2 : SDTypeProfile<1, 2, [ SDTCisSameAs<0,2>, SDTCisInt<0>, SDTCisPtrTy<1> ]>; + +def SDTFPAtomic2 : SDTypeProfile<1, 2, [ + SDTCisSameAs<0,2>, SDTCisFP<0>, SDTCisPtrTy<1> +]>; + def SDTAtomicStore : SDTypeProfile<0, 2, [ SDTCisPtrTy<0>, SDTCisInt<1> ]>; @@ -385,7 +389,10 @@ def saddsat : SDNode<"ISD::SADDSAT" , SDTIntBinOp, [SDNPCommutative]>; def uaddsat : SDNode<"ISD::UADDSAT" , SDTIntBinOp, [SDNPCommutative]>; def ssubsat : SDNode<"ISD::SSUBSAT" , SDTIntBinOp>; def usubsat : SDNode<"ISD::USUBSAT" , SDTIntBinOp>; + def smulfix : SDNode<"ISD::SMULFIX" , SDTIntScaledBinOp, [SDNPCommutative]>; +def smulfixsat : SDNode<"ISD::SMULFIXSAT", SDTIntScaledBinOp, [SDNPCommutative]>; +def umulfix : SDNode<"ISD::UMULFIX" , SDTIntScaledBinOp, [SDNPCommutative]>; def sext_inreg : SDNode<"ISD::SIGN_EXTEND_INREG", SDTExtInreg>; def sext_invec : SDNode<"ISD::SIGN_EXTEND_VECTOR_INREG", SDTExtInvec>; @@ -444,6 +451,11 @@ def ffloor : SDNode<"ISD::FFLOOR" , SDTFPUnaryOp>; def fnearbyint : SDNode<"ISD::FNEARBYINT" , SDTFPUnaryOp>; def fround : SDNode<"ISD::FROUND" , SDTFPUnaryOp>; +def lround : SDNode<"ISD::LROUND" , SDTFPToIntOp>; +def llround : SDNode<"ISD::LLROUND" , SDTFPToIntOp>; +def lrint : SDNode<"ISD::LRINT" , SDTFPToIntOp>; +def llrint : SDNode<"ISD::LLRINT" , SDTFPToIntOp>; + def fpround : SDNode<"ISD::FP_ROUND" , SDTFPRoundOp>; def fpextend : SDNode<"ISD::FP_EXTEND" , SDTFPExtendOp>; def fcopysign : SDNode<"ISD::FCOPYSIGN" , SDTFPSignOp>; @@ -455,6 +467,53 @@ def fp_to_uint : SDNode<"ISD::FP_TO_UINT" , SDTFPToIntOp>; def f16_to_fp : SDNode<"ISD::FP16_TO_FP" , SDTIntToFPOp>; def fp_to_f16 : SDNode<"ISD::FP_TO_FP16" , SDTFPToIntOp>; +def strict_fadd : SDNode<"ISD::STRICT_FADD", + SDTFPBinOp, [SDNPHasChain, SDNPCommutative]>; +def strict_fsub : SDNode<"ISD::STRICT_FSUB", + SDTFPBinOp, [SDNPHasChain]>; +def strict_fmul : SDNode<"ISD::STRICT_FMUL", + SDTFPBinOp, [SDNPHasChain, SDNPCommutative]>; +def strict_fdiv : SDNode<"ISD::STRICT_FDIV", + SDTFPBinOp, [SDNPHasChain]>; +def strict_frem : SDNode<"ISD::STRICT_FREM", + SDTFPBinOp, [SDNPHasChain]>; +def strict_fma : SDNode<"ISD::STRICT_FMA", + SDTFPTernaryOp, [SDNPHasChain]>; +def strict_fsqrt : SDNode<"ISD::STRICT_FSQRT", + SDTFPUnaryOp, [SDNPHasChain]>; +def strict_fsin : SDNode<"ISD::STRICT_FSIN", + SDTFPUnaryOp, [SDNPHasChain]>; +def strict_fcos : SDNode<"ISD::STRICT_FCOS", + SDTFPUnaryOp, [SDNPHasChain]>; +def strict_fexp2 : SDNode<"ISD::STRICT_FEXP2", + SDTFPUnaryOp, [SDNPHasChain]>; +def strict_fpow : SDNode<"ISD::STRICT_FPOW", + SDTFPBinOp, [SDNPHasChain]>; +def strict_flog2 : SDNode<"ISD::STRICT_FLOG2", + SDTFPUnaryOp, [SDNPHasChain]>; +def strict_frint : SDNode<"ISD::STRICT_FRINT", + SDTFPUnaryOp, [SDNPHasChain]>; +def strict_fnearbyint : SDNode<"ISD::STRICT_FNEARBYINT", + SDTFPUnaryOp, [SDNPHasChain]>; +def strict_fceil : SDNode<"ISD::STRICT_FCEIL", + SDTFPUnaryOp, [SDNPHasChain]>; +def strict_ffloor : SDNode<"ISD::STRICT_FFLOOR", + SDTFPUnaryOp, [SDNPHasChain]>; +def strict_fround : SDNode<"ISD::STRICT_FROUND", + SDTFPUnaryOp, [SDNPHasChain]>; +def strict_ftrunc : SDNode<"ISD::STRICT_FTRUNC", + SDTFPUnaryOp, [SDNPHasChain]>; +def strict_fminnum : SDNode<"ISD::STRICT_FMINNUM", + SDTFPBinOp, [SDNPHasChain, + SDNPCommutative, SDNPAssociative]>; +def strict_fmaxnum : SDNode<"ISD::STRICT_FMAXNUM", + SDTFPBinOp, [SDNPHasChain, + SDNPCommutative, SDNPAssociative]>; +def strict_fpround : SDNode<"ISD::STRICT_FP_ROUND", + SDTFPRoundOp, [SDNPHasChain]>; +def strict_fpextend : SDNode<"ISD::STRICT_FP_EXTEND", + SDTFPExtendOp, [SDNPHasChain]>; + def setcc : SDNode<"ISD::SETCC" , SDTSetCC>; def select : SDNode<"ISD::SELECT" , SDTSelect>; def vselect : SDNode<"ISD::VSELECT" , SDTVSelect>; @@ -511,14 +570,19 @@ def atomic_load_umin : SDNode<"ISD::ATOMIC_LOAD_UMIN", SDTAtomic2, [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>; def atomic_load_umax : SDNode<"ISD::ATOMIC_LOAD_UMAX", SDTAtomic2, [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>; +def atomic_load_fadd : SDNode<"ISD::ATOMIC_LOAD_FADD" , SDTFPAtomic2, + [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>; +def atomic_load_fsub : SDNode<"ISD::ATOMIC_LOAD_FSUB" , SDTFPAtomic2, + [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand]>; + def atomic_load : SDNode<"ISD::ATOMIC_LOAD", SDTAtomicLoad, [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>; def atomic_store : SDNode<"ISD::ATOMIC_STORE", SDTAtomicStore, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>; -def masked_store : SDNode<"ISD::MSTORE", SDTMaskedStore, +def masked_st : SDNode<"ISD::MSTORE", SDTMaskedStore, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>; -def masked_load : SDNode<"ISD::MLOAD", SDTMaskedLoad, +def masked_ld : SDNode<"ISD::MLOAD", SDTMaskedLoad, [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>; // Do not use ld, st directly. Use load, extload, sextload, zextload, store, @@ -673,6 +737,10 @@ class PatFrags frags, code pred = [{}], // cast(N)->isTruncatingStore(); bit IsTruncStore = ?; + // cast(N)->getAddressSpace() == + // If this empty, accept any address space. + list AddressSpaces = ?; + // cast(N)->getOrdering() == AtomicOrdering::Monotonic bit IsAtomicOrderingMonotonic = ?; // cast(N)->getOrdering() == AtomicOrdering::Acquire @@ -698,6 +766,8 @@ class PatFrags frags, code pred = [{}], // cast(N)->getMemoryVT().getScalarType() == MVT::; // cast(N)->getMemoryVT().getScalarType() == MVT::; ValueType ScalarMemoryVT = ?; + + // TODO: Add alignment } // PatFrag - A version of PatFrags matching only a single fragment. @@ -771,14 +841,11 @@ class FPImmLeaf def vtInt : PatLeaf<(vt), [{ return N->getVT().isInteger(); }]>; def vtFP : PatLeaf<(vt), [{ return N->getVT().isFloatingPoint(); }]>; -def immAllOnesV: PatLeaf<(build_vector), [{ - return ISD::isBuildVectorAllOnes(N); -}]>; -def immAllZerosV: PatLeaf<(build_vector), [{ - return ISD::isBuildVectorAllZeros(N); -}]>; - - +// Use ISD::isBuildVectorAllOnes or ISD::isBuildVectorAllZeros to look for +// the corresponding build_vector. Will look through bitcasts except when used +// as a pattern root. +def immAllOnesV; // ISD::isBuildVectorAllOnes +def immAllZerosV; // ISD::isBuildVectorAllZeros // Other helper fragments. def not : PatFrag<(ops node:$in), (xor node:$in, -1)>; @@ -1163,6 +1230,87 @@ def setle : PatFrag<(ops node:$lhs, node:$rhs), def setne : PatFrag<(ops node:$lhs, node:$rhs), (setcc node:$lhs, node:$rhs, SETNE)>; +// We don't have strict FP extended loads as single DAG nodes, but we can +// still provide convenience fragments to match those operations. +def strict_extloadf32 : PatFrag<(ops node:$ptr), + (strict_fpextend (f32 (load node:$ptr)))>; +def strict_extloadf64 : PatFrag<(ops node:$ptr), + (strict_fpextend (f64 (load node:$ptr)))>; + +// Convenience fragments to match both strict and non-strict fp operations +def any_fadd : PatFrags<(ops node:$lhs, node:$rhs), + [(strict_fadd node:$lhs, node:$rhs), + (fadd node:$lhs, node:$rhs)]>; +def any_fsub : PatFrags<(ops node:$lhs, node:$rhs), + [(strict_fsub node:$lhs, node:$rhs), + (fsub node:$lhs, node:$rhs)]>; +def any_fmul : PatFrags<(ops node:$lhs, node:$rhs), + [(strict_fmul node:$lhs, node:$rhs), + (fmul node:$lhs, node:$rhs)]>; +def any_fdiv : PatFrags<(ops node:$lhs, node:$rhs), + [(strict_fdiv node:$lhs, node:$rhs), + (fdiv node:$lhs, node:$rhs)]>; +def any_frem : PatFrags<(ops node:$lhs, node:$rhs), + [(strict_frem node:$lhs, node:$rhs), + (frem node:$lhs, node:$rhs)]>; +def any_fma : PatFrags<(ops node:$src1, node:$src2, node:$src3), + [(strict_fma node:$src1, node:$src2, node:$src3), + (fma node:$src1, node:$src2, node:$src3)]>; +def any_fsqrt : PatFrags<(ops node:$src), + [(strict_fsqrt node:$src), + (fsqrt node:$src)]>; +def any_fsin : PatFrags<(ops node:$src), + [(strict_fsin node:$src), + (fsin node:$src)]>; +def any_fcos : PatFrags<(ops node:$src), + [(strict_fcos node:$src), + (fcos node:$src)]>; +def any_fexp2 : PatFrags<(ops node:$src), + [(strict_fexp2 node:$src), + (fexp2 node:$src)]>; +def any_fpow : PatFrags<(ops node:$lhs, node:$rhs), + [(strict_fpow node:$lhs, node:$rhs), + (fpow node:$lhs, node:$rhs)]>; +def any_flog2 : PatFrags<(ops node:$src), + [(strict_flog2 node:$src), + (flog2 node:$src)]>; +def any_frint : PatFrags<(ops node:$src), + [(strict_frint node:$src), + (frint node:$src)]>; +def any_fnearbyint : PatFrags<(ops node:$src), + [(strict_fnearbyint node:$src), + (fnearbyint node:$src)]>; +def any_fceil : PatFrags<(ops node:$src), + [(strict_fceil node:$src), + (fceil node:$src)]>; +def any_ffloor : PatFrags<(ops node:$src), + [(strict_ffloor node:$src), + (ffloor node:$src)]>; +def any_fround : PatFrags<(ops node:$src), + [(strict_fround node:$src), + (fround node:$src)]>; +def any_ftrunc : PatFrags<(ops node:$src), + [(strict_ftrunc node:$src), + (ftrunc node:$src)]>; +def any_fmaxnum : PatFrags<(ops node:$lhs, node:$rhs), + [(strict_fmaxnum node:$lhs, node:$rhs), + (fmaxnum node:$lhs, node:$rhs)]>; +def any_fminnum : PatFrags<(ops node:$lhs, node:$rhs), + [(strict_fminnum node:$lhs, node:$rhs), + (fminnum node:$lhs, node:$rhs)]>; +def any_fpround : PatFrags<(ops node:$src), + [(strict_fpround node:$src), + (fpround node:$src)]>; +def any_fpextend : PatFrags<(ops node:$src), + [(strict_fpextend node:$src), + (fpextend node:$src)]>; +def any_extloadf32 : PatFrags<(ops node:$ptr), + [(strict_extloadf32 node:$ptr), + (extloadf32 node:$ptr)]>; +def any_extloadf64 : PatFrags<(ops node:$ptr), + [(strict_extloadf64 node:$ptr), + (extloadf64 node:$ptr)]>; + multiclass binary_atomic_op_ord { def #NAME#_monotonic : PatFrag<(ops node:$ptr, node:$val), (!cast(#NAME) node:$ptr, node:$val)> { diff --git a/include/llvm/Testing/Support/Annotations.h b/include/llvm/Testing/Support/Annotations.h new file mode 100644 index 000000000000..aad1a44f4ec9 --- /dev/null +++ b/include/llvm/Testing/Support/Annotations.h @@ -0,0 +1,90 @@ +//===--- Annotations.h - Annotated source code for tests ---------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_TESTING_SUPPORT_ANNOTATIONS_H +#define LLVM_TESTING_SUPPORT_ANNOTATIONS_H + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringMap.h" +#include "llvm/ADT/StringRef.h" +#include +#include + +namespace llvm { + +/// Annotations lets you mark points and ranges inside source code, for tests: +/// +/// Annotations Example(R"cpp( +/// int complete() { x.pri^ } // ^ indicates a point +/// void err() { [["hello" == 42]]; } // [[this is a range]] +/// $definition^class Foo{}; // points can be named: "definition" +/// $fail[[static_assert(false, "")]] // ranges can be named too: "fail" +/// )cpp"); +/// +/// StringRef Code = Example.code(); // annotations stripped. +/// std::vector PP = Example.points(); // all unnamed points +/// size_t P = Example.point(); // there must be exactly one +/// llvm::Range R = Example.range("fail"); // find named ranges +/// +/// Points/ranges are coordinated into `code()` which is stripped of +/// annotations. +/// +/// Ranges may be nested (and points can be inside ranges), but there's no way +/// to define general overlapping ranges. +/// +/// FIXME: the choice of the marking syntax makes it impossible to represent +/// some of the C++ and Objective C constructs (including common ones +/// like C++ attributes). We can fix this by: +/// 1. introducing an escaping mechanism for the special characters, +/// 2. making characters for marking points and ranges configurable, +/// 3. changing the syntax to something less commonly used, +/// 4. ... +class Annotations { +public: + /// Two offsets pointing to a continuous substring. End is not included, i.e. + /// represents a half-open range. + struct Range { + size_t Begin = 0; + size_t End = 0; + + friend bool operator==(const Range &L, const Range &R) { + return std::tie(L.Begin, L.End) == std::tie(R.Begin, R.End); + } + friend bool operator!=(const Range &L, const Range &R) { return !(L == R); } + }; + + /// Parses the annotations from Text. Crashes if it's malformed. + Annotations(llvm::StringRef Text); + + /// The input text with all annotations stripped. + /// All points and ranges are relative to this stripped text. + llvm::StringRef code() const { return Code; } + + /// Returns the position of the point marked by ^ (or $name^) in the text. + /// Crashes if there isn't exactly one. + size_t point(llvm::StringRef Name = "") const; + /// Returns the position of all points marked by ^ (or $name^) in the text. + std::vector points(llvm::StringRef Name = "") const; + + /// Returns the location of the range marked by [[ ]] (or $name[[ ]]). + /// Crashes if there isn't exactly one. + Range range(llvm::StringRef Name = "") const; + /// Returns the location of all ranges marked by [[ ]] (or $name[[ ]]). + std::vector ranges(llvm::StringRef Name = "") const; + +private: + std::string Code; + llvm::StringMap> Points; + llvm::StringMap> Ranges; +}; + +llvm::raw_ostream &operator<<(llvm::raw_ostream &O, + const llvm::Annotations::Range &R); + +} // namespace llvm + +#endif diff --git a/include/llvm/Testing/Support/Error.h b/include/llvm/Testing/Support/Error.h index 0e5b5403ce87..85328f26440b 100644 --- a/include/llvm/Testing/Support/Error.h +++ b/include/llvm/Testing/Support/Error.h @@ -1,9 +1,8 @@ //===- llvm/Testing/Support/Error.h ---------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Testing/Support/SupportHelpers.h b/include/llvm/Testing/Support/SupportHelpers.h index b2975ec395d5..38726b1cfaf7 100644 --- a/include/llvm/Testing/Support/SupportHelpers.h +++ b/include/llvm/Testing/Support/SupportHelpers.h @@ -1,18 +1,19 @@ //===- Testing/Support/SupportHelpers.h -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef LLVM_TESTING_SUPPORT_SUPPORTHELPERS_H #define LLVM_TESTING_SUPPORT_SUPPORTHELPERS_H +#include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallString.h" #include "llvm/Support/Error.h" #include "llvm/Support/raw_os_ostream.h" +#include "gmock/gmock-matchers.h" #include "gtest/gtest-printers.h" #include @@ -54,11 +55,56 @@ void PrintTo(const ExpectedHolder &Item, std::ostream *Out) { PrintTo(static_cast(Item), Out); } } + +template class ValueIsMatcher { +public: + explicit ValueIsMatcher(InnerMatcher ValueMatcher) + : ValueMatcher(ValueMatcher) {} + + template + operator ::testing::Matcher &>() const { + return ::testing::MakeMatcher( + new Impl(::testing::SafeMatcherCast(ValueMatcher))); + } + + template + class Impl : public ::testing::MatcherInterface &> { + public: + explicit Impl(const ::testing::Matcher &ValueMatcher) + : ValueMatcher(ValueMatcher) {} + + bool MatchAndExplain(const llvm::Optional &Input, + testing::MatchResultListener *L) const override { + return Input && ValueMatcher.MatchAndExplain(Input.getValue(), L); + } + + void DescribeTo(std::ostream *OS) const override { + *OS << "has a value that "; + ValueMatcher.DescribeTo(OS); + } + void DescribeNegationTo(std::ostream *OS) const override { + *OS << "does not have a value that "; + ValueMatcher.DescribeTo(OS); + } + + private: + testing::Matcher ValueMatcher; + }; + +private: + InnerMatcher ValueMatcher; +}; } // namespace detail +/// Matches an llvm::Optional with a value that conforms to an inner matcher. +/// To match llvm::None you could use Eq(llvm::None). +template +detail::ValueIsMatcher ValueIs(const InnerMatcher &ValueMatcher) { + return detail::ValueIsMatcher(ValueMatcher); +} namespace unittest { SmallString<128> getInputFileDirectory(const char *Argv0); -} +} // namespace unittest } // namespace llvm #endif diff --git a/include/llvm/TextAPI/ELF/ELFStub.h b/include/llvm/TextAPI/ELF/ELFStub.h index fa54e6f8b711..76b2af121662 100644 --- a/include/llvm/TextAPI/ELF/ELFStub.h +++ b/include/llvm/TextAPI/ELF/ELFStub.h @@ -1,9 +1,8 @@ //===- ELFStub.h ------------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===-----------------------------------------------------------------------===/ /// diff --git a/include/llvm/TextAPI/ELF/TBEHandler.h b/include/llvm/TextAPI/ELF/TBEHandler.h index 91521c656fa2..1748fd13f3dc 100644 --- a/include/llvm/TextAPI/ELF/TBEHandler.h +++ b/include/llvm/TextAPI/ELF/TBEHandler.h @@ -1,9 +1,8 @@ //===- TBEHandler.h ---------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===-----------------------------------------------------------------------===/ /// diff --git a/include/llvm/TextAPI/MachO/Architecture.def b/include/llvm/TextAPI/MachO/Architecture.def new file mode 100644 index 000000000000..4c695fe18eec --- /dev/null +++ b/include/llvm/TextAPI/MachO/Architecture.def @@ -0,0 +1,38 @@ +//===- llvm/TextAPI/MachO/Architecture.def - Architecture -----------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef ARCHINFO +#define ARCHINFO(arch) +#endif + +/// +/// X86 architectures sorted by cpu type and sub type id. +/// +ARCHINFO(i386, MachO::CPU_TYPE_I386, MachO::CPU_SUBTYPE_I386_ALL) +ARCHINFO(x86_64, MachO::CPU_TYPE_X86_64, MachO::CPU_SUBTYPE_X86_64_ALL) +ARCHINFO(x86_64h, MachO::CPU_TYPE_X86_64, MachO::CPU_SUBTYPE_X86_64_H) + + +/// +/// ARM architectures sorted by cpu sub type id. +/// +ARCHINFO(armv4t, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V4T) +ARCHINFO(armv6, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V6) +ARCHINFO(armv5, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V5TEJ) +ARCHINFO(armv7, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V7) +ARCHINFO(armv7s, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V7S) +ARCHINFO(armv7k, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V7K) +ARCHINFO(armv6m, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V6M) +ARCHINFO(armv7m, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V7M) +ARCHINFO(armv7em, MachO::CPU_TYPE_ARM, MachO::CPU_SUBTYPE_ARM_V7EM) + + +/// +/// ARM64 architectures sorted by cpu sub type id. +/// +ARCHINFO(arm64, MachO::CPU_TYPE_ARM64, MachO::CPU_SUBTYPE_ARM64_ALL) diff --git a/include/llvm/TextAPI/MachO/Architecture.h b/include/llvm/TextAPI/MachO/Architecture.h new file mode 100644 index 000000000000..055baeb0c0f0 --- /dev/null +++ b/include/llvm/TextAPI/MachO/Architecture.h @@ -0,0 +1,47 @@ +//===- llvm/TextAPI/MachO/Architecture.h - Architecture ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Defines the architecture enum and helper methods. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TEXTAPI_MACHO_ARCHITECTURE_H +#define LLVM_TEXTAPI_MACHO_ARCHITECTURE_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { +namespace MachO { + +/// Defines the architecture slices that are supported by Text-based Stub files. +enum Architecture : uint8_t { +#define ARCHINFO(Arch, Type, SubType) AK_##Arch, +#include "llvm/TextAPI/MachO/Architecture.def" +#undef ARCHINFO + AK_unknown, // this has to go last. +}; + +/// Convert a CPU Type and Subtype pair to an architecture slice. +Architecture getArchitectureFromCpuType(uint32_t CPUType, uint32_t CPUSubType); + +/// Convert a name to an architecture slice. +Architecture getArchitectureFromName(StringRef Name); + +/// Convert an architecture slice to a string. +StringRef getArchitectureName(Architecture Arch); + +/// Convert an architecture slice to a CPU Type and Subtype pair. +std::pair getCPUTypeFromArchitecture(Architecture Arch); + +raw_ostream &operator<<(raw_ostream &OS, Architecture Arch); + +} // end namespace MachO. +} // end namespace llvm. + +#endif // LLVM_TEXTAPI_MACHO_ARCHITECTURE_H diff --git a/include/llvm/TextAPI/MachO/ArchitectureSet.h b/include/llvm/TextAPI/MachO/ArchitectureSet.h new file mode 100644 index 000000000000..d8dfc7f1af21 --- /dev/null +++ b/include/llvm/TextAPI/MachO/ArchitectureSet.h @@ -0,0 +1,159 @@ +//===- llvm/TextAPI/MachO/ArchitectureSet.h - ArchitectureSet ---*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Defines the architecture set. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TEXTAPI_MACHO_ARCHITECTURE_SET_H +#define LLVM_TEXTAPI_MACHO_ARCHITECTURE_SET_H + +#include "llvm/Support/raw_ostream.h" +#include "llvm/TextAPI/MachO/Architecture.h" +#include +#include +#include +#include + +namespace llvm { +namespace MachO { + +class ArchitectureSet { +private: + using ArchSetType = uint32_t; + + const static ArchSetType EndIndexVal = + std::numeric_limits::max(); + ArchSetType ArchSet{0}; + +public: + constexpr ArchitectureSet() = default; + constexpr ArchitectureSet(ArchSetType Raw) : ArchSet(Raw) {} + ArchitectureSet(Architecture Arch) : ArchitectureSet() { set(Arch); } + ArchitectureSet(const std::vector &Archs); + + void set(Architecture Arch) { + if (Arch == AK_unknown) + return; + ArchSet |= 1U << static_cast(Arch); + } + + void clear(Architecture Arch) { ArchSet &= ~(1U << static_cast(Arch)); } + + bool has(Architecture Arch) const { + return ArchSet & (1U << static_cast(Arch)); + } + + bool contains(ArchitectureSet Archs) const { + return (ArchSet & Archs.ArchSet) == Archs.ArchSet; + } + + size_t count() const; + + bool empty() const { return ArchSet == 0; } + + ArchSetType rawValue() const { return ArchSet; } + + template + class arch_iterator + : public std::iterator { + private: + ArchSetType Index; + Ty *ArchSet; + + void findNextSetBit() { + if (Index == EndIndexVal) + return; + while (++Index < sizeof(Ty) * 8) { + if (*ArchSet & (1UL << Index)) + return; + } + + Index = EndIndexVal; + } + + public: + arch_iterator(Ty *ArchSet, ArchSetType Index = 0) + : Index(Index), ArchSet(ArchSet) { + if (Index != EndIndexVal && !(*ArchSet & (1UL << Index))) + findNextSetBit(); + } + + Architecture operator*() const { return static_cast(Index); } + + arch_iterator &operator++() { + findNextSetBit(); + return *this; + } + + arch_iterator operator++(int) { + auto tmp = *this; + findNextSetBit(); + return tmp; + } + + bool operator==(const arch_iterator &o) const { + return std::tie(Index, ArchSet) == std::tie(o.Index, o.ArchSet); + } + + bool operator!=(const arch_iterator &o) const { return !(*this == o); } + }; + + ArchitectureSet operator&(const ArchitectureSet &o) { + return {ArchSet & o.ArchSet}; + } + + ArchitectureSet operator|(const ArchitectureSet &o) { + return {ArchSet | o.ArchSet}; + } + + ArchitectureSet &operator|=(const ArchitectureSet &o) { + ArchSet |= o.ArchSet; + return *this; + } + + ArchitectureSet &operator|=(const Architecture &Arch) { + set(Arch); + return *this; + } + + bool operator==(const ArchitectureSet &o) const { + return ArchSet == o.ArchSet; + } + + bool operator!=(const ArchitectureSet &o) const { + return ArchSet != o.ArchSet; + } + + bool operator<(const ArchitectureSet &o) const { return ArchSet < o.ArchSet; } + + using iterator = arch_iterator; + using const_iterator = arch_iterator; + + iterator begin() { return {&ArchSet}; } + iterator end() { return {&ArchSet, EndIndexVal}; } + + const_iterator begin() const { return {&ArchSet}; } + const_iterator end() const { return {&ArchSet, EndIndexVal}; } + + operator std::string() const; + operator std::vector() const; + void print(raw_ostream &OS) const; +}; + +inline ArchitectureSet operator|(const Architecture &lhs, + const Architecture &rhs) { + return ArchitectureSet(lhs) | ArchitectureSet(rhs); +} + +raw_ostream &operator<<(raw_ostream &OS, ArchitectureSet Set); + +} // end namespace MachO. +} // end namespace llvm. + +#endif // LLVM_TEXTAPI_MACHO_ARCHITECTURE_SET_H diff --git a/include/llvm/TextAPI/MachO/InterfaceFile.h b/include/llvm/TextAPI/MachO/InterfaceFile.h new file mode 100644 index 000000000000..e722449d52f1 --- /dev/null +++ b/include/llvm/TextAPI/MachO/InterfaceFile.h @@ -0,0 +1,436 @@ +//===- llvm/TextAPI/MachO/IntefaceFile.h - TAPI Interface File --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// A generic and abstract interface representation for linkable objects. This +// could be an MachO executable, bundle, dylib, or text-based stub file. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TEXTAPI_MACHO_INTERFACE_FILE_H +#define LLVM_TEXTAPI_MACHO_INTERFACE_FILE_H + +#include "llvm/ADT/BitmaskEnum.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/Hashing.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/ADT/iterator.h" +#include "llvm/BinaryFormat/MachO.h" +#include "llvm/BinaryFormat/Magic.h" +#include "llvm/Support/Allocator.h" +#include "llvm/Support/Error.h" +#include "llvm/TextAPI/MachO/Architecture.h" +#include "llvm/TextAPI/MachO/ArchitectureSet.h" +#include "llvm/TextAPI/MachO/PackedVersion.h" +#include "llvm/TextAPI/MachO/Symbol.h" + +namespace llvm { +namespace MachO { + +/// Defines the list of MachO platforms. +enum class PlatformKind : unsigned { + unknown, + macOS = MachO::PLATFORM_MACOS, + iOS = MachO::PLATFORM_IOS, + tvOS = MachO::PLATFORM_TVOS, + watchOS = MachO::PLATFORM_WATCHOS, + bridgeOS = MachO::PLATFORM_BRIDGEOS, +}; + +/// Defines a list of Objective-C constraints. +enum class ObjCConstraintType : unsigned { + /// No constraint. + None = 0, + + /// Retain/Release. + Retain_Release = 1, + + /// Retain/Release for Simulator. + Retain_Release_For_Simulator = 2, + + /// Retain/Release or Garbage Collection. + Retain_Release_Or_GC = 3, + + /// Garbage Collection. + GC = 4, +}; + +// clang-format off + +/// Defines the file type this file represents. +enum FileType : unsigned { + /// Invalid file type. + Invalid = 0U, + + /// Text-based stub file (.tbd) version 1.0 + TBD_V1 = 1U << 0, + + /// Text-based stub file (.tbd) version 2.0 + TBD_V2 = 1U << 1, + + /// Text-based stub file (.tbd) version 3.0 + TBD_V3 = 1U << 2, + + All = ~0U, + + LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/All), +}; + +// clang-format on + +/// Reference to an interface file. +class InterfaceFileRef { +public: + InterfaceFileRef() = default; + + InterfaceFileRef(StringRef InstallName) : InstallName(InstallName) {} + + InterfaceFileRef(StringRef InstallName, ArchitectureSet Archs) + : InstallName(InstallName), Architectures(Archs) {} + + StringRef getInstallName() const { return InstallName; }; + void addArchitectures(ArchitectureSet Archs) { Architectures |= Archs; } + ArchitectureSet getArchitectures() const { return Architectures; } + bool hasArchitecture(Architecture Arch) const { + return Architectures.has(Arch); + } + + bool operator==(const InterfaceFileRef &O) const { + return std::tie(InstallName, Architectures) == + std::tie(O.InstallName, O.Architectures); + } + + bool operator<(const InterfaceFileRef &O) const { + return std::tie(InstallName, Architectures) < + std::tie(O.InstallName, O.Architectures); + } + +private: + std::string InstallName; + ArchitectureSet Architectures; +}; + +} // end namespace MachO. + +struct SymbolsMapKey { + MachO::SymbolKind Kind; + StringRef Name; + + SymbolsMapKey(MachO::SymbolKind Kind, StringRef Name) + : Kind(Kind), Name(Name) {} +}; +template <> struct DenseMapInfo { + static inline SymbolsMapKey getEmptyKey() { + return SymbolsMapKey(MachO::SymbolKind::GlobalSymbol, StringRef{}); + } + + static inline SymbolsMapKey getTombstoneKey() { + return SymbolsMapKey(MachO::SymbolKind::ObjectiveCInstanceVariable, + StringRef{}); + } + + static unsigned getHashValue(const SymbolsMapKey &Key) { + return hash_combine(hash_value(Key.Kind), hash_value(Key.Name)); + } + + static bool isEqual(const SymbolsMapKey &LHS, const SymbolsMapKey &RHS) { + return std::tie(LHS.Kind, LHS.Name) == std::tie(RHS.Kind, RHS.Name); + } +}; + +namespace MachO { + +/// Defines the interface file. +class InterfaceFile { +public: + /// Set the path from which this file was generated (if applicable). + /// + /// \param Path_ The path to the source file. + void setPath(StringRef Path_) { Path = Path_; } + + /// Get the path from which this file was generated (if applicable). + /// + /// \return The path to the source file or empty. + StringRef getPath() const { return Path; } + + /// Set the file type. + /// + /// This is used by the YAML writer to identify the specification it should + /// use for writing the file. + /// + /// \param Kind The file type. + void setFileType(FileType Kind) { FileKind = Kind; } + + /// Get the file type. + /// + /// \return The file type. + FileType getFileType() const { return FileKind; } + + /// Set the platform. + void setPlatform(PlatformKind Platform_) { Platform = Platform_; } + + /// Get the platform. + PlatformKind getPlatform() const { return Platform; } + + /// Specify the set of supported architectures by this file. + void setArchitectures(ArchitectureSet Architectures_) { + Architectures = Architectures_; + } + + /// Add the set of supported architectures by this file. + void addArchitectures(ArchitectureSet Architectures_) { + Architectures |= Architectures_; + } + + /// Add supported architecture by this file.. + void addArch(Architecture Arch) { Architectures.set(Arch); } + + /// Get the set of supported architectures. + ArchitectureSet getArchitectures() const { return Architectures; } + + /// Set the install name of the library. + void setInstallName(StringRef InstallName_) { InstallName = InstallName_; } + + /// Get the install name of the library. + StringRef getInstallName() const { return InstallName; } + + /// Set the current version of the library. + void setCurrentVersion(PackedVersion Version) { CurrentVersion = Version; } + + /// Get the current version of the library. + PackedVersion getCurrentVersion() const { return CurrentVersion; } + + /// Set the compatibility version of the library. + void setCompatibilityVersion(PackedVersion Version) { + CompatibilityVersion = Version; + } + + /// Get the compatibility version of the library. + PackedVersion getCompatibilityVersion() const { return CompatibilityVersion; } + + /// Set the Swift ABI version of the library. + void setSwiftABIVersion(uint8_t Version) { SwiftABIVersion = Version; } + + /// Get the Swift ABI version of the library. + uint8_t getSwiftABIVersion() const { return SwiftABIVersion; } + + /// Specify if the library uses two-level namespace (or flat namespace). + void setTwoLevelNamespace(bool V = true) { IsTwoLevelNamespace = V; } + + /// Check if the library uses two-level namespace. + bool isTwoLevelNamespace() const { return IsTwoLevelNamespace; } + + /// Specify if the library is application extension safe (or not). + void setApplicationExtensionSafe(bool V = true) { IsAppExtensionSafe = V; } + + /// Check if the library is application extension safe. + bool isApplicationExtensionSafe() const { return IsAppExtensionSafe; } + + /// Set the Objective-C constraint. + void setObjCConstraint(ObjCConstraintType Constraint) { + ObjcConstraint = Constraint; + } + + /// Get the Objective-C constraint. + ObjCConstraintType getObjCConstraint() const { return ObjcConstraint; } + + /// Specify if this file was generated during InstallAPI (or not). + void setInstallAPI(bool V = true) { IsInstallAPI = V; } + + /// Check if this file was generated during InstallAPI. + bool isInstallAPI() const { return IsInstallAPI; } + + /// Set the parent umbrella framework. + void setParentUmbrella(StringRef Parent) { ParentUmbrella = Parent; } + + /// Get the parent umbrella framework. + StringRef getParentUmbrella() const { return ParentUmbrella; } + + /// Add an allowable client. + /// + /// Mach-O Dynamic libraries have the concept of allowable clients that are + /// checked during static link time. The name of the application or library + /// that is being generated needs to match one of the allowable clients or the + /// linker refuses to link this library. + /// + /// \param Name The name of the client that is allowed to link this library. + /// \param Architectures The set of architecture for which this applies. + void addAllowableClient(StringRef Name, ArchitectureSet Architectures); + + /// Get the list of allowable clients. + /// + /// \return Returns a list of allowable clients. + const std::vector &allowableClients() const { + return AllowableClients; + } + + /// Add a re-exported library. + /// + /// \param InstallName The name of the library to re-export. + /// \param Architectures The set of architecture for which this applies. + void addReexportedLibrary(StringRef InstallName, + ArchitectureSet Architectures); + + /// Get the list of re-exported libraries. + /// + /// \return Returns a list of re-exported libraries. + const std::vector &reexportedLibraries() const { + return ReexportedLibraries; + } + + /// Add an architecture/UUID pair. + /// + /// \param Arch The architecture for which this applies. + /// \param UUID The UUID of the library for the specified architecture. + void addUUID(Architecture Arch, StringRef UUID); + + /// Add an architecture/UUID pair. + /// + /// \param Arch The architecture for which this applies. + /// \param UUID The UUID of the library for the specified architecture. + void addUUID(Architecture Arch, uint8_t UUID[16]); + + /// Get the list of architecture/UUID pairs. + /// + /// \return Returns a list of architecture/UUID pairs. + const std::vector> &uuids() const { + return UUIDs; + } + + /// Add a symbol to the symbols list or extend an existing one. + void addSymbol(SymbolKind Kind, StringRef Name, ArchitectureSet Architectures, + SymbolFlags Flags = SymbolFlags::None); + + using SymbolMapType = DenseMap; + struct const_symbol_iterator + : public iterator_adaptor_base< + const_symbol_iterator, SymbolMapType::const_iterator, + std::forward_iterator_tag, const Symbol *, ptrdiff_t, + const Symbol *, const Symbol *> { + const_symbol_iterator() = default; + + template + const_symbol_iterator(U &&u) + : iterator_adaptor_base(std::forward(u)) {} + + reference operator*() const { return I->second; } + pointer operator->() const { return I->second; } + }; + using const_symbol_range = iterator_range; + + // Custom iterator to return only exported symbols. + struct const_export_iterator + : public iterator_adaptor_base< + const_export_iterator, const_symbol_iterator, + std::forward_iterator_tag, const Symbol *> { + const_symbol_iterator _end; + + void skipToNextSymbol() { + while (I != _end && I->isUndefined()) + ++I; + } + + const_export_iterator() = default; + template + const_export_iterator(U &&it, U &&end) + : iterator_adaptor_base(std::forward(it)), + _end(std::forward(end)) { + skipToNextSymbol(); + } + + const_export_iterator &operator++() { + ++I; + skipToNextSymbol(); + return *this; + } + + const_export_iterator operator++(int) { + const_export_iterator tmp(*this); + ++(*this); + return tmp; + } + }; + using const_export_range = llvm::iterator_range; + + // Custom iterator to return only undefined symbols. + struct const_undefined_iterator + : public iterator_adaptor_base< + const_undefined_iterator, const_symbol_iterator, + std::forward_iterator_tag, const Symbol *> { + const_symbol_iterator _end; + + void skipToNextSymbol() { + while (I != _end && !I->isUndefined()) + ++I; + } + + const_undefined_iterator() = default; + template + const_undefined_iterator(U &&it, U &&end) + : iterator_adaptor_base(std::forward(it)), + _end(std::forward(end)) { + skipToNextSymbol(); + } + + const_undefined_iterator &operator++() { + ++I; + skipToNextSymbol(); + return *this; + } + + const_undefined_iterator operator++(int) { + const_undefined_iterator tmp(*this); + ++(*this); + return tmp; + } + }; + using const_undefined_range = llvm::iterator_range; + + const_symbol_range symbols() const { + return {Symbols.begin(), Symbols.end()}; + } + const_export_range exports() const { + return {{Symbols.begin(), Symbols.end()}, {Symbols.end(), Symbols.end()}}; + } + const_undefined_range undefineds() const { + return {{Symbols.begin(), Symbols.end()}, {Symbols.end(), Symbols.end()}}; + } + +private: + llvm::BumpPtrAllocator Allocator; + StringRef copyString(StringRef String) { + if (String.empty()) + return {}; + + void *Ptr = Allocator.Allocate(String.size(), 1); + memcpy(Ptr, String.data(), String.size()); + return StringRef(reinterpret_cast(Ptr), String.size()); + } + + std::string Path; + FileType FileKind; + PlatformKind Platform; + ArchitectureSet Architectures; + std::string InstallName; + PackedVersion CurrentVersion; + PackedVersion CompatibilityVersion; + uint8_t SwiftABIVersion{0}; + bool IsTwoLevelNamespace{false}; + bool IsAppExtensionSafe{false}; + bool IsInstallAPI{false}; + ObjCConstraintType ObjcConstraint = ObjCConstraintType::None; + std::string ParentUmbrella; + std::vector AllowableClients; + std::vector ReexportedLibraries; + std::vector> UUIDs; + SymbolMapType Symbols; +}; + +} // end namespace MachO. +} // end namespace llvm. + +#endif // LLVM_TEXTAPI_MACHO_INTERFACE_FILE_H diff --git a/include/llvm/TextAPI/MachO/PackedVersion.h b/include/llvm/TextAPI/MachO/PackedVersion.h new file mode 100644 index 000000000000..2d0138097dd9 --- /dev/null +++ b/include/llvm/TextAPI/MachO/PackedVersion.h @@ -0,0 +1,64 @@ +//===- llvm/TextAPI/MachO/PackedVersion.h - PackedVersion -------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Defines the Mach-O packed version format. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TEXTAPI_MACHO_PACKED_VERSION_H +#define LLVM_TEXTAPI_MACHO_PACKED_VERSION_H + +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/raw_ostream.h" + +namespace llvm { +namespace MachO { + +class PackedVersion { + uint32_t Version{0}; + +public: + constexpr PackedVersion() = default; + explicit constexpr PackedVersion(uint32_t RawVersion) : Version(RawVersion) {} + PackedVersion(unsigned Major, unsigned Minor, unsigned Subminor) + : Version((Major << 16) | ((Minor & 0xff) << 8) | (Subminor & 0xff)) {} + + bool empty() const { return Version == 0; } + + /// Retrieve the major version number. + unsigned getMajor() const { return Version >> 16; } + + /// Retrieve the minor version number, if provided. + unsigned getMinor() const { return (Version >> 8) & 0xff; } + + /// Retrieve the subminor version number, if provided. + unsigned getSubminor() const { return Version & 0xff; } + + bool parse32(StringRef Str); + std::pair parse64(StringRef Str); + + bool operator<(const PackedVersion &O) const { return Version < O.Version; } + + bool operator==(const PackedVersion &O) const { return Version == O.Version; } + + bool operator!=(const PackedVersion &O) const { return Version != O.Version; } + + uint32_t rawValue() const { return Version; } + + void print(raw_ostream &OS) const; +}; + +inline raw_ostream &operator<<(raw_ostream &OS, const PackedVersion &Version) { + Version.print(OS); + return OS; +} + +} // end namespace MachO. +} // end namespace llvm. + +#endif // LLVM_TEXTAPI_MACHO_PACKED_VERSION_H diff --git a/include/llvm/TextAPI/MachO/Symbol.h b/include/llvm/TextAPI/MachO/Symbol.h new file mode 100644 index 000000000000..3c7ff5e0f4ea --- /dev/null +++ b/include/llvm/TextAPI/MachO/Symbol.h @@ -0,0 +1,96 @@ +//===- llvm/TextAPI/Symbol.h - TAPI Symbol ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TEXTAPI_MACHO_SYMBOL_H +#define LLVM_TEXTAPI_MACHO_SYMBOL_H + +#include "llvm/ADT/BitmaskEnum.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/raw_ostream.h" +#include "llvm/TextAPI/MachO/ArchitectureSet.h" + +namespace llvm { +namespace MachO { + +// clang-format off + +/// Symbol flags. +enum class SymbolFlags : uint8_t { + /// No flags + None = 0, + + /// Thread-local value symbol + ThreadLocalValue = 1U << 0, + + /// Weak defined symbol + WeakDefined = 1U << 1, + + /// Weak referenced symbol + WeakReferenced = 1U << 2, + + /// Undefined + Undefined = 1U << 3, + + LLVM_MARK_AS_BITMASK_ENUM(/*LargestValue=*/Undefined), +}; + +// clang-format on + +enum class SymbolKind : uint8_t { + GlobalSymbol, + ObjectiveCClass, + ObjectiveCClassEHType, + ObjectiveCInstanceVariable, +}; + +class Symbol { +public: + constexpr Symbol(SymbolKind Kind, StringRef Name, + ArchitectureSet Architectures, SymbolFlags Flags) + : Name(Name), Architectures(Architectures), Kind(Kind), Flags(Flags) {} + + SymbolKind getKind() const { return Kind; } + StringRef getName() const { return Name; } + ArchitectureSet getArchitectures() const { return Architectures; } + void addArchitectures(ArchitectureSet Archs) { Architectures |= Archs; } + SymbolFlags getFlags() const { return Flags; } + + bool isWeakDefined() const { + return (Flags & SymbolFlags::WeakDefined) == SymbolFlags::WeakDefined; + } + + bool isWeakReferenced() const { + return (Flags & SymbolFlags::WeakReferenced) == SymbolFlags::WeakReferenced; + } + + bool isThreadLocalValue() const { + return (Flags & SymbolFlags::ThreadLocalValue) == + SymbolFlags::ThreadLocalValue; + } + + bool isUndefined() const { + return (Flags & SymbolFlags::Undefined) == SymbolFlags::Undefined; + } + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + void dump(raw_ostream &OS) const; + void dump() const { dump(llvm::errs()); } +#endif + +private: + StringRef Name; + ArchitectureSet Architectures; + SymbolKind Kind; + SymbolFlags Flags; +}; + +} // end namespace MachO. +} // end namespace llvm. + +#endif // LLVM_TEXTAPI_MACHO_SYMBOL_H diff --git a/include/llvm/TextAPI/MachO/TextAPIReader.h b/include/llvm/TextAPI/MachO/TextAPIReader.h new file mode 100644 index 000000000000..6d9c09de5294 --- /dev/null +++ b/include/llvm/TextAPI/MachO/TextAPIReader.h @@ -0,0 +1,34 @@ +//===--- TextAPIReader.h - Text API Reader ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TEXTAPI_MACHO_READER_H +#define LLVM_TEXTAPI_MACHO_READER_H + +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { +namespace MachO { + +class InterfaceFile; + +class TextAPIReader { +public: + static Expected> + get(std::unique_ptr InputBuffer); + + static Expected> + getUnmanaged(llvm::MemoryBuffer *InputBuffer); + + TextAPIReader() = delete; +}; + +} // end namespace MachO. +} // end namespace llvm. + +#endif // LLVM_TEXTAPI_MACHO_READER_H diff --git a/include/llvm/TextAPI/MachO/TextAPIWriter.h b/include/llvm/TextAPI/MachO/TextAPIWriter.h new file mode 100644 index 000000000000..2a45bb86a332 --- /dev/null +++ b/include/llvm/TextAPI/MachO/TextAPIWriter.h @@ -0,0 +1,29 @@ +//===--- TextAPIWriter.h - Text API Writer ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TEXTAPI_MACHO_WRITER_H +#define LLVM_TEXTAPI_MACHO_WRITER_H + +#include "llvm/Support/MemoryBuffer.h" + +namespace llvm { +namespace MachO { + +class InterfaceFile; + +class TextAPIWriter { +public: + TextAPIWriter() = delete; + + static Error writeToStream(raw_ostream &os, const InterfaceFile &); +}; + +} // end namespace MachO. +} // end namespace llvm. + +#endif // LLVM_TEXTAPI_MACHO_WRITER_H diff --git a/include/llvm/ToolDrivers/llvm-dlltool/DlltoolDriver.h b/include/llvm/ToolDrivers/llvm-dlltool/DlltoolDriver.h index 964b0f7620a2..d144f62f1cc1 100644 --- a/include/llvm/ToolDrivers/llvm-dlltool/DlltoolDriver.h +++ b/include/llvm/ToolDrivers/llvm-dlltool/DlltoolDriver.h @@ -1,9 +1,8 @@ //===- DlltoolDriver.h - dlltool.exe-compatible driver ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/ToolDrivers/llvm-lib/LibDriver.h b/include/llvm/ToolDrivers/llvm-lib/LibDriver.h index a4806ac4ad69..23a2fc348a89 100644 --- a/include/llvm/ToolDrivers/llvm-lib/LibDriver.h +++ b/include/llvm/ToolDrivers/llvm-lib/LibDriver.h @@ -1,9 +1,8 @@ //===- llvm-lib/LibDriver.h - lib.exe-compatible driver ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,6 +18,7 @@ namespace llvm { template class ArrayRef; int libDriverMain(ArrayRef ARgs); + } #endif diff --git a/include/llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h b/include/llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h index f970acdc741f..887c8807904e 100644 --- a/include/llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h +++ b/include/llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h @@ -1,9 +1,8 @@ //===- AggressiveInstCombine.h - AggressiveInstCombine pass -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/Transforms/Coroutines.h b/include/llvm/Transforms/Coroutines.h index 51beb44fdc56..9df3ec0f3ef4 100644 --- a/include/llvm/Transforms/Coroutines.h +++ b/include/llvm/Transforms/Coroutines.h @@ -1,9 +1,8 @@ //===-- Coroutines.h - Coroutine Transformations ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Declare accessor functions for coroutine lowering passes. diff --git a/include/llvm/Transforms/IPO.h b/include/llvm/Transforms/IPO.h index 11d363b1200b..de0c80f5b19a 100644 --- a/include/llvm/Transforms/IPO.h +++ b/include/llvm/Transforms/IPO.h @@ -1,9 +1,8 @@ //===- llvm/Transforms/IPO.h - Interprocedural Transformations --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -183,6 +182,10 @@ ModulePass *createBlockExtractorPass(); ModulePass * createBlockExtractorPass(const SmallVectorImpl &BlocksToExtract, bool EraseFunctions); +ModulePass * +createBlockExtractorPass(const SmallVectorImpl> + &GroupsOfBlocksToExtract, + bool EraseFunctions); /// createStripDeadPrototypesPass - This pass removes any function declarations /// (prototypes) that are not used. diff --git a/include/llvm/Transforms/IPO/AlwaysInliner.h b/include/llvm/Transforms/IPO/AlwaysInliner.h index b52c0fdbd2c9..64e25230f6da 100644 --- a/include/llvm/Transforms/IPO/AlwaysInliner.h +++ b/include/llvm/Transforms/IPO/AlwaysInliner.h @@ -1,9 +1,8 @@ //===-- AlwaysInliner.h - Pass to inline "always_inline" functions --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/include/llvm/Transforms/IPO/ArgumentPromotion.h b/include/llvm/Transforms/IPO/ArgumentPromotion.h index 49ca6cc73393..c8afb7bdcd65 100644 --- a/include/llvm/Transforms/IPO/ArgumentPromotion.h +++ b/include/llvm/Transforms/IPO/ArgumentPromotion.h @@ -1,9 +1,8 @@ //===- ArgumentPromotion.h - Promote by-reference arguments -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Transforms/IPO/Attributor.h b/include/llvm/Transforms/IPO/Attributor.h new file mode 100644 index 000000000000..5dbe21ac5e4e --- /dev/null +++ b/include/llvm/Transforms/IPO/Attributor.h @@ -0,0 +1,789 @@ +//===- Attributor.h --- Module-wide attribute deduction ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Attributor: An inter procedural (abstract) "attribute" deduction framework. +// +// The Attributor framework is an inter procedural abstract analysis (fixpoint +// iteration analysis). The goal is to allow easy deduction of new attributes as +// well as information exchange between abstract attributes in-flight. +// +// The Attributor class is the driver and the link between the various abstract +// attributes. The Attributor will iterate until a fixpoint state is reached by +// all abstract attributes in-flight, or until it will enforce a pessimistic fix +// point because an iteration limit is reached. +// +// Abstract attributes, derived from the AbstractAttribute class, actually +// describe properties of the code. They can correspond to actual LLVM-IR +// attributes, or they can be more general, ultimately unrelated to LLVM-IR +// attributes. The latter is useful when an abstract attributes provides +// information to other abstract attributes in-flight but we might not want to +// manifest the information. The Attributor allows to query in-flight abstract +// attributes through the `Attributor::getAAFor` method (see the method +// description for an example). If the method is used by an abstract attribute +// P, and it results in an abstract attribute Q, the Attributor will +// automatically capture a potential dependence from Q to P. This dependence +// will cause P to be reevaluated whenever Q changes in the future. +// +// The Attributor will only reevaluated abstract attributes that might have +// changed since the last iteration. That means that the Attribute will not +// revisit all instructions/blocks/functions in the module but only query +// an update from a subset of the abstract attributes. +// +// The update method `AbstractAttribute::updateImpl` is implemented by the +// specific "abstract attribute" subclasses. The method is invoked whenever the +// currently assumed state (see the AbstractState class) might not be valid +// anymore. This can, for example, happen if the state was dependent on another +// abstract attribute that changed. In every invocation, the update method has +// to adjust the internal state of an abstract attribute to a point that is +// justifiable by the underlying IR and the current state of abstract attributes +// in-flight. Since the IR is given and assumed to be valid, the information +// derived from it can be assumed to hold. However, information derived from +// other abstract attributes is conditional on various things. If the justifying +// state changed, the `updateImpl` has to revisit the situation and potentially +// find another justification or limit the optimistic assumes made. +// +// Change is the key in this framework. Until a state of no-change, thus a +// fixpoint, is reached, the Attributor will query the abstract attributes +// in-flight to re-evaluate their state. If the (current) state is too +// optimistic, hence it cannot be justified anymore through other abstract +// attributes or the state of the IR, the state of the abstract attribute will +// have to change. Generally, we assume abstract attribute state to be a finite +// height lattice and the update function to be monotone. However, these +// conditions are not enforced because the iteration limit will guarantee +// termination. If an optimistic fixpoint is reached, or a pessimistic fix +// point is enforced after a timeout, the abstract attributes are tasked to +// manifest their result in the IR for passes to come. +// +// Attribute manifestation is not mandatory. If desired, there is support to +// generate a single LLVM-IR attribute already in the AbstractAttribute base +// class. In the simplest case, a subclass overloads +// `AbstractAttribute::getManifestPosition()` and +// `AbstractAttribute::getAttrKind()` to return the appropriate values. The +// Attributor manifestation framework will then create and place a new attribute +// if it is allowed to do so (based on the abstract state). Other use cases can +// be achieved by overloading other abstract attribute methods. +// +// +// The "mechanics" of adding a new "abstract attribute": +// - Define a class (transitively) inheriting from AbstractAttribute and one +// (which could be the same) that (transitively) inherits from AbstractState. +// For the latter, consider the already available BooleanState and +// IntegerState if they fit your needs, e.g., you require only a bit-encoding. +// - Implement all pure methods. Also use overloading if the attribute is not +// conforming with the "default" behavior: A (set of) LLVM-IR attribute(s) for +// an argument, call site argument, function return value, or function. See +// the class and method descriptions for more information on the two +// "Abstract" classes and their respective methods. +// - Register opportunities for the new abstract attribute in the +// `Attributor::identifyDefaultAbstractAttributes` method if it should be +// counted as a 'default' attribute. +// - Add sufficient tests. +// - Add a Statistics object for bookkeeping. If it is a simple (set of) +// attribute(s) manifested through the Attributor manifestation framework, see +// the bookkeeping function in Attributor.cpp. +// - If instructions with a certain opcode are interesting to the attribute, add +// that opcode to the switch in `Attributor::identifyAbstractAttributes`. This +// will make it possible to query all those instructions through the +// `InformationCache::getOpcodeInstMapForFunction` interface and eliminate the +// need to traverse the IR repeatedly. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_IPO_ATTRIBUTOR_H +#define LLVM_TRANSFORMS_IPO_ATTRIBUTOR_H + +#include "llvm/Analysis/LazyCallGraph.h" +#include "llvm/IR/CallSite.h" +#include "llvm/IR/PassManager.h" + +namespace llvm { + +struct AbstractAttribute; +struct InformationCache; + +class Function; + +/// Simple enum class that forces the status to be spelled out explicitly. +/// +///{ +enum class ChangeStatus { + CHANGED, + UNCHANGED, +}; + +ChangeStatus operator|(ChangeStatus l, ChangeStatus r); +ChangeStatus operator&(ChangeStatus l, ChangeStatus r); +///} + +/// The fixpoint analysis framework that orchestrates the attribute deduction. +/// +/// The Attributor provides a general abstract analysis framework (guided +/// fixpoint iteration) as well as helper functions for the deduction of +/// (LLVM-IR) attributes. However, also other code properties can be deduced, +/// propagated, and ultimately manifested through the Attributor framework. This +/// is particularly useful if these properties interact with attributes and a +/// co-scheduled deduction allows to improve the solution. Even if not, thus if +/// attributes/properties are completely isolated, they should use the +/// Attributor framework to reduce the number of fixpoint iteration frameworks +/// in the code base. Note that the Attributor design makes sure that isolated +/// attributes are not impacted, in any way, by others derived at the same time +/// if there is no cross-reasoning performed. +/// +/// The public facing interface of the Attributor is kept simple and basically +/// allows abstract attributes to one thing, query abstract attributes +/// in-flight. There are two reasons to do this: +/// a) The optimistic state of one abstract attribute can justify an +/// optimistic state of another, allowing to framework to end up with an +/// optimistic (=best possible) fixpoint instead of one based solely on +/// information in the IR. +/// b) This avoids reimplementing various kinds of lookups, e.g., to check +/// for existing IR attributes, in favor of a single lookups interface +/// provided by an abstract attribute subclass. +/// +/// NOTE: The mechanics of adding a new "concrete" abstract attribute are +/// described in the file comment. +struct Attributor { + ~Attributor() { DeleteContainerPointers(AllAbstractAttributes); } + + /// Run the analyses until a fixpoint is reached or enforced (timeout). + /// + /// The attributes registered with this Attributor can be used after as long + /// as the Attributor is not destroyed (it owns the attributes now). + /// + /// \Returns CHANGED if the IR was changed, otherwise UNCHANGED. + ChangeStatus run(); + + /// Lookup an abstract attribute of type \p AAType anchored at value \p V and + /// argument number \p ArgNo. If no attribute is found and \p V is a call base + /// instruction, the called function is tried as a value next. Thus, the + /// returned abstract attribute might be anchored at the callee of \p V. + /// + /// This method is the only (supported) way an abstract attribute can retrieve + /// information from another abstract attribute. As an example, take an + /// abstract attribute that determines the memory access behavior for a + /// argument (readnone, readonly, ...). It should use `getAAFor` to get the + /// most optimistic information for other abstract attributes in-flight, e.g. + /// the one reasoning about the "captured" state for the argument or the one + /// reasoning on the memory access behavior of the function as a whole. + template + const AAType *getAAFor(AbstractAttribute &QueryingAA, const Value &V, + int ArgNo = -1) { + static_assert(std::is_base_of::value, + "Cannot query an attribute with a type not derived from " + "'AbstractAttribute'!"); + assert(AAType::ID != Attribute::None && + "Cannot lookup generic abstract attributes!"); + + // Determine the argument number automatically for llvm::Arguments if none + // is set. Do not override a given one as it could be a use of the argument + // in a call site. + if (ArgNo == -1) + if (auto *Arg = dyn_cast(&V)) + ArgNo = Arg->getArgNo(); + + // If a function was given together with an argument number, perform the + // lookup for the actual argument instead. Don't do it for variadic + // arguments. + if (ArgNo >= 0 && isa(&V) && + cast(&V)->arg_size() > (size_t)ArgNo) + return getAAFor( + QueryingAA, *(cast(&V)->arg_begin() + ArgNo), ArgNo); + + // Lookup the abstract attribute of type AAType. If found, return it after + // registering a dependence of QueryingAA on the one returned attribute. + const auto &KindToAbstractAttributeMap = AAMap.lookup({&V, ArgNo}); + if (AAType *AA = static_cast( + KindToAbstractAttributeMap.lookup(AAType::ID))) { + // Do not return an attribute with an invalid state. This minimizes checks + // at the calls sites and allows the fallback below to kick in. + if (AA->getState().isValidState()) { + QueryMap[AA].insert(&QueryingAA); + return AA; + } + } + + // If no abstract attribute was found and we look for a call site argument, + // defer to the actual argument instead. + ImmutableCallSite ICS(&V); + if (ICS && ICS.getCalledValue()) + return getAAFor(QueryingAA, *ICS.getCalledValue(), ArgNo); + + // No matching attribute found + return nullptr; + } + + /// Introduce a new abstract attribute into the fixpoint analysis. + /// + /// Note that ownership of the attribute is given to the Attributor. It will + /// invoke delete for the Attributor on destruction of the Attributor. + /// + /// Attributes are identified by + /// (1) their anchored value (see AA.getAnchoredValue()), + /// (2) their argument number (\p ArgNo, or Argument::getArgNo()), and + /// (3) their default attribute kind (see AAType::ID). + template AAType ®isterAA(AAType &AA, int ArgNo = -1) { + static_assert(std::is_base_of::value, + "Cannot register an attribute with a type not derived from " + "'AbstractAttribute'!"); + + // Determine the anchor value and the argument number which are used to + // lookup the attribute together with AAType::ID. If passed an argument, + // use its argument number but do not override a given one as it could be a + // use of the argument at a call site. + Value &AnchoredVal = AA.getAnchoredValue(); + if (ArgNo == -1) + if (auto *Arg = dyn_cast(&AnchoredVal)) + ArgNo = Arg->getArgNo(); + + // Put the attribute in the lookup map structure and the container we use to + // keep track of all attributes. + AAMap[{&AnchoredVal, ArgNo}][AAType::ID] = &AA; + AllAbstractAttributes.push_back(&AA); + return AA; + } + + /// Determine opportunities to derive 'default' attributes in \p F and create + /// abstract attribute objects for them. + /// + /// \param F The function that is checked for attribute opportunities. + /// \param InfoCache A cache for information queryable by the new attributes. + /// \param Whitelist If not null, a set limiting the attribute opportunities. + /// + /// Note that abstract attribute instances are generally created even if the + /// IR already contains the information they would deduce. The most important + /// reason for this is the single interface, the one of the abstract attribute + /// instance, which can be queried without the need to look at the IR in + /// various places. + void identifyDefaultAbstractAttributes( + Function &F, InformationCache &InfoCache, + DenseSet *Whitelist = nullptr); + + /// Check \p Pred on all function call sites. + /// + /// This method will evaluate \p Pred on call sites and return + /// true if \p Pred holds in every call sites. However, this is only possible + /// all call sites are known, hence the function has internal linkage. + bool checkForAllCallSites(Function &F, std::function &Pred, + bool RequireAllCallSites); + +private: + /// The set of all abstract attributes. + ///{ + using AAVector = SmallVector; + AAVector AllAbstractAttributes; + ///} + + /// A nested map to lookup abstract attributes based on the anchored value and + /// an argument positions (or -1) on the outer level, and attribute kinds + /// (Attribute::AttrKind) on the inner level. + ///{ + using KindToAbstractAttributeMap = DenseMap; + DenseMap, KindToAbstractAttributeMap> AAMap; + ///} + + /// A map from abstract attributes to the ones that queried them through calls + /// to the getAAFor<...>(...) method. + ///{ + using QueryMapTy = + DenseMap>; + QueryMapTy QueryMap; + ///} +}; + +/// Data structure to hold cached (LLVM-IR) information. +/// +/// All attributes are given an InformationCache object at creation time to +/// avoid inspection of the IR by all of them individually. This default +/// InformationCache will hold information required by 'default' attributes, +/// thus the ones deduced when Attributor::identifyDefaultAbstractAttributes(..) +/// is called. +/// +/// If custom abstract attributes, registered manually through +/// Attributor::registerAA(...), need more information, especially if it is not +/// reusable, it is advised to inherit from the InformationCache and cast the +/// instance down in the abstract attributes. +struct InformationCache { + /// A map type from opcodes to instructions with this opcode. + using OpcodeInstMapTy = DenseMap>; + + /// Return the map that relates "interesting" opcodes with all instructions + /// with that opcode in \p F. + OpcodeInstMapTy &getOpcodeInstMapForFunction(Function &F) { + return FuncInstOpcodeMap[&F]; + } + + /// A vector type to hold instructions. + using InstructionVectorTy = std::vector; + + /// Return the instructions in \p F that may read or write memory. + InstructionVectorTy &getReadOrWriteInstsForFunction(Function &F) { + return FuncRWInstsMap[&F]; + } + +private: + /// A map type from functions to opcode to instruction maps. + using FuncInstOpcodeMapTy = DenseMap; + + /// A map type from functions to their read or write instructions. + using FuncRWInstsMapTy = DenseMap; + + /// A nested map that remembers all instructions in a function with a certain + /// instruction opcode (Instruction::getOpcode()). + FuncInstOpcodeMapTy FuncInstOpcodeMap; + + /// A map from functions to their instructions that may read or write memory. + FuncRWInstsMapTy FuncRWInstsMap; + + /// Give the Attributor access to the members so + /// Attributor::identifyDefaultAbstractAttributes(...) can initialize them. + friend struct Attributor; +}; + +/// An interface to query the internal state of an abstract attribute. +/// +/// The abstract state is a minimal interface that allows the Attributor to +/// communicate with the abstract attributes about their internal state without +/// enforcing or exposing implementation details, e.g., the (existence of an) +/// underlying lattice. +/// +/// It is sufficient to be able to query if a state is (1) valid or invalid, (2) +/// at a fixpoint, and to indicate to the state that (3) an optimistic fixpoint +/// was reached or (4) a pessimistic fixpoint was enforced. +/// +/// All methods need to be implemented by the subclass. For the common use case, +/// a single boolean state or a bit-encoded state, the BooleanState and +/// IntegerState classes are already provided. An abstract attribute can inherit +/// from them to get the abstract state interface and additional methods to +/// directly modify the state based if needed. See the class comments for help. +struct AbstractState { + virtual ~AbstractState() {} + + /// Return if this abstract state is in a valid state. If false, no + /// information provided should be used. + virtual bool isValidState() const = 0; + + /// Return if this abstract state is fixed, thus does not need to be updated + /// if information changes as it cannot change itself. + virtual bool isAtFixpoint() const = 0; + + /// Indicate that the abstract state should converge to the optimistic state. + /// + /// This will usually make the optimistically assumed state the known to be + /// true state. + virtual void indicateOptimisticFixpoint() = 0; + + /// Indicate that the abstract state should converge to the pessimistic state. + /// + /// This will usually revert the optimistically assumed state to the known to + /// be true state. + virtual void indicatePessimisticFixpoint() = 0; +}; + +/// Simple state with integers encoding. +/// +/// The interface ensures that the assumed bits are always a subset of the known +/// bits. Users can only add known bits and, except through adding known bits, +/// they can only remove assumed bits. This should guarantee monotoniticy and +/// thereby the existence of a fixpoint (if used corretly). The fixpoint is +/// reached when the assumed and known state/bits are equal. Users can +/// force/inidicate a fixpoint. If an optimistic one is indicated, the known +/// state will catch up with the assumed one, for a pessimistic fixpoint it is +/// the other way around. +struct IntegerState : public AbstractState { + /// Underlying integer type, we assume 32 bits to be enough. + using base_t = uint32_t; + + /// Initialize the (best) state. + IntegerState(base_t BestState = ~0) : Assumed(BestState) {} + + /// Return the worst possible representable state. + static constexpr base_t getWorstState() { return 0; } + + /// See AbstractState::isValidState() + /// NOTE: For now we simply pretend that the worst possible state is invalid. + bool isValidState() const override { return Assumed != getWorstState(); } + + /// See AbstractState::isAtFixpoint() + bool isAtFixpoint() const override { return Assumed == Known; } + + /// See AbstractState::indicateOptimisticFixpoint(...) + void indicateOptimisticFixpoint() override { Known = Assumed; } + + /// See AbstractState::indicatePessimisticFixpoint(...) + void indicatePessimisticFixpoint() override { Assumed = Known; } + + /// Return the known state encoding + base_t getKnown() const { return Known; } + + /// Return the assumed state encoding. + base_t getAssumed() const { return Assumed; } + + /// Return true if the bits set in \p BitsEncoding are "known bits". + bool isKnown(base_t BitsEncoding) const { + return (Known & BitsEncoding) == BitsEncoding; + } + + /// Return true if the bits set in \p BitsEncoding are "assumed bits". + bool isAssumed(base_t BitsEncoding) const { + return (Assumed & BitsEncoding) == BitsEncoding; + } + + /// Add the bits in \p BitsEncoding to the "known bits". + IntegerState &addKnownBits(base_t Bits) { + // Make sure we never miss any "known bits". + Assumed |= Bits; + Known |= Bits; + return *this; + } + + /// Remove the bits in \p BitsEncoding from the "assumed bits" if not known. + IntegerState &removeAssumedBits(base_t BitsEncoding) { + // Make sure we never loose any "known bits". + Assumed = (Assumed & ~BitsEncoding) | Known; + return *this; + } + + /// Keep only "assumed bits" also set in \p BitsEncoding but all known ones. + IntegerState &intersectAssumedBits(base_t BitsEncoding) { + // Make sure we never loose any "known bits". + Assumed = (Assumed & BitsEncoding) | Known; + return *this; + } + +private: + /// The known state encoding in an integer of type base_t. + base_t Known = getWorstState(); + + /// The assumed state encoding in an integer of type base_t. + base_t Assumed; +}; + +/// Simple wrapper for a single bit (boolean) state. +struct BooleanState : public IntegerState { + BooleanState() : IntegerState(1){}; +}; + +/// Base struct for all "concrete attribute" deductions. +/// +/// The abstract attribute is a minimal interface that allows the Attributor to +/// orchestrate the abstract/fixpoint analysis. The design allows to hide away +/// implementation choices made for the subclasses but also to structure their +/// implementation and simplify the use of other abstract attributes in-flight. +/// +/// To allow easy creation of new attributes, most methods have default +/// implementations. The ones that do not are generally straight forward, except +/// `AbstractAttribute::updateImpl` which is the location of most reasoning +/// associated with the abstract attribute. The update is invoked by the +/// Attributor in case the situation used to justify the current optimistic +/// state might have changed. The Attributor determines this automatically +/// by monitoring the `Attributor::getAAFor` calls made by abstract attributes. +/// +/// The `updateImpl` method should inspect the IR and other abstract attributes +/// in-flight to justify the best possible (=optimistic) state. The actual +/// implementation is, similar to the underlying abstract state encoding, not +/// exposed. In the most common case, the `updateImpl` will go through a list of +/// reasons why its optimistic state is valid given the current information. If +/// any combination of them holds and is sufficient to justify the current +/// optimistic state, the method shall return UNCHAGED. If not, the optimistic +/// state is adjusted to the situation and the method shall return CHANGED. +/// +/// If the manifestation of the "concrete attribute" deduced by the subclass +/// differs from the "default" behavior, which is a (set of) LLVM-IR +/// attribute(s) for an argument, call site argument, function return value, or +/// function, the `AbstractAttribute::manifest` method should be overloaded. +/// +/// NOTE: If the state obtained via getState() is INVALID, thus if +/// AbstractAttribute::getState().isValidState() returns false, no +/// information provided by the methods of this class should be used. +/// NOTE: The Attributor currently has certain limitations to what we can do. +/// As a general rule of thumb, "concrete" abstract attributes should *for +/// now* only perform "backward" information propagation. That means +/// optimistic information obtained through abstract attributes should +/// only be used at positions that precede the origin of the information +/// with regards to the program flow. More practically, information can +/// *now* be propagated from instructions to their enclosing function, but +/// *not* from call sites to the called function. The mechanisms to allow +/// both directions will be added in the future. +/// NOTE: The mechanics of adding a new "concrete" abstract attribute are +/// described in the file comment. +struct AbstractAttribute { + + /// The positions attributes can be manifested in. + enum ManifestPosition { + MP_ARGUMENT, ///< An attribute for a function argument. + MP_CALL_SITE_ARGUMENT, ///< An attribute for a call site argument. + MP_FUNCTION, ///< An attribute for a function as a whole. + MP_RETURNED, ///< An attribute for the function return value. + }; + + /// An abstract attribute associated with \p AssociatedVal and anchored at + /// \p AnchoredVal. + /// + /// \param AssociatedVal The value this abstract attribute is associated with. + /// \param AnchoredVal The value this abstract attributes is anchored at. + /// \param InfoCache Cached information accessible to the abstract attribute. + AbstractAttribute(Value *AssociatedVal, Value &AnchoredVal, + InformationCache &InfoCache) + : AssociatedVal(AssociatedVal), AnchoredVal(AnchoredVal), + InfoCache(InfoCache) {} + + /// An abstract attribute associated with and anchored at \p V. + AbstractAttribute(Value &V, InformationCache &InfoCache) + : AbstractAttribute(&V, V, InfoCache) {} + + /// Virtual destructor. + virtual ~AbstractAttribute() {} + + /// Initialize the state with the information in the Attributor \p A. + /// + /// This function is called by the Attributor once all abstract attributes + /// have been identified. It can and shall be used for task like: + /// - identify existing knowledge in the IR and use it for the "known state" + /// - perform any work that is not going to change over time, e.g., determine + /// a subset of the IR, or attributes in-flight, that have to be looked at + /// in the `updateImpl` method. + virtual void initialize(Attributor &A) {} + + /// Return the internal abstract state for inspection. + virtual const AbstractState &getState() const = 0; + + /// Return the value this abstract attribute is anchored with. + /// + /// The anchored value might not be the associated value if the latter is not + /// sufficient to determine where arguments will be manifested. This is mostly + /// the case for call site arguments as the value is not sufficient to + /// pinpoint them. Instead, we can use the call site as an anchor. + /// + ///{ + Value &getAnchoredValue() { return AnchoredVal; } + const Value &getAnchoredValue() const { return AnchoredVal; } + ///} + + /// Return the llvm::Function surrounding the anchored value. + /// + ///{ + Function &getAnchorScope(); + const Function &getAnchorScope() const; + ///} + + /// Return the value this abstract attribute is associated with. + /// + /// The abstract state usually represents this value. + /// + ///{ + virtual Value *getAssociatedValue() { return AssociatedVal; } + virtual const Value *getAssociatedValue() const { return AssociatedVal; } + ///} + + /// Return the position this abstract state is manifested in. + virtual ManifestPosition getManifestPosition() const = 0; + + /// Return the kind that identifies the abstract attribute implementation. + virtual Attribute::AttrKind getAttrKind() const = 0; + + /// Return the deduced attributes in \p Attrs. + virtual void getDeducedAttributes(SmallVectorImpl &Attrs) const { + LLVMContext &Ctx = AnchoredVal.getContext(); + Attrs.emplace_back(Attribute::get(Ctx, getAttrKind())); + } + + /// Helper functions, for debug purposes only. + ///{ + virtual void print(raw_ostream &OS) const; + void dump() const { print(dbgs()); } + + /// This function should return the "summarized" assumed state as string. + virtual const std::string getAsStr() const = 0; + ///} + + /// Allow the Attributor access to the protected methods. + friend struct Attributor; + +protected: + /// Hook for the Attributor to trigger an update of the internal state. + /// + /// If this attribute is already fixed, this method will return UNCHANGED, + /// otherwise it delegates to `AbstractAttribute::updateImpl`. + /// + /// \Return CHANGED if the internal state changed, otherwise UNCHANGED. + ChangeStatus update(Attributor &A); + + /// Hook for the Attributor to trigger the manifestation of the information + /// represented by the abstract attribute in the LLVM-IR. + /// + /// \Return CHANGED if the IR was altered, otherwise UNCHANGED. + virtual ChangeStatus manifest(Attributor &A); + + /// Return the internal abstract state for careful modification. + virtual AbstractState &getState() = 0; + + /// The actual update/transfer function which has to be implemented by the + /// derived classes. + /// + /// If it is called, the environment has changed and we have to determine if + /// the current information is still valid or adjust it otherwise. + /// + /// \Return CHANGED if the internal state changed, otherwise UNCHANGED. + virtual ChangeStatus updateImpl(Attributor &A) = 0; + + /// The value this abstract attribute is associated with. + Value *AssociatedVal; + + /// The value this abstract attribute is anchored at. + Value &AnchoredVal; + + /// The information cache accessible to this abstract attribute. + InformationCache &InfoCache; +}; + +/// Forward declarations of output streams for debug purposes. +/// +///{ +raw_ostream &operator<<(raw_ostream &OS, const AbstractAttribute &AA); +raw_ostream &operator<<(raw_ostream &OS, ChangeStatus S); +raw_ostream &operator<<(raw_ostream &OS, AbstractAttribute::ManifestPosition); +raw_ostream &operator<<(raw_ostream &OS, const AbstractState &State); +///} + +struct AttributorPass : public PassInfoMixin { + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); +}; + +Pass *createAttributorLegacyPass(); + +/// ---------------------------------------------------------------------------- +/// Abstract Attribute Classes +/// ---------------------------------------------------------------------------- + +/// An abstract attribute for the returned values of a function. +struct AAReturnedValues : public AbstractAttribute { + /// See AbstractAttribute::AbstractAttribute(...). + AAReturnedValues(Function &F, InformationCache &InfoCache) + : AbstractAttribute(F, InfoCache) {} + + /// Check \p Pred on all returned values. + /// + /// This method will evaluate \p Pred on returned values and return + /// true if (1) all returned values are known, and (2) \p Pred returned true + /// for all returned values. + virtual bool + checkForallReturnedValues(std::function &Pred) const = 0; + + /// See AbstractAttribute::getAttrKind() + Attribute::AttrKind getAttrKind() const override { return ID; } + + /// The identifier used by the Attributor for this class of attributes. + static constexpr Attribute::AttrKind ID = Attribute::Returned; +}; + +struct AANoUnwind : public AbstractAttribute { + /// An abstract interface for all nosync attributes. + AANoUnwind(Value &V, InformationCache &InfoCache) + : AbstractAttribute(V, InfoCache) {} + + /// See AbstractAttribute::getAttrKind()/ + Attribute::AttrKind getAttrKind() const override { return ID; } + + static constexpr Attribute::AttrKind ID = Attribute::NoUnwind; + + /// Returns true if nounwind is assumed. + virtual bool isAssumedNoUnwind() const = 0; + + /// Returns true if nounwind is known. + virtual bool isKnownNoUnwind() const = 0; +}; + +struct AANoSync : public AbstractAttribute { + /// An abstract interface for all nosync attributes. + AANoSync(Value &V, InformationCache &InfoCache) + : AbstractAttribute(V, InfoCache) {} + + /// See AbstractAttribute::getAttrKind(). + Attribute::AttrKind getAttrKind() const override { return ID; } + + static constexpr Attribute::AttrKind ID = + Attribute::AttrKind(Attribute::NoSync); + + /// Returns true if "nosync" is assumed. + virtual bool isAssumedNoSync() const = 0; + + /// Returns true if "nosync" is known. + virtual bool isKnownNoSync() const = 0; +}; + +/// An abstract interface for all nonnull attributes. +struct AANonNull : public AbstractAttribute { + + /// See AbstractAttribute::AbstractAttribute(...). + AANonNull(Value &V, InformationCache &InfoCache) + : AbstractAttribute(V, InfoCache) {} + + /// See AbstractAttribute::AbstractAttribute(...). + AANonNull(Value *AssociatedVal, Value &AnchoredValue, + InformationCache &InfoCache) + : AbstractAttribute(AssociatedVal, AnchoredValue, InfoCache) {} + + /// Return true if we assume that the underlying value is nonnull. + virtual bool isAssumedNonNull() const = 0; + + /// Return true if we know that underlying value is nonnull. + virtual bool isKnownNonNull() const = 0; + + /// See AbastractState::getAttrKind(). + Attribute::AttrKind getAttrKind() const override { return ID; } + + /// The identifier used by the Attributor for this class of attributes. + static constexpr Attribute::AttrKind ID = Attribute::NonNull; +}; + +/// An abstract attribute for norecurse. +struct AANoRecurse : public AbstractAttribute { + + /// See AbstractAttribute::AbstractAttribute(...). + AANoRecurse(Value &V, InformationCache &InfoCache) + : AbstractAttribute(V, InfoCache) {} + + /// See AbstractAttribute::getAttrKind() + virtual Attribute::AttrKind getAttrKind() const override { + return Attribute::NoRecurse; + } + + /// Return true if "norecurse" is known. + virtual bool isKnownNoRecurse() const = 0; + + /// Return true if "norecurse" is assumed. + virtual bool isAssumedNoRecurse() const = 0; + + /// The identifier used by the Attributor for this class of attributes. + static constexpr Attribute::AttrKind ID = Attribute::NoRecurse; +}; + +/// An abstract attribute for willreturn. +struct AAWillReturn : public AbstractAttribute { + + /// See AbstractAttribute::AbstractAttribute(...). + AAWillReturn(Value &V, InformationCache &InfoCache) + : AbstractAttribute(V, InfoCache) {} + + /// See AbstractAttribute::getAttrKind() + virtual Attribute::AttrKind getAttrKind() const override { + return Attribute::WillReturn; + } + + /// Return true if "willreturn" is known. + virtual bool isKnownWillReturn() const = 0; + + /// Return true if "willreturn" is assumed. + virtual bool isAssumedWillReturn() const = 0; + + /// The identifier used by the Attributor for this class of attributes. + static constexpr Attribute::AttrKind ID = Attribute::WillReturn; +}; +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_IPO_FUNCTIONATTRS_H diff --git a/include/llvm/Transforms/IPO/CalledValuePropagation.h b/include/llvm/Transforms/IPO/CalledValuePropagation.h index 352bdc7ac17f..c2626d0867b4 100644 --- a/include/llvm/Transforms/IPO/CalledValuePropagation.h +++ b/include/llvm/Transforms/IPO/CalledValuePropagation.h @@ -1,9 +1,8 @@ //===- CalledValuePropagation.h - Propagate called values -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/IPO/ConstantMerge.h b/include/llvm/Transforms/IPO/ConstantMerge.h index e04d3ae1a40e..12d38b5f58fa 100644 --- a/include/llvm/Transforms/IPO/ConstantMerge.h +++ b/include/llvm/Transforms/IPO/ConstantMerge.h @@ -1,9 +1,8 @@ //===- ConstantMerge.h - Merge duplicate global constants -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/IPO/CrossDSOCFI.h b/include/llvm/Transforms/IPO/CrossDSOCFI.h index 0979f5b79e86..8440df639729 100644 --- a/include/llvm/Transforms/IPO/CrossDSOCFI.h +++ b/include/llvm/Transforms/IPO/CrossDSOCFI.h @@ -1,9 +1,8 @@ //===-- CrossDSOCFI.cpp - Externalize this module's CFI checks --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/IPO/DeadArgumentElimination.h b/include/llvm/Transforms/IPO/DeadArgumentElimination.h index ba5666f20a9b..73797bc10017 100644 --- a/include/llvm/Transforms/IPO/DeadArgumentElimination.h +++ b/include/llvm/Transforms/IPO/DeadArgumentElimination.h @@ -1,9 +1,8 @@ //===- DeadArgumentElimination.h - Eliminate Dead Args ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/IPO/ElimAvailExtern.h b/include/llvm/Transforms/IPO/ElimAvailExtern.h index 94cb954fd2d5..92c319b3cce3 100644 --- a/include/llvm/Transforms/IPO/ElimAvailExtern.h +++ b/include/llvm/Transforms/IPO/ElimAvailExtern.h @@ -1,9 +1,8 @@ //===- ElimAvailExtern.h - Optimize Global Variables ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/IPO/ForceFunctionAttrs.h b/include/llvm/Transforms/IPO/ForceFunctionAttrs.h index ff8a6546f059..7379009b2592 100644 --- a/include/llvm/Transforms/IPO/ForceFunctionAttrs.h +++ b/include/llvm/Transforms/IPO/ForceFunctionAttrs.h @@ -1,9 +1,8 @@ //===-- ForceFunctionAttrs.h - Force function attrs for debugging ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/Transforms/IPO/FunctionAttrs.h b/include/llvm/Transforms/IPO/FunctionAttrs.h index 901fed7a0fa4..ce61eea05c79 100644 --- a/include/llvm/Transforms/IPO/FunctionAttrs.h +++ b/include/llvm/Transforms/IPO/FunctionAttrs.h @@ -1,9 +1,8 @@ //===- FunctionAttrs.h - Compute function attributes ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/IPO/FunctionImport.h b/include/llvm/Transforms/IPO/FunctionImport.h index c2103b637266..bbf270c400af 100644 --- a/include/llvm/Transforms/IPO/FunctionImport.h +++ b/include/llvm/Transforms/IPO/FunctionImport.h @@ -1,9 +1,8 @@ //===- llvm/Transforms/IPO/FunctionImport.h - ThinLTO importing -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Transforms/IPO/GlobalDCE.h b/include/llvm/Transforms/IPO/GlobalDCE.h index 7ca241f4645a..c434484d1ae3 100644 --- a/include/llvm/Transforms/IPO/GlobalDCE.h +++ b/include/llvm/Transforms/IPO/GlobalDCE.h @@ -1,9 +1,8 @@ //===-- GlobalDCE.h - DCE unreachable internal functions ------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/IPO/GlobalOpt.h b/include/llvm/Transforms/IPO/GlobalOpt.h index 5b4878604eab..48a861ff2cf8 100644 --- a/include/llvm/Transforms/IPO/GlobalOpt.h +++ b/include/llvm/Transforms/IPO/GlobalOpt.h @@ -1,9 +1,8 @@ //===- GlobalOpt.h - Optimize Global Variables ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/IPO/GlobalSplit.h b/include/llvm/Transforms/IPO/GlobalSplit.h index 56cefb7886fe..690b23a2d785 100644 --- a/include/llvm/Transforms/IPO/GlobalSplit.h +++ b/include/llvm/Transforms/IPO/GlobalSplit.h @@ -1,9 +1,8 @@ //===- GlobalSplit.h - global variable splitter -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/IPO/HotColdSplitting.h b/include/llvm/Transforms/IPO/HotColdSplitting.h index 57e9a9e69187..73668844590d 100644 --- a/include/llvm/Transforms/IPO/HotColdSplitting.h +++ b/include/llvm/Transforms/IPO/HotColdSplitting.h @@ -1,9 +1,8 @@ //===- HotColdSplitting.h ---- Outline Cold Regions -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception //===----------------------------------------------------------------------===// // // This pass outlines cold regions to a separate function. diff --git a/include/llvm/Transforms/IPO/InferFunctionAttrs.h b/include/llvm/Transforms/IPO/InferFunctionAttrs.h index 54e1c243ae27..bb7907fb8ac8 100644 --- a/include/llvm/Transforms/IPO/InferFunctionAttrs.h +++ b/include/llvm/Transforms/IPO/InferFunctionAttrs.h @@ -1,9 +1,8 @@ //===-- InferFunctionAttrs.h - Infer implicit function attributes ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/include/llvm/Transforms/IPO/Inliner.h b/include/llvm/Transforms/IPO/Inliner.h index 610e4500e4b1..8202b94d5a93 100644 --- a/include/llvm/Transforms/IPO/Inliner.h +++ b/include/llvm/Transforms/IPO/Inliner.h @@ -1,9 +1,8 @@ //===- Inliner.h - Inliner pass and infrastructure --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Transforms/IPO/Internalize.h b/include/llvm/Transforms/IPO/Internalize.h index 45d676d9f77b..6c1e19ef9fe4 100644 --- a/include/llvm/Transforms/IPO/Internalize.h +++ b/include/llvm/Transforms/IPO/Internalize.h @@ -1,9 +1,8 @@ //====- Internalize.h - Internalization API ---------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -22,11 +21,11 @@ #ifndef LLVM_TRANSFORMS_IPO_INTERNALIZE_H #define LLVM_TRANSFORMS_IPO_INTERNALIZE_H +#include "llvm/ADT/DenseSet.h" #include "llvm/ADT/StringSet.h" #include "llvm/IR/GlobalValue.h" #include "llvm/IR/PassManager.h" #include -#include namespace llvm { class Module; @@ -45,11 +44,11 @@ class InternalizePass : public PassInfoMixin { /// Internalize GV if it is possible to do so, i.e. it is not externally /// visible and is not a member of an externally visible comdat. bool maybeInternalize(GlobalValue &GV, - const std::set &ExternalComdats); + const DenseSet &ExternalComdats); /// If GV is part of a comdat and is externally visible, keep track of its /// comdat so that we don't internalize any of its members. void checkComdatVisibility(GlobalValue &GV, - std::set &ExternalComdats); + DenseSet &ExternalComdats); public: InternalizePass(); diff --git a/include/llvm/Transforms/IPO/LowerTypeTests.h b/include/llvm/Transforms/IPO/LowerTypeTests.h index bc448386b63d..39b23f5957db 100644 --- a/include/llvm/Transforms/IPO/LowerTypeTests.h +++ b/include/llvm/Transforms/IPO/LowerTypeTests.h @@ -1,9 +1,8 @@ //===- LowerTypeTests.h - type metadata lowering pass -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/IPO/PartialInlining.h b/include/llvm/Transforms/IPO/PartialInlining.h index ec6dd36dae06..3b8297d65987 100644 --- a/include/llvm/Transforms/IPO/PartialInlining.h +++ b/include/llvm/Transforms/IPO/PartialInlining.h @@ -1,9 +1,8 @@ //===- PartialInlining.h - Inline parts of functions ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/IPO/PassManagerBuilder.h b/include/llvm/Transforms/IPO/PassManagerBuilder.h index 276306f686ff..63ff00afc2ae 100644 --- a/include/llvm/Transforms/IPO/PassManagerBuilder.h +++ b/include/llvm/Transforms/IPO/PassManagerBuilder.h @@ -1,9 +1,8 @@ // llvm/Transforms/IPO/PassManagerBuilder.h - Build Standard Pass -*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -58,7 +57,7 @@ class PassManagerBase; /// ... class PassManagerBuilder { public: - /// Extensions are passed the builder itself (so they can see how it is + /// Extensions are passed to the builder itself (so they can see how it is /// configured) as well as the pass manager to add stuff to. typedef std::function @@ -113,6 +112,16 @@ public: /// passes at the end of the main CallGraphSCC passes and before any /// function simplification passes run by CGPassManager. EP_CGSCCOptimizerLate, + + /// EP_FullLinkTimeOptimizationEarly - This extensions point allow adding + /// passes that + /// run at Link Time, before Full Link Time Optimization. + EP_FullLinkTimeOptimizationEarly, + + /// EP_FullLinkTimeOptimizationLast - This extensions point allow adding + /// passes that + /// run at Link Time, after Full Link Time Optimization. + EP_FullLinkTimeOptimizationLast, }; /// The Optimization Level - Specify the basic optimization level. @@ -143,13 +152,14 @@ public: const ModuleSummaryIndex *ImportSummary = nullptr; bool DisableTailCalls; - bool DisableUnitAtATime; bool DisableUnrollLoops; bool SLPVectorize; bool LoopVectorize; + bool LoopsInterleaved; bool RerollLoops; bool NewGVN; bool DisableGVNLoadPRE; + bool ForgetAllSCEVInLoopUnroll; bool VerifyInput; bool VerifyOutput; bool MergeFunctions; @@ -157,9 +167,15 @@ public: bool PrepareForThinLTO; bool PerformThinLTO; bool DivergentTarget; + unsigned LicmMssaOptCap; + unsigned LicmMssaNoAccForPromotionCap; /// Enable profile instrumentation pass. bool EnablePGOInstrGen; + /// Enable profile context sensitive instrumentation pass. + bool EnablePGOCSInstrGen; + /// Enable profile context sensitive profile use pass. + bool EnablePGOCSInstrUse; /// Profile data file name that the instrumentation will be written to. std::string PGOInstrGen; /// Path of the profile data file. @@ -186,7 +202,7 @@ private: void addInitialAliasAnalysisPasses(legacy::PassManagerBase &PM) const; void addLTOOptimizationPasses(legacy::PassManagerBase &PM); void addLateLTOOptimizationPasses(legacy::PassManagerBase &PM); - void addPGOInstrPasses(legacy::PassManagerBase &MPM); + void addPGOInstrPasses(legacy::PassManagerBase &MPM, bool IsCS); void addFunctionSimplificationPasses(legacy::PassManagerBase &MPM); void addInstructionCombiningPass(legacy::PassManagerBase &MPM) const; diff --git a/include/llvm/Transforms/IPO/SCCP.h b/include/llvm/Transforms/IPO/SCCP.h index fdb7865fbac3..3c40d44ca9de 100644 --- a/include/llvm/Transforms/IPO/SCCP.h +++ b/include/llvm/Transforms/IPO/SCCP.h @@ -1,9 +1,8 @@ //===- SCCP.h - Sparse Conditional Constant Propagation ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/IPO/SampleProfile.h b/include/llvm/Transforms/IPO/SampleProfile.h index af4a933ec1f6..a5ad44551bf6 100644 --- a/include/llvm/Transforms/IPO/SampleProfile.h +++ b/include/llvm/Transforms/IPO/SampleProfile.h @@ -1,9 +1,8 @@ //===- SampleProfile.h - SamplePGO pass ---------- --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/IPO/StripDeadPrototypes.h b/include/llvm/Transforms/IPO/StripDeadPrototypes.h index 5a05cd75c9d5..f4a15c36afc9 100644 --- a/include/llvm/Transforms/IPO/StripDeadPrototypes.h +++ b/include/llvm/Transforms/IPO/StripDeadPrototypes.h @@ -1,9 +1,8 @@ //===-- StripDeadPrototypes.h - Remove unused function declarations -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/IPO/ThinLTOBitcodeWriter.h b/include/llvm/Transforms/IPO/ThinLTOBitcodeWriter.h index bf04bbfe92d8..7acb922b37e1 100644 --- a/include/llvm/Transforms/IPO/ThinLTOBitcodeWriter.h +++ b/include/llvm/Transforms/IPO/ThinLTOBitcodeWriter.h @@ -1,9 +1,8 @@ //===- ThinLTOBitcodeWriter.h - Bitcode writing pass for ThinLTO ----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/IPO/WholeProgramDevirt.h b/include/llvm/Transforms/IPO/WholeProgramDevirt.h index bf2c79b0751e..509fcc867060 100644 --- a/include/llvm/Transforms/IPO/WholeProgramDevirt.h +++ b/include/llvm/Transforms/IPO/WholeProgramDevirt.h @@ -1,9 +1,8 @@ //===- WholeProgramDevirt.h - Whole-program devirt pass ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/InstCombine/InstCombine.h b/include/llvm/Transforms/InstCombine/InstCombine.h index ab25fe08553a..8894d96e591f 100644 --- a/include/llvm/Transforms/InstCombine/InstCombine.h +++ b/include/llvm/Transforms/InstCombine/InstCombine.h @@ -1,9 +1,8 @@ //===- InstCombine.h - InstCombine pass -------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/Transforms/InstCombine/InstCombineWorklist.h b/include/llvm/Transforms/InstCombine/InstCombineWorklist.h index f860b4b86555..6c33bdbafbd2 100644 --- a/include/llvm/Transforms/InstCombine/InstCombineWorklist.h +++ b/include/llvm/Transforms/InstCombine/InstCombineWorklist.h @@ -1,9 +1,8 @@ //===- InstCombineWorklist.h - Worklist for InstCombine pass ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Transforms/Instrumentation.h b/include/llvm/Transforms/Instrumentation.h index 017cab0a7750..8b70d2926ae9 100644 --- a/include/llvm/Transforms/Instrumentation.h +++ b/include/llvm/Transforms/Instrumentation.h @@ -1,9 +1,8 @@ //===- Transforms/Instrumentation.h - Instrumentation passes ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -88,10 +87,14 @@ struct GCOVOptions { ModulePass *createGCOVProfilerPass(const GCOVOptions &Options = GCOVOptions::getDefault()); -// PGO Instrumention -ModulePass *createPGOInstrumentationGenLegacyPass(); +// PGO Instrumention. Parameter IsCS indicates if this is the context senstive +// instrumentation. +ModulePass *createPGOInstrumentationGenLegacyPass(bool IsCS = false); ModulePass * -createPGOInstrumentationUseLegacyPass(StringRef Filename = StringRef("")); +createPGOInstrumentationUseLegacyPass(StringRef Filename = StringRef(""), + bool IsCS = false); +ModulePass *createPGOInstrumentationGenCreateVarLegacyPass( + StringRef CSInstrName = StringRef("")); ModulePass *createPGOIndirectCallPromotionLegacyPass(bool InLTO = false, bool SamplePGO = false); FunctionPass *createPGOMemOPSizeOptLegacyPass(); @@ -133,48 +136,27 @@ struct InstrProfOptions { // Use atomic profile counter increments. bool Atomic = false; + // Use BFI to guide register promotion + bool UseBFIInPromotion = false; + // Name of the profile file to use as output std::string InstrProfileOutput; InstrProfOptions() = default; }; -/// Insert frontend instrumentation based profiling. +/// Insert frontend instrumentation based profiling. Parameter IsCS indicates if +// this is the context senstive instrumentation. ModulePass *createInstrProfilingLegacyPass( - const InstrProfOptions &Options = InstrProfOptions()); + const InstrProfOptions &Options = InstrProfOptions(), bool IsCS = false); -// Insert AddressSanitizer (address sanity checking) instrumentation -FunctionPass *createAddressSanitizerFunctionPass(bool CompileKernel = false, - bool Recover = false, - bool UseAfterScope = false); -ModulePass *createAddressSanitizerModulePass(bool CompileKernel = false, - bool Recover = false, - bool UseGlobalsGC = true, - bool UseOdrIndicator = true); - -FunctionPass *createHWAddressSanitizerPass(bool CompileKernel = false, - bool Recover = false); +ModulePass *createInstrOrderFilePass(); // Insert DataFlowSanitizer (dynamic data flow analysis) instrumentation ModulePass *createDataFlowSanitizerPass( const std::vector &ABIListFiles = std::vector(), void *(*getArgTLS)() = nullptr, void *(*getRetValTLS)() = nullptr); -// Options for EfficiencySanitizer sub-tools. -struct EfficiencySanitizerOptions { - enum Type { - ESAN_None = 0, - ESAN_CacheFrag, - ESAN_WorkingSet, - } ToolType = ESAN_None; - - EfficiencySanitizerOptions() = default; -}; - -// Insert EfficiencySanitizer instrumentation. -ModulePass *createEfficiencySanitizerPass( - const EfficiencySanitizerOptions &Options = EfficiencySanitizerOptions()); - // Options for sanitizer coverage instrumentation. struct SanitizerCoverageOptions { enum Type { diff --git a/include/llvm/Transforms/Instrumentation/AddressSanitizer.h b/include/llvm/Transforms/Instrumentation/AddressSanitizer.h new file mode 100644 index 000000000000..40007a9b8c53 --- /dev/null +++ b/include/llvm/Transforms/Instrumentation/AddressSanitizer.h @@ -0,0 +1,143 @@ +//===--------- Definition of the AddressSanitizer class ---------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file declares the AddressSanitizer class which is a port of the legacy +// AddressSanitizer pass to use the new PassManager infrastructure. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_ADDRESSSANITIZERPASS_H +#define LLVM_TRANSFORMS_INSTRUMENTATION_ADDRESSSANITIZERPASS_H + +#include "llvm/IR/Function.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/PassManager.h" + +namespace llvm { + +/// Frontend-provided metadata for source location. +struct LocationMetadata { + StringRef Filename; + int LineNo = 0; + int ColumnNo = 0; + + LocationMetadata() = default; + + bool empty() const { return Filename.empty(); } + void parse(MDNode *MDN); +}; + +/// Frontend-provided metadata for global variables. +class GlobalsMetadata { +public: + struct Entry { + LocationMetadata SourceLoc; + StringRef Name; + bool IsDynInit = false; + bool IsBlacklisted = false; + + Entry() = default; + }; + + /// Create a default uninitialized GlobalsMetadata instance. + GlobalsMetadata() = default; + + /// Create an initialized GlobalsMetadata instance. + GlobalsMetadata(Module &M); + + /// Returns metadata entry for a given global. + Entry get(GlobalVariable *G) const { + auto Pos = Entries.find(G); + return (Pos != Entries.end()) ? Pos->second : Entry(); + } + + /// Handle invalidation from the pass manager. + /// These results are never invalidated. + bool invalidate(Module &, const PreservedAnalyses &, + ModuleAnalysisManager::Invalidator &) { + return false; + } + bool invalidate(Function &, const PreservedAnalyses &, + FunctionAnalysisManager::Invalidator &) { + return false; + } + +private: + DenseMap Entries; +}; + +/// The ASanGlobalsMetadataAnalysis initializes and returns a GlobalsMetadata +/// object. More specifically, ASan requires looking at all globals registered +/// in 'llvm.asan.globals' before running, which only depends on reading module +/// level metadata. This analysis is required to run before running the +/// AddressSanitizerPass since it collects that metadata. +/// The legacy pass manager equivalent of this is ASanGlobalsMetadataLegacyPass. +class ASanGlobalsMetadataAnalysis + : public AnalysisInfoMixin { +public: + using Result = GlobalsMetadata; + + Result run(Module &, ModuleAnalysisManager &); + +private: + friend AnalysisInfoMixin; + static AnalysisKey Key; +}; + +/// Public interface to the address sanitizer pass for instrumenting code to +/// check for various memory errors at runtime. +/// +/// The sanitizer itself is a function pass that works by inserting various +/// calls to the ASan runtime library functions. The runtime library essentially +/// replaces malloc() and free() with custom implementations that allow regions +/// surrounding requested memory to be checked for invalid accesses. +class AddressSanitizerPass : public PassInfoMixin { +public: + explicit AddressSanitizerPass(bool CompileKernel = false, + bool Recover = false, + bool UseAfterScope = false); + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); + +private: + bool CompileKernel; + bool Recover; + bool UseAfterScope; +}; + +/// Public interface to the address sanitizer module pass for instrumenting code +/// to check for various memory errors. +/// +/// This adds 'asan.module_ctor' to 'llvm.global_ctors'. This pass may also +/// run intependently of the function address sanitizer. +class ModuleAddressSanitizerPass + : public PassInfoMixin { +public: + explicit ModuleAddressSanitizerPass(bool CompileKernel = false, + bool Recover = false, + bool UseGlobalGC = true, + bool UseOdrIndicator = false); + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); + +private: + bool CompileKernel; + bool Recover; + bool UseGlobalGC; + bool UseOdrIndicator; +}; + +// Insert AddressSanitizer (address sanity checking) instrumentation +FunctionPass *createAddressSanitizerFunctionPass(bool CompileKernel = false, + bool Recover = false, + bool UseAfterScope = false); +ModulePass *createModuleAddressSanitizerLegacyPassPass( + bool CompileKernel = false, bool Recover = false, bool UseGlobalsGC = true, + bool UseOdrIndicator = true); + +} // namespace llvm + +#endif diff --git a/include/llvm/Transforms/Instrumentation/BoundsChecking.h b/include/llvm/Transforms/Instrumentation/BoundsChecking.h index 3d4f62c121c2..120c6a8fb09f 100644 --- a/include/llvm/Transforms/Instrumentation/BoundsChecking.h +++ b/include/llvm/Transforms/Instrumentation/BoundsChecking.h @@ -1,9 +1,8 @@ //===- BoundsChecking.h - Bounds checking instrumentation -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Transforms/Instrumentation/CGProfile.h b/include/llvm/Transforms/Instrumentation/CGProfile.h index c06c1a28715e..28fd3804dec9 100644 --- a/include/llvm/Transforms/Instrumentation/CGProfile.h +++ b/include/llvm/Transforms/Instrumentation/CGProfile.h @@ -1,9 +1,8 @@ //===- Transforms/Instrumentation/CGProfile.h -------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/Transforms/Instrumentation/ControlHeightReduction.h b/include/llvm/Transforms/Instrumentation/ControlHeightReduction.h index 460342d1631b..18b428582046 100644 --- a/include/llvm/Transforms/Instrumentation/ControlHeightReduction.h +++ b/include/llvm/Transforms/Instrumentation/ControlHeightReduction.h @@ -1,9 +1,8 @@ //===- ControlHeightReduction.h - Control Height Reduction ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Instrumentation/GCOVProfiler.h b/include/llvm/Transforms/Instrumentation/GCOVProfiler.h index dd55fbe29eed..b3971e49754e 100644 --- a/include/llvm/Transforms/Instrumentation/GCOVProfiler.h +++ b/include/llvm/Transforms/Instrumentation/GCOVProfiler.h @@ -1,9 +1,8 @@ //===- Transforms/Instrumentation/GCOVProfiler.h ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/Transforms/Instrumentation/HWAddressSanitizer.h b/include/llvm/Transforms/Instrumentation/HWAddressSanitizer.h new file mode 100644 index 000000000000..e3104eeb1d36 --- /dev/null +++ b/include/llvm/Transforms/Instrumentation/HWAddressSanitizer.h @@ -0,0 +1,41 @@ +//===--------- Definition of the HWAddressSanitizer class -------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file declares the Hardware AddressSanitizer class which is a port of the +// legacy HWAddressSanitizer pass to use the new PassManager infrastructure. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_HWADDRESSSANITIZERPASS_H +#define LLVM_TRANSFORMS_INSTRUMENTATION_HWADDRESSSANITIZERPASS_H + +#include "llvm/IR/Function.h" +#include "llvm/IR/PassManager.h" + +namespace llvm { + +/// This is a public interface to the hardware address sanitizer pass for +/// instrumenting code to check for various memory errors at runtime, similar to +/// AddressSanitizer but based on partial hardware assistance. +class HWAddressSanitizerPass : public PassInfoMixin { +public: + explicit HWAddressSanitizerPass(bool CompileKernel = false, + bool Recover = false); + PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); + +private: + bool CompileKernel; + bool Recover; +}; + +FunctionPass *createHWAddressSanitizerLegacyPassPass(bool CompileKernel = false, + bool Recover = false); + +} // namespace llvm + +#endif diff --git a/include/llvm/Transforms/Instrumentation/InstrOrderFile.h b/include/llvm/Transforms/Instrumentation/InstrOrderFile.h new file mode 100644 index 000000000000..f1245d8fd785 --- /dev/null +++ b/include/llvm/Transforms/Instrumentation/InstrOrderFile.h @@ -0,0 +1,28 @@ +//===- InstrOrderFile.h ---- Late IR instrumentation for order file ----===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_INSTRORDERFILE_H +#define LLVM_TRANSFORMS_INSTRORDERFILE_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { +class Module; + +/// The instrumentation pass for recording function order. +class InstrOrderFilePass : public PassInfoMixin { +public: + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); +}; + +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_INSTRORDERFILE_H diff --git a/include/llvm/Transforms/Instrumentation/InstrProfiling.h b/include/llvm/Transforms/Instrumentation/InstrProfiling.h index 13fb3db4ae6f..8f76d4a1ce55 100644 --- a/include/llvm/Transforms/Instrumentation/InstrProfiling.h +++ b/include/llvm/Transforms/Instrumentation/InstrProfiling.h @@ -1,9 +1,8 @@ //===- Transforms/Instrumentation/InstrProfiling.h --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -35,8 +34,9 @@ using LoadStorePair = std::pair; /// instrumentation pass. class InstrProfiling : public PassInfoMixin { public: - InstrProfiling() = default; - InstrProfiling(const InstrProfOptions &Options) : Options(Options) {} + InstrProfiling() : IsCS(false) {} + InstrProfiling(const InstrProfOptions &Options, bool IsCS = false) + : Options(Options), IsCS(IsCS) {} PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); bool run(Module &M, const TargetLibraryInfo &TLI); @@ -61,6 +61,9 @@ private: GlobalVariable *NamesVar; size_t NamesSize; + // Is this lowering for the context-sensitive instrumentation. + bool IsCS; + // vector of counter load/store pairs to be register promoted. std::vector PromotionCandidates; diff --git a/include/llvm/Transforms/Instrumentation/MemorySanitizer.h b/include/llvm/Transforms/Instrumentation/MemorySanitizer.h index 54f0e2f78230..0739d9e58a61 100644 --- a/include/llvm/Transforms/Instrumentation/MemorySanitizer.h +++ b/include/llvm/Transforms/Instrumentation/MemorySanitizer.h @@ -1,9 +1,8 @@ //===- Transforms/Instrumentation/MemorySanitizer.h - MSan Pass -----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,10 +18,18 @@ namespace llvm { +struct MemorySanitizerOptions { + MemorySanitizerOptions() = default; + MemorySanitizerOptions(int TrackOrigins, bool Recover, bool Kernel) + : TrackOrigins(TrackOrigins), Recover(Recover), Kernel(Kernel) {} + int TrackOrigins = 0; + bool Recover = false; + bool Kernel = false; +}; + // Insert MemorySanitizer instrumentation (detection of uninitialized reads) -FunctionPass *createMemorySanitizerLegacyPassPass(int TrackOrigins = 0, - bool Recover = false, - bool EnableKmsan = false); +FunctionPass * +createMemorySanitizerLegacyPassPass(MemorySanitizerOptions Options = {}); /// A function pass for msan instrumentation. /// @@ -31,17 +38,12 @@ FunctionPass *createMemorySanitizerLegacyPassPass(int TrackOrigins = 0, /// yet, the pass inserts the declarations. Otherwise the existing globals are /// used. struct MemorySanitizerPass : public PassInfoMixin { - MemorySanitizerPass(int TrackOrigins = 0, bool Recover = false, - bool EnableKmsan = false) - : TrackOrigins(TrackOrigins), Recover(Recover), EnableKmsan(EnableKmsan) { - } + MemorySanitizerPass(MemorySanitizerOptions Options) : Options(Options) {} PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); private: - int TrackOrigins; - bool Recover; - bool EnableKmsan; + MemorySanitizerOptions Options; }; } diff --git a/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h b/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h index fdc5df68a669..21cf291d82d1 100644 --- a/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h +++ b/include/llvm/Transforms/Instrumentation/PGOInstrumentation.h @@ -1,9 +1,8 @@ //===- Transforms/Instrumentation/PGOInstrumentation.h ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -27,23 +26,47 @@ class Function; class Instruction; class Module; +/// The instrumentation (profile-instr-gen) pass for IR based PGO. +// We use this pass to create COMDAT profile variables for context +// sensitive PGO (CSPGO). The reason to have a pass for this is CSPGO +// can be run after LTO/ThinLTO linking. Lld linker needs to see +// all the COMDAT variables before linking. So we have this pass +// always run before linking for CSPGO. +class PGOInstrumentationGenCreateVar + : public PassInfoMixin { +public: + PGOInstrumentationGenCreateVar(std::string CSInstrName = "") + : CSInstrName(CSInstrName) {} + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); + +private: + std::string CSInstrName; +}; + /// The instrumentation (profile-instr-gen) pass for IR based PGO. class PGOInstrumentationGen : public PassInfoMixin { public: + PGOInstrumentationGen(bool IsCS = false) : IsCS(IsCS) {} PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); + +private: + // If this is a context sensitive instrumentation. + bool IsCS; }; /// The profile annotation (profile-instr-use) pass for IR based PGO. class PGOInstrumentationUse : public PassInfoMixin { public: PGOInstrumentationUse(std::string Filename = "", - std::string RemappingFilename = ""); + std::string RemappingFilename = "", bool IsCS = false); PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); private: std::string ProfileFileName; std::string ProfileRemappingFileName; + // If this is a context sensitive instrumentation. + bool IsCS; }; /// The indirect function call promotion pass. diff --git a/include/llvm/Transforms/Instrumentation/PoisonChecking.h b/include/llvm/Transforms/Instrumentation/PoisonChecking.h new file mode 100644 index 000000000000..606d3c255359 --- /dev/null +++ b/include/llvm/Transforms/Instrumentation/PoisonChecking.h @@ -0,0 +1,25 @@ +//===- PoisonChecking.h - ---------------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + + +#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_POISON_CHECKING_H +#define LLVM_TRANSFORMS_INSTRUMENTATION_POISON_CHECKING_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +struct PoisonCheckingPass : public PassInfoMixin { + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; + +} + + +#endif // LLVM_TRANSFORMS_INSTRUMENTATION_POISON_CHECKING_H diff --git a/include/llvm/Transforms/Instrumentation/ThreadSanitizer.h b/include/llvm/Transforms/Instrumentation/ThreadSanitizer.h index 701e2e6ec89e..b4e7d9924ff6 100644 --- a/include/llvm/Transforms/Instrumentation/ThreadSanitizer.h +++ b/include/llvm/Transforms/Instrumentation/ThreadSanitizer.h @@ -1,9 +1,8 @@ -//===- Transforms/Instrumentation/MemorySanitizer.h - TSan Pass -----------===// +//===- Transforms/Instrumentation/ThreadSanitizer.h - TSan Pass -----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/ObjCARC.h b/include/llvm/Transforms/ObjCARC.h index 1897adc2ffbf..2f114c75e2e2 100644 --- a/include/llvm/Transforms/ObjCARC.h +++ b/include/llvm/Transforms/ObjCARC.h @@ -1,9 +1,8 @@ //===-- ObjCARC.h - ObjCARC Scalar Transformations --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar.h b/include/llvm/Transforms/Scalar.h index 8fcf9296ba47..f9360b5ee2c8 100644 --- a/include/llvm/Transforms/Scalar.h +++ b/include/llvm/Transforms/Scalar.h @@ -1,9 +1,8 @@ //===-- Scalar.h - Scalar Transformations -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -138,6 +137,8 @@ Pass *createIndVarSimplifyPass(); // LICM - This pass is a loop invariant code motion and memory promotion pass. // Pass *createLICMPass(); +Pass *createLICMPass(unsigned LicmMssaOptCap, + unsigned LicmMssaNoAccForPromotionCap); //===----------------------------------------------------------------------===// // @@ -184,11 +185,13 @@ Pass *createLoopInstSimplifyPass(); // LoopUnroll - This pass is a simple loop unrolling pass. // Pass *createLoopUnrollPass(int OptLevel = 2, bool OnlyWhenForced = false, - int Threshold = -1, int Count = -1, - int AllowPartial = -1, int Runtime = -1, - int UpperBound = -1, int AllowPeeling = -1); + bool ForgetAllSCEV = false, int Threshold = -1, + int Count = -1, int AllowPartial = -1, + int Runtime = -1, int UpperBound = -1, + int AllowPeeling = -1); // Create an unrolling pass for full unrolling that uses exact trip count only. -Pass *createSimpleLoopUnrollPass(int OptLevel = 2, bool OnlyWhenForced = false); +Pass *createSimpleLoopUnrollPass(int OptLevel = 2, bool OnlyWhenForced = false, + bool ForgetAllSCEV = false); //===----------------------------------------------------------------------===// // @@ -358,11 +361,17 @@ Pass *createLowerAtomicPass(); // Pass *createLowerGuardIntrinsicPass(); +//===----------------------------------------------------------------------===// +// +// LowerWidenableCondition - Lower widenable condition to i1 true. +// +Pass *createLowerWidenableConditionPass(); + //===----------------------------------------------------------------------===// // // MergeICmps - Merge integer comparison chains into a memcmp // -Pass *createMergeICmpsPass(); +Pass *createMergeICmpsLegacyPass(); //===----------------------------------------------------------------------===// // @@ -374,9 +383,10 @@ Pass *createCorrelatedValuePropagationPass(); // // InferAddressSpaces - Modify users of addrspacecast instructions with values // in the source address space if using the destination address space is slower -// on the target. +// on the target. If AddressSpace is left to its default value, it will be +// obtained from the TargetTransformInfo. // -FunctionPass *createInferAddressSpacesPass(); +FunctionPass *createInferAddressSpacesPass(unsigned AddressSpace = ~0u); extern char &InferAddressSpacesID; //===----------------------------------------------------------------------===// @@ -451,6 +461,12 @@ FunctionPass *createNaryReassociatePass(); // FunctionPass *createLoopDistributePass(); +//===----------------------------------------------------------------------===// +// +// LoopFuse - Fuse loops. +// +FunctionPass *createLoopFusePass(); + //===----------------------------------------------------------------------===// // // LoopLoadElimination - Perform loop-aware load elimination. diff --git a/include/llvm/Transforms/Scalar/ADCE.h b/include/llvm/Transforms/Scalar/ADCE.h index f98af62c1a76..7d8b7ae68c00 100644 --- a/include/llvm/Transforms/Scalar/ADCE.h +++ b/include/llvm/Transforms/Scalar/ADCE.h @@ -1,9 +1,8 @@ //===- ADCE.h - Aggressive dead code elimination ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/AlignmentFromAssumptions.h b/include/llvm/Transforms/Scalar/AlignmentFromAssumptions.h index 61975036e9ff..fb1687e1ac5d 100644 --- a/include/llvm/Transforms/Scalar/AlignmentFromAssumptions.h +++ b/include/llvm/Transforms/Scalar/AlignmentFromAssumptions.h @@ -1,9 +1,8 @@ //===---- AlignmentFromAssumptions.h ----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/BDCE.h b/include/llvm/Transforms/Scalar/BDCE.h index d7d2730a8033..996622bccdba 100644 --- a/include/llvm/Transforms/Scalar/BDCE.h +++ b/include/llvm/Transforms/Scalar/BDCE.h @@ -1,9 +1,8 @@ //===---- BDCE.cpp - Bit-tracking dead code elimination ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/CallSiteSplitting.h b/include/llvm/Transforms/Scalar/CallSiteSplitting.h index b2ca2a1c09ae..b6055639e8a8 100644 --- a/include/llvm/Transforms/Scalar/CallSiteSplitting.h +++ b/include/llvm/Transforms/Scalar/CallSiteSplitting.h @@ -1,9 +1,8 @@ //===- CallSiteSplitting..h - Callsite Splitting ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Transforms/Scalar/ConstantHoisting.h b/include/llvm/Transforms/Scalar/ConstantHoisting.h index ba32e122fa10..6b0fc9c1dd07 100644 --- a/include/llvm/Transforms/Scalar/ConstantHoisting.h +++ b/include/llvm/Transforms/Scalar/ConstantHoisting.h @@ -1,9 +1,8 @@ //==- ConstantHoisting.h - Prepare code for expensive constants --*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -56,6 +55,7 @@ class DominatorTree; class Function; class GlobalVariable; class Instruction; +class ProfileSummaryInfo; class TargetTransformInfo; /// A private "module" namespace for types and utilities used by @@ -125,9 +125,10 @@ public: // Glue for old PM. bool runImpl(Function &F, TargetTransformInfo &TTI, DominatorTree &DT, - BlockFrequencyInfo *BFI, BasicBlock &Entry); + BlockFrequencyInfo *BFI, BasicBlock &Entry, + ProfileSummaryInfo *PSI); - void releaseMemory() { + void cleanup() { ClonedCastMap.clear(); ConstIntCandVec.clear(); for (auto MapEntry : ConstGEPCandMap) @@ -149,6 +150,7 @@ private: LLVMContext *Ctx; const DataLayout *DL; BasicBlock *Entry; + ProfileSummaryInfo *PSI; /// Keeps track of constant candidates found in the function. using ConstCandVecType = std::vector; diff --git a/include/llvm/Transforms/Scalar/CorrelatedValuePropagation.h b/include/llvm/Transforms/Scalar/CorrelatedValuePropagation.h index 20930699b557..25795de5d951 100644 --- a/include/llvm/Transforms/Scalar/CorrelatedValuePropagation.h +++ b/include/llvm/Transforms/Scalar/CorrelatedValuePropagation.h @@ -1,9 +1,8 @@ //===- CorrelatedValuePropagation.h -----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Transforms/Scalar/DCE.h b/include/llvm/Transforms/Scalar/DCE.h index 273346cf81d9..974e4b20d152 100644 --- a/include/llvm/Transforms/Scalar/DCE.h +++ b/include/llvm/Transforms/Scalar/DCE.h @@ -1,9 +1,8 @@ //===- DCE.h - Dead code elimination ----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/DeadStoreElimination.h b/include/llvm/Transforms/Scalar/DeadStoreElimination.h index cfeb21814232..b66b0de90c79 100644 --- a/include/llvm/Transforms/Scalar/DeadStoreElimination.h +++ b/include/llvm/Transforms/Scalar/DeadStoreElimination.h @@ -1,9 +1,8 @@ //===- DeadStoreElimination.h - Fast Dead Store Elimination -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/DivRemPairs.h b/include/llvm/Transforms/Scalar/DivRemPairs.h index 0a4346f33b12..7401e02cb4ab 100644 --- a/include/llvm/Transforms/Scalar/DivRemPairs.h +++ b/include/llvm/Transforms/Scalar/DivRemPairs.h @@ -1,9 +1,8 @@ //===- DivRemPairs.h - Hoist/decompose integer division and remainder -----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/EarlyCSE.h b/include/llvm/Transforms/Scalar/EarlyCSE.h index faf03a4ec489..1e7fd71dcbf4 100644 --- a/include/llvm/Transforms/Scalar/EarlyCSE.h +++ b/include/llvm/Transforms/Scalar/EarlyCSE.h @@ -1,9 +1,8 @@ //===- EarlyCSE.h - Simple and fast CSE pass --------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/Float2Int.h b/include/llvm/Transforms/Scalar/Float2Int.h index 206ee980109b..06aeb8322527 100644 --- a/include/llvm/Transforms/Scalar/Float2Int.h +++ b/include/llvm/Transforms/Scalar/Float2Int.h @@ -1,9 +1,8 @@ //===-- Float2Int.h - Demote floating point ops to work on integers -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/GVN.h b/include/llvm/Transforms/Scalar/GVN.h index 9827678b89f2..9fe00a9e7f2d 100644 --- a/include/llvm/Transforms/Scalar/GVN.h +++ b/include/llvm/Transforms/Scalar/GVN.h @@ -1,9 +1,8 @@ //===- GVN.h - Eliminate redundant values and loads -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/Transforms/Scalar/GVNExpression.h b/include/llvm/Transforms/Scalar/GVNExpression.h index 8b346969b1e9..3dc4515f85a1 100644 --- a/include/llvm/Transforms/Scalar/GVNExpression.h +++ b/include/llvm/Transforms/Scalar/GVNExpression.h @@ -1,9 +1,8 @@ //===- GVNExpression.h - GVN Expression classes -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/GuardWidening.h b/include/llvm/Transforms/Scalar/GuardWidening.h index 2bc0940ac715..06dc9ac97bec 100644 --- a/include/llvm/Transforms/Scalar/GuardWidening.h +++ b/include/llvm/Transforms/Scalar/GuardWidening.h @@ -1,9 +1,8 @@ //===- GuardWidening.h - ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -17,7 +16,9 @@ #ifndef LLVM_TRANSFORMS_SCALAR_GUARD_WIDENING_H #define LLVM_TRANSFORMS_SCALAR_GUARD_WIDENING_H +#include "llvm/Analysis/LoopInfo.h" #include "llvm/IR/PassManager.h" +#include "llvm/Transforms/Scalar/LoopPassManager.h" namespace llvm { @@ -25,6 +26,8 @@ class Function; struct GuardWideningPass : public PassInfoMixin { PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); + PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, + LoopStandardAnalysisResults &AR, LPMUpdater &U); }; } diff --git a/include/llvm/Transforms/Scalar/IVUsersPrinter.h b/include/llvm/Transforms/Scalar/IVUsersPrinter.h index fad00d86a95f..a1f20d9ca983 100644 --- a/include/llvm/Transforms/Scalar/IVUsersPrinter.h +++ b/include/llvm/Transforms/Scalar/IVUsersPrinter.h @@ -1,9 +1,8 @@ //===- IVUsersPrinter.h - Induction Variable Users Printing -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Transforms/Scalar/IndVarSimplify.h b/include/llvm/Transforms/Scalar/IndVarSimplify.h index e321c8fc6e9c..3c20537ab76a 100644 --- a/include/llvm/Transforms/Scalar/IndVarSimplify.h +++ b/include/llvm/Transforms/Scalar/IndVarSimplify.h @@ -1,9 +1,8 @@ //===- IndVarSimplify.h - Induction Variable Simplification -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/InductiveRangeCheckElimination.h b/include/llvm/Transforms/Scalar/InductiveRangeCheckElimination.h index 311c549b8326..b1e700714e51 100644 --- a/include/llvm/Transforms/Scalar/InductiveRangeCheckElimination.h +++ b/include/llvm/Transforms/Scalar/InductiveRangeCheckElimination.h @@ -1,9 +1,8 @@ //===- InductiveRangeCheckElimination.h - IRCE ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/InstSimplifyPass.h b/include/llvm/Transforms/Scalar/InstSimplifyPass.h index da79a13eb7cf..0c30b6260536 100644 --- a/include/llvm/Transforms/Scalar/InstSimplifyPass.h +++ b/include/llvm/Transforms/Scalar/InstSimplifyPass.h @@ -1,9 +1,8 @@ //===- InstSimplifyPass.h ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/Transforms/Scalar/JumpThreading.h b/include/llvm/Transforms/Scalar/JumpThreading.h index 9894345645a1..0464d40c45e6 100644 --- a/include/llvm/Transforms/Scalar/JumpThreading.h +++ b/include/llvm/Transforms/Scalar/JumpThreading.h @@ -1,9 +1,8 @@ //===- JumpThreading.h - thread control through conditional BBs -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -23,7 +22,7 @@ #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/BlockFrequencyInfo.h" #include "llvm/Analysis/BranchProbabilityInfo.h" -#include "llvm/IR/DomTreeUpdater.h" +#include "llvm/Analysis/DomTreeUpdater.h" #include "llvm/IR/ValueHandle.h" #include #include diff --git a/include/llvm/Transforms/Scalar/LICM.h b/include/llvm/Transforms/Scalar/LICM.h index 68ad190c7647..f0ea928abd49 100644 --- a/include/llvm/Transforms/Scalar/LICM.h +++ b/include/llvm/Transforms/Scalar/LICM.h @@ -1,9 +1,8 @@ //===- LICM.h - Loop Invariant Code Motion Pass -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -39,9 +38,21 @@ namespace llvm { +extern cl::opt SetLicmMssaOptCap; +extern cl::opt SetLicmMssaNoAccForPromotionCap; + /// Performs Loop Invariant Code Motion Pass. class LICMPass : public PassInfoMixin { + unsigned LicmMssaOptCap; + unsigned LicmMssaNoAccForPromotionCap; + public: + LICMPass() + : LicmMssaOptCap(SetLicmMssaOptCap), + LicmMssaNoAccForPromotionCap(SetLicmMssaNoAccForPromotionCap) {} + LICMPass(unsigned LicmMssaOptCap, unsigned LicmMssaNoAccForPromotionCap) + : LicmMssaOptCap(LicmMssaOptCap), + LicmMssaNoAccForPromotionCap(LicmMssaNoAccForPromotionCap) {} PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR, LPMUpdater &U); }; diff --git a/include/llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h b/include/llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h index e1b33799578b..3f250fc1ce8c 100644 --- a/include/llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h +++ b/include/llvm/Transforms/Scalar/LoopAccessAnalysisPrinter.h @@ -1,9 +1,8 @@ //===- llvm/Analysis/LoopAccessAnalysisPrinter.h ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Transforms/Scalar/LoopDataPrefetch.h b/include/llvm/Transforms/Scalar/LoopDataPrefetch.h index e1ad67ac6fff..9ebd5984cea9 100644 --- a/include/llvm/Transforms/Scalar/LoopDataPrefetch.h +++ b/include/llvm/Transforms/Scalar/LoopDataPrefetch.h @@ -1,10 +1,9 @@ //===-------- LoopDataPrefetch.h - Loop Data Prefetching Pass ---*- C++ -*-===// // // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/Transforms/Scalar/LoopDeletion.h b/include/llvm/Transforms/Scalar/LoopDeletion.h index 7b8cb1e115c9..557616e2e6ba 100644 --- a/include/llvm/Transforms/Scalar/LoopDeletion.h +++ b/include/llvm/Transforms/Scalar/LoopDeletion.h @@ -1,9 +1,8 @@ //===- LoopDeletion.h - Loop Deletion ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/LoopDistribute.h b/include/llvm/Transforms/Scalar/LoopDistribute.h index 2bf1c9d696d5..1a82176490c5 100644 --- a/include/llvm/Transforms/Scalar/LoopDistribute.h +++ b/include/llvm/Transforms/Scalar/LoopDistribute.h @@ -1,9 +1,8 @@ //===- LoopDistribute.cpp - Loop Distribution Pass --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/LoopFuse.h b/include/llvm/Transforms/Scalar/LoopFuse.h new file mode 100644 index 000000000000..d3a02db6bd28 --- /dev/null +++ b/include/llvm/Transforms/Scalar/LoopFuse.h @@ -0,0 +1,30 @@ +//===- LoopFuse.h - Loop Fusion Pass ----------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file implements the Loop Fusion pass. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_SCALAR_LOOPFUSE_H +#define LLVM_TRANSFORMS_SCALAR_LOOPFUSE_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class Function; + +class LoopFusePass : public PassInfoMixin { +public: + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; + +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_SCALAR_LOOPFUSE_H diff --git a/include/llvm/Transforms/Scalar/LoopIdiomRecognize.h b/include/llvm/Transforms/Scalar/LoopIdiomRecognize.h index 7added8d2c61..d2fff8bb5743 100644 --- a/include/llvm/Transforms/Scalar/LoopIdiomRecognize.h +++ b/include/llvm/Transforms/Scalar/LoopIdiomRecognize.h @@ -1,9 +1,8 @@ //===- LoopIdiomRecognize.h - Loop Idiom Recognize Pass ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/LoopInstSimplify.h b/include/llvm/Transforms/Scalar/LoopInstSimplify.h index 04dc79c3fa57..f6e86d11ed95 100644 --- a/include/llvm/Transforms/Scalar/LoopInstSimplify.h +++ b/include/llvm/Transforms/Scalar/LoopInstSimplify.h @@ -1,9 +1,8 @@ //===- LoopInstSimplify.h - Loop Inst Simplify Pass -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/LoopLoadElimination.h b/include/llvm/Transforms/Scalar/LoopLoadElimination.h index b0514a4a7c98..65b9aabb8f51 100644 --- a/include/llvm/Transforms/Scalar/LoopLoadElimination.h +++ b/include/llvm/Transforms/Scalar/LoopLoadElimination.h @@ -1,9 +1,8 @@ //===- LoopLoadElimination.h ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/LoopPassManager.h b/include/llvm/Transforms/Scalar/LoopPassManager.h index 46ebb74c413c..61ec58585fd0 100644 --- a/include/llvm/Transforms/Scalar/LoopPassManager.h +++ b/include/llvm/Transforms/Scalar/LoopPassManager.h @@ -1,9 +1,8 @@ //===- LoopPassManager.h - Loop pass management -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/Transforms/Scalar/LoopPredication.h b/include/llvm/Transforms/Scalar/LoopPredication.h index 57398bdb6bd1..252daafab7a3 100644 --- a/include/llvm/Transforms/Scalar/LoopPredication.h +++ b/include/llvm/Transforms/Scalar/LoopPredication.h @@ -1,9 +1,8 @@ //===- LoopPredication.h - Guard based loop predication pass ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/LoopRotation.h b/include/llvm/Transforms/Scalar/LoopRotation.h index ea8d5618e6f7..254e6072906a 100644 --- a/include/llvm/Transforms/Scalar/LoopRotation.h +++ b/include/llvm/Transforms/Scalar/LoopRotation.h @@ -1,9 +1,8 @@ //===- LoopRotation.h - Loop Rotation -------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/LoopSimplifyCFG.h b/include/llvm/Transforms/Scalar/LoopSimplifyCFG.h index 7628c7413eac..2d718592aef5 100644 --- a/include/llvm/Transforms/Scalar/LoopSimplifyCFG.h +++ b/include/llvm/Transforms/Scalar/LoopSimplifyCFG.h @@ -1,9 +1,8 @@ //===- LoopSimplifyCFG.cpp - Loop CFG Simplification Pass -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/LoopSink.h b/include/llvm/Transforms/Scalar/LoopSink.h index 371a7c8d2c44..234c48cbebc5 100644 --- a/include/llvm/Transforms/Scalar/LoopSink.h +++ b/include/llvm/Transforms/Scalar/LoopSink.h @@ -1,9 +1,8 @@ //===- LoopSink.h - Loop Sink Pass ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/LoopStrengthReduce.h b/include/llvm/Transforms/Scalar/LoopStrengthReduce.h index 62c038a3857d..5cf805bc4939 100644 --- a/include/llvm/Transforms/Scalar/LoopStrengthReduce.h +++ b/include/llvm/Transforms/Scalar/LoopStrengthReduce.h @@ -1,9 +1,8 @@ //===- LoopStrengthReduce.h - Loop Strength Reduce Pass ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h b/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h index fc69aa361059..7920269b0fb2 100644 --- a/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h +++ b/include/llvm/Transforms/Scalar/LoopUnrollAndJamPass.h @@ -1,9 +1,8 @@ //===- LoopUnrollAndJamPass.h -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Transforms/Scalar/LoopUnrollPass.h b/include/llvm/Transforms/Scalar/LoopUnrollPass.h index e38e983cc9eb..a84d889a83ad 100644 --- a/include/llvm/Transforms/Scalar/LoopUnrollPass.h +++ b/include/llvm/Transforms/Scalar/LoopUnrollPass.h @@ -1,9 +1,8 @@ //===- LoopUnrollPass.h -----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -16,6 +15,8 @@ namespace llvm { +extern cl::opt ForgetSCEVInLoopUnroll; + class Function; class Loop; class LPMUpdater; @@ -29,9 +30,16 @@ class LoopFullUnrollPass : public PassInfoMixin { /// metadata are considered. All other loops are skipped. const bool OnlyWhenForced; + /// If true, forget all loops when unrolling. If false, forget top-most loop + /// of the currently processed loops, which removes one entry at a time from + /// the internal SCEV records. For large loops, the former is faster. + const bool ForgetSCEV; + public: - explicit LoopFullUnrollPass(int OptLevel = 2, bool OnlyWhenForced = false) - : OptLevel(OptLevel), OnlyWhenForced(OnlyWhenForced) {} + explicit LoopFullUnrollPass(int OptLevel = 2, bool OnlyWhenForced = false, + bool ForgetSCEV = false) + : OptLevel(OptLevel), OnlyWhenForced(OnlyWhenForced), + ForgetSCEV(ForgetSCEV) {} PreservedAnalyses run(Loop &L, LoopAnalysisManager &AM, LoopStandardAnalysisResults &AR, LPMUpdater &U); @@ -61,8 +69,15 @@ struct LoopUnrollOptions { /// metadata are considered. All other loops are skipped. bool OnlyWhenForced; - LoopUnrollOptions(int OptLevel = 2, bool OnlyWhenForced = false) - : OptLevel(OptLevel), OnlyWhenForced(OnlyWhenForced) {} + /// If true, forget all loops when unrolling. If false, forget top-most loop + /// of the currently processed loops, which removes one entry at a time from + /// the internal SCEV records. For large loops, the former is faster. + const bool ForgetSCEV; + + LoopUnrollOptions(int OptLevel = 2, bool OnlyWhenForced = false, + bool ForgetSCEV = false) + : OptLevel(OptLevel), OnlyWhenForced(OnlyWhenForced), + ForgetSCEV(ForgetSCEV) {} /// Enables or disables partial unrolling. When disabled only full unrolling /// is allowed. diff --git a/include/llvm/Transforms/Scalar/LowerAtomic.h b/include/llvm/Transforms/Scalar/LowerAtomic.h index a4a2e7aafe44..40f8ca571f19 100644 --- a/include/llvm/Transforms/Scalar/LowerAtomic.h +++ b/include/llvm/Transforms/Scalar/LowerAtomic.h @@ -1,9 +1,8 @@ //===- LowerAtomic.cpp - Lower atomic intrinsics ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h b/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h index b6ee6523697c..4e47ff70d557 100644 --- a/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h +++ b/include/llvm/Transforms/Scalar/LowerExpectIntrinsic.h @@ -1,9 +1,8 @@ //===- LowerExpectIntrinsic.h - LowerExpectIntrinsic pass -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/Transforms/Scalar/LowerGuardIntrinsic.h b/include/llvm/Transforms/Scalar/LowerGuardIntrinsic.h index a9f19f6b84b4..ce97b9e4c386 100644 --- a/include/llvm/Transforms/Scalar/LowerGuardIntrinsic.h +++ b/include/llvm/Transforms/Scalar/LowerGuardIntrinsic.h @@ -1,9 +1,8 @@ //===--- LowerGuardIntrinsic.h - Lower the guard intrinsic ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/LowerWidenableCondition.h b/include/llvm/Transforms/Scalar/LowerWidenableCondition.h new file mode 100644 index 000000000000..7c1e64b8f3a9 --- /dev/null +++ b/include/llvm/Transforms/Scalar/LowerWidenableCondition.h @@ -0,0 +1,26 @@ +//===--- LowerWidenableCondition.h - Lower the guard intrinsic ---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This pass lowers the llvm.widenable.condition intrinsic to default value +// which is i1 true. +// +//===----------------------------------------------------------------------===// +#ifndef LLVM_TRANSFORMS_SCALAR_LOWERWIDENABLECONDITION_H +#define LLVM_TRANSFORMS_SCALAR_LOWERWIDENABLECONDITION_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +struct LowerWidenableConditionPass : PassInfoMixin { + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; + +} + +#endif //LLVM_TRANSFORMS_SCALAR_LOWERWIDENABLECONDITION_H diff --git a/include/llvm/Transforms/Scalar/MakeGuardsExplicit.h b/include/llvm/Transforms/Scalar/MakeGuardsExplicit.h index 41b4aada2baa..525174734303 100644 --- a/include/llvm/Transforms/Scalar/MakeGuardsExplicit.h +++ b/include/llvm/Transforms/Scalar/MakeGuardsExplicit.h @@ -1,9 +1,8 @@ //===-- MakeGuardsExplicit.h - Turn guard intrinsics into guard branches --===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/MemCpyOptimizer.h b/include/llvm/Transforms/Scalar/MemCpyOptimizer.h index 046c808bd051..5386f58b2b82 100644 --- a/include/llvm/Transforms/Scalar/MemCpyOptimizer.h +++ b/include/llvm/Transforms/Scalar/MemCpyOptimizer.h @@ -1,9 +1,8 @@ //===- MemCpyOptimizer.h - memcpy optimization ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/MergeICmps.h b/include/llvm/Transforms/Scalar/MergeICmps.h new file mode 100644 index 000000000000..63bdbf8f4d09 --- /dev/null +++ b/include/llvm/Transforms/Scalar/MergeICmps.h @@ -0,0 +1,25 @@ +//===- MergeICmps.h -----------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_SCALAR_MERGEICMPS_H +#define LLVM_TRANSFORMS_SCALAR_MERGEICMPS_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class Function; + +struct MergeICmpsPass + : PassInfoMixin { + PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); +}; + +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_SCALAR_MERGEICMPS_H diff --git a/include/llvm/Transforms/Scalar/MergedLoadStoreMotion.h b/include/llvm/Transforms/Scalar/MergedLoadStoreMotion.h index 48df09cdec9e..9071a56532f8 100644 --- a/include/llvm/Transforms/Scalar/MergedLoadStoreMotion.h +++ b/include/llvm/Transforms/Scalar/MergedLoadStoreMotion.h @@ -1,9 +1,8 @@ //===- MergedLoadStoreMotion.h - merge and hoist/sink load/stores ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/NaryReassociate.h b/include/llvm/Transforms/Scalar/NaryReassociate.h index e835bd5f0761..26f5fe185dd5 100644 --- a/include/llvm/Transforms/Scalar/NaryReassociate.h +++ b/include/llvm/Transforms/Scalar/NaryReassociate.h @@ -1,9 +1,8 @@ //===- NaryReassociate.h - Reassociate n-ary expressions --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/NewGVN.h b/include/llvm/Transforms/Scalar/NewGVN.h index 3f7541863a19..1f3680fec79c 100644 --- a/include/llvm/Transforms/Scalar/NewGVN.h +++ b/include/llvm/Transforms/Scalar/NewGVN.h @@ -1,9 +1,8 @@ //===- NewGVN.h - Global Value Numbering Pass -------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/PartiallyInlineLibCalls.h b/include/llvm/Transforms/Scalar/PartiallyInlineLibCalls.h index 7f73831e0eb3..fd5a06c5051d 100644 --- a/include/llvm/Transforms/Scalar/PartiallyInlineLibCalls.h +++ b/include/llvm/Transforms/Scalar/PartiallyInlineLibCalls.h @@ -1,9 +1,8 @@ //===--- PartiallyInlineLibCalls.h - Partially inline libcalls --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/Reassociate.h b/include/llvm/Transforms/Scalar/Reassociate.h index ba7586dffd9d..2db8d8ce309c 100644 --- a/include/llvm/Transforms/Scalar/Reassociate.h +++ b/include/llvm/Transforms/Scalar/Reassociate.h @@ -1,9 +1,8 @@ //===- Reassociate.h - Reassociate binary expressions -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -83,7 +82,14 @@ protected: static const unsigned GlobalReassociateLimit = 10; static const unsigned NumBinaryOps = Instruction::BinaryOpsEnd - Instruction::BinaryOpsBegin; - DenseMap, unsigned> PairMap[NumBinaryOps]; + + struct PairMapValue { + WeakVH Value1; + WeakVH Value2; + unsigned Score; + bool isValid() const { return Value1 && Value2; } + }; + DenseMap, PairMapValue> PairMap[NumBinaryOps]; bool MadeChange; diff --git a/include/llvm/Transforms/Scalar/RewriteStatepointsForGC.h b/include/llvm/Transforms/Scalar/RewriteStatepointsForGC.h index 128f176f4420..12773c16dcc2 100644 --- a/include/llvm/Transforms/Scalar/RewriteStatepointsForGC.h +++ b/include/llvm/Transforms/Scalar/RewriteStatepointsForGC.h @@ -1,9 +1,8 @@ //===- RewriteStatepointsForGC.h - ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/SCCP.h b/include/llvm/Transforms/Scalar/SCCP.h index 0abbb32fde6a..0ffd983eb3e0 100644 --- a/include/llvm/Transforms/Scalar/SCCP.h +++ b/include/llvm/Transforms/Scalar/SCCP.h @@ -1,9 +1,8 @@ //===- SCCP.cpp - Sparse Conditional Constant Propagation -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/SROA.h b/include/llvm/Transforms/Scalar/SROA.h index b36c6f492be1..864a0cbd9db1 100644 --- a/include/llvm/Transforms/Scalar/SROA.h +++ b/include/llvm/Transforms/Scalar/SROA.h @@ -1,9 +1,8 @@ //===- SROA.h - Scalar Replacement Of Aggregates ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -19,7 +18,6 @@ #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/IR/PassManager.h" -#include "llvm/Support/Compiler.h" #include namespace llvm { diff --git a/include/llvm/Transforms/Scalar/Scalarizer.h b/include/llvm/Transforms/Scalar/Scalarizer.h index 1a0b9a2b638c..81363130e2e3 100644 --- a/include/llvm/Transforms/Scalar/Scalarizer.h +++ b/include/llvm/Transforms/Scalar/Scalarizer.h @@ -1,9 +1,8 @@ //===- Scalarizer.h --- Scalarize vector operations -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h b/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h index eed50ec96161..33c1faaeee0b 100644 --- a/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h +++ b/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h @@ -1,9 +1,8 @@ //===- SimpleLoopUnswitch.h - Hoist loop-invariant control flow -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Transforms/Scalar/SimplifyCFG.h b/include/llvm/Transforms/Scalar/SimplifyCFG.h index ce0a35fc06bd..f9792d38bbe6 100644 --- a/include/llvm/Transforms/Scalar/SimplifyCFG.h +++ b/include/llvm/Transforms/Scalar/SimplifyCFG.h @@ -1,9 +1,8 @@ //===- SimplifyCFG.h - Simplify and canonicalize the CFG --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/include/llvm/Transforms/Scalar/Sink.h b/include/llvm/Transforms/Scalar/Sink.h index f9b3cb0fae39..6cbe964d1580 100644 --- a/include/llvm/Transforms/Scalar/Sink.h +++ b/include/llvm/Transforms/Scalar/Sink.h @@ -1,9 +1,8 @@ //===-- Sink.h - Code Sinking -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/SpeculateAroundPHIs.h b/include/llvm/Transforms/Scalar/SpeculateAroundPHIs.h index 4a0bfd754723..3c7dafe71e8e 100644 --- a/include/llvm/Transforms/Scalar/SpeculateAroundPHIs.h +++ b/include/llvm/Transforms/Scalar/SpeculateAroundPHIs.h @@ -1,9 +1,8 @@ //===- SpeculateAroundPHIs.h - Speculate around PHIs ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Transforms/Scalar/SpeculativeExecution.h b/include/llvm/Transforms/Scalar/SpeculativeExecution.h index d00e950222a0..14da86483213 100644 --- a/include/llvm/Transforms/Scalar/SpeculativeExecution.h +++ b/include/llvm/Transforms/Scalar/SpeculativeExecution.h @@ -1,9 +1,8 @@ //===- SpeculativeExecution.h -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/TailRecursionElimination.h b/include/llvm/Transforms/Scalar/TailRecursionElimination.h index 793f9bc152ed..906867644504 100644 --- a/include/llvm/Transforms/Scalar/TailRecursionElimination.h +++ b/include/llvm/Transforms/Scalar/TailRecursionElimination.h @@ -1,9 +1,8 @@ //===---- TailRecursionElimination.h ----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Scalar/WarnMissedTransforms.h b/include/llvm/Transforms/Scalar/WarnMissedTransforms.h index 018b22a932e6..2d5942a3f569 100644 --- a/include/llvm/Transforms/Scalar/WarnMissedTransforms.h +++ b/include/llvm/Transforms/Scalar/WarnMissedTransforms.h @@ -1,9 +1,8 @@ //===- WarnMissedTransforms.h -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils.h b/include/llvm/Transforms/Utils.h index 378552775c77..6e03453babf1 100644 --- a/include/llvm/Transforms/Utils.h +++ b/include/llvm/Transforms/Utils.h @@ -1,9 +1,8 @@ //===- llvm/Transforms/Utils.h - Utility Transformations --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/ASanStackFrameLayout.h b/include/llvm/Transforms/Utils/ASanStackFrameLayout.h index eaad06a10819..0b570c0d1342 100644 --- a/include/llvm/Transforms/Utils/ASanStackFrameLayout.h +++ b/include/llvm/Transforms/Utils/ASanStackFrameLayout.h @@ -1,9 +1,8 @@ //===- ASanStackFrameLayout.h - ComputeASanStackFrameLayout -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/AddDiscriminators.h b/include/llvm/Transforms/Utils/AddDiscriminators.h index 4dad06e6c125..f512c6c06331 100644 --- a/include/llvm/Transforms/Utils/AddDiscriminators.h +++ b/include/llvm/Transforms/Utils/AddDiscriminators.h @@ -1,9 +1,8 @@ //===- AddDiscriminators.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/BasicBlockUtils.h b/include/llvm/Transforms/Utils/BasicBlockUtils.h index 5b16a2c0d0b1..4d861ffe9a31 100644 --- a/include/llvm/Transforms/Utils/BasicBlockUtils.h +++ b/include/llvm/Transforms/Utils/BasicBlockUtils.h @@ -1,9 +1,8 @@ //===- Transform/Utils/BasicBlockUtils.h - BasicBlock Utils -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,9 +17,9 @@ // FIXME: Move to this file: BasicBlock::removePredecessor, BB::splitBasicBlock #include "llvm/ADT/ArrayRef.h" +#include "llvm/Analysis/DomTreeUpdater.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" -#include "llvm/IR/DomTreeUpdater.h" #include "llvm/IR/InstrTypes.h" #include @@ -36,19 +35,38 @@ class LoopInfo; class MDNode; class MemoryDependenceResults; class MemorySSAUpdater; +class PostDominatorTree; class ReturnInst; class TargetLibraryInfo; class Value; +/// Replace contents of every block in \p BBs with single unreachable +/// instruction. If \p Updates is specified, collect all necessary DT updates +/// into this vector. If \p KeepOneInputPHIs is true, one-input Phis in +/// successors of blocks being deleted will be preserved. +void DetatchDeadBlocks(ArrayRef BBs, + SmallVectorImpl *Updates, + bool KeepOneInputPHIs = false); + /// Delete the specified block, which must have no predecessors. -void DeleteDeadBlock(BasicBlock *BB, DomTreeUpdater *DTU = nullptr); +void DeleteDeadBlock(BasicBlock *BB, DomTreeUpdater *DTU = nullptr, + bool KeepOneInputPHIs = false); /// Delete the specified blocks from \p BB. The set of deleted blocks must have /// no predecessors that are not being deleted themselves. \p BBs must have no /// duplicating blocks. If there are loops among this set of blocks, all /// relevant loop info updates should be done before this function is called. -void DeleteDeadBlocks(SmallVectorImpl &BBs, - DomTreeUpdater *DTU = nullptr); +/// If \p KeepOneInputPHIs is true, one-input Phis in successors of blocks +/// being deleted will be preserved. +void DeleteDeadBlocks(ArrayRef BBs, + DomTreeUpdater *DTU = nullptr, + bool KeepOneInputPHIs = false); + +/// Delete all basic blocks from \p F that are not reachable from its entry +/// node. If \p KeepOneInputPHIs is true, one-input Phis in successors of +/// blocks being deleted will be preserved. +bool EliminateUnreachableBlocks(Function &F, DomTreeUpdater *DTU = nullptr, + bool KeepOneInputPHIs = false); /// We know that BB has one predecessor. If there are any single-entry PHI nodes /// in it, fold them away. This handles the case when all entries to the PHI @@ -92,24 +110,27 @@ void ReplaceInstWithInst(Instruction *From, Instruction *To); /// during critical edge splitting. struct CriticalEdgeSplittingOptions { DominatorTree *DT; + PostDominatorTree *PDT; LoopInfo *LI; MemorySSAUpdater *MSSAU; bool MergeIdenticalEdges = false; - bool DontDeleteUselessPHIs = false; + bool KeepOneInputPHIs = false; bool PreserveLCSSA = false; + bool IgnoreUnreachableDests = false; CriticalEdgeSplittingOptions(DominatorTree *DT = nullptr, LoopInfo *LI = nullptr, - MemorySSAUpdater *MSSAU = nullptr) - : DT(DT), LI(LI), MSSAU(MSSAU) {} + MemorySSAUpdater *MSSAU = nullptr, + PostDominatorTree *PDT = nullptr) + : DT(DT), PDT(PDT), LI(LI), MSSAU(MSSAU) {} CriticalEdgeSplittingOptions &setMergeIdenticalEdges() { MergeIdenticalEdges = true; return *this; } - CriticalEdgeSplittingOptions &setDontDeleteUselessPHIs() { - DontDeleteUselessPHIs = true; + CriticalEdgeSplittingOptions &setKeepOneInputPHIs() { + KeepOneInputPHIs = true; return *this; } @@ -117,6 +138,11 @@ struct CriticalEdgeSplittingOptions { PreserveLCSSA = true; return *this; } + + CriticalEdgeSplittingOptions &setIgnoreUnreachableDests() { + IgnoreUnreachableDests = true; + return *this; + } }; /// If this edge is a critical edge, insert a new node to split the critical @@ -259,7 +285,8 @@ ReturnInst *FoldReturnIntoUncondBranch(ReturnInst *RI, BasicBlock *BB, /// SplitBefore /// Tail /// -/// If Unreachable is true, then ThenBlock ends with +/// If \p ThenBlock is not specified, a new block will be created for it. +/// If \p Unreachable is true, the newly created block will end with /// UnreachableInst, otherwise it branches to Tail. /// Returns the NewBasicBlock's terminator. /// @@ -268,7 +295,8 @@ Instruction *SplitBlockAndInsertIfThen(Value *Cond, Instruction *SplitBefore, bool Unreachable, MDNode *BranchWeights = nullptr, DominatorTree *DT = nullptr, - LoopInfo *LI = nullptr); + LoopInfo *LI = nullptr, + BasicBlock *ThenBlock = nullptr); /// SplitBlockAndInsertIfThenElse is similar to SplitBlockAndInsertIfThen, /// but also creates the ElseBlock. diff --git a/include/llvm/Transforms/Utils/BreakCriticalEdges.h b/include/llvm/Transforms/Utils/BreakCriticalEdges.h index 9cc81a176cb6..3644f1ed7a13 100644 --- a/include/llvm/Transforms/Utils/BreakCriticalEdges.h +++ b/include/llvm/Transforms/Utils/BreakCriticalEdges.h @@ -1,9 +1,8 @@ //===- BreakCriticalEdges.h - Critical Edge Elimination Pass --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/BuildLibCalls.h b/include/llvm/Transforms/Utils/BuildLibCalls.h index 28efce6ac3fb..8421c31a36da 100644 --- a/include/llvm/Transforms/Utils/BuildLibCalls.h +++ b/include/llvm/Transforms/Utils/BuildLibCalls.h @@ -1,9 +1,8 @@ //===- BuildLibCalls.h - Utility builder for libcalls -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -71,12 +70,22 @@ namespace llvm { /// Emit a call to the strcpy function to the builder, for the specified /// pointer arguments. Value *emitStrCpy(Value *Dst, Value *Src, IRBuilder<> &B, - const TargetLibraryInfo *TLI, StringRef Name = "strcpy"); + const TargetLibraryInfo *TLI); + + /// Emit a call to the stpcpy function to the builder, for the specified + /// pointer arguments. + Value *emitStpCpy(Value *Dst, Value *Src, IRBuilder<> &B, + const TargetLibraryInfo *TLI); /// Emit a call to the strncpy function to the builder, for the specified /// pointer arguments and length. Value *emitStrNCpy(Value *Dst, Value *Src, Value *Len, IRBuilder<> &B, - const TargetLibraryInfo *TLI, StringRef Name = "strncpy"); + const TargetLibraryInfo *TLI); + + /// Emit a call to the stpncpy function to the builder, for the specified + /// pointer arguments and length. + Value *emitStpNCpy(Value *Dst, Value *Src, Value *Len, IRBuilder<> &B, + const TargetLibraryInfo *TLI); /// Emit a call to the __memcpy_chk function to the builder. This expects that /// the Len and ObjSize have type 'intptr_t' and Dst/Src are pointers. @@ -93,6 +102,47 @@ namespace llvm { Value *emitMemCmp(Value *Ptr1, Value *Ptr2, Value *Len, IRBuilder<> &B, const DataLayout &DL, const TargetLibraryInfo *TLI); + /// Emit a call to the bcmp function. + Value *emitBCmp(Value *Ptr1, Value *Ptr2, Value *Len, IRBuilder<> &B, + const DataLayout &DL, const TargetLibraryInfo *TLI); + + /// Emit a call to the memccpy function. + Value *emitMemCCpy(Value *Ptr1, Value *Ptr2, Value *Val, Value *Len, + IRBuilder<> &B, const TargetLibraryInfo *TLI); + + /// Emit a call to the snprintf function. + Value *emitSNPrintf(Value *Dest, Value *Size, Value *Fmt, + ArrayRef Args, IRBuilder<> &B, + const TargetLibraryInfo *TLI); + + /// Emit a call to the sprintf function. + Value *emitSPrintf(Value *Dest, Value *Fmt, ArrayRef VariadicArgs, + IRBuilder<> &B, const TargetLibraryInfo *TLI); + + /// Emit a call to the strcat function. + Value *emitStrCat(Value *Dest, Value *Src, IRBuilder<> &B, + const TargetLibraryInfo *TLI); + + /// Emit a call to the strlcpy function. + Value *emitStrLCpy(Value *Dest, Value *Src, Value *Size, IRBuilder<> &B, + const TargetLibraryInfo *TLI); + + /// Emit a call to the strlcat function. + Value *emitStrLCat(Value *Dest, Value *Src, Value *Size, IRBuilder<> &B, + const TargetLibraryInfo *TLI); + + /// Emit a call to the strncat function. + Value *emitStrNCat(Value *Dest, Value *Src, Value *Size, IRBuilder<> &B, + const TargetLibraryInfo *TLI); + + /// Emit a call to the vsnprintf function. + Value *emitVSNPrintf(Value *Dest, Value *Size, Value *Fmt, Value *VAList, + IRBuilder<> &B, const TargetLibraryInfo *TLI); + + /// Emit a call to the vsprintf function. + Value *emitVSPrintf(Value *Dest, Value *Fmt, Value *VAList, IRBuilder<> &B, + const TargetLibraryInfo *TLI); + /// Emit a call to the unary function named 'Name' (e.g. 'floor'). This /// function is known to take a single of type matching 'Op' and returns one /// value with the same type. If 'Op' is a long double, 'l' is added as the diff --git a/include/llvm/Transforms/Utils/BypassSlowDivision.h b/include/llvm/Transforms/Utils/BypassSlowDivision.h index 6eca5ed2154e..471055921fa8 100644 --- a/include/llvm/Transforms/Utils/BypassSlowDivision.h +++ b/include/llvm/Transforms/Utils/BypassSlowDivision.h @@ -1,9 +1,8 @@ //===- llvm/Transforms/Utils/BypassSlowDivision.h ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/CallPromotionUtils.h b/include/llvm/Transforms/Utils/CallPromotionUtils.h index 6e8ece723638..d9d171c6d8bd 100644 --- a/include/llvm/Transforms/Utils/CallPromotionUtils.h +++ b/include/llvm/Transforms/Utils/CallPromotionUtils.h @@ -1,9 +1,8 @@ //===- CallPromotionUtils.h - Utilities for call promotion ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/CanonicalizeAliases.h b/include/llvm/Transforms/Utils/CanonicalizeAliases.h index f23263783fec..8f23a041a24e 100644 --- a/include/llvm/Transforms/Utils/CanonicalizeAliases.h +++ b/include/llvm/Transforms/Utils/CanonicalizeAliases.h @@ -1,9 +1,8 @@ //===-- CanonicalizeAliases.h - Alias Canonicalization Pass -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/Cloning.h b/include/llvm/Transforms/Utils/Cloning.h index f5e997324fc8..872ab9cab85c 100644 --- a/include/llvm/Transforms/Utils/Cloning.h +++ b/include/llvm/Transforms/Utils/Cloning.h @@ -1,9 +1,8 @@ //===- Cloning.h - Clone various parts of LLVM programs ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -230,10 +229,7 @@ public: /// and all varargs at the callsite will be passed to any calls to /// ForwardVarArgsTo. The caller of InlineFunction has to make sure any varargs /// are only used by ForwardVarArgsTo. -InlineResult InlineFunction(CallInst *C, InlineFunctionInfo &IFI, - AAResults *CalleeAAR = nullptr, - bool InsertLifetime = true); -InlineResult InlineFunction(InvokeInst *II, InlineFunctionInfo &IFI, +InlineResult InlineFunction(CallBase *CB, InlineFunctionInfo &IFI, AAResults *CalleeAAR = nullptr, bool InsertLifetime = true); InlineResult InlineFunction(CallSite CS, InlineFunctionInfo &IFI, @@ -269,6 +265,13 @@ BasicBlock *DuplicateInstructionsInSplitBetween(BasicBlock *BB, ValueToValueMapTy &ValueMapping, DomTreeUpdater &DTU); +/// Updates profile information by adjusting the entry count by adding +/// entryDelta then scaling callsite information by the new count divided by the +/// old count. VMap is used during inlinng to also update the new clone +void updateProfileCallee( + Function *Callee, int64_t entryDelta, + const ValueMap *VMap = nullptr); + } // end namespace llvm #endif // LLVM_TRANSFORMS_UTILS_CLONING_H diff --git a/include/llvm/Transforms/Utils/CodeExtractor.h b/include/llvm/Transforms/Utils/CodeExtractor.h index fee79fdc3bff..9d79ee1633f6 100644 --- a/include/llvm/Transforms/Utils/CodeExtractor.h +++ b/include/llvm/Transforms/Utils/CodeExtractor.h @@ -1,9 +1,8 @@ //===- Transform/Utils/CodeExtractor.h - Code extraction util ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -27,6 +26,7 @@ class BasicBlock; class BlockFrequency; class BlockFrequencyInfo; class BranchProbabilityInfo; +class AssumptionCache; class CallInst; class DominatorTree; class Function; @@ -57,6 +57,7 @@ class Value; const bool AggregateArgs; BlockFrequencyInfo *BFI; BranchProbabilityInfo *BPI; + AssumptionCache *AC; // If true, varargs functions can be extracted. bool AllowVarArgs; @@ -85,6 +86,7 @@ class Value; CodeExtractor(ArrayRef BBs, DominatorTree *DT = nullptr, bool AggregateArgs = false, BlockFrequencyInfo *BFI = nullptr, BranchProbabilityInfo *BPI = nullptr, + AssumptionCache *AC = nullptr, bool AllowVarArgs = false, bool AllowAlloca = false, std::string Suffix = ""); @@ -95,6 +97,7 @@ class Value; CodeExtractor(DominatorTree &DT, Loop &L, bool AggregateArgs = false, BlockFrequencyInfo *BFI = nullptr, BranchProbabilityInfo *BPI = nullptr, + AssumptionCache *AC = nullptr, std::string Suffix = ""); /// Perform the extraction, returning the new function. @@ -148,6 +151,16 @@ class Value; BasicBlock *findOrCreateBlockForHoisting(BasicBlock *CommonExitBlock); private: + struct LifetimeMarkerInfo { + bool SinkLifeStart = false; + bool HoistLifeEnd = false; + Instruction *LifeStart = nullptr; + Instruction *LifeEnd = nullptr; + }; + + LifetimeMarkerInfo getLifetimeMarkers(Instruction *Addr, + BasicBlock *ExitBlock) const; + void severSplitPHINodesOfEntry(BasicBlock *&Header); void severSplitPHINodesOfExits(const SmallPtrSetImpl &Exits); void splitReturnBlocks(); diff --git a/include/llvm/Transforms/Utils/CtorUtils.h b/include/llvm/Transforms/Utils/CtorUtils.h index 63e564dcb87a..3625ee662b1c 100644 --- a/include/llvm/Transforms/Utils/CtorUtils.h +++ b/include/llvm/Transforms/Utils/CtorUtils.h @@ -1,9 +1,8 @@ //===- CtorUtils.h - Helpers for working with global_ctors ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/EntryExitInstrumenter.h b/include/llvm/Transforms/Utils/EntryExitInstrumenter.h index f50c5c922081..3913693af359 100644 --- a/include/llvm/Transforms/Utils/EntryExitInstrumenter.h +++ b/include/llvm/Transforms/Utils/EntryExitInstrumenter.h @@ -1,9 +1,8 @@ //===- EntryExitInstrumenter.h - Function Entry/Exit Instrumentation ------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/EscapeEnumerator.h b/include/llvm/Transforms/Utils/EscapeEnumerator.h index 1256dfdaca17..e667796c841b 100644 --- a/include/llvm/Transforms/Utils/EscapeEnumerator.h +++ b/include/llvm/Transforms/Utils/EscapeEnumerator.h @@ -1,9 +1,8 @@ //===-- EscapeEnumerator.h --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/Evaluator.h b/include/llvm/Transforms/Utils/Evaluator.h index 9908ae6fd393..bffd65f71b2e 100644 --- a/include/llvm/Transforms/Utils/Evaluator.h +++ b/include/llvm/Transforms/Utils/Evaluator.h @@ -1,9 +1,8 @@ //===- Evaluator.h - LLVM IR evaluator --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/FunctionComparator.h b/include/llvm/Transforms/Utils/FunctionComparator.h index 35ba0950343c..4e2571b1d0b6 100644 --- a/include/llvm/Transforms/Utils/FunctionComparator.h +++ b/include/llvm/Transforms/Utils/FunctionComparator.h @@ -1,9 +1,8 @@ //===- FunctionComparator.h - Function Comparator ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/FunctionImportUtils.h b/include/llvm/Transforms/Utils/FunctionImportUtils.h index e24398b90012..9c2a9ea531ea 100644 --- a/include/llvm/Transforms/Utils/FunctionImportUtils.h +++ b/include/llvm/Transforms/Utils/FunctionImportUtils.h @@ -1,9 +1,8 @@ //===- FunctionImportUtils.h - Importing support utilities -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -44,6 +43,11 @@ class FunctionImportGlobalProcessing { /// to promote any non-renamable values. SmallPtrSet Used; + /// Keep track of any COMDATs that require renaming (because COMDAT + /// leader was promoted and renamed). Maps from original COMDAT to one + /// with new name. + DenseMap RenamedComdats; + /// Check if we should promote the given local value to global scope. bool shouldPromoteLocalToGlobal(const GlobalValue *SGV); diff --git a/include/llvm/Transforms/Utils/GlobalStatus.h b/include/llvm/Transforms/Utils/GlobalStatus.h index 8cc265bdf81d..519593c96766 100644 --- a/include/llvm/Transforms/Utils/GlobalStatus.h +++ b/include/llvm/Transforms/Utils/GlobalStatus.h @@ -1,9 +1,8 @@ //===- GlobalStatus.h - Compute status info for globals ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Transforms/Utils/GuardUtils.h b/include/llvm/Transforms/Utils/GuardUtils.h index 537045edafe4..3b365c56a5c0 100644 --- a/include/llvm/Transforms/Utils/GuardUtils.h +++ b/include/llvm/Transforms/Utils/GuardUtils.h @@ -1,9 +1,8 @@ //===-- GuardUtils.h - Utils for work with guards ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Utils that are used to perform transformations related to guards and their diff --git a/include/llvm/Transforms/Utils/ImportedFunctionsInliningStatistics.h b/include/llvm/Transforms/Utils/ImportedFunctionsInliningStatistics.h index b55a9893bcf7..033ea05b77fa 100644 --- a/include/llvm/Transforms/Utils/ImportedFunctionsInliningStatistics.h +++ b/include/llvm/Transforms/Utils/ImportedFunctionsInliningStatistics.h @@ -1,9 +1,8 @@ -//===-- ImportedFunctionsInliningStats.h ------------------------*- C++ -*-===// +//===-- ImportedFunctionsInliningStatistics.h -------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Generating inliner statistics for imported functions, mostly useful for diff --git a/include/llvm/Transforms/Utils/IntegerDivision.h b/include/llvm/Transforms/Utils/IntegerDivision.h index 5d9927eb51b2..35cae9aa2269 100644 --- a/include/llvm/Transforms/Utils/IntegerDivision.h +++ b/include/llvm/Transforms/Utils/IntegerDivision.h @@ -1,9 +1,8 @@ //===- llvm/Transforms/Utils/IntegerDivision.h ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/LCSSA.h b/include/llvm/Transforms/Utils/LCSSA.h index fe717e5f6635..b01c8022a65b 100644 --- a/include/llvm/Transforms/Utils/LCSSA.h +++ b/include/llvm/Transforms/Utils/LCSSA.h @@ -1,9 +1,8 @@ //===- LCSSA.h - Loop-closed SSA transform Pass -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/LibCallsShrinkWrap.h b/include/llvm/Transforms/Utils/LibCallsShrinkWrap.h index c9df532e5794..ff1537ace329 100644 --- a/include/llvm/Transforms/Utils/LibCallsShrinkWrap.h +++ b/include/llvm/Transforms/Utils/LibCallsShrinkWrap.h @@ -1,9 +1,8 @@ //===- LibCallsShrinkWrap.h - Shrink Wrap Library Calls -------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/Local.h b/include/llvm/Transforms/Utils/Local.h index ec8b0eda3641..ff516f230979 100644 --- a/include/llvm/Transforms/Utils/Local.h +++ b/include/llvm/Transforms/Utils/Local.h @@ -1,9 +1,8 @@ //===- Local.h - Functions to perform local transformations -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -21,12 +20,11 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/TinyPtrVector.h" #include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/DomTreeUpdater.h" #include "llvm/Analysis/Utils/Local.h" -#include "llvm/IR/CallSite.h" #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" -#include "llvm/IR/DomTreeUpdater.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/GetElementPtrTypeIterator.h" #include "llvm/IR/Operator.h" @@ -233,7 +231,8 @@ bool FlattenCFG(BasicBlock *BB, AliasAnalysis *AA = nullptr); /// If this basic block is ONLY a setcc and a branch, and if a predecessor /// branches to us and one of our successors, fold the setcc into the /// predecessor and use logical operations to pick the right destination. -bool FoldBranchToCommonDest(BranchInst *BI, unsigned BonusInstThreshold = 1); +bool FoldBranchToCommonDest(BranchInst *BI, MemorySSAUpdater *MSSAU = nullptr, + unsigned BonusInstThreshold = 1); /// This function takes a virtual register computed by an Instruction and /// replaces it with a slot in the stack frame, allocated via alloca. @@ -317,7 +316,7 @@ void findDbgUsers(SmallVectorImpl &DbgInsts, Value *V); /// (between the optional Deref operations). Offset can be negative. bool replaceDbgDeclare(Value *Address, Value *NewAddress, Instruction *InsertBefore, DIBuilder &Builder, - bool DerefBefore, int Offset, bool DerefAfter); + uint8_t DIExprFlags, int Offset); /// Replaces llvm.dbg.declare instruction when the alloca it describes /// is replaced with a new value. If Deref is true, an additional @@ -326,8 +325,8 @@ bool replaceDbgDeclare(Value *Address, Value *NewAddress, /// optional Deref operations). Offset can be negative. The new /// llvm.dbg.declare is inserted immediately after AI. bool replaceDbgDeclareForAlloca(AllocaInst *AI, Value *NewAllocaAddress, - DIBuilder &Builder, bool DerefBefore, - int Offset, bool DerefAfter); + DIBuilder &Builder, uint8_t DIExprFlags, + int Offset); /// Replaces multiple llvm.dbg.value instructions when the alloca it describes /// is replaced with a new value. If Offset is non-zero, a constant displacement @@ -337,11 +336,27 @@ bool replaceDbgDeclareForAlloca(AllocaInst *AI, Value *NewAllocaAddress, void replaceDbgValueForAlloca(AllocaInst *AI, Value *NewAllocaAddress, DIBuilder &Builder, int Offset = 0); +/// Finds alloca where the value comes from. +AllocaInst *findAllocaForValue(Value *V, + DenseMap &AllocaForValue); + /// Assuming the instruction \p I is going to be deleted, attempt to salvage /// debug users of \p I by writing the effect of \p I in a DIExpression. /// Returns true if any debug users were updated. bool salvageDebugInfo(Instruction &I); +/// Implementation of salvageDebugInfo, applying only to instructions in +/// \p Insns, rather than all debug users of \p I. +bool salvageDebugInfoForDbgValues(Instruction &I, + ArrayRef Insns); + +/// Given an instruction \p I and DIExpression \p DIExpr operating on it, write +/// the effects of \p I into the returned DIExpression, or return nullptr if +/// it cannot be salvaged. \p StackVal: whether DW_OP_stack_value should be +/// appended to the expression. +DIExpression *salvageDebugInfoImpl(Instruction &I, DIExpression *DIExpr, + bool StackVal); + /// Point debug users of \p From to \p To or salvage them. Use this function /// only when replacing all uses of \p From with \p To, with a guarantee that /// \p From is going to be deleted. @@ -367,7 +382,8 @@ unsigned removeAllNonTerminatorAndEHPadInstructions(BasicBlock *BB); /// instruction, making it and the rest of the code in the block dead. unsigned changeToUnreachable(Instruction *I, bool UseLLVMTrap, bool PreserveLCSSA = false, - DomTreeUpdater *DTU = nullptr); + DomTreeUpdater *DTU = nullptr, + MemorySSAUpdater *MSSAU = nullptr); /// Convert the CallInst to InvokeInst with the specified unwind edge basic /// block. This also splits the basic block where CI is located, because @@ -426,7 +442,7 @@ unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT, unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT, const BasicBlock *BB); -/// Return true if the CallSite CS calls a gc leaf function. +/// Return true if this call calls a gc leaf function. /// /// A leaf function is a function that does not safepoint the thread during its /// execution. During a call or invoke to such a function, the callers stack @@ -434,7 +450,7 @@ unsigned replaceDominatedUsesWith(Value *From, Value *To, DominatorTree &DT, /// /// Most passes can and should ignore this information, and it is only used /// during lowering by the GC infrastructure. -bool callsGCLeafFunction(ImmutableCallSite CS, const TargetLibraryInfo &TLI); +bool callsGCLeafFunction(const CallBase *Call, const TargetLibraryInfo &TLI); /// Copy a nonnull metadata node to a new load instruction. /// @@ -456,8 +472,7 @@ void dropDebugUsers(Instruction &I); /// \p DomBlock, by moving its instructions to the insertion point \p InsertPt. /// /// The moved instructions receive the insertion point debug location values -/// (DILocations) and their debug intrinsic instructions (dbg.values) are -/// removed. +/// (DILocations) and their debug intrinsic instructions are removed. void hoistAllInstructionsInto(BasicBlock *DomBlock, Instruction *InsertPt, BasicBlock *BB); diff --git a/include/llvm/Transforms/Utils/LoopRotationUtils.h b/include/llvm/Transforms/Utils/LoopRotationUtils.h index cd5bc4301018..1e80722ed8b8 100644 --- a/include/llvm/Transforms/Utils/LoopRotationUtils.h +++ b/include/llvm/Transforms/Utils/LoopRotationUtils.h @@ -1,9 +1,8 @@ //===- LoopRotationUtils.h - Utilities to perform loop rotation -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/LoopSimplify.h b/include/llvm/Transforms/Utils/LoopSimplify.h index 166da2738ffd..2c1df7942f63 100644 --- a/include/llvm/Transforms/Utils/LoopSimplify.h +++ b/include/llvm/Transforms/Utils/LoopSimplify.h @@ -1,9 +1,8 @@ //===- LoopSimplify.h - Loop Canonicalization Pass --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -46,6 +45,8 @@ namespace llvm { +class MemorySSAUpdater; + /// This pass is responsible for loop canonicalization. class LoopSimplifyPass : public PassInfoMixin { public: @@ -56,9 +57,11 @@ public: /// /// This takes a potentially un-simplified loop L (and its children) and turns /// it into a simplified loop nest with preheaders and single backedges. It will -/// update \c AliasAnalysis and \c ScalarEvolution analyses if they're non-null. +/// update \c DominatorTree, \c LoopInfo, \c ScalarEvolution and \c MemorySSA +/// analyses if they're non-null, and LCSSA if \c PreserveLCSSA is true. bool simplifyLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, ScalarEvolution *SE, - AssumptionCache *AC, bool PreserveLCSSA); + AssumptionCache *AC, MemorySSAUpdater *MSSAU, + bool PreserveLCSSA); } // end namespace llvm diff --git a/include/llvm/Transforms/Utils/LoopUtils.h b/include/llvm/Transforms/Utils/LoopUtils.h index 8c2527b6ae68..68bdded5cf93 100644 --- a/include/llvm/Transforms/Utils/LoopUtils.h +++ b/include/llvm/Transforms/Utils/LoopUtils.h @@ -1,9 +1,8 @@ //===- llvm/Transforms/Utils/LoopUtils.h - Loop utilities -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -41,6 +40,7 @@ class BasicBlock; class DataLayout; class Loop; class LoopInfo; +class MemoryAccess; class MemorySSAUpdater; class OptimizationRemarkEmitter; class PredicatedScalarEvolution; @@ -51,7 +51,7 @@ class TargetLibraryInfo; class TargetTransformInfo; BasicBlock *InsertPreheaderForLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, - bool PreserveLCSSA); + MemorySSAUpdater *MSSAU, bool PreserveLCSSA); /// Ensure that all exit blocks of the loop are dedicated exits. /// @@ -59,7 +59,7 @@ BasicBlock *InsertPreheaderForLoop(Loop *L, DominatorTree *DT, LoopInfo *LI, /// predecessors to use a dedicated loop exit block. We update the dominator /// tree and loop info if provided, and will preserve LCSSA if requested. bool formDedicatedExitBlocks(Loop *L, DominatorTree *DT, LoopInfo *LI, - bool PreserveLCSSA); + MemorySSAUpdater *MSSAU, bool PreserveLCSSA); /// Ensures LCSSA form for every instruction from the Worklist in the scope of /// innermost containing loop. @@ -79,7 +79,8 @@ bool formLCSSAForInstructions(SmallVectorImpl &Worklist, /// /// Looks at all instructions in the loop which have uses outside of the /// current loop. For each, an LCSSA PHI node is inserted and the uses outside -/// the loop are rewritten to use this node. +/// the loop are rewritten to use this node. Sub-loops must be in LCSSA form +/// already. /// /// LoopInfo and DominatorTree are required and preserved. /// @@ -100,6 +101,14 @@ bool formLCSSA(Loop &L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution *SE); bool formLCSSARecursively(Loop &L, DominatorTree &DT, LoopInfo *LI, ScalarEvolution *SE); +struct SinkAndHoistLICMFlags { + bool NoOfMemAccTooLarge; + unsigned LicmMssaOptCounter; + unsigned LicmMssaOptCap; + unsigned LicmMssaNoAccForPromotionCap; + bool IsSink; +}; + /// Walk the specified region of the CFG (defined by all blocks /// dominated by the specified block, and that are in the current loop) in /// reverse depth first order w.r.t the DominatorTree. This allows us to visit @@ -111,7 +120,7 @@ bool formLCSSARecursively(Loop &L, DominatorTree &DT, LoopInfo *LI, bool sinkRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *, TargetLibraryInfo *, TargetTransformInfo *, Loop *, AliasSetTracker *, MemorySSAUpdater *, ICFLoopSafetyInfo *, - OptimizationRemarkEmitter *ORE); + SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *); /// Walk the specified region of the CFG (defined by all blocks /// dominated by the specified block, and that are in the current loop) in depth @@ -124,7 +133,7 @@ bool sinkRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *, bool hoistRegion(DomTreeNode *, AliasAnalysis *, LoopInfo *, DominatorTree *, TargetLibraryInfo *, Loop *, AliasSetTracker *, MemorySSAUpdater *, ICFLoopSafetyInfo *, - OptimizationRemarkEmitter *ORE); + SinkAndHoistLICMFlags &, OptimizationRemarkEmitter *); /// This function deletes dead loops. The caller of this function needs to /// guarantee that the loop is infact dead. @@ -148,14 +157,12 @@ void deleteDeadLoop(Loop *L, DominatorTree *DT, ScalarEvolution *SE, /// LoopInfo, DominatorTree, Loop, AliasSet information for all instructions /// of the loop and loop safety information as arguments. /// Diagnostics is emitted via \p ORE. It returns changed status. -bool promoteLoopAccessesToScalars(const SmallSetVector &, - SmallVectorImpl &, - SmallVectorImpl &, - PredIteratorCache &, LoopInfo *, - DominatorTree *, const TargetLibraryInfo *, - Loop *, AliasSetTracker *, - ICFLoopSafetyInfo *, - OptimizationRemarkEmitter *); +bool promoteLoopAccessesToScalars( + const SmallSetVector &, SmallVectorImpl &, + SmallVectorImpl &, SmallVectorImpl &, + PredIteratorCache &, LoopInfo *, DominatorTree *, const TargetLibraryInfo *, + Loop *, AliasSetTracker *, MemorySSAUpdater *, ICFLoopSafetyInfo *, + OptimizationRemarkEmitter *); /// Does a BFS from a given node to all of its children inside a given loop. /// The returned vector of nodes includes the starting point. @@ -277,6 +284,7 @@ void getLoopAnalysisUsage(AnalysisUsage &AU); bool canSinkOrHoistInst(Instruction &I, AAResults *AA, DominatorTree *DT, Loop *CurLoop, AliasSetTracker *CurAST, MemorySSAUpdater *MSSAU, bool TargetExecutesOncePerLoop, + SinkAndHoistLICMFlags *LICMFlags = nullptr, OptimizationRemarkEmitter *ORE = nullptr); /// Returns a Min/Max operation corresponding to MinMaxRecurrenceKind. @@ -292,6 +300,7 @@ getOrderedReduction(IRBuilder<> &Builder, Value *Acc, Value *Src, unsigned Op, ArrayRef RedOps = None); /// Generates a vector reduction using shufflevectors to reduce the value. +/// Fast-math-flags are propagated using the IRBuilder's setting. Value *getShuffleReduction(IRBuilder<> &Builder, Value *Src, unsigned Op, RecurrenceDescriptor::MinMaxRecurrenceKind MinMaxKind = RecurrenceDescriptor::MRK_Invalid, @@ -302,6 +311,7 @@ Value *getShuffleReduction(IRBuilder<> &Builder, Value *Src, unsigned Op, /// additional information supplied in \p Flags. /// The target is queried to determine if intrinsics or shuffle sequences are /// required to implement the reduction. +/// Fast-math-flags are propagated using the IRBuilder's setting. Value *createSimpleTargetReduction(IRBuilder<> &B, const TargetTransformInfo *TTI, unsigned Opcode, Value *Src, @@ -312,6 +322,7 @@ Value *createSimpleTargetReduction(IRBuilder<> &B, /// Create a generic target reduction using a recurrence descriptor \p Desc /// The target is queried to determine if intrinsics or shuffle sequences are /// required to implement the reduction. +/// Fast-math-flags are propagated using the RecurrenceDescriptor. Value *createTargetReduction(IRBuilder<> &B, const TargetTransformInfo *TTI, RecurrenceDescriptor &Desc, Value *Src, bool NoNaN = false); diff --git a/include/llvm/Transforms/Utils/LoopVersioning.h b/include/llvm/Transforms/Utils/LoopVersioning.h index fcd734b37a1f..355c4d7dc6d8 100644 --- a/include/llvm/Transforms/Utils/LoopVersioning.h +++ b/include/llvm/Transforms/Utils/LoopVersioning.h @@ -1,9 +1,8 @@ //===- LoopVersioning.h - Utility to version a loop -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/LowerInvoke.h b/include/llvm/Transforms/Utils/LowerInvoke.h index 12774c7fd1f7..c1198b08d3de 100644 --- a/include/llvm/Transforms/Utils/LowerInvoke.h +++ b/include/llvm/Transforms/Utils/LowerInvoke.h @@ -1,9 +1,8 @@ //===- LowerInvoke.h - Eliminate Invoke instructions ----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/LowerMemIntrinsics.h b/include/llvm/Transforms/Utils/LowerMemIntrinsics.h index 2b7d0f67a324..8e9d7b522c78 100644 --- a/include/llvm/Transforms/Utils/LowerMemIntrinsics.h +++ b/include/llvm/Transforms/Utils/LowerMemIntrinsics.h @@ -1,9 +1,8 @@ -//===- llvm/Transforms/Utils/LowerMemintrinsics.h ---------------*- C++ -*-===// +//===- llvm/Transforms/Utils/LowerMemIntrinsics.h ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/Mem2Reg.h b/include/llvm/Transforms/Utils/Mem2Reg.h index 407684338a3b..76c1c2c5bffe 100644 --- a/include/llvm/Transforms/Utils/Mem2Reg.h +++ b/include/llvm/Transforms/Utils/Mem2Reg.h @@ -1,9 +1,8 @@ //===- Mem2Reg.h - The -mem2reg pass, a wrapper around the Utils lib ------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/ModuleUtils.h b/include/llvm/Transforms/Utils/ModuleUtils.h index fee492be2a90..c69af5588741 100644 --- a/include/llvm/Transforms/Utils/ModuleUtils.h +++ b/include/llvm/Transforms/Utils/ModuleUtils.h @@ -1,9 +1,8 @@ //===-- ModuleUtils.h - Functions to manipulate Modules ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -22,6 +21,7 @@ namespace llvm { template class ArrayRef; class Module; class Function; +class FunctionCallee; class GlobalValue; class GlobalVariable; class Constant; @@ -40,20 +40,14 @@ void appendToGlobalCtors(Module &M, Function *F, int Priority, void appendToGlobalDtors(Module &M, Function *F, int Priority, Constant *Data = nullptr); -// Validate the result of Module::getOrInsertFunction called for an interface -// function of given sanitizer. If the instrumented module defines a function -// with the same name, their prototypes must match, otherwise -// getOrInsertFunction returns a bitcast. -Function *checkSanitizerInterfaceFunction(Constant *FuncOrBitcast); - -Function *declareSanitizerInitFunction(Module &M, StringRef InitName, - ArrayRef InitArgTypes); +FunctionCallee declareSanitizerInitFunction(Module &M, StringRef InitName, + ArrayRef InitArgTypes); /// Creates sanitizer constructor function, and calls sanitizer's init /// function from it. /// \return Returns pair of pointers to constructor, and init functions /// respectively. -std::pair createSanitizerCtorAndInitFunctions( +std::pair createSanitizerCtorAndInitFunctions( Module &M, StringRef CtorName, StringRef InitName, ArrayRef InitArgTypes, ArrayRef InitArgs, StringRef VersionCheckName = StringRef()); @@ -65,10 +59,10 @@ std::pair createSanitizerCtorAndInitFunctions( /// /// \return Returns pair of pointers to constructor, and init functions /// respectively. -std::pair getOrCreateSanitizerCtorAndInitFunctions( +std::pair getOrCreateSanitizerCtorAndInitFunctions( Module &M, StringRef CtorName, StringRef InitName, ArrayRef InitArgTypes, ArrayRef InitArgs, - function_ref FunctionsCreatedCallback, + function_ref FunctionsCreatedCallback, StringRef VersionCheckName = StringRef()); // Creates and returns a sanitizer init function without argument if it doesn't diff --git a/include/llvm/Transforms/Utils/NameAnonGlobals.h b/include/llvm/Transforms/Utils/NameAnonGlobals.h index 17fc902eebf8..659ebe33ffa6 100644 --- a/include/llvm/Transforms/Utils/NameAnonGlobals.h +++ b/include/llvm/Transforms/Utils/NameAnonGlobals.h @@ -1,9 +1,8 @@ //===-- NameAnonGlobals.h - Anonymous Global Naming Pass --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/PredicateInfo.h b/include/llvm/Transforms/Utils/PredicateInfo.h index 2fc38089f3f1..da4a5dcc28c0 100644 --- a/include/llvm/Transforms/Utils/PredicateInfo.h +++ b/include/llvm/Transforms/Utils/PredicateInfo.h @@ -1,9 +1,8 @@ //===- PredicateInfo.h - Build PredicateInfo ----------------------*-C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/include/llvm/Transforms/Utils/PromoteMemToReg.h b/include/llvm/Transforms/Utils/PromoteMemToReg.h index 5ddfbe2bf058..b2b4507bbc74 100644 --- a/include/llvm/Transforms/Utils/PromoteMemToReg.h +++ b/include/llvm/Transforms/Utils/PromoteMemToReg.h @@ -1,9 +1,8 @@ //===- PromoteMemToReg.h - Promote Allocas to Scalars -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/SSAUpdater.h b/include/llvm/Transforms/Utils/SSAUpdater.h index d02607acbbb5..22b2295cc9d7 100644 --- a/include/llvm/Transforms/Utils/SSAUpdater.h +++ b/include/llvm/Transforms/Utils/SSAUpdater.h @@ -1,9 +1,8 @@ //===- SSAUpdater.h - Unstructured SSA Update Tool --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -148,7 +147,7 @@ public: /// Insts is a list of loads and stores to promote, and Name is the basename /// for the PHIs to insert. After this is complete, the loads and stores are /// removed from the code. - void run(const SmallVectorImpl &Insts) const; + void run(const SmallVectorImpl &Insts); /// Return true if the specified instruction is in the Inst list. /// @@ -159,7 +158,7 @@ public: /// This hook is invoked after all the stores are found and inserted as /// available values. - virtual void doExtraRewritesBeforeFinalDeletion() const {} + virtual void doExtraRewritesBeforeFinalDeletion() {} /// Clients can choose to implement this to get notified right before /// a load is RAUW'd another value. diff --git a/include/llvm/Transforms/Utils/SSAUpdaterBulk.h b/include/llvm/Transforms/Utils/SSAUpdaterBulk.h index 53a608f01804..5d17d6f3d285 100644 --- a/include/llvm/Transforms/Utils/SSAUpdaterBulk.h +++ b/include/llvm/Transforms/Utils/SSAUpdaterBulk.h @@ -1,9 +1,8 @@ //===- SSAUpdaterBulk.h - Unstructured SSA Update Tool ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/SSAUpdaterImpl.h b/include/llvm/Transforms/Utils/SSAUpdaterImpl.h index cab0f3e71575..ee06893ca660 100644 --- a/include/llvm/Transforms/Utils/SSAUpdaterImpl.h +++ b/include/llvm/Transforms/Utils/SSAUpdaterImpl.h @@ -1,9 +1,8 @@ //===- SSAUpdaterImpl.h - SSA Updater Implementation ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/SanitizerStats.h b/include/llvm/Transforms/Utils/SanitizerStats.h index d36e34258a3f..14e8ae045cdd 100644 --- a/include/llvm/Transforms/Utils/SanitizerStats.h +++ b/include/llvm/Transforms/Utils/SanitizerStats.h @@ -1,9 +1,8 @@ //===- SanitizerStats.h - Sanitizer statistics gathering -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/SimplifyIndVar.h b/include/llvm/Transforms/Utils/SimplifyIndVar.h index a1dfed29a22d..dec73ef057e8 100644 --- a/include/llvm/Transforms/Utils/SimplifyIndVar.h +++ b/include/llvm/Transforms/Utils/SimplifyIndVar.h @@ -1,9 +1,8 @@ //===-- llvm/Transforms/Utils/SimplifyIndVar.h - Indvar Utils ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/SimplifyLibCalls.h b/include/llvm/Transforms/Utils/SimplifyLibCalls.h index 025bcd44e310..2572094ddac8 100644 --- a/include/llvm/Transforms/Utils/SimplifyLibCalls.h +++ b/include/llvm/Transforms/Utils/SimplifyLibCalls.h @@ -1,9 +1,8 @@ //===- SimplifyLibCalls.h - Library call simplifier -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -29,6 +28,8 @@ class TargetLibraryInfo; class BasicBlock; class Function; class OptimizationRemarkEmitter; +class BlockFrequencyInfo; +class ProfileSummaryInfo; /// This class implements simplifications for calls to fortified library /// functions (__st*cpy_chk, __memcpy_chk, __memmove_chk, __memset_chk), to, @@ -56,14 +57,41 @@ private: Value *optimizeMemMoveChk(CallInst *CI, IRBuilder<> &B); Value *optimizeMemSetChk(CallInst *CI, IRBuilder<> &B); - // Str/Stp cpy are similar enough to be handled in the same functions. + /// Str/Stp cpy are similar enough to be handled in the same functions. Value *optimizeStrpCpyChk(CallInst *CI, IRBuilder<> &B, LibFunc Func); Value *optimizeStrpNCpyChk(CallInst *CI, IRBuilder<> &B, LibFunc Func); + Value *optimizeMemCCpyChk(CallInst *CI, IRBuilder<> &B); + Value *optimizeSNPrintfChk(CallInst *CI, IRBuilder<> &B); + Value *optimizeSPrintfChk(CallInst *CI,IRBuilder<> &B); + Value *optimizeStrCatChk(CallInst *CI, IRBuilder<> &B); + Value *optimizeStrLCat(CallInst *CI, IRBuilder<> &B); + Value *optimizeStrNCatChk(CallInst *CI, IRBuilder<> &B); + Value *optimizeStrLCpyChk(CallInst *CI, IRBuilder<> &B); + Value *optimizeVSNPrintfChk(CallInst *CI, IRBuilder<> &B); + Value *optimizeVSPrintfChk(CallInst *CI, IRBuilder<> &B); /// Checks whether the call \p CI to a fortified libcall is foldable /// to the non-fortified version. + /// + /// \param CI the call to the fortified libcall. + /// + /// \param ObjSizeOp the index of the object size parameter of this chk + /// function. Not optional since this is mandatory. + /// + /// \param SizeOp optionally set to the parameter index of an explicit buffer + /// size argument. For instance, set to '2' for __strncpy_chk. + /// + /// \param StrOp optionally set to the parameter index of the source string + /// parameter to strcpy-like functions, where only the strlen of the source + /// will be writtin into the destination. + /// + /// \param FlagsOp optionally set to the parameter index of a 'flags' + /// parameter. These are used by an implementation to opt-into stricter + /// checking. bool isFortifiedCallFoldable(CallInst *CI, unsigned ObjSizeOp, - unsigned SizeOp, bool isString); + Optional SizeOp = None, + Optional StrOp = None, + Optional FlagsOp = None); }; /// LibCallSimplifier - This class implements a collection of optimizations @@ -75,6 +103,8 @@ private: const DataLayout &DL; const TargetLibraryInfo *TLI; OptimizationRemarkEmitter &ORE; + BlockFrequencyInfo *BFI; + ProfileSummaryInfo *PSI; bool UnsafeFPShrink; function_ref Replacer; function_ref Eraser; @@ -102,6 +132,7 @@ public: LibCallSimplifier( const DataLayout &DL, const TargetLibraryInfo *TLI, OptimizationRemarkEmitter &ORE, + BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, function_ref Replacer = &replaceAllUsesWithDefault, function_ref Eraser = &eraseFromParentDefault); @@ -134,6 +165,8 @@ private: Value *optimizeStrStr(CallInst *CI, IRBuilder<> &B); Value *optimizeMemChr(CallInst *CI, IRBuilder<> &B); Value *optimizeMemCmp(CallInst *CI, IRBuilder<> &B); + Value *optimizeBCmp(CallInst *CI, IRBuilder<> &B); + Value *optimizeMemCmpBCmpCommon(CallInst *CI, IRBuilder<> &B); Value *optimizeMemCpy(CallInst *CI, IRBuilder<> &B); Value *optimizeMemMove(CallInst *CI, IRBuilder<> &B); Value *optimizeMemSet(CallInst *CI, IRBuilder<> &B); diff --git a/include/llvm/Transforms/Utils/SizeOpts.h b/include/llvm/Transforms/Utils/SizeOpts.h new file mode 100644 index 000000000000..1a052c694e6d --- /dev/null +++ b/include/llvm/Transforms/Utils/SizeOpts.h @@ -0,0 +1,34 @@ +//===- llvm/Transforms/Utils/SizeOpts.h - size optimization -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains some shared code size optimization related code. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_UTILS_SIZEOPTS_H +#define LLVM_TRANSFORMS_UTILS_SIZEOPTS_H + +namespace llvm { + +class BasicBlock; +class BlockFrequencyInfo; +class Function; +class ProfileSummaryInfo; + +/// Returns true if function \p F is suggested to be size-optimized base on the +/// profile. +bool shouldOptimizeForSize(Function *F, ProfileSummaryInfo *PSI, + BlockFrequencyInfo *BFI); +/// Returns true if basic block \p BB is suggested to be size-optimized base +/// on the profile. +bool shouldOptimizeForSize(BasicBlock *BB, ProfileSummaryInfo *PSI, + BlockFrequencyInfo *BFI); + +} // end namespace llvm + +#endif // LLVM_TRANSFORMS_UTILS_SIZEOPTS_H diff --git a/include/llvm/Transforms/Utils/SplitModule.h b/include/llvm/Transforms/Utils/SplitModule.h index d2c31f2701ac..7839c5d9a589 100644 --- a/include/llvm/Transforms/Utils/SplitModule.h +++ b/include/llvm/Transforms/Utils/SplitModule.h @@ -1,9 +1,8 @@ //===- SplitModule.h - Split a module into partitions -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/SymbolRewriter.h b/include/llvm/Transforms/Utils/SymbolRewriter.h index 5f6488e08b5a..ce9dcaf2b74f 100644 --- a/include/llvm/Transforms/Utils/SymbolRewriter.h +++ b/include/llvm/Transforms/Utils/SymbolRewriter.h @@ -1,9 +1,8 @@ //===- SymbolRewriter.h - Symbol Rewriting Pass -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/UnifyFunctionExitNodes.h b/include/llvm/Transforms/Utils/UnifyFunctionExitNodes.h index 222c601ad608..f68534ecd2eb 100644 --- a/include/llvm/Transforms/Utils/UnifyFunctionExitNodes.h +++ b/include/llvm/Transforms/Utils/UnifyFunctionExitNodes.h @@ -1,9 +1,8 @@ //===-- UnifyFunctionExitNodes.h - Ensure fn's have one return --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Utils/UnrollLoop.h b/include/llvm/Transforms/Utils/UnrollLoop.h index 70e936d75008..593ca26feb98 100644 --- a/include/llvm/Transforms/Utils/UnrollLoop.h +++ b/include/llvm/Transforms/Utils/UnrollLoop.h @@ -1,9 +1,8 @@ //===- llvm/Transforms/Utils/UnrollLoop.h - Unrolling utilities -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,11 +24,13 @@ namespace llvm { class AssumptionCache; class BasicBlock; +class BlockFrequencyInfo; class DependenceInfo; class DominatorTree; class Loop; class LoopInfo; class MDNode; +class ProfileSummaryInfo; class OptimizationRemarkEmitter; class ScalarEvolution; @@ -63,22 +64,31 @@ enum class LoopUnrollResult { FullyUnrolled }; -LoopUnrollResult UnrollLoop(Loop *L, unsigned Count, unsigned TripCount, - bool Force, bool AllowRuntime, - bool AllowExpensiveTripCount, bool PreserveCondBr, - bool PreserveOnlyFirst, unsigned TripMultiple, - unsigned PeelCount, bool UnrollRemainder, - LoopInfo *LI, ScalarEvolution *SE, - DominatorTree *DT, AssumptionCache *AC, - OptimizationRemarkEmitter *ORE, bool PreserveLCSSA, - Loop **RemainderLoop = nullptr); +struct UnrollLoopOptions { + unsigned Count; + unsigned TripCount; + bool Force; + bool AllowRuntime; + bool AllowExpensiveTripCount; + bool PreserveCondBr; + bool PreserveOnlyFirst; + unsigned TripMultiple; + unsigned PeelCount; + bool UnrollRemainder; + bool ForgetAllSCEV; +}; + +LoopUnrollResult UnrollLoop(Loop *L, UnrollLoopOptions ULO, LoopInfo *LI, + ScalarEvolution *SE, DominatorTree *DT, + AssumptionCache *AC, OptimizationRemarkEmitter *ORE, + bool PreserveLCSSA, Loop **RemainderLoop = nullptr); bool UnrollRuntimeLoopRemainder(Loop *L, unsigned Count, bool AllowExpensiveTripCount, bool UseEpilogRemainder, bool UnrollRemainder, - LoopInfo *LI, ScalarEvolution *SE, - DominatorTree *DT, AssumptionCache *AC, - bool PreserveLCSSA, + bool ForgetAllSCEV, LoopInfo *LI, + ScalarEvolution *SE, DominatorTree *DT, + AssumptionCache *AC, bool PreserveLCSSA, Loop **ResultLoop = nullptr); void computePeelCount(Loop *L, unsigned LoopSize, @@ -109,9 +119,6 @@ bool computeUnrollCount(Loop *L, const TargetTransformInfo &TTI, TargetTransformInfo::UnrollingPreferences &UP, bool &UseUpperBound); -BasicBlock *foldBlockIntoPredecessor(BasicBlock *BB, LoopInfo *LI, - ScalarEvolution *SE, DominatorTree *DT); - void remapInstruction(Instruction *I, ValueToValueMapTy &VMap); void simplifyLoopAfterUnroll(Loop *L, bool SimplifyIVs, LoopInfo *LI, @@ -121,7 +128,8 @@ void simplifyLoopAfterUnroll(Loop *L, bool SimplifyIVs, LoopInfo *LI, MDNode *GetUnrollMetadata(MDNode *LoopID, StringRef Name); TargetTransformInfo::UnrollingPreferences gatherUnrollingPreferences( - Loop *L, ScalarEvolution &SE, const TargetTransformInfo &TTI, int OptLevel, + Loop *L, ScalarEvolution &SE, const TargetTransformInfo &TTI, + BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, int OptLevel, Optional UserThreshold, Optional UserCount, Optional UserAllowPartial, Optional UserRuntime, Optional UserUpperBound, Optional UserAllowPeeling); diff --git a/include/llvm/Transforms/Utils/VNCoercion.h b/include/llvm/Transforms/Utils/VNCoercion.h index 1baa9b66e491..f67b9ed0afdd 100644 --- a/include/llvm/Transforms/Utils/VNCoercion.h +++ b/include/llvm/Transforms/Utils/VNCoercion.h @@ -1,9 +1,8 @@ //===- VNCoercion.h - Value Numbering Coercion Utilities --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file / This file provides routines used by LLVM's value numbering passes to diff --git a/include/llvm/Transforms/Utils/ValueMapper.h b/include/llvm/Transforms/Utils/ValueMapper.h index 4ecb23ea1951..1952a210291e 100644 --- a/include/llvm/Transforms/Utils/ValueMapper.h +++ b/include/llvm/Transforms/Utils/ValueMapper.h @@ -1,9 +1,8 @@ //===- ValueMapper.h - Remapping for constants and metadata -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Vectorize.h b/include/llvm/Transforms/Vectorize.h index 70f9a2e0741b..88a0e49d0fae 100644 --- a/include/llvm/Transforms/Vectorize.h +++ b/include/llvm/Transforms/Vectorize.h @@ -1,9 +1,8 @@ //===-- Vectorize.h - Vectorization Transformations -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -110,8 +109,9 @@ struct VectorizeConfig { // // LoopVectorize - Create a loop vectorization pass. // -Pass *createLoopVectorizePass(bool InterleaveOnlyWhenForced = false, - bool VectorizeOnlyWhenForced = false); +Pass *createLoopVectorizePass(); +Pass *createLoopVectorizePass(bool InterleaveOnlyWhenForced, + bool VectorizeOnlyWhenForced); //===----------------------------------------------------------------------===// // diff --git a/include/llvm/Transforms/Vectorize/LoadStoreVectorizer.h b/include/llvm/Transforms/Vectorize/LoadStoreVectorizer.h index 6b37d7093c44..f72c76c6f0f2 100644 --- a/include/llvm/Transforms/Vectorize/LoadStoreVectorizer.h +++ b/include/llvm/Transforms/Vectorize/LoadStoreVectorizer.h @@ -1,9 +1,8 @@ //===- LoadStoreVectorizer.cpp - GPU Load & Store Vectorizer --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h b/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h index 5c7bba048607..b144006e2628 100644 --- a/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h +++ b/include/llvm/Transforms/Vectorize/LoopVectorizationLegality.h @@ -1,9 +1,8 @@ //===- llvm/Transforms/Vectorize/LoopVectorizationLegality.h ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -99,11 +98,7 @@ public: OptimizationRemarkEmitter &ORE); /// Mark the loop L as already vectorized by setting the width to 1. - void setAlreadyVectorized() { - IsVectorized.Value = 1; - Hint Hints[] = {IsVectorized}; - writeHintsToMetadata(Hints); - } + void setAlreadyVectorized(); bool allowVectorization(Function *F, Loop *L, bool VectorizeOnlyWhenForced) const; @@ -152,15 +147,6 @@ private: /// Checks string hint with one operand and set value if valid. void setHint(StringRef Name, Metadata *Arg); - /// Create a new hint from name / value pair. - MDNode *createHintMetadata(StringRef Name, unsigned V) const; - - /// Matches metadata with hint name. - bool matchesHintMetadataName(MDNode *Node, ArrayRef HintTypes); - - /// Sets current hints into loop metadata, keeping other values intact. - void writeHintsToMetadata(ArrayRef HintTypes); - /// The loop these hints belong to. const Loop *TheLoop; @@ -219,12 +205,13 @@ class LoopVectorizationLegality { public: LoopVectorizationLegality( Loop *L, PredicatedScalarEvolution &PSE, DominatorTree *DT, - TargetLibraryInfo *TLI, AliasAnalysis *AA, Function *F, - std::function *GetLAA, LoopInfo *LI, - OptimizationRemarkEmitter *ORE, LoopVectorizationRequirements *R, - LoopVectorizeHints *H, DemandedBits *DB, AssumptionCache *AC) - : TheLoop(L), LI(LI), PSE(PSE), TLI(TLI), DT(DT), GetLAA(GetLAA), - ORE(ORE), Requirements(R), Hints(H), DB(DB), AC(AC) {} + TargetTransformInfo *TTI, TargetLibraryInfo *TLI, AliasAnalysis *AA, + Function *F, std::function *GetLAA, + LoopInfo *LI, OptimizationRemarkEmitter *ORE, + LoopVectorizationRequirements *R, LoopVectorizeHints *H, DemandedBits *DB, + AssumptionCache *AC) + : TheLoop(L), LI(LI), PSE(PSE), TTI(TTI), TLI(TLI), DT(DT), + GetLAA(GetLAA), ORE(ORE), Requirements(R), Hints(H), DB(DB), AC(AC) {} /// ReductionList contains the reduction descriptors for all /// of the reductions that were found in the loop. @@ -385,18 +372,6 @@ private: void addInductionPhi(PHINode *Phi, const InductionDescriptor &ID, SmallPtrSetImpl &AllowedExit); - /// Create an analysis remark that explains why vectorization failed - /// - /// \p RemarkName is the identifier for the remark. If \p I is passed it is - /// an instruction that prevents vectorization. Otherwise the loop is used - /// for the location of the remark. \return the remark object that can be - /// streamed to. - OptimizationRemarkAnalysis - createMissedAnalysis(StringRef RemarkName, Instruction *I = nullptr) const { - return createLVMissedAnalysis(Hints->vectorizeAnalysisPassName(), - RemarkName, TheLoop, I); - } - /// If an access has a symbolic strides, this maps the pointer value to /// the stride symbol. const ValueToValueMap *getSymbolicStrides() { @@ -407,6 +382,14 @@ private: return LAI ? &LAI->getSymbolicStrides() : nullptr; } + /// Reports a vectorization illegality: print \p DebugMsg for debugging + /// purposes along with the corresponding optimization remark \p RemarkName. + /// If \p I is passed it is an instruction that prevents vectorization. + /// Otherwise the loop is used for the location of the remark. + void reportVectorizationFailure(const StringRef DebugMsg, + const StringRef OREMsg, const StringRef ORETag, + Instruction *I = nullptr) const; + /// The loop that we evaluate. Loop *TheLoop; @@ -420,6 +403,9 @@ private: /// unrolling. PredicatedScalarEvolution &PSE; + /// Target Transform Info. + TargetTransformInfo *TTI; + /// Target Library Info. TargetLibraryInfo *TLI; @@ -479,7 +465,7 @@ private: /// Used to emit an analysis of any legality issues. LoopVectorizeHints *Hints; - /// The demanded bits analsyis is used to compute the minimum type size in + /// The demanded bits analysis is used to compute the minimum type size in /// which a reduction can be computed. DemandedBits *DB; diff --git a/include/llvm/Transforms/Vectorize/LoopVectorize.h b/include/llvm/Transforms/Vectorize/LoopVectorize.h index d9c4f7b023c1..d1ec06afb02a 100644 --- a/include/llvm/Transforms/Vectorize/LoopVectorize.h +++ b/include/llvm/Transforms/Vectorize/LoopVectorize.h @@ -1,9 +1,8 @@ //===- LoopVectorize.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -72,19 +71,63 @@ class Loop; class LoopAccessInfo; class LoopInfo; class OptimizationRemarkEmitter; +class ProfileSummaryInfo; class ScalarEvolution; class TargetLibraryInfo; class TargetTransformInfo; +extern cl::opt EnableLoopInterleaving; +extern cl::opt EnableLoopVectorization; + +struct LoopVectorizeOptions { + /// If false, consider all loops for interleaving. + /// If true, only loops that explicitly request interleaving are considered. + bool InterleaveOnlyWhenForced; + + /// If false, consider all loops for vectorization. + /// If true, only loops that explicitly request vectorization are considered. + bool VectorizeOnlyWhenForced; + + /// The current defaults when creating the pass with no arguments are: + /// EnableLoopInterleaving = true and EnableLoopVectorization = true. This + /// means that interleaving default is consistent with the cl::opt flag, while + /// vectorization is not. + /// FIXME: The default for EnableLoopVectorization in the cl::opt should be + /// set to true, and the corresponding change to account for this be made in + /// opt.cpp. The initializations below will become: + /// InterleaveOnlyWhenForced(!EnableLoopInterleaving) + /// VectorizeOnlyWhenForced(!EnableLoopVectorization). + LoopVectorizeOptions() + : InterleaveOnlyWhenForced(false), VectorizeOnlyWhenForced(false) {} + LoopVectorizeOptions(bool InterleaveOnlyWhenForced, + bool VectorizeOnlyWhenForced) + : InterleaveOnlyWhenForced(InterleaveOnlyWhenForced), + VectorizeOnlyWhenForced(VectorizeOnlyWhenForced) {} + + LoopVectorizeOptions &setInterleaveOnlyWhenForced(bool Value) { + InterleaveOnlyWhenForced = Value; + return *this; + } + + LoopVectorizeOptions &setVectorizeOnlyWhenForced(bool Value) { + VectorizeOnlyWhenForced = Value; + return *this; + } +}; + /// The LoopVectorize Pass. struct LoopVectorizePass : public PassInfoMixin { /// If false, consider all loops for interleaving. /// If true, only loops that explicitly request interleaving are considered. - bool InterleaveOnlyWhenForced = false; + bool InterleaveOnlyWhenForced; /// If false, consider all loops for vectorization. /// If true, only loops that explicitly request vectorization are considered. - bool VectorizeOnlyWhenForced = false; + bool VectorizeOnlyWhenForced; + + LoopVectorizePass(LoopVectorizeOptions Opts = {}) + : InterleaveOnlyWhenForced(Opts.InterleaveOnlyWhenForced), + VectorizeOnlyWhenForced(Opts.VectorizeOnlyWhenForced) {} ScalarEvolution *SE; LoopInfo *LI; @@ -97,6 +140,7 @@ struct LoopVectorizePass : public PassInfoMixin { AssumptionCache *AC; std::function *GetLAA; OptimizationRemarkEmitter *ORE; + ProfileSummaryInfo *PSI; PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM); @@ -106,7 +150,7 @@ struct LoopVectorizePass : public PassInfoMixin { BlockFrequencyInfo &BFI_, TargetLibraryInfo *TLI_, DemandedBits &DB_, AliasAnalysis &AA_, AssumptionCache &AC_, std::function &GetLAA_, - OptimizationRemarkEmitter &ORE); + OptimizationRemarkEmitter &ORE_, ProfileSummaryInfo *PSI_); bool processLoop(Loop *L); }; diff --git a/include/llvm/Transforms/Vectorize/SLPVectorizer.h b/include/llvm/Transforms/Vectorize/SLPVectorizer.h index 3152e8192fc5..ac6afb761d4d 100644 --- a/include/llvm/Transforms/Vectorize/SLPVectorizer.h +++ b/include/llvm/Transforms/Vectorize/SLPVectorizer.h @@ -1,9 +1,8 @@ //===- SLPVectorizer.h ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This pass implements the Bottom Up SLP vectorizer. It detects consecutive @@ -56,6 +55,8 @@ class BoUpSLP; } // end namespace slpvectorizer +extern cl::opt RunSLPVectorization; + struct SLPVectorizerPass : public PassInfoMixin { using StoreList = SmallVector; using StoreListMap = MapVector; diff --git a/include/llvm/WindowsManifest/WindowsManifestMerger.h b/include/llvm/WindowsManifest/WindowsManifestMerger.h index 302d3705887b..935c930ad91d 100644 --- a/include/llvm/WindowsManifest/WindowsManifestMerger.h +++ b/include/llvm/WindowsManifest/WindowsManifestMerger.h @@ -1,9 +1,8 @@ //===-- WindowsManifestMerger.h ---------------------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===---------------------------------------------------------------------===// // diff --git a/include/llvm/WindowsResource/ResourceProcessor.h b/include/llvm/WindowsResource/ResourceProcessor.h index 4ca0a4b05bd0..4e99c05f4cd9 100644 --- a/include/llvm/WindowsResource/ResourceProcessor.h +++ b/include/llvm/WindowsResource/ResourceProcessor.h @@ -1,9 +1,8 @@ //===-- ResourceProcessor.h -------------------------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===---------------------------------------------------------------------===// diff --git a/include/llvm/WindowsResource/ResourceScriptToken.h b/include/llvm/WindowsResource/ResourceScriptToken.h index 494ae3222a4b..254121cd318a 100644 --- a/include/llvm/WindowsResource/ResourceScriptToken.h +++ b/include/llvm/WindowsResource/ResourceScriptToken.h @@ -1,9 +1,8 @@ //===-- ResourceScriptToken.h -----------------------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===---------------------------------------------------------------------===// // diff --git a/include/llvm/WindowsResource/ResourceScriptTokenList.h b/include/llvm/WindowsResource/ResourceScriptTokenList.h index 0beed117c3e7..6b44dccaa35f 100644 --- a/include/llvm/WindowsResource/ResourceScriptTokenList.h +++ b/include/llvm/WindowsResource/ResourceScriptTokenList.h @@ -1,9 +1,8 @@ //===-- ResourceScriptTokenList.h -------------------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===---------------------------------------------------------------------===// // diff --git a/include/llvm/XRay/BlockIndexer.h b/include/llvm/XRay/BlockIndexer.h index b42fa17f3fb7..dafd2b5a5230 100644 --- a/include/llvm/XRay/BlockIndexer.h +++ b/include/llvm/XRay/BlockIndexer.h @@ -1,9 +1,8 @@ //===- BlockIndexer.h - FDR Block Indexing Visitor ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/XRay/BlockPrinter.h b/include/llvm/XRay/BlockPrinter.h index bfb21e239517..949258085332 100644 --- a/include/llvm/XRay/BlockPrinter.h +++ b/include/llvm/XRay/BlockPrinter.h @@ -1,9 +1,8 @@ //===- BlockPrinter.h - FDR Block Pretty Printer -------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/XRay/BlockVerifier.h b/include/llvm/XRay/BlockVerifier.h index 46371c13891a..c848fdf084bc 100644 --- a/include/llvm/XRay/BlockVerifier.h +++ b/include/llvm/XRay/BlockVerifier.h @@ -1,9 +1,8 @@ //===- BlockVerifier.h - FDR Block Verifier -------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/XRay/FDRLogBuilder.h b/include/llvm/XRay/FDRLogBuilder.h index b5e9ed5c406b..3f49dc6dcb9d 100644 --- a/include/llvm/XRay/FDRLogBuilder.h +++ b/include/llvm/XRay/FDRLogBuilder.h @@ -1,9 +1,8 @@ //===- FDRLogBuilder.h - XRay FDR Log Building Utility --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef LLVM_INCLUDE_LLVM_XRAY_FDRLOGBUILDER_H_ diff --git a/include/llvm/XRay/FDRRecordConsumer.h b/include/llvm/XRay/FDRRecordConsumer.h index e856e1540558..4fbb1d41d0da 100644 --- a/include/llvm/XRay/FDRRecordConsumer.h +++ b/include/llvm/XRay/FDRRecordConsumer.h @@ -1,9 +1,8 @@ //===- FDRRecordConsumer.h - XRay Flight Data Recorder Mode Records -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef LLVM_INCLUDE_LLVM_XRAY_FDRRECORDCONSUMER_H_ diff --git a/include/llvm/XRay/FDRRecordProducer.h b/include/llvm/XRay/FDRRecordProducer.h index efdba2a67b7b..b530a85bc7e1 100644 --- a/include/llvm/XRay/FDRRecordProducer.h +++ b/include/llvm/XRay/FDRRecordProducer.h @@ -1,9 +1,8 @@ //===- FDRRecordProducer.h - XRay FDR Mode Record Producer ----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #ifndef LLVM_INCLUDE_LLVM_XRAY_FDRRECORDPRODUCER_H_ diff --git a/include/llvm/XRay/FDRRecords.h b/include/llvm/XRay/FDRRecords.h index 8a84f4d0c1fb..a8ce74bd88fb 100644 --- a/include/llvm/XRay/FDRRecords.h +++ b/include/llvm/XRay/FDRRecords.h @@ -1,9 +1,8 @@ //===- FDRRecords.h - XRay Flight Data Recorder Mode Records --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/XRay/FDRTraceExpander.h b/include/llvm/XRay/FDRTraceExpander.h index 02a21bed5ce9..f3c36cf4cf66 100644 --- a/include/llvm/XRay/FDRTraceExpander.h +++ b/include/llvm/XRay/FDRTraceExpander.h @@ -1,9 +1,8 @@ //===- FDRTraceExpander.h - XRay FDR Mode Log Expander --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/XRay/FDRTraceWriter.h b/include/llvm/XRay/FDRTraceWriter.h index 7b3b5fa25eff..2498877e27c1 100644 --- a/include/llvm/XRay/FDRTraceWriter.h +++ b/include/llvm/XRay/FDRTraceWriter.h @@ -1,9 +1,8 @@ //===- FDRTraceWriter.h - XRay FDR Trace Writer -----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/XRay/FileHeaderReader.h b/include/llvm/XRay/FileHeaderReader.h index 3b8809bdbb34..1c9681cfd9af 100644 --- a/include/llvm/XRay/FileHeaderReader.h +++ b/include/llvm/XRay/FileHeaderReader.h @@ -1,9 +1,8 @@ //===- FileHeaderReader.h - XRay Trace File Header Reading Function -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/XRay/Graph.h b/include/llvm/XRay/Graph.h index a4d34a8a4be3..004681512800 100644 --- a/include/llvm/XRay/Graph.h +++ b/include/llvm/XRay/Graph.h @@ -1,9 +1,8 @@ //===-- Graph.h - XRay Graph Class ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/XRay/InstrumentationMap.h b/include/llvm/XRay/InstrumentationMap.h index 42bfca36a20b..5cbe5c44893b 100644 --- a/include/llvm/XRay/InstrumentationMap.h +++ b/include/llvm/XRay/InstrumentationMap.h @@ -1,9 +1,8 @@ //===- InstrumentationMap.h - XRay Instrumentation Map ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/XRay/Profile.h b/include/llvm/XRay/Profile.h index 9365630358e8..79d9b53387f3 100644 --- a/include/llvm/XRay/Profile.h +++ b/include/llvm/XRay/Profile.h @@ -1,9 +1,8 @@ //===- Profile.h - XRay Profile Abstraction -------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/XRay/RecordPrinter.h b/include/llvm/XRay/RecordPrinter.h index 649c64ab6f5c..7c7b7a32c56d 100644 --- a/include/llvm/XRay/RecordPrinter.h +++ b/include/llvm/XRay/RecordPrinter.h @@ -1,9 +1,8 @@ //===- RecordPrinter.h - FDR Record Printer -------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/XRay/Trace.h b/include/llvm/XRay/Trace.h index 924addd1560d..eb1f03b2a0d4 100644 --- a/include/llvm/XRay/Trace.h +++ b/include/llvm/XRay/Trace.h @@ -1,9 +1,8 @@ //===- Trace.h - XRay Trace Abstraction -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/XRay/XRayRecord.h b/include/llvm/XRay/XRayRecord.h index 7685ec95838a..546b02303b6a 100644 --- a/include/llvm/XRay/XRayRecord.h +++ b/include/llvm/XRay/XRayRecord.h @@ -1,9 +1,8 @@ //===- XRayRecord.h - XRay Trace Record -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/XRay/YAMLXRayRecord.h b/include/llvm/XRay/YAMLXRayRecord.h index 6150196ed98d..bc8b03548d6e 100644 --- a/include/llvm/XRay/YAMLXRayRecord.h +++ b/include/llvm/XRay/YAMLXRayRecord.h @@ -1,9 +1,8 @@ //===- YAMLXRayRecord.h - XRay Record YAML Support Definitions ------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/include/llvm/module.modulemap b/include/llvm/module.modulemap index bcc12534ec85..9c4668e1473c 100644 --- a/include/llvm/module.modulemap +++ b/include/llvm/module.modulemap @@ -5,6 +5,7 @@ module LLVM_Analysis { // This is intended for (repeated) textual inclusion. textual header "Analysis/TargetLibraryInfo.def" + textual header "Analysis/VecFuncs.def" } module LLVM_AsmParser { @@ -53,6 +54,7 @@ module LLVM_BinaryFormat { textual header "BinaryFormat/Dwarf.def" textual header "BinaryFormat/DynamicTags.def" textual header "BinaryFormat/MachO.def" + textual header "BinaryFormat/MinidumpConstants.def" textual header "BinaryFormat/ELFRelocs/AArch64.def" textual header "BinaryFormat/ELFRelocs/AMDGPU.def" textual header "BinaryFormat/ELFRelocs/ARM.def" @@ -220,7 +222,7 @@ module LLVM_intrinsic_gen { module IR_ConstantRange { header "IR/ConstantRange.h" export * } module IR_Dominators { header "IR/Dominators.h" export * } module Analysis_PostDominators { header "Analysis/PostDominators.h" export * } - module IR_DomTreeUpdater { header "IR/DomTreeUpdater.h" export * } + module Analysis_DomTreeUpdater { header "Analysis/DomTreeUpdater.h" export * } module IR_IRBuilder { header "IR/IRBuilder.h" export * } module IR_PassManager { header "IR/PassManager.h" export * } module IR_PredIteratorCache { header "IR/PredIteratorCache.h" export * } @@ -235,6 +237,7 @@ module LLVM_intrinsic_gen { } module IR_IntrinsicInst { header "IR/IntrinsicInst.h" export * } module IR_PatternMatch { header "IR/PatternMatch.h" export * } + module IR_SafepointIRVerifier { header "IR/SafepointIRVerifier.h" export * } module IR_Statepoint { header "IR/Statepoint.h" export * } export * diff --git a/lib/Analysis/AliasAnalysis.cpp b/lib/Analysis/AliasAnalysis.cpp index 3446aef39938..32241e355eb8 100644 --- a/lib/Analysis/AliasAnalysis.cpp +++ b/lib/Analysis/AliasAnalysis.cpp @@ -1,9 +1,8 @@ //==- AliasAnalysis.cpp - Generic Alias Analysis Interface Implementation --==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -80,12 +79,16 @@ AAResults::~AAResults() { bool AAResults::invalidate(Function &F, const PreservedAnalyses &PA, FunctionAnalysisManager::Invalidator &Inv) { - // Check if the AA manager itself has been invalidated. + // AAResults preserves the AAManager by default, due to the stateless nature + // of AliasAnalysis. There is no need to check whether it has been preserved + // explicitly. Check if any module dependency was invalidated and caused the + // AAManager to be invalidated. Invalidate ourselves in that case. auto PAC = PA.getChecker(); - if (!PAC.preserved() && !PAC.preservedSet>()) - return true; // The manager needs to be blown away, clear everything. + if (!PAC.preservedWhenStateless()) + return true; - // Check all of the dependencies registered. + // Check if any of the function dependencies were invalidated, and invalidate + // ourselves in that case. for (AnalysisKey *ID : AADeps) if (Inv.invalidate(ID, F, PA)) return true; @@ -100,8 +103,14 @@ bool AAResults::invalidate(Function &F, const PreservedAnalyses &PA, AliasResult AAResults::alias(const MemoryLocation &LocA, const MemoryLocation &LocB) { + AAQueryInfo AAQIP; + return alias(LocA, LocB, AAQIP); +} + +AliasResult AAResults::alias(const MemoryLocation &LocA, + const MemoryLocation &LocB, AAQueryInfo &AAQI) { for (const auto &AA : AAs) { - auto Result = AA->alias(LocA, LocB); + auto Result = AA->alias(LocA, LocB, AAQI); if (Result != MayAlias) return Result; } @@ -110,8 +119,14 @@ AliasResult AAResults::alias(const MemoryLocation &LocA, bool AAResults::pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal) { + AAQueryInfo AAQIP; + return pointsToConstantMemory(Loc, AAQIP, OrLocal); +} + +bool AAResults::pointsToConstantMemory(const MemoryLocation &Loc, + AAQueryInfo &AAQI, bool OrLocal) { for (const auto &AA : AAs) - if (AA->pointsToConstantMemory(Loc, OrLocal)) + if (AA->pointsToConstantMemory(Loc, AAQI, OrLocal)) return true; return false; @@ -132,10 +147,16 @@ ModRefInfo AAResults::getArgModRefInfo(const CallBase *Call, unsigned ArgIdx) { } ModRefInfo AAResults::getModRefInfo(Instruction *I, const CallBase *Call2) { + AAQueryInfo AAQIP; + return getModRefInfo(I, Call2, AAQIP); +} + +ModRefInfo AAResults::getModRefInfo(Instruction *I, const CallBase *Call2, + AAQueryInfo &AAQI) { // We may have two calls. if (const auto *Call1 = dyn_cast(I)) { // Check if the two calls modify the same memory. - return getModRefInfo(Call1, Call2); + return getModRefInfo(Call1, Call2, AAQI); } else if (I->isFenceLike()) { // If this is a fence, just return ModRef. return ModRefInfo::ModRef; @@ -145,7 +166,7 @@ ModRefInfo AAResults::getModRefInfo(Instruction *I, const CallBase *Call2) { // is that if the call references what this instruction // defines, it must be clobbered by this location. const MemoryLocation DefLoc = MemoryLocation::get(I); - ModRefInfo MR = getModRefInfo(Call2, DefLoc); + ModRefInfo MR = getModRefInfo(Call2, DefLoc, AAQI); if (isModOrRefSet(MR)) return setModAndRef(MR); } @@ -154,10 +175,17 @@ ModRefInfo AAResults::getModRefInfo(Instruction *I, const CallBase *Call2) { ModRefInfo AAResults::getModRefInfo(const CallBase *Call, const MemoryLocation &Loc) { + AAQueryInfo AAQIP; + return getModRefInfo(Call, Loc, AAQIP); +} + +ModRefInfo AAResults::getModRefInfo(const CallBase *Call, + const MemoryLocation &Loc, + AAQueryInfo &AAQI) { ModRefInfo Result = ModRefInfo::ModRef; for (const auto &AA : AAs) { - Result = intersectModRef(Result, AA->getModRefInfo(Call, Loc)); + Result = intersectModRef(Result, AA->getModRefInfo(Call, Loc, AAQI)); // Early-exit the moment we reach the bottom of the lattice. if (isNoModRef(Result)) @@ -215,10 +243,16 @@ ModRefInfo AAResults::getModRefInfo(const CallBase *Call, ModRefInfo AAResults::getModRefInfo(const CallBase *Call1, const CallBase *Call2) { + AAQueryInfo AAQIP; + return getModRefInfo(Call1, Call2, AAQIP); +} + +ModRefInfo AAResults::getModRefInfo(const CallBase *Call1, + const CallBase *Call2, AAQueryInfo &AAQI) { ModRefInfo Result = ModRefInfo::ModRef; for (const auto &AA : AAs) { - Result = intersectModRef(Result, AA->getModRefInfo(Call1, Call2)); + Result = intersectModRef(Result, AA->getModRefInfo(Call1, Call2, AAQI)); // Early-exit the moment we reach the bottom of the lattice. if (isNoModRef(Result)) @@ -397,6 +431,12 @@ raw_ostream &llvm::operator<<(raw_ostream &OS, AliasResult AR) { ModRefInfo AAResults::getModRefInfo(const LoadInst *L, const MemoryLocation &Loc) { + AAQueryInfo AAQIP; + return getModRefInfo(L, Loc, AAQIP); +} +ModRefInfo AAResults::getModRefInfo(const LoadInst *L, + const MemoryLocation &Loc, + AAQueryInfo &AAQI) { // Be conservative in the face of atomic. if (isStrongerThan(L->getOrdering(), AtomicOrdering::Unordered)) return ModRefInfo::ModRef; @@ -404,7 +444,7 @@ ModRefInfo AAResults::getModRefInfo(const LoadInst *L, // If the load address doesn't alias the given address, it doesn't read // or write the specified memory. if (Loc.Ptr) { - AliasResult AR = alias(MemoryLocation::get(L), Loc); + AliasResult AR = alias(MemoryLocation::get(L), Loc, AAQI); if (AR == NoAlias) return ModRefInfo::NoModRef; if (AR == MustAlias) @@ -416,12 +456,18 @@ ModRefInfo AAResults::getModRefInfo(const LoadInst *L, ModRefInfo AAResults::getModRefInfo(const StoreInst *S, const MemoryLocation &Loc) { + AAQueryInfo AAQIP; + return getModRefInfo(S, Loc, AAQIP); +} +ModRefInfo AAResults::getModRefInfo(const StoreInst *S, + const MemoryLocation &Loc, + AAQueryInfo &AAQI) { // Be conservative in the face of atomic. if (isStrongerThan(S->getOrdering(), AtomicOrdering::Unordered)) return ModRefInfo::ModRef; if (Loc.Ptr) { - AliasResult AR = alias(MemoryLocation::get(S), Loc); + AliasResult AR = alias(MemoryLocation::get(S), Loc, AAQI); // If the store address cannot alias the pointer in question, then the // specified memory cannot be modified by the store. if (AR == NoAlias) @@ -429,7 +475,7 @@ ModRefInfo AAResults::getModRefInfo(const StoreInst *S, // If the pointer is a pointer to constant memory, then it could not have // been modified by this store. - if (pointsToConstantMemory(Loc)) + if (pointsToConstantMemory(Loc, AAQI)) return ModRefInfo::NoModRef; // If the store address aliases the pointer as must alias, set Must. @@ -442,17 +488,31 @@ ModRefInfo AAResults::getModRefInfo(const StoreInst *S, } ModRefInfo AAResults::getModRefInfo(const FenceInst *S, const MemoryLocation &Loc) { + AAQueryInfo AAQIP; + return getModRefInfo(S, Loc, AAQIP); +} + +ModRefInfo AAResults::getModRefInfo(const FenceInst *S, + const MemoryLocation &Loc, + AAQueryInfo &AAQI) { // If we know that the location is a constant memory location, the fence // cannot modify this location. - if (Loc.Ptr && pointsToConstantMemory(Loc)) + if (Loc.Ptr && pointsToConstantMemory(Loc, AAQI)) return ModRefInfo::Ref; return ModRefInfo::ModRef; } ModRefInfo AAResults::getModRefInfo(const VAArgInst *V, const MemoryLocation &Loc) { + AAQueryInfo AAQIP; + return getModRefInfo(V, Loc, AAQIP); +} + +ModRefInfo AAResults::getModRefInfo(const VAArgInst *V, + const MemoryLocation &Loc, + AAQueryInfo &AAQI) { if (Loc.Ptr) { - AliasResult AR = alias(MemoryLocation::get(V), Loc); + AliasResult AR = alias(MemoryLocation::get(V), Loc, AAQI); // If the va_arg address cannot alias the pointer in question, then the // specified memory cannot be accessed by the va_arg. if (AR == NoAlias) @@ -460,7 +520,7 @@ ModRefInfo AAResults::getModRefInfo(const VAArgInst *V, // If the pointer is a pointer to constant memory, then it could not have // been modified by this va_arg. - if (pointsToConstantMemory(Loc)) + if (pointsToConstantMemory(Loc, AAQI)) return ModRefInfo::NoModRef; // If the va_arg aliases the pointer as must alias, set Must. @@ -474,10 +534,17 @@ ModRefInfo AAResults::getModRefInfo(const VAArgInst *V, ModRefInfo AAResults::getModRefInfo(const CatchPadInst *CatchPad, const MemoryLocation &Loc) { + AAQueryInfo AAQIP; + return getModRefInfo(CatchPad, Loc, AAQIP); +} + +ModRefInfo AAResults::getModRefInfo(const CatchPadInst *CatchPad, + const MemoryLocation &Loc, + AAQueryInfo &AAQI) { if (Loc.Ptr) { // If the pointer is a pointer to constant memory, // then it could not have been modified by this catchpad. - if (pointsToConstantMemory(Loc)) + if (pointsToConstantMemory(Loc, AAQI)) return ModRefInfo::NoModRef; } @@ -487,10 +554,17 @@ ModRefInfo AAResults::getModRefInfo(const CatchPadInst *CatchPad, ModRefInfo AAResults::getModRefInfo(const CatchReturnInst *CatchRet, const MemoryLocation &Loc) { + AAQueryInfo AAQIP; + return getModRefInfo(CatchRet, Loc, AAQIP); +} + +ModRefInfo AAResults::getModRefInfo(const CatchReturnInst *CatchRet, + const MemoryLocation &Loc, + AAQueryInfo &AAQI) { if (Loc.Ptr) { // If the pointer is a pointer to constant memory, // then it could not have been modified by this catchpad. - if (pointsToConstantMemory(Loc)) + if (pointsToConstantMemory(Loc, AAQI)) return ModRefInfo::NoModRef; } @@ -500,12 +574,19 @@ ModRefInfo AAResults::getModRefInfo(const CatchReturnInst *CatchRet, ModRefInfo AAResults::getModRefInfo(const AtomicCmpXchgInst *CX, const MemoryLocation &Loc) { + AAQueryInfo AAQIP; + return getModRefInfo(CX, Loc, AAQIP); +} + +ModRefInfo AAResults::getModRefInfo(const AtomicCmpXchgInst *CX, + const MemoryLocation &Loc, + AAQueryInfo &AAQI) { // Acquire/Release cmpxchg has properties that matter for arbitrary addresses. if (isStrongerThanMonotonic(CX->getSuccessOrdering())) return ModRefInfo::ModRef; if (Loc.Ptr) { - AliasResult AR = alias(MemoryLocation::get(CX), Loc); + AliasResult AR = alias(MemoryLocation::get(CX), Loc, AAQI); // If the cmpxchg address does not alias the location, it does not access // it. if (AR == NoAlias) @@ -521,12 +602,19 @@ ModRefInfo AAResults::getModRefInfo(const AtomicCmpXchgInst *CX, ModRefInfo AAResults::getModRefInfo(const AtomicRMWInst *RMW, const MemoryLocation &Loc) { + AAQueryInfo AAQIP; + return getModRefInfo(RMW, Loc, AAQIP); +} + +ModRefInfo AAResults::getModRefInfo(const AtomicRMWInst *RMW, + const MemoryLocation &Loc, + AAQueryInfo &AAQI) { // Acquire/Release atomicrmw has properties that matter for arbitrary addresses. if (isStrongerThanMonotonic(RMW->getOrdering())) return ModRefInfo::ModRef; if (Loc.Ptr) { - AliasResult AR = alias(MemoryLocation::get(RMW), Loc); + AliasResult AR = alias(MemoryLocation::get(RMW), Loc, AAQI); // If the atomicrmw address does not alias the location, it does not access // it. if (AR == NoAlias) diff --git a/lib/Analysis/AliasAnalysisEvaluator.cpp b/lib/Analysis/AliasAnalysisEvaluator.cpp index 85dd4fe95b33..e83703867e09 100644 --- a/lib/Analysis/AliasAnalysisEvaluator.cpp +++ b/lib/Analysis/AliasAnalysisEvaluator.cpp @@ -1,9 +1,8 @@ //===- AliasAnalysisEvaluator.cpp - Alias Analysis Accuracy Evaluator -----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/Analysis/AliasAnalysisSummary.cpp b/lib/Analysis/AliasAnalysisSummary.cpp index 2b4879453beb..2f3396a44117 100644 --- a/lib/Analysis/AliasAnalysisSummary.cpp +++ b/lib/Analysis/AliasAnalysisSummary.cpp @@ -73,28 +73,28 @@ AliasAttrs getExternallyVisibleAttrs(AliasAttrs Attr) { } Optional instantiateInterfaceValue(InterfaceValue IValue, - CallSite CS) { + CallBase &Call) { auto Index = IValue.Index; - auto Value = (Index == 0) ? CS.getInstruction() : CS.getArgument(Index - 1); - if (Value->getType()->isPointerTy()) - return InstantiatedValue{Value, IValue.DerefLevel}; + auto *V = (Index == 0) ? &Call : Call.getArgOperand(Index - 1); + if (V->getType()->isPointerTy()) + return InstantiatedValue{V, IValue.DerefLevel}; return None; } Optional -instantiateExternalRelation(ExternalRelation ERelation, CallSite CS) { - auto From = instantiateInterfaceValue(ERelation.From, CS); +instantiateExternalRelation(ExternalRelation ERelation, CallBase &Call) { + auto From = instantiateInterfaceValue(ERelation.From, Call); if (!From) return None; - auto To = instantiateInterfaceValue(ERelation.To, CS); + auto To = instantiateInterfaceValue(ERelation.To, Call); if (!To) return None; return InstantiatedRelation{*From, *To, ERelation.Offset}; } Optional instantiateExternalAttribute(ExternalAttribute EAttr, - CallSite CS) { - auto Value = instantiateInterfaceValue(EAttr.IValue, CS); + CallBase &Call) { + auto Value = instantiateInterfaceValue(EAttr.IValue, Call); if (!Value) return None; return InstantiatedAttr{*Value, EAttr.Attr}; diff --git a/lib/Analysis/AliasAnalysisSummary.h b/lib/Analysis/AliasAnalysisSummary.h index fb93a12420f8..fe75b03cedef 100644 --- a/lib/Analysis/AliasAnalysisSummary.h +++ b/lib/Analysis/AliasAnalysisSummary.h @@ -1,9 +1,8 @@ //=====- CFLSummary.h - Abstract stratified sets implementation. --------=====// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -38,7 +37,7 @@ #include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/Optional.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/IR/CallSite.h" +#include "llvm/IR/InstrTypes.h" #include namespace llvm { @@ -196,12 +195,13 @@ struct AliasSummary { SmallVector RetParamAttributes; }; -/// This is the result of instantiating InterfaceValue at a particular callsite +/// This is the result of instantiating InterfaceValue at a particular call struct InstantiatedValue { Value *Val; unsigned DerefLevel; }; -Optional instantiateInterfaceValue(InterfaceValue, CallSite); +Optional instantiateInterfaceValue(InterfaceValue IValue, + CallBase &Call); inline bool operator==(InstantiatedValue LHS, InstantiatedValue RHS) { return LHS.Val == RHS.Val && LHS.DerefLevel == RHS.DerefLevel; @@ -229,8 +229,8 @@ struct InstantiatedRelation { InstantiatedValue From, To; int64_t Offset; }; -Optional instantiateExternalRelation(ExternalRelation, - CallSite); +Optional +instantiateExternalRelation(ExternalRelation ERelation, CallBase &Call); /// This is the result of instantiating ExternalAttribute at a particular /// callsite @@ -238,8 +238,8 @@ struct InstantiatedAttr { InstantiatedValue IValue; AliasAttrs Attr; }; -Optional instantiateExternalAttribute(ExternalAttribute, - CallSite); +Optional instantiateExternalAttribute(ExternalAttribute EAttr, + CallBase &Call); } template <> struct DenseMapInfo { diff --git a/lib/Analysis/AliasSetTracker.cpp b/lib/Analysis/AliasSetTracker.cpp index f6ad704cc914..a6e5b9fab558 100644 --- a/lib/Analysis/AliasSetTracker.cpp +++ b/lib/Analysis/AliasSetTracker.cpp @@ -1,9 +1,8 @@ //===- AliasSetTracker.cpp - Alias Sets Tracker implementation-------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,7 +13,9 @@ #include "llvm/Analysis/AliasSetTracker.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/GuardUtils.h" +#include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/MemoryLocation.h" +#include "llvm/Analysis/MemorySSA.h" #include "llvm/Config/llvm-config.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" @@ -127,24 +128,24 @@ void AliasSet::removeFromTracker(AliasSetTracker &AST) { void AliasSet::addPointer(AliasSetTracker &AST, PointerRec &Entry, LocationSize Size, const AAMDNodes &AAInfo, - bool KnownMustAlias) { + bool KnownMustAlias, bool SkipSizeUpdate) { assert(!Entry.hasAliasSet() && "Entry already in set!"); // Check to see if we have to downgrade to _may_ alias. - if (isMustAlias() && !KnownMustAlias) + if (isMustAlias()) if (PointerRec *P = getSomePointer()) { - AliasAnalysis &AA = AST.getAliasAnalysis(); - AliasResult Result = - AA.alias(MemoryLocation(P->getValue(), P->getSize(), P->getAAInfo()), - MemoryLocation(Entry.getValue(), Size, AAInfo)); - if (Result != MustAlias) { - Alias = SetMayAlias; - AST.TotalMayAliasSetSize += size(); - } else { - // First entry of must alias must have maximum size! + if (!KnownMustAlias) { + AliasAnalysis &AA = AST.getAliasAnalysis(); + AliasResult Result = AA.alias( + MemoryLocation(P->getValue(), P->getSize(), P->getAAInfo()), + MemoryLocation(Entry.getValue(), Size, AAInfo)); + if (Result != MustAlias) { + Alias = SetMayAlias; + AST.TotalMayAliasSetSize += size(); + } + assert(Result != NoAlias && "Cannot be part of must set!"); + } else if (!SkipSizeUpdate) P->updateSizeAndAAInfo(Size, AAInfo); - } - assert(Result != NoAlias && "Cannot be part of must set!"); } Entry.setAliasSet(this); @@ -184,14 +185,15 @@ void AliasSet::addUnknownInst(Instruction *I, AliasAnalysis &AA) { Access = ModRefAccess; } -/// aliasesPointer - Return true if the specified pointer "may" (or must) -/// alias one of the members in the set. +/// aliasesPointer - If the specified pointer "may" (or must) alias one of the +/// members in the set return the appropriate AliasResult. Otherwise return +/// NoAlias. /// -bool AliasSet::aliasesPointer(const Value *Ptr, LocationSize Size, - const AAMDNodes &AAInfo, - AliasAnalysis &AA) const { +AliasResult AliasSet::aliasesPointer(const Value *Ptr, LocationSize Size, + const AAMDNodes &AAInfo, + AliasAnalysis &AA) const { if (AliasAny) - return true; + return MayAlias; if (Alias == SetMustAlias) { assert(UnknownInsts.empty() && "Illegal must alias set!"); @@ -208,9 +210,10 @@ bool AliasSet::aliasesPointer(const Value *Ptr, LocationSize Size, // If this is a may-alias set, we have to check all of the pointers in the set // to be sure it doesn't alias the set... for (iterator I = begin(), E = end(); I != E; ++I) - if (AA.alias(MemoryLocation(Ptr, Size, AAInfo), - MemoryLocation(I.getPointer(), I.getSize(), I.getAAInfo()))) - return true; + if (AliasResult AR = AA.alias( + MemoryLocation(Ptr, Size, AAInfo), + MemoryLocation(I.getPointer(), I.getSize(), I.getAAInfo()))) + return AR; // Check the unknown instructions... if (!UnknownInsts.empty()) { @@ -218,10 +221,10 @@ bool AliasSet::aliasesPointer(const Value *Ptr, LocationSize Size, if (auto *Inst = getUnknownInst(i)) if (isModOrRefSet( AA.getModRefInfo(Inst, MemoryLocation(Ptr, Size, AAInfo)))) - return true; + return MayAlias; } - return false; + return NoAlias; } bool AliasSet::aliasesUnknownInst(const Instruction *Inst, @@ -288,25 +291,38 @@ void AliasSetTracker::clear() { AliasSets.clear(); } - /// mergeAliasSetsForPointer - Given a pointer, merge all alias sets that may /// alias the pointer. Return the unified set, or nullptr if no set that aliases -/// the pointer was found. +/// the pointer was found. MustAliasAll is updated to true/false if the pointer +/// is found to MustAlias all the sets it merged. AliasSet *AliasSetTracker::mergeAliasSetsForPointer(const Value *Ptr, LocationSize Size, - const AAMDNodes &AAInfo) { + const AAMDNodes &AAInfo, + bool &MustAliasAll) { AliasSet *FoundSet = nullptr; + AliasResult AllAR = MustAlias; for (iterator I = begin(), E = end(); I != E;) { iterator Cur = I++; - if (Cur->Forward || !Cur->aliasesPointer(Ptr, Size, AAInfo, AA)) continue; + if (Cur->Forward) + continue; + + AliasResult AR = Cur->aliasesPointer(Ptr, Size, AAInfo, AA); + if (AR == NoAlias) + continue; + + AllAR = + AliasResult(AllAR & AR); // Possible downgrade to May/Partial, even No - if (!FoundSet) { // If this is the first alias set ptr can go into. - FoundSet = &*Cur; // Remember it. - } else { // Otherwise, we must merge the sets. - FoundSet->mergeSetIn(*Cur, *this); // Merge in contents. + if (!FoundSet) { + // If this is the first alias set ptr can go into, remember it. + FoundSet = &*Cur; + } else { + // Otherwise, we must merge the sets. + FoundSet->mergeSetIn(*Cur, *this); } } + MustAliasAll = (AllAR == MustAlias); return FoundSet; } @@ -316,10 +332,13 @@ AliasSet *AliasSetTracker::findAliasSetForUnknownInst(Instruction *Inst) { iterator Cur = I++; if (Cur->Forward || !Cur->aliasesUnknownInst(Inst, AA)) continue; - if (!FoundSet) // If this is the first alias set ptr can go into. - FoundSet = &*Cur; // Remember it. - else // Otherwise, we must merge the sets. - FoundSet->mergeSetIn(*Cur, *this); // Merge in contents. + if (!FoundSet) { + // If this is the first alias set ptr can go into, remember it. + FoundSet = &*Cur; + } else { + // Otherwise, we must merge the sets. + FoundSet->mergeSetIn(*Cur, *this); + } } return FoundSet; } @@ -329,7 +348,7 @@ AliasSet &AliasSetTracker::getAliasSetFor(const MemoryLocation &MemLoc) { Value * const Pointer = const_cast(MemLoc.Ptr); const LocationSize Size = MemLoc.Size; const AAMDNodes &AAInfo = MemLoc.AATags; - + AliasSet::PointerRec &Entry = getEntryFor(Pointer); if (AliasAnyAS) { @@ -348,6 +367,7 @@ AliasSet &AliasSetTracker::getAliasSetFor(const MemoryLocation &MemLoc) { return *AliasAnyAS; } + bool MustAliasAll = false; // Check to see if the pointer is already known. if (Entry.hasAliasSet()) { // If the size changed, we may need to merge several alias sets. @@ -356,20 +376,21 @@ AliasSet &AliasSetTracker::getAliasSetFor(const MemoryLocation &MemLoc) { // is NoAlias, mergeAliasSetsForPointer(undef, ...) will not find the // the right set for undef, even if it exists. if (Entry.updateSizeAndAAInfo(Size, AAInfo)) - mergeAliasSetsForPointer(Pointer, Size, AAInfo); + mergeAliasSetsForPointer(Pointer, Size, AAInfo, MustAliasAll); // Return the set! return *Entry.getAliasSet(*this)->getForwardedTarget(*this); } - if (AliasSet *AS = mergeAliasSetsForPointer(Pointer, Size, AAInfo)) { + if (AliasSet *AS = + mergeAliasSetsForPointer(Pointer, Size, AAInfo, MustAliasAll)) { // Add it to the alias set it aliases. - AS->addPointer(*this, Entry, Size, AAInfo); + AS->addPointer(*this, Entry, Size, AAInfo, MustAliasAll); return *AS; } // Otherwise create a new alias set to hold the loaded pointer. AliasSets.push_back(new AliasSet()); - AliasSets.back().addPointer(*this, Entry, Size, AAInfo); + AliasSets.back().addPointer(*this, Entry, Size, AAInfo, true); return AliasSets.back(); } @@ -422,14 +443,12 @@ void AliasSetTracker::addUnknown(Instruction *Inst) { if (!Inst->mayReadOrWriteMemory()) return; // doesn't alias anything - AliasSet *AS = findAliasSetForUnknownInst(Inst); - if (AS) { + if (AliasSet *AS = findAliasSetForUnknownInst(Inst)) { AS->addUnknownInst(Inst, AA); return; } AliasSets.push_back(new AliasSet()); - AS = &AliasSets.back(); - AS->addUnknownInst(Inst, AA); + AliasSets.back().addUnknownInst(Inst, AA); } void AliasSetTracker::add(Instruction *I) { @@ -516,6 +535,15 @@ void AliasSetTracker::add(const AliasSetTracker &AST) { } } +void AliasSetTracker::addAllInstructionsInLoopUsingMSSA() { + assert(MSSA && L && "MSSA and L must be available"); + for (const BasicBlock *BB : L->blocks()) + if (auto *Accesses = MSSA->getBlockAccesses(BB)) + for (auto &Access : *Accesses) + if (auto *MUD = dyn_cast(&Access)) + add(MUD->getMemoryInst()); +} + // deleteValue method - This method is used to remove a pointer value from the // AliasSetTracker entirely. It should be used when an instruction is deleted // from the program to update the AST. If you don't use this, you would have @@ -563,9 +591,8 @@ void AliasSetTracker::copyValue(Value *From, Value *To) { I = PointerMap.find_as(From); // Add it to the alias set it aliases... AliasSet *AS = I->second->getAliasSet(*this); - AS->addPointer(*this, Entry, I->second->getSize(), - I->second->getAAInfo(), - true); + AS->addPointer(*this, Entry, I->second->getSize(), I->second->getAAInfo(), + true, true); } AliasSet &AliasSetTracker::mergeAllAliasSets() { diff --git a/lib/Analysis/Analysis.cpp b/lib/Analysis/Analysis.cpp index bb8742123a0f..d46a8d8e306c 100644 --- a/lib/Analysis/Analysis.cpp +++ b/lib/Analysis/Analysis.cpp @@ -1,9 +1,8 @@ //===-- Analysis.cpp ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/Analysis/AssumptionCache.cpp b/lib/Analysis/AssumptionCache.cpp index 8bfd24ccf77b..cf2f845dee0a 100644 --- a/lib/Analysis/AssumptionCache.cpp +++ b/lib/Analysis/AssumptionCache.cpp @@ -1,9 +1,8 @@ //===- AssumptionCache.cpp - Cache finding @llvm.assume calls -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -54,11 +53,11 @@ AssumptionCache::getOrInsertAffectedValues(Value *V) { return AVIP.first->second; } -void AssumptionCache::updateAffectedValues(CallInst *CI) { +static void findAffectedValues(CallInst *CI, + SmallVectorImpl &Affected) { // Note: This code must be kept in-sync with the code in // computeKnownBitsFromAssume in ValueTracking. - SmallVector Affected; auto AddAffected = [&Affected](Value *V) { if (isa(V)) { Affected.push_back(V); @@ -109,6 +108,11 @@ void AssumptionCache::updateAffectedValues(CallInst *CI) { AddAffectedFromEq(B); } } +} + +void AssumptionCache::updateAffectedValues(CallInst *CI) { + SmallVector Affected; + findAffectedValues(CI, Affected); for (auto &AV : Affected) { auto &AVV = getOrInsertAffectedValues(AV); @@ -117,6 +121,18 @@ void AssumptionCache::updateAffectedValues(CallInst *CI) { } } +void AssumptionCache::unregisterAssumption(CallInst *CI) { + SmallVector Affected; + findAffectedValues(CI, Affected); + + for (auto &AV : Affected) { + auto AVI = AffectedValues.find_as(AV); + if (AVI != AffectedValues.end()) + AffectedValues.erase(AVI); + } + remove_if(AssumeHandles, [CI](WeakTrackingVH &VH) { return CI == VH; }); +} + void AssumptionCache::AffectedValueCallbackVH::deleted() { auto AVI = AC->AffectedValues.find(getValPtr()); if (AVI != AC->AffectedValues.end()) @@ -241,6 +257,13 @@ AssumptionCache &AssumptionCacheTracker::getAssumptionCache(Function &F) { return *IP.first->second; } +AssumptionCache *AssumptionCacheTracker::lookupAssumptionCache(Function &F) { + auto I = AssumptionCaches.find_as(&F); + if (I != AssumptionCaches.end()) + return I->second.get(); + return nullptr; +} + void AssumptionCacheTracker::verifyAnalysis() const { // FIXME: In the long term the verifier should not be controllable with a // flag. We should either fix all passes to correctly update the assumption diff --git a/lib/Analysis/BasicAliasAnalysis.cpp b/lib/Analysis/BasicAliasAnalysis.cpp index 332eeaa00e73..3721c99883b8 100644 --- a/lib/Analysis/BasicAliasAnalysis.cpp +++ b/lib/Analysis/BasicAliasAnalysis.cpp @@ -1,9 +1,8 @@ //===- BasicAliasAnalysis.cpp - Stateless Alias Analysis Impl -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -117,25 +116,44 @@ bool BasicAAResult::invalidate(Function &Fn, const PreservedAnalyses &PA, /// Returns true if the pointer is to a function-local object that never /// escapes from the function. -static bool isNonEscapingLocalObject(const Value *V) { +static bool isNonEscapingLocalObject( + const Value *V, + SmallDenseMap *IsCapturedCache = nullptr) { + SmallDenseMap::iterator CacheIt; + if (IsCapturedCache) { + bool Inserted; + std::tie(CacheIt, Inserted) = IsCapturedCache->insert({V, false}); + if (!Inserted) + // Found cached result, return it! + return CacheIt->second; + } + // If this is a local allocation, check to see if it escapes. - if (isa(V) || isNoAliasCall(V)) + if (isa(V) || isNoAliasCall(V)) { // Set StoreCaptures to True so that we can assume in our callers that the // pointer is not the result of a load instruction. Currently // PointerMayBeCaptured doesn't have any special analysis for the // StoreCaptures=false case; if it did, our callers could be refined to be // more precise. - return !PointerMayBeCaptured(V, false, /*StoreCaptures=*/true); + auto Ret = !PointerMayBeCaptured(V, false, /*StoreCaptures=*/true); + if (IsCapturedCache) + CacheIt->second = Ret; + return Ret; + } // If this is an argument that corresponds to a byval or noalias argument, // then it has not escaped before entering the function. Check if it escapes // inside the function. if (const Argument *A = dyn_cast(V)) - if (A->hasByValAttr() || A->hasNoAliasAttr()) + if (A->hasByValAttr() || A->hasNoAliasAttr()) { // Note even if the argument is marked nocapture, we still need to check // for copies made inside the function. The nocapture attribute only // specifies that there are no copies made that outlive the function. - return !PointerMayBeCaptured(V, false, /*StoreCaptures=*/true); + auto Ret = !PointerMayBeCaptured(V, false, /*StoreCaptures=*/true); + if (IsCapturedCache) + CacheIt->second = Ret; + return Ret; + } return false; } @@ -613,7 +631,7 @@ bool BasicAAResult::DecomposeGEPExpression(const Value *V, /// the function, with global constants being considered local to all /// functions. bool BasicAAResult::pointsToConstantMemory(const MemoryLocation &Loc, - bool OrLocal) { + AAQueryInfo &AAQI, bool OrLocal) { assert(Visited.empty() && "Visited must be cleared after use!"); unsigned MaxLookup = 8; @@ -623,7 +641,7 @@ bool BasicAAResult::pointsToConstantMemory(const MemoryLocation &Loc, const Value *V = GetUnderlyingObject(Worklist.pop_back_val(), DL); if (!Visited.insert(V).second) { Visited.clear(); - return AAResultBase::pointsToConstantMemory(Loc, OrLocal); + return AAResultBase::pointsToConstantMemory(Loc, AAQI, OrLocal); } // An alloca instruction defines local memory. @@ -637,7 +655,7 @@ bool BasicAAResult::pointsToConstantMemory(const MemoryLocation &Loc, // others. GV may even be a declaration, not a definition. if (!GV->isConstant()) { Visited.clear(); - return AAResultBase::pointsToConstantMemory(Loc, OrLocal); + return AAResultBase::pointsToConstantMemory(Loc, AAQI, OrLocal); } continue; } @@ -655,7 +673,7 @@ bool BasicAAResult::pointsToConstantMemory(const MemoryLocation &Loc, // Don't bother inspecting phi nodes with many operands. if (PN->getNumIncomingValues() > MaxLookup) { Visited.clear(); - return AAResultBase::pointsToConstantMemory(Loc, OrLocal); + return AAResultBase::pointsToConstantMemory(Loc, AAQI, OrLocal); } for (Value *IncValue : PN->incoming_values()) Worklist.push_back(IncValue); @@ -664,7 +682,7 @@ bool BasicAAResult::pointsToConstantMemory(const MemoryLocation &Loc, // Otherwise be conservative. Visited.clear(); - return AAResultBase::pointsToConstantMemory(Loc, OrLocal); + return AAResultBase::pointsToConstantMemory(Loc, AAQI, OrLocal); } while (!Worklist.empty() && --MaxLookup); Visited.clear(); @@ -799,24 +817,25 @@ static bool notDifferentParent(const Value *O1, const Value *O2) { #endif AliasResult BasicAAResult::alias(const MemoryLocation &LocA, - const MemoryLocation &LocB) { + const MemoryLocation &LocB, + AAQueryInfo &AAQI) { assert(notDifferentParent(LocA.Ptr, LocB.Ptr) && "BasicAliasAnalysis doesn't support interprocedural queries."); // If we have a directly cached entry for these locations, we have recursed // through this once, so just return the cached results. Notably, when this // happens, we don't clear the cache. - auto CacheIt = AliasCache.find(LocPair(LocA, LocB)); - if (CacheIt != AliasCache.end()) + auto CacheIt = AAQI.AliasCache.find(AAQueryInfo::LocPair(LocA, LocB)); + if (CacheIt != AAQI.AliasCache.end()) + return CacheIt->second; + + CacheIt = AAQI.AliasCache.find(AAQueryInfo::LocPair(LocB, LocA)); + if (CacheIt != AAQI.AliasCache.end()) return CacheIt->second; AliasResult Alias = aliasCheck(LocA.Ptr, LocA.Size, LocA.AATags, LocB.Ptr, - LocB.Size, LocB.AATags); - // AliasCache rarely has more than 1 or 2 elements, always use - // shrink_and_clear so it quickly returns to the inline capacity of the - // SmallDenseMap if it ever grows larger. - // FIXME: This should really be shrink_to_inline_capacity_and_clear(). - AliasCache.shrink_and_clear(); + LocB.Size, LocB.AATags, AAQI); + VisitedPhiBBs.clear(); return Alias; } @@ -828,7 +847,8 @@ AliasResult BasicAAResult::alias(const MemoryLocation &LocA, /// say much about this query. We do, however, use simple "address taken" /// analysis on local objects. ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call, - const MemoryLocation &Loc) { + const MemoryLocation &Loc, + AAQueryInfo &AAQI) { assert(notDifferentParent(Call, Loc.Ptr) && "AliasAnalysis query involving multiple functions!"); @@ -855,7 +875,7 @@ ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call, // then the call can not mod/ref the pointer unless the call takes the pointer // as an argument, and itself doesn't capture it. if (!isa(Object) && Call != Object && - isNonEscapingLocalObject(Object)) { + isNonEscapingLocalObject(Object, &AAQI.IsCapturedCache)) { // Optimistically assume that call doesn't touch Object and check this // assumption in the following loop. @@ -881,11 +901,11 @@ ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call, // If this is a no-capture pointer argument, see if we can tell that it // is impossible to alias the pointer we're checking. - AliasResult AR = - getBestAAResults().alias(MemoryLocation(*CI), MemoryLocation(Object)); + AliasResult AR = getBestAAResults().alias(MemoryLocation(*CI), + MemoryLocation(Object), AAQI); if (AR != MustAlias) IsMustAlias = false; - // Operand doesnt alias 'Object', continue looking for other aliases + // Operand doesn't alias 'Object', continue looking for other aliases if (AR == NoAlias) continue; // Operand aliases 'Object', but call doesn't modify it. Strengthen @@ -928,7 +948,7 @@ ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call, if (isMallocOrCallocLikeFn(Call, &TLI)) { // Be conservative if the accessed pointer may alias the allocation - // fallback to the generic handling below. - if (getBestAAResults().alias(MemoryLocation(Call), Loc) == NoAlias) + if (getBestAAResults().alias(MemoryLocation(Call), Loc, AAQI) == NoAlias) return ModRefInfo::NoModRef; } @@ -940,11 +960,11 @@ ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call, AliasResult SrcAA, DestAA; if ((SrcAA = getBestAAResults().alias(MemoryLocation::getForSource(Inst), - Loc)) == MustAlias) + Loc, AAQI)) == MustAlias) // Loc is exactly the memcpy source thus disjoint from memcpy dest. return ModRefInfo::Ref; if ((DestAA = getBestAAResults().alias(MemoryLocation::getForDest(Inst), - Loc)) == MustAlias) + Loc, AAQI)) == MustAlias) // The converse case. return ModRefInfo::Mod; @@ -1000,11 +1020,12 @@ ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call, return ModRefInfo::Ref; // The AAResultBase base class has some smarts, lets use them. - return AAResultBase::getModRefInfo(Call, Loc); + return AAResultBase::getModRefInfo(Call, Loc, AAQI); } ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call1, - const CallBase *Call2) { + const CallBase *Call2, + AAQueryInfo &AAQI) { // While the assume intrinsic is marked as arbitrarily writing so that // proper control dependencies will be maintained, it never aliases any // particular memory location. @@ -1020,7 +1041,7 @@ ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call1, // heap state at the point the guard is issued needs to be consistent in case // the guard invokes the "deopt" continuation. - // NB! This function is *not* commutative, so we specical case two + // NB! This function is *not* commutative, so we special case two // possibilities for guard intrinsics. if (isIntrinsicCall(Call1, Intrinsic::experimental_guard)) @@ -1034,7 +1055,7 @@ ModRefInfo BasicAAResult::getModRefInfo(const CallBase *Call1, : ModRefInfo::NoModRef; // The AAResultBase base class has some smarts, lets use them. - return AAResultBase::getModRefInfo(Call1, Call2); + return AAResultBase::getModRefInfo(Call1, Call2, AAQI); } /// Provide ad-hoc rules to disambiguate accesses through two GEP operators, @@ -1266,11 +1287,10 @@ bool BasicAAResult::isGEPBaseAtNegativeOffset(const GEPOperator *GEPOp, /// We know that V1 is a GEP, but we don't know anything about V2. /// UnderlyingV1 is GetUnderlyingObject(GEP1, DL), UnderlyingV2 is the same for /// V2. -AliasResult -BasicAAResult::aliasGEP(const GEPOperator *GEP1, LocationSize V1Size, - const AAMDNodes &V1AAInfo, const Value *V2, - LocationSize V2Size, const AAMDNodes &V2AAInfo, - const Value *UnderlyingV1, const Value *UnderlyingV2) { +AliasResult BasicAAResult::aliasGEP( + const GEPOperator *GEP1, LocationSize V1Size, const AAMDNodes &V1AAInfo, + const Value *V2, LocationSize V2Size, const AAMDNodes &V2AAInfo, + const Value *UnderlyingV1, const Value *UnderlyingV2, AAQueryInfo &AAQI) { DecomposedGEP DecompGEP1, DecompGEP2; unsigned MaxPointerSize = getMaxPointerSize(DL); DecompGEP1.StructOffset = DecompGEP1.OtherOffset = APInt(MaxPointerSize, 0); @@ -1306,14 +1326,14 @@ BasicAAResult::aliasGEP(const GEPOperator *GEP1, LocationSize V1Size, // Do the base pointers alias? AliasResult BaseAlias = aliasCheck(UnderlyingV1, LocationSize::unknown(), AAMDNodes(), - UnderlyingV2, LocationSize::unknown(), AAMDNodes()); + UnderlyingV2, LocationSize::unknown(), AAMDNodes(), AAQI); // Check for geps of non-aliasing underlying pointers where the offsets are // identical. if ((BaseAlias == MayAlias) && V1Size == V2Size) { // Do the base pointers alias assuming type and size. - AliasResult PreciseBaseAlias = aliasCheck(UnderlyingV1, V1Size, V1AAInfo, - UnderlyingV2, V2Size, V2AAInfo); + AliasResult PreciseBaseAlias = aliasCheck( + UnderlyingV1, V1Size, V1AAInfo, UnderlyingV2, V2Size, V2AAInfo, AAQI); if (PreciseBaseAlias == NoAlias) { // See if the computed offset from the common pointer tells us about the // relation of the resulting pointer. @@ -1368,9 +1388,9 @@ BasicAAResult::aliasGEP(const GEPOperator *GEP1, LocationSize V1Size, if (V1Size == LocationSize::unknown() && V2Size == LocationSize::unknown()) return MayAlias; - AliasResult R = - aliasCheck(UnderlyingV1, LocationSize::unknown(), AAMDNodes(), V2, - LocationSize::unknown(), V2AAInfo, nullptr, UnderlyingV2); + AliasResult R = aliasCheck(UnderlyingV1, LocationSize::unknown(), + AAMDNodes(), V2, LocationSize::unknown(), + V2AAInfo, AAQI, nullptr, UnderlyingV2); if (R != MustAlias) { // If V2 may alias GEP base pointer, conservatively returns MayAlias. // If V2 is known not to alias GEP base pointer, then the two values @@ -1504,37 +1524,35 @@ static AliasResult MergeAliasResults(AliasResult A, AliasResult B) { /// Provides a bunch of ad-hoc rules to disambiguate a Select instruction /// against another. -AliasResult BasicAAResult::aliasSelect(const SelectInst *SI, - LocationSize SISize, - const AAMDNodes &SIAAInfo, - const Value *V2, LocationSize V2Size, - const AAMDNodes &V2AAInfo, - const Value *UnderV2) { +AliasResult +BasicAAResult::aliasSelect(const SelectInst *SI, LocationSize SISize, + const AAMDNodes &SIAAInfo, const Value *V2, + LocationSize V2Size, const AAMDNodes &V2AAInfo, + const Value *UnderV2, AAQueryInfo &AAQI) { // If the values are Selects with the same condition, we can do a more precise // check: just check for aliases between the values on corresponding arms. if (const SelectInst *SI2 = dyn_cast(V2)) if (SI->getCondition() == SI2->getCondition()) { - AliasResult Alias = aliasCheck(SI->getTrueValue(), SISize, SIAAInfo, - SI2->getTrueValue(), V2Size, V2AAInfo); + AliasResult Alias = + aliasCheck(SI->getTrueValue(), SISize, SIAAInfo, SI2->getTrueValue(), + V2Size, V2AAInfo, AAQI); if (Alias == MayAlias) return MayAlias; AliasResult ThisAlias = aliasCheck(SI->getFalseValue(), SISize, SIAAInfo, - SI2->getFalseValue(), V2Size, V2AAInfo); + SI2->getFalseValue(), V2Size, V2AAInfo, AAQI); return MergeAliasResults(ThisAlias, Alias); } // If both arms of the Select node NoAlias or MustAlias V2, then returns // NoAlias / MustAlias. Otherwise, returns MayAlias. - AliasResult Alias = - aliasCheck(V2, V2Size, V2AAInfo, SI->getTrueValue(), - SISize, SIAAInfo, UnderV2); + AliasResult Alias = aliasCheck(V2, V2Size, V2AAInfo, SI->getTrueValue(), + SISize, SIAAInfo, AAQI, UnderV2); if (Alias == MayAlias) return MayAlias; - AliasResult ThisAlias = - aliasCheck(V2, V2Size, V2AAInfo, SI->getFalseValue(), SISize, SIAAInfo, - UnderV2); + AliasResult ThisAlias = aliasCheck(V2, V2Size, V2AAInfo, SI->getFalseValue(), + SISize, SIAAInfo, AAQI, UnderV2); return MergeAliasResults(ThisAlias, Alias); } @@ -1544,7 +1562,7 @@ AliasResult BasicAAResult::aliasPHI(const PHINode *PN, LocationSize PNSize, const AAMDNodes &PNAAInfo, const Value *V2, LocationSize V2Size, const AAMDNodes &V2AAInfo, - const Value *UnderV2) { + const Value *UnderV2, AAQueryInfo &AAQI) { // Track phi nodes we have visited. We use this information when we determine // value equivalence. VisitedPhiBBs.insert(PN->getParent()); @@ -1554,8 +1572,8 @@ AliasResult BasicAAResult::aliasPHI(const PHINode *PN, LocationSize PNSize, // on corresponding edges. if (const PHINode *PN2 = dyn_cast(V2)) if (PN2->getParent() == PN->getParent()) { - LocPair Locs(MemoryLocation(PN, PNSize, PNAAInfo), - MemoryLocation(V2, V2Size, V2AAInfo)); + AAQueryInfo::LocPair Locs(MemoryLocation(PN, PNSize, PNAAInfo), + MemoryLocation(V2, V2Size, V2AAInfo)); if (PN > V2) std::swap(Locs.first, Locs.second); // Analyse the PHIs' inputs under the assumption that the PHIs are @@ -1566,25 +1584,33 @@ AliasResult BasicAAResult::aliasPHI(const PHINode *PN, LocationSize PNSize, // that causes a MayAlias. // Pretend the phis do not alias. AliasResult Alias = NoAlias; - assert(AliasCache.count(Locs) && - "There must exist an entry for the phi node"); - AliasResult OrigAliasResult = AliasCache[Locs]; - AliasCache[Locs] = NoAlias; + AliasResult OrigAliasResult; + { + // Limited lifetime iterator invalidated by the aliasCheck call below. + auto CacheIt = AAQI.AliasCache.find(Locs); + assert((CacheIt != AAQI.AliasCache.end()) && + "There must exist an entry for the phi node"); + OrigAliasResult = CacheIt->second; + CacheIt->second = NoAlias; + } for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i) { AliasResult ThisAlias = aliasCheck(PN->getIncomingValue(i), PNSize, PNAAInfo, PN2->getIncomingValueForBlock(PN->getIncomingBlock(i)), - V2Size, V2AAInfo); + V2Size, V2AAInfo, AAQI); Alias = MergeAliasResults(ThisAlias, Alias); if (Alias == MayAlias) break; } // Reset if speculation failed. - if (Alias != NoAlias) - AliasCache[Locs] = OrigAliasResult; - + if (Alias != NoAlias) { + auto Pair = + AAQI.AliasCache.insert(std::make_pair(Locs, OrigAliasResult)); + assert(!Pair.second && "Entry must have existed"); + Pair.first->second = OrigAliasResult; + } return Alias; } @@ -1658,9 +1684,8 @@ AliasResult BasicAAResult::aliasPHI(const PHINode *PN, LocationSize PNSize, if (isRecursive) PNSize = LocationSize::unknown(); - AliasResult Alias = - aliasCheck(V2, V2Size, V2AAInfo, V1Srcs[0], - PNSize, PNAAInfo, UnderV2); + AliasResult Alias = aliasCheck(V2, V2Size, V2AAInfo, V1Srcs[0], PNSize, + PNAAInfo, AAQI, UnderV2); // Early exit if the check of the first PHI source against V2 is MayAlias. // Other results are not possible. @@ -1673,7 +1698,7 @@ AliasResult BasicAAResult::aliasPHI(const PHINode *PN, LocationSize PNSize, Value *V = V1Srcs[i]; AliasResult ThisAlias = - aliasCheck(V2, V2Size, V2AAInfo, V, PNSize, PNAAInfo, UnderV2); + aliasCheck(V2, V2Size, V2AAInfo, V, PNSize, PNAAInfo, AAQI, UnderV2); Alias = MergeAliasResults(ThisAlias, Alias); if (Alias == MayAlias) break; @@ -1687,7 +1712,8 @@ AliasResult BasicAAResult::aliasPHI(const PHINode *PN, LocationSize PNSize, AliasResult BasicAAResult::aliasCheck(const Value *V1, LocationSize V1Size, AAMDNodes V1AAInfo, const Value *V2, LocationSize V2Size, AAMDNodes V2AAInfo, - const Value *O1, const Value *O2) { + AAQueryInfo &AAQI, const Value *O1, + const Value *O2) { // If either of the memory references is empty, it doesn't matter what the // pointer values are. if (V1Size.isZero() || V2Size.isZero()) @@ -1755,9 +1781,11 @@ AliasResult BasicAAResult::aliasCheck(const Value *V1, LocationSize V1Size, // temporary store the nocapture argument's value in a temporary memory // location if that memory location doesn't escape. Or it may pass a // nocapture value to other functions as long as they don't capture it. - if (isEscapeSource(O1) && isNonEscapingLocalObject(O2)) + if (isEscapeSource(O1) && + isNonEscapingLocalObject(O2, &AAQI.IsCapturedCache)) return NoAlias; - if (isEscapeSource(O2) && isNonEscapingLocalObject(O1)) + if (isEscapeSource(O2) && + isNonEscapingLocalObject(O1, &AAQI.IsCapturedCache)) return NoAlias; } @@ -1772,12 +1800,12 @@ AliasResult BasicAAResult::aliasCheck(const Value *V1, LocationSize V1Size, // Check the cache before climbing up use-def chains. This also terminates // otherwise infinitely recursive queries. - LocPair Locs(MemoryLocation(V1, V1Size, V1AAInfo), - MemoryLocation(V2, V2Size, V2AAInfo)); + AAQueryInfo::LocPair Locs(MemoryLocation(V1, V1Size, V1AAInfo), + MemoryLocation(V2, V2Size, V2AAInfo)); if (V1 > V2) std::swap(Locs.first, Locs.second); - std::pair Pair = - AliasCache.insert(std::make_pair(Locs, MayAlias)); + std::pair Pair = + AAQI.AliasCache.try_emplace(Locs, MayAlias); if (!Pair.second) return Pair.first->second; @@ -1791,9 +1819,13 @@ AliasResult BasicAAResult::aliasCheck(const Value *V1, LocationSize V1Size, } if (const GEPOperator *GV1 = dyn_cast(V1)) { AliasResult Result = - aliasGEP(GV1, V1Size, V1AAInfo, V2, V2Size, V2AAInfo, O1, O2); - if (Result != MayAlias) - return AliasCache[Locs] = Result; + aliasGEP(GV1, V1Size, V1AAInfo, V2, V2Size, V2AAInfo, O1, O2, AAQI); + if (Result != MayAlias) { + auto ItInsPair = AAQI.AliasCache.insert(std::make_pair(Locs, Result)); + assert(!ItInsPair.second && "Entry must have existed"); + ItInsPair.first->second = Result; + return Result; + } } if (isa(V2) && !isa(V1)) { @@ -1803,10 +1835,13 @@ AliasResult BasicAAResult::aliasCheck(const Value *V1, LocationSize V1Size, std::swap(V1AAInfo, V2AAInfo); } if (const PHINode *PN = dyn_cast(V1)) { - AliasResult Result = aliasPHI(PN, V1Size, V1AAInfo, - V2, V2Size, V2AAInfo, O2); - if (Result != MayAlias) - return AliasCache[Locs] = Result; + AliasResult Result = + aliasPHI(PN, V1Size, V1AAInfo, V2, V2Size, V2AAInfo, O2, AAQI); + if (Result != MayAlias) { + Pair = AAQI.AliasCache.try_emplace(Locs, Result); + assert(!Pair.second && "Entry must have existed"); + return Pair.first->second = Result; + } } if (isa(V2) && !isa(V1)) { @@ -1817,9 +1852,12 @@ AliasResult BasicAAResult::aliasCheck(const Value *V1, LocationSize V1Size, } if (const SelectInst *S1 = dyn_cast(V1)) { AliasResult Result = - aliasSelect(S1, V1Size, V1AAInfo, V2, V2Size, V2AAInfo, O2); - if (Result != MayAlias) - return AliasCache[Locs] = Result; + aliasSelect(S1, V1Size, V1AAInfo, V2, V2Size, V2AAInfo, O2, AAQI); + if (Result != MayAlias) { + Pair = AAQI.AliasCache.try_emplace(Locs, Result); + assert(!Pair.second && "Entry must have existed"); + return Pair.first->second = Result; + } } // If both pointers are pointing into the same object and one of them @@ -1827,14 +1865,19 @@ AliasResult BasicAAResult::aliasCheck(const Value *V1, LocationSize V1Size, if (O1 == O2) if (V1Size.isPrecise() && V2Size.isPrecise() && (isObjectSize(O1, V1Size.getValue(), DL, TLI, NullIsValidLocation) || - isObjectSize(O2, V2Size.getValue(), DL, TLI, NullIsValidLocation))) - return AliasCache[Locs] = PartialAlias; + isObjectSize(O2, V2Size.getValue(), DL, TLI, NullIsValidLocation))) { + Pair = AAQI.AliasCache.try_emplace(Locs, PartialAlias); + assert(!Pair.second && "Entry must have existed"); + return Pair.first->second = PartialAlias; + } // Recurse back into the best AA results we have, potentially with refined // memory locations. We have already ensured that BasicAA has a MayAlias // cache result for these, so any recursion back into BasicAA won't loop. - AliasResult Result = getBestAAResults().alias(Locs.first, Locs.second); - return AliasCache[Locs] = Result; + AliasResult Result = getBestAAResults().alias(Locs.first, Locs.second, AAQI); + Pair = AAQI.AliasCache.try_emplace(Locs, Result); + assert(!Pair.second && "Entry must have existed"); + return Pair.first->second = Result; } /// Check whether two Values can be considered equivalent. @@ -1863,7 +1906,7 @@ bool BasicAAResult::isValueEqualInPotentialCycles(const Value *V, // the Values cannot come from different iterations of a potential cycle the // phi nodes could be involved in. for (auto *P : VisitedPhiBBs) - if (isPotentiallyReachable(&P->front(), Inst, DT, LI)) + if (isPotentiallyReachable(&P->front(), Inst, nullptr, DT, LI)) return false; return true; diff --git a/lib/Analysis/BlockFrequencyInfo.cpp b/lib/Analysis/BlockFrequencyInfo.cpp index ef27c36517ea..de183bbde173 100644 --- a/lib/Analysis/BlockFrequencyInfo.cpp +++ b/lib/Analysis/BlockFrequencyInfo.cpp @@ -1,9 +1,8 @@ //===- BlockFrequencyInfo.cpp - Block Frequency Analysis ------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -204,11 +203,12 @@ BlockFrequency BlockFrequencyInfo::getBlockFreq(const BasicBlock *BB) const { } Optional -BlockFrequencyInfo::getBlockProfileCount(const BasicBlock *BB) const { +BlockFrequencyInfo::getBlockProfileCount(const BasicBlock *BB, + bool AllowSynthetic) const { if (!BFI) return None; - return BFI->getBlockProfileCount(*getFunction(), BB); + return BFI->getBlockProfileCount(*getFunction(), BB, AllowSynthetic); } Optional diff --git a/lib/Analysis/BlockFrequencyInfoImpl.cpp b/lib/Analysis/BlockFrequencyInfoImpl.cpp index 08ebcc47a807..0db6dd04a7e8 100644 --- a/lib/Analysis/BlockFrequencyInfoImpl.cpp +++ b/lib/Analysis/BlockFrequencyInfoImpl.cpp @@ -1,9 +1,8 @@ //===- BlockFrequencyImplInfo.cpp - Block Frequency Info Implementation ---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -558,14 +557,17 @@ BlockFrequencyInfoImplBase::getBlockFreq(const BlockNode &Node) const { Optional BlockFrequencyInfoImplBase::getBlockProfileCount(const Function &F, - const BlockNode &Node) const { - return getProfileCountFromFreq(F, getBlockFreq(Node).getFrequency()); + const BlockNode &Node, + bool AllowSynthetic) const { + return getProfileCountFromFreq(F, getBlockFreq(Node).getFrequency(), + AllowSynthetic); } Optional BlockFrequencyInfoImplBase::getProfileCountFromFreq(const Function &F, - uint64_t Freq) const { - auto EntryCount = F.getEntryCount(); + uint64_t Freq, + bool AllowSynthetic) const { + auto EntryCount = F.getEntryCount(AllowSynthetic); if (!EntryCount) return None; // Use 128 bit APInt to do the arithmetic to avoid overflow. diff --git a/lib/Analysis/BranchProbabilityInfo.cpp b/lib/Analysis/BranchProbabilityInfo.cpp index 7f544b27fe9d..5eb95003f5d8 100644 --- a/lib/Analysis/BranchProbabilityInfo.cpp +++ b/lib/Analysis/BranchProbabilityInfo.cpp @@ -1,9 +1,8 @@ //===- BranchProbabilityInfo.cpp - Branch Probability Analysis ------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -661,8 +660,14 @@ bool BranchProbabilityInfo::calcZeroHeuristics(const BasicBlock *BB, if (!CI) return false; + auto GetConstantInt = [](Value *V) { + if (auto *I = dyn_cast(V)) + return dyn_cast(I->getOperand(0)); + return dyn_cast(V); + }; + Value *RHS = CI->getOperand(1); - ConstantInt *CV = dyn_cast(RHS); + ConstantInt *CV = GetConstantInt(RHS); if (!CV) return false; diff --git a/lib/Analysis/CFG.cpp b/lib/Analysis/CFG.cpp index aa880a62b754..18b83d6838cc 100644 --- a/lib/Analysis/CFG.cpp +++ b/lib/Analysis/CFG.cpp @@ -1,9 +1,8 @@ //===-- CFG.cpp - BasicBlock analysis --------------------------------------==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -13,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/CFG.h" +#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/IR/Dominators.h" @@ -120,22 +120,33 @@ static const Loop *getOutermostLoop(const LoopInfo *LI, const BasicBlock *BB) { return L; } -// True if there is a loop which contains both BB1 and BB2. -static bool loopContainsBoth(const LoopInfo *LI, - const BasicBlock *BB1, const BasicBlock *BB2) { - const Loop *L1 = getOutermostLoop(LI, BB1); - const Loop *L2 = getOutermostLoop(LI, BB2); - return L1 != nullptr && L1 == L2; -} - bool llvm::isPotentiallyReachableFromMany( SmallVectorImpl &Worklist, BasicBlock *StopBB, - const DominatorTree *DT, const LoopInfo *LI) { + const SmallPtrSetImpl *ExclusionSet, const DominatorTree *DT, + const LoopInfo *LI) { // When the stop block is unreachable, it's dominated from everywhere, // regardless of whether there's a path between the two blocks. if (DT && !DT->isReachableFromEntry(StopBB)) DT = nullptr; + // We can't skip directly from a block that dominates the stop block if the + // exclusion block is potentially in between. + if (ExclusionSet && !ExclusionSet->empty()) + DT = nullptr; + + // Normally any block in a loop is reachable from any other block in a loop, + // however excluded blocks might partition the body of a loop to make that + // untrue. + SmallPtrSet LoopsWithHoles; + if (LI && ExclusionSet) { + for (auto BB : *ExclusionSet) { + if (const Loop *L = getOutermostLoop(LI, BB)) + LoopsWithHoles.insert(L); + } + } + + const Loop *StopLoop = LI ? getOutermostLoop(LI, StopBB) : nullptr; + // Limit the number of blocks we visit. The goal is to avoid run-away compile // times on large CFGs without hampering sensible code. Arbitrarily chosen. unsigned Limit = 32; @@ -146,10 +157,23 @@ bool llvm::isPotentiallyReachableFromMany( continue; if (BB == StopBB) return true; + if (ExclusionSet && ExclusionSet->count(BB)) + continue; if (DT && DT->dominates(BB, StopBB)) return true; - if (LI && loopContainsBoth(LI, BB, StopBB)) - return true; + + const Loop *Outer = nullptr; + if (LI) { + Outer = getOutermostLoop(LI, BB); + // If we're in a loop with a hole, not all blocks in the loop are + // reachable from all other blocks. That implies we can't simply jump to + // the loop's exit blocks, as that exit might need to pass through an + // excluded block. Clear Outer so we process BB's successors. + if (LoopsWithHoles.count(Outer)) + Outer = nullptr; + if (StopLoop && Outer == StopLoop) + return true; + } if (!--Limit) { // We haven't been able to prove it one way or the other. Conservatively @@ -157,7 +181,7 @@ bool llvm::isPotentiallyReachableFromMany( return true; } - if (const Loop *Outer = LI ? getOutermostLoop(LI, BB) : nullptr) { + if (Outer) { // All blocks in a single loop are reachable from all other blocks. From // any of these blocks, we can skip directly to the exits of the loop, // ignoring any other blocks inside the loop body. @@ -181,11 +205,13 @@ bool llvm::isPotentiallyReachable(const BasicBlock *A, const BasicBlock *B, Worklist.push_back(const_cast(A)); return isPotentiallyReachableFromMany(Worklist, const_cast(B), - DT, LI); + nullptr, DT, LI); } -bool llvm::isPotentiallyReachable(const Instruction *A, const Instruction *B, - const DominatorTree *DT, const LoopInfo *LI) { +bool llvm::isPotentiallyReachable( + const Instruction *A, const Instruction *B, + const SmallPtrSetImpl *ExclusionSet, const DominatorTree *DT, + const LoopInfo *LI) { assert(A->getParent()->getParent() == B->getParent()->getParent() && "This analysis is function-local!"); @@ -227,11 +253,20 @@ bool llvm::isPotentiallyReachable(const Instruction *A, const Instruction *B, Worklist.push_back(const_cast(A->getParent())); } - if (A->getParent() == &A->getParent()->getParent()->getEntryBlock()) - return true; - if (B->getParent() == &A->getParent()->getParent()->getEntryBlock()) - return false; + if (DT) { + if (DT->isReachableFromEntry(A->getParent()) && + !DT->isReachableFromEntry(B->getParent())) + return false; + if (!ExclusionSet || ExclusionSet->empty()) { + if (A->getParent() == &A->getParent()->getParent()->getEntryBlock() && + DT->isReachableFromEntry(B->getParent())) + return true; + if (B->getParent() == &A->getParent()->getParent()->getEntryBlock() && + DT->isReachableFromEntry(A->getParent())) + return false; + } + } return isPotentiallyReachableFromMany( - Worklist, const_cast(B->getParent()), DT, LI); + Worklist, const_cast(B->getParent()), ExclusionSet, DT, LI); } diff --git a/lib/Analysis/CFGPrinter.cpp b/lib/Analysis/CFGPrinter.cpp index 6d01e9d5d447..619b675b58d8 100644 --- a/lib/Analysis/CFGPrinter.cpp +++ b/lib/Analysis/CFGPrinter.cpp @@ -1,9 +1,8 @@ //===- CFGPrinter.cpp - DOT printer for the control flow graph ------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/CFLAndersAliasAnalysis.cpp b/lib/Analysis/CFLAndersAliasAnalysis.cpp index 1c61dd369a05..690e514d4f5c 100644 --- a/lib/Analysis/CFLAndersAliasAnalysis.cpp +++ b/lib/Analysis/CFLAndersAliasAnalysis.cpp @@ -1,9 +1,8 @@ //===- CFLAndersAliasAnalysis.cpp - Unification-based Alias Analysis ------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -613,7 +612,7 @@ static void initializeWorkList(std::vector &WorkList, for (unsigned I = 0, E = ValueInfo.getNumLevels(); I < E; ++I) { auto Src = InstantiatedValue{Val, I}; // If there's an assignment edge from X to Y, it means Y is reachable from - // X at S2 and X is reachable from Y at S1 + // X at S3 and X is reachable from Y at S1 for (auto &Edge : ValueInfo.getNodeInfoAtLevel(I).Edges) { propagate(Edge.Other, Src, MatchState::FlowFromReadOnly, ReachSet, WorkList); @@ -876,7 +875,8 @@ AliasResult CFLAndersAAResult::query(const MemoryLocation &LocA, } AliasResult CFLAndersAAResult::alias(const MemoryLocation &LocA, - const MemoryLocation &LocB) { + const MemoryLocation &LocB, + AAQueryInfo &AAQI) { if (LocA.Ptr == LocB.Ptr) return MustAlias; @@ -886,11 +886,11 @@ AliasResult CFLAndersAAResult::alias(const MemoryLocation &LocA, // ConstantExpr, but every query needs to have at least one Value tied to a // Function, and neither GlobalValues nor ConstantExprs are. if (isa(LocA.Ptr) && isa(LocB.Ptr)) - return AAResultBase::alias(LocA, LocB); + return AAResultBase::alias(LocA, LocB, AAQI); AliasResult QueryResult = query(LocA, LocB); if (QueryResult == MayAlias) - return AAResultBase::alias(LocA, LocB); + return AAResultBase::alias(LocA, LocB, AAQI); return QueryResult; } diff --git a/lib/Analysis/CFLGraph.h b/lib/Analysis/CFLGraph.h index 12121d717433..21842ed36487 100644 --- a/lib/Analysis/CFLGraph.h +++ b/lib/Analysis/CFLGraph.h @@ -1,9 +1,8 @@ //===- CFLGraph.h - Abstract stratified sets implementation. -----*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,7 +24,6 @@ #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/Argument.h" #include "llvm/IR/BasicBlock.h" -#include "llvm/IR/CallSite.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Function.h" @@ -154,7 +152,7 @@ public: } }; -///A builder class used to create CFLGraph instance from a given function +/// A builder class used to create CFLGraph instance from a given function /// The CFL-AA that uses this builder must provide its own type as a template /// argument. This is necessary for interprocedural processing: CFLGraphBuilder /// needs a way of obtaining the summary of other functions when callinsts are @@ -183,24 +181,23 @@ template class CFLGraphBuilder { static bool hasUsefulEdges(ConstantExpr *CE) { // ConstantExpr doesn't have terminators, invokes, or fences, so only - // needs - // to check for compares. + // needs to check for compares. return CE->getOpcode() != Instruction::ICmp && CE->getOpcode() != Instruction::FCmp; } // Returns possible functions called by CS into the given SmallVectorImpl. // Returns true if targets found, false otherwise. - static bool getPossibleTargets(CallSite CS, + static bool getPossibleTargets(CallBase &Call, SmallVectorImpl &Output) { - if (auto *Fn = CS.getCalledFunction()) { + if (auto *Fn = Call.getCalledFunction()) { Output.push_back(Fn); return true; } // TODO: If the call is indirect, we might be able to enumerate all - // potential - // targets of the call and return them, rather than just failing. + // potential targets of the call and return them, rather than just + // failing. return false; } @@ -294,6 +291,11 @@ template class CFLGraphBuilder { addAssignEdge(Op2, &Inst); } + void visitUnaryOperator(UnaryOperator &Inst) { + auto *Src = Inst.getOperand(0); + addAssignEdge(Src, &Inst); + } + void visitAtomicCmpXchgInst(AtomicCmpXchgInst &Inst) { auto *Ptr = Inst.getPointerOperand(); auto *Val = Inst.getNewValOperand(); @@ -370,11 +372,11 @@ template class CFLGraphBuilder { return !Fn->hasExactDefinition(); } - bool tryInterproceduralAnalysis(CallSite CS, + bool tryInterproceduralAnalysis(CallBase &Call, const SmallVectorImpl &Fns) { assert(Fns.size() > 0); - if (CS.arg_size() > MaxSupportedArgsInSummary) + if (Call.arg_size() > MaxSupportedArgsInSummary) return false; // Exit early if we'll fail anyway @@ -382,7 +384,7 @@ template class CFLGraphBuilder { if (isFunctionExternal(Fn) || Fn->isVarArg()) return false; // Fail if the caller does not provide enough arguments - assert(Fn->arg_size() <= CS.arg_size()); + assert(Fn->arg_size() <= Call.arg_size()); if (!AA.getAliasSummary(*Fn)) return false; } @@ -393,7 +395,7 @@ template class CFLGraphBuilder { auto &RetParamRelations = Summary->RetParamRelations; for (auto &Relation : RetParamRelations) { - auto IRelation = instantiateExternalRelation(Relation, CS); + auto IRelation = instantiateExternalRelation(Relation, Call); if (IRelation.hasValue()) { Graph.addNode(IRelation->From); Graph.addNode(IRelation->To); @@ -403,7 +405,7 @@ template class CFLGraphBuilder { auto &RetParamAttributes = Summary->RetParamAttributes; for (auto &Attribute : RetParamAttributes) { - auto IAttr = instantiateExternalAttribute(Attribute, CS); + auto IAttr = instantiateExternalAttribute(Attribute, Call); if (IAttr.hasValue()) Graph.addNode(IAttr->IValue, IAttr->Attr); } @@ -412,37 +414,35 @@ template class CFLGraphBuilder { return true; } - void visitCallSite(CallSite CS) { - auto Inst = CS.getInstruction(); - + void visitCallBase(CallBase &Call) { // Make sure all arguments and return value are added to the graph first - for (Value *V : CS.args()) + for (Value *V : Call.args()) if (V->getType()->isPointerTy()) addNode(V); - if (Inst->getType()->isPointerTy()) - addNode(Inst); + if (Call.getType()->isPointerTy()) + addNode(&Call); // Check if Inst is a call to a library function that // allocates/deallocates on the heap. Those kinds of functions do not // introduce any aliases. // TODO: address other common library functions such as realloc(), // strdup(), etc. - if (isMallocOrCallocLikeFn(Inst, &TLI) || isFreeCall(Inst, &TLI)) + if (isMallocOrCallocLikeFn(&Call, &TLI) || isFreeCall(&Call, &TLI)) return; // TODO: Add support for noalias args/all the other fun function // attributes that we can tack on. SmallVector Targets; - if (getPossibleTargets(CS, Targets)) - if (tryInterproceduralAnalysis(CS, Targets)) + if (getPossibleTargets(Call, Targets)) + if (tryInterproceduralAnalysis(Call, Targets)) return; // Because the function is opaque, we need to note that anything // could have happened to the arguments (unless the function is marked // readonly or readnone), and that the result could alias just about // anything, too (unless the result is marked noalias). - if (!CS.onlyReadsMemory()) - for (Value *V : CS.args()) { + if (!Call.onlyReadsMemory()) + for (Value *V : Call.args()) { if (V->getType()->isPointerTy()) { // The argument itself escapes. Graph.addAttr(InstantiatedValue{V, 0}, getAttrEscaped()); @@ -453,12 +453,12 @@ template class CFLGraphBuilder { } } - if (Inst->getType()->isPointerTy()) { - auto *Fn = CS.getCalledFunction(); + if (Call.getType()->isPointerTy()) { + auto *Fn = Call.getCalledFunction(); if (Fn == nullptr || !Fn->returnDoesNotAlias()) // No need to call addNode() since we've added Inst at the // beginning of this function and we know it is not a global. - Graph.addAttr(InstantiatedValue{Inst, 0}, getAttrUnknown()); + Graph.addAttr(InstantiatedValue{&Call, 0}, getAttrUnknown()); } } @@ -559,6 +559,7 @@ template class CFLGraphBuilder { } case Instruction::Add: + case Instruction::FAdd: case Instruction::Sub: case Instruction::FSub: case Instruction::Mul: @@ -583,6 +584,11 @@ template class CFLGraphBuilder { break; } + case Instruction::FNeg: { + addAssignEdge(CE->getOperand(0), CE); + break; + } + default: llvm_unreachable("Unknown instruction type encountered!"); } diff --git a/lib/Analysis/CFLSteensAliasAnalysis.cpp b/lib/Analysis/CFLSteensAliasAnalysis.cpp index 30ce13578e54..44b1834f70bf 100644 --- a/lib/Analysis/CFLSteensAliasAnalysis.cpp +++ b/lib/Analysis/CFLSteensAliasAnalysis.cpp @@ -1,9 +1,8 @@ //===- CFLSteensAliasAnalysis.cpp - Unification-based Alias Analysis ------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/CGSCCPassManager.cpp b/lib/Analysis/CGSCCPassManager.cpp index fd2292ced017..a0b3f83cca6a 100644 --- a/lib/Analysis/CGSCCPassManager.cpp +++ b/lib/Analysis/CGSCCPassManager.cpp @@ -1,9 +1,8 @@ //===- CGSCCPassManager.cpp - Managing & running CGSCC passes -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -111,6 +110,12 @@ PassManagerhasLocalLinkage() || F->hasAddressTaken()) - ExternalCallingNode->addCalledFunction(CallSite(), Node); + ExternalCallingNode->addCalledFunction(nullptr, Node); // If this function is not defined in this translation unit, it could call // anything. if (F->isDeclaration() && !F->isIntrinsic()) - Node->addCalledFunction(CallSite(), CallsExternalNode.get()); + Node->addCalledFunction(nullptr, CallsExternalNode.get()); // Look for calls by this function. for (BasicBlock &BB : *F) for (Instruction &I : BB) { - if (auto CS = CallSite(&I)) { - const Function *Callee = CS.getCalledFunction(); + if (auto *Call = dyn_cast(&I)) { + const Function *Callee = Call->getCalledFunction(); if (!Callee || !Intrinsic::isLeaf(Callee->getIntrinsicID())) // Indirect calls of intrinsics are not allowed so no need to check. // We can be more precise here by using TargetArg returned by // Intrinsic::isLeaf. - Node->addCalledFunction(CS, CallsExternalNode.get()); + Node->addCalledFunction(Call, CallsExternalNode.get()); else if (!Callee->isIntrinsic()) - Node->addCalledFunction(CS, getOrInsertFunction(Callee)); + Node->addCalledFunction(Call, getOrInsertFunction(Callee)); } } } @@ -185,10 +183,10 @@ LLVM_DUMP_METHOD void CallGraphNode::dump() const { print(dbgs()); } /// removeCallEdgeFor - This method removes the edge in the node for the /// specified call site. Note that this method takes linear time, so it /// should be used sparingly. -void CallGraphNode::removeCallEdgeFor(CallSite CS) { +void CallGraphNode::removeCallEdgeFor(CallBase &Call) { for (CalledFunctionsVector::iterator I = CalledFunctions.begin(); ; ++I) { assert(I != CalledFunctions.end() && "Cannot find callsite to remove!"); - if (I->first == CS.getInstruction()) { + if (I->first == &Call) { I->second->DropRef(); *I = CalledFunctions.back(); CalledFunctions.pop_back(); @@ -228,13 +226,13 @@ void CallGraphNode::removeOneAbstractEdgeTo(CallGraphNode *Callee) { /// replaceCallEdge - This method replaces the edge in the node for the /// specified call site with a new one. Note that this method takes linear /// time, so it should be used sparingly. -void CallGraphNode::replaceCallEdge(CallSite CS, - CallSite NewCS, CallGraphNode *NewNode){ +void CallGraphNode::replaceCallEdge(CallBase &Call, CallBase &NewCall, + CallGraphNode *NewNode) { for (CalledFunctionsVector::iterator I = CalledFunctions.begin(); ; ++I) { assert(I != CalledFunctions.end() && "Cannot find callsite to remove!"); - if (I->first == CS.getInstruction()) { + if (I->first == &Call) { I->second->DropRef(); - I->first = NewCS.getInstruction(); + I->first = &NewCall; I->second = NewNode; NewNode->AddRef(); return; diff --git a/lib/Analysis/CallGraphSCCPass.cpp b/lib/Analysis/CallGraphSCCPass.cpp index 0aed57a39387..196ef400bc4e 100644 --- a/lib/Analysis/CallGraphSCCPass.cpp +++ b/lib/Analysis/CallGraphSCCPass.cpp @@ -1,9 +1,8 @@ //===- CallGraphSCCPass.cpp - Pass that operates BU on call graph ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -20,7 +19,6 @@ #include "llvm/ADT/SCCIterator.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/CallGraph.h" -#include "llvm/IR/CallSite.h" #include "llvm/IR/Function.h" #include "llvm/IR/IRPrintingPasses.h" #include "llvm/IR/Intrinsics.h" @@ -202,7 +200,7 @@ bool CGPassManager::RunPassOnSCC(Pass *P, CallGraphSCC &CurSCC, /// This never happens in checking mode. bool CGPassManager::RefreshCallGraph(const CallGraphSCC &CurSCC, CallGraph &CG, bool CheckingMode) { - DenseMap CallSites; + DenseMap Calls; LLVM_DEBUG(dbgs() << "CGSCCPASSMGR: Refreshing SCC with " << CurSCC.size() << " nodes:\n"; @@ -231,21 +229,21 @@ bool CGPassManager::RefreshCallGraph(const CallGraphSCC &CurSCC, CallGraph &CG, for (CallGraphNode::iterator I = CGN->begin(), E = CGN->end(); I != E; ) { // If this call site is null, then the function pass deleted the call // entirely and the WeakTrackingVH nulled it out. + auto *Call = dyn_cast_or_null(I->first); if (!I->first || // If we've already seen this call site, then the FunctionPass RAUW'd // one call with another, which resulted in two "uses" in the edge // list of the same call. - CallSites.count(I->first) || + Calls.count(I->first) || // If the call edge is not from a call or invoke, or it is a // instrinsic call, then the function pass RAUW'd a call with // another value. This can happen when constant folding happens // of well known functions etc. - !CallSite(I->first) || - (CallSite(I->first).getCalledFunction() && - CallSite(I->first).getCalledFunction()->isIntrinsic() && - Intrinsic::isLeaf( - CallSite(I->first).getCalledFunction()->getIntrinsicID()))) { + !Call || + (Call->getCalledFunction() && + Call->getCalledFunction()->isIntrinsic() && + Intrinsic::isLeaf(Call->getCalledFunction()->getIntrinsicID()))) { assert(!CheckingMode && "CallGraphSCCPass did not update the CallGraph correctly!"); @@ -269,15 +267,14 @@ bool CGPassManager::RefreshCallGraph(const CallGraphSCC &CurSCC, CallGraph &CG, continue; } - assert(!CallSites.count(I->first) && + assert(!Calls.count(I->first) && "Call site occurs in node multiple times"); - CallSite CS(I->first); - if (CS) { - Function *Callee = CS.getCalledFunction(); + if (Call) { + Function *Callee = Call->getCalledFunction(); // Ignore intrinsics because they're not really function calls. if (!Callee || !(Callee->isIntrinsic())) - CallSites.insert(std::make_pair(I->first, I->second)); + Calls.insert(std::make_pair(I->first, I->second)); } ++I; } @@ -288,23 +285,25 @@ bool CGPassManager::RefreshCallGraph(const CallGraphSCC &CurSCC, CallGraph &CG, for (BasicBlock &BB : *F) for (Instruction &I : BB) { - CallSite CS(&I); - if (!CS) continue; - Function *Callee = CS.getCalledFunction(); - if (Callee && Callee->isIntrinsic()) continue; + auto *Call = dyn_cast(&I); + if (!Call) + continue; + Function *Callee = Call->getCalledFunction(); + if (Callee && Callee->isIntrinsic()) + continue; // If this call site already existed in the callgraph, just verify it - // matches up to expectations and remove it from CallSites. - DenseMap::iterator ExistingIt = - CallSites.find(CS.getInstruction()); - if (ExistingIt != CallSites.end()) { + // matches up to expectations and remove it from Calls. + DenseMap::iterator ExistingIt = + Calls.find(Call); + if (ExistingIt != Calls.end()) { CallGraphNode *ExistingNode = ExistingIt->second; - // Remove from CallSites since we have now seen it. - CallSites.erase(ExistingIt); + // Remove from Calls since we have now seen it. + Calls.erase(ExistingIt); // Verify that the callee is right. - if (ExistingNode->getFunction() == CS.getCalledFunction()) + if (ExistingNode->getFunction() == Call->getCalledFunction()) continue; // If we are in checking mode, we are not allowed to actually mutate @@ -312,7 +311,7 @@ bool CGPassManager::RefreshCallGraph(const CallGraphSCC &CurSCC, CallGraph &CG, // callgraph is less precise than it could be (e.g. an indirect call // site could be turned direct), don't reject it in checking mode, and // don't tweak it to be more precise. - if (CheckingMode && CS.getCalledFunction() && + if (CheckingMode && Call->getCalledFunction() && ExistingNode->getFunction() == nullptr) continue; @@ -322,7 +321,7 @@ bool CGPassManager::RefreshCallGraph(const CallGraphSCC &CurSCC, CallGraph &CG, // If not, we either went from a direct call to indirect, indirect to // direct, or direct to different direct. CallGraphNode *CalleeNode; - if (Function *Callee = CS.getCalledFunction()) { + if (Function *Callee = Call->getCalledFunction()) { CalleeNode = CG.getOrInsertFunction(Callee); // Keep track of whether we turned an indirect call into a direct // one. @@ -336,7 +335,7 @@ bool CGPassManager::RefreshCallGraph(const CallGraphSCC &CurSCC, CallGraph &CG, } // Update the edge target in CGN. - CGN->replaceCallEdge(CS, CS, CalleeNode); + CGN->replaceCallEdge(*Call, *Call, CalleeNode); MadeChange = true; continue; } @@ -346,7 +345,7 @@ bool CGPassManager::RefreshCallGraph(const CallGraphSCC &CurSCC, CallGraph &CG, // If the call site didn't exist in the CGN yet, add it. CallGraphNode *CalleeNode; - if (Function *Callee = CS.getCalledFunction()) { + if (Function *Callee = Call->getCalledFunction()) { CalleeNode = CG.getOrInsertFunction(Callee); ++NumDirectAdded; } else { @@ -354,7 +353,7 @@ bool CGPassManager::RefreshCallGraph(const CallGraphSCC &CurSCC, CallGraph &CG, ++NumIndirectAdded; } - CGN->addCalledFunction(CS, CalleeNode); + CGN->addCalledFunction(Call, CalleeNode); MadeChange = true; } @@ -376,12 +375,12 @@ bool CGPassManager::RefreshCallGraph(const CallGraphSCC &CurSCC, CallGraph &CG, // they are dangling pointers. WeakTrackingVH should save us for this, so // abort if // this happens. - assert(CallSites.empty() && "Dangling pointers found in call sites map"); + assert(Calls.empty() && "Dangling pointers found in call sites map"); // Periodically do an explicit clear to remove tombstones when processing // large scc's. if ((FunctionNo & 15) == 15) - CallSites.clear(); + Calls.clear(); } LLVM_DEBUG(if (MadeChange) { @@ -682,11 +681,28 @@ Pass *CallGraphSCCPass::createPrinterPass(raw_ostream &OS, return new PrintCallGraphPass(Banner, OS); } +static std::string getDescription(const CallGraphSCC &SCC) { + std::string Desc = "SCC ("; + bool First = true; + for (CallGraphNode *CGN : SCC) { + if (First) + First = false; + else + Desc += ", "; + Function *F = CGN->getFunction(); + if (F) + Desc += F->getName(); + else + Desc += "<>"; + } + Desc += ")"; + return Desc; +} + bool CallGraphSCCPass::skipSCC(CallGraphSCC &SCC) const { - return !SCC.getCallGraph().getModule() - .getContext() - .getOptPassGate() - .shouldRunPass(this, SCC); + OptPassGate &Gate = + SCC.getCallGraph().getModule().getContext().getOptPassGate(); + return Gate.isEnabled() && !Gate.shouldRunPass(this, getDescription(SCC)); } char DummyCGSCCPass::ID = 0; diff --git a/lib/Analysis/CallPrinter.cpp b/lib/Analysis/CallPrinter.cpp index e7017e77652a..d24cbd104bf6 100644 --- a/lib/Analysis/CallPrinter.cpp +++ b/lib/Analysis/CallPrinter.cpp @@ -1,9 +1,8 @@ //===- CallPrinter.cpp - DOT printer for call graph -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/CaptureTracking.cpp b/lib/Analysis/CaptureTracking.cpp index 669f4f2835fa..adaa83a6c443 100644 --- a/lib/Analysis/CaptureTracking.cpp +++ b/lib/Analysis/CaptureTracking.cpp @@ -1,9 +1,8 @@ //===--- CaptureTracking.cpp - Determine whether a pointer is captured ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -102,14 +101,14 @@ namespace { SmallVector Worklist; Worklist.append(succ_begin(BB), succ_end(BB)); - return !isPotentiallyReachableFromMany(Worklist, BB, DT); + return !isPotentiallyReachableFromMany(Worklist, BB, nullptr, DT); } // If the value is defined in the same basic block as use and BeforeHere, // there is no need to explore the use if BeforeHere dominates use. // Check whether there is a path from I to BeforeHere. if (BeforeHere != I && DT->dominates(BeforeHere, I) && - !isPotentiallyReachable(I, BeforeHere, DT)) + !isPotentiallyReachable(I, BeforeHere, nullptr, DT)) return true; return false; @@ -331,14 +330,32 @@ void llvm::PointerMayBeCaptured(const Value *V, CaptureTracker *Tracker, AddUses(I); break; case Instruction::ICmp: { - // Don't count comparisons of a no-alias return value against null as - // captures. This allows us to ignore comparisons of malloc results - // with null, for example. - if (ConstantPointerNull *CPN = - dyn_cast(I->getOperand(1))) + if (auto *CPN = dyn_cast(I->getOperand(1))) { + // Don't count comparisons of a no-alias return value against null as + // captures. This allows us to ignore comparisons of malloc results + // with null, for example. if (CPN->getType()->getAddressSpace() == 0) if (isNoAliasCall(V->stripPointerCasts())) break; + if (!I->getFunction()->nullPointerIsDefined()) { + auto *O = I->getOperand(0)->stripPointerCastsSameRepresentation(); + // An inbounds GEP can either be a valid pointer (pointing into + // or to the end of an allocation), or be null in the default + // address space. So for an inbounds GEPs there is no way to let + // the pointer escape using clever GEP hacking because doing so + // would make the pointer point outside of the allocated object + // and thus make the GEP result a poison value. + if (auto *GEP = dyn_cast(O)) + if (GEP->isInBounds()) + break; + // Comparing a dereferenceable_or_null argument against null + // cannot lead to pointer escapes, because if it is not null it + // must be a valid (in-bounds) pointer. + bool CanBeNull; + if (O->getPointerDereferenceableBytes(I->getModule()->getDataLayout(), CanBeNull)) + break; + } + } // Comparison against value stored in global variable. Given the pointer // does not escape, its value cannot be guessed and stored separately in a // global variable. diff --git a/lib/Analysis/CmpInstAnalysis.cpp b/lib/Analysis/CmpInstAnalysis.cpp index 27071babec5c..a5757be2c4f4 100644 --- a/lib/Analysis/CmpInstAnalysis.cpp +++ b/lib/Analysis/CmpInstAnalysis.cpp @@ -1,9 +1,8 @@ //===- CmpInstAnalysis.cpp - Utils to help fold compares ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/CodeMetrics.cpp b/lib/Analysis/CodeMetrics.cpp index 46cc87d2b178..627d955c865f 100644 --- a/lib/Analysis/CodeMetrics.cpp +++ b/lib/Analysis/CodeMetrics.cpp @@ -1,9 +1,8 @@ //===- CodeMetrics.cpp - Code cost measurements ---------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -16,7 +15,6 @@ #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/ValueTracking.h" -#include "llvm/IR/CallSite.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Function.h" #include "llvm/Support/Debug.h" @@ -126,14 +124,12 @@ void CodeMetrics::analyzeBasicBlock(const BasicBlock *BB, continue; // Special handling for calls. - if (isa(I) || isa(I)) { - ImmutableCallSite CS(&I); - - if (const Function *F = CS.getCalledFunction()) { + if (const auto *Call = dyn_cast(&I)) { + if (const Function *F = Call->getCalledFunction()) { // If a function is both internal and has a single use, then it is // extremely likely to get inlined in the future (it was probably // exposed by an interleaved devirtualization pass). - if (!CS.isNoInline() && F->hasInternalLinkage() && F->hasOneUse()) + if (!Call->isNoInline() && F->hasInternalLinkage() && F->hasOneUse()) ++NumInlineCandidates; // If this call is to function itself, then the function is recursive. @@ -148,7 +144,7 @@ void CodeMetrics::analyzeBasicBlock(const BasicBlock *BB, } else { // We don't want inline asm to count as a call - that would prevent loop // unrolling. The argument setup cost is still real, though. - if (!isa(CS.getCalledValue())) + if (!Call->isInlineAsm()) ++NumCalls; } } diff --git a/lib/Analysis/ConstantFolding.cpp b/lib/Analysis/ConstantFolding.cpp index 5da29d6d2372..20231ca78b45 100644 --- a/lib/Analysis/ConstantFolding.cpp +++ b/lib/Analysis/ConstantFolding.cpp @@ -1,9 +1,8 @@ //===-- ConstantFolding.cpp - Fold instructions into constants ------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -26,6 +25,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/Analysis/ValueTracking.h" +#include "llvm/Analysis/VectorUtils.h" #include "llvm/Config/config.h" #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" @@ -516,7 +516,7 @@ Constant *FoldReinterpretLoadFromConstPtr(Constant *C, Type *LoadTy, MapTy = Type::getInt64Ty(C->getContext()); else if (LoadTy->isVectorTy()) { MapTy = PointerType::getIntNTy(C->getContext(), - DL.getTypeAllocSizeInBits(LoadTy)); + DL.getTypeSizeInBits(LoadTy)); } else return nullptr; @@ -1000,7 +1000,9 @@ Constant *ConstantFoldInstOperandsImpl(const Value *InstOrCE, unsigned Opcode, const TargetLibraryInfo *TLI) { Type *DestTy = InstOrCE->getType(); - // Handle easy binops first. + if (Instruction::isUnaryOp(Opcode)) + return ConstantFoldUnaryOpOperand(Opcode, Ops[0], DL); + if (Instruction::isBinaryOp(Opcode)) return ConstantFoldBinaryOpOperands(Opcode, Ops[0], Ops[1], DL); @@ -1025,15 +1027,18 @@ Constant *ConstantFoldInstOperandsImpl(const Value *InstOrCE, unsigned Opcode, case Instruction::FCmp: llvm_unreachable("Invalid for compares"); case Instruction::Call: if (auto *F = dyn_cast(Ops.back())) { - ImmutableCallSite CS(cast(InstOrCE)); - if (canConstantFoldCallTo(CS, F)) - return ConstantFoldCall(CS, F, Ops.slice(0, Ops.size() - 1), TLI); + const auto *Call = cast(InstOrCE); + if (canConstantFoldCallTo(Call, F)) + return ConstantFoldCall(Call, F, Ops.slice(0, Ops.size() - 1), TLI); } return nullptr; case Instruction::Select: return ConstantExpr::getSelect(Ops[0], Ops[1], Ops[2]); case Instruction::ExtractElement: return ConstantExpr::getExtractElement(Ops[0], Ops[1]); + case Instruction::ExtractValue: + return ConstantExpr::getExtractValue( + Ops[0], dyn_cast(InstOrCE)->getIndices()); case Instruction::InsertElement: return ConstantExpr::getInsertElement(Ops[0], Ops[1], Ops[2]); case Instruction::ShuffleVector: @@ -1263,6 +1268,13 @@ Constant *llvm::ConstantFoldCompareInstOperands(unsigned Predicate, return ConstantExpr::getCompare(Predicate, Ops0, Ops1); } +Constant *llvm::ConstantFoldUnaryOpOperand(unsigned Opcode, Constant *Op, + const DataLayout &DL) { + assert(Instruction::isUnaryOp(Opcode)); + + return ConstantExpr::get(Opcode, Op); +} + Constant *llvm::ConstantFoldBinaryOpOperands(unsigned Opcode, Constant *LHS, Constant *RHS, const DataLayout &DL) { @@ -1367,8 +1379,8 @@ llvm::ConstantFoldLoadThroughGEPIndices(Constant *C, // Constant Folding for Calls // -bool llvm::canConstantFoldCallTo(ImmutableCallSite CS, const Function *F) { - if (CS.isNoBuiltin() || CS.isStrictFP()) +bool llvm::canConstantFoldCallTo(const CallBase *Call, const Function *F) { + if (Call->isNoBuiltin() || Call->isStrictFP()) return false; switch (F->getIntrinsicID()) { case Intrinsic::fabs: @@ -1414,6 +1426,8 @@ bool llvm::canConstantFoldCallTo(ImmutableCallSite CS, const Function *F) { case Intrinsic::uadd_sat: case Intrinsic::ssub_sat: case Intrinsic::usub_sat: + case Intrinsic::smul_fix: + case Intrinsic::smul_fix_sat: case Intrinsic::convert_from_fp16: case Intrinsic::convert_to_fp16: case Intrinsic::bitreverse: @@ -1518,14 +1532,12 @@ bool llvm::canConstantFoldCallTo(ImmutableCallSite CS, const Function *F) { namespace { Constant *GetConstantFoldFPValue(double V, Type *Ty) { - if (Ty->isHalfTy()) { + if (Ty->isHalfTy() || Ty->isFloatTy()) { APFloat APF(V); bool unused; - APF.convert(APFloat::IEEEhalf(), APFloat::rmNearestTiesToEven, &unused); + APF.convert(Ty->getFltSemantics(), APFloat::rmNearestTiesToEven, &unused); return ConstantFP::get(Ty->getContext(), APF); } - if (Ty->isFloatTy()) - return ConstantFP::get(Ty->getContext(), APFloat((float)V)); if (Ty->isDoubleTy()) return ConstantFP::get(Ty->getContext(), APFloat(V)); llvm_unreachable("Can only constant fold half/float/double"); @@ -1641,522 +1653,538 @@ static bool getConstIntOrUndef(Value *Op, const APInt *&C) { return false; } -Constant *ConstantFoldScalarCall(StringRef Name, unsigned IntrinsicID, Type *Ty, - ArrayRef Operands, - const TargetLibraryInfo *TLI, - ImmutableCallSite CS) { - if (Operands.size() == 1) { - if (IntrinsicID == Intrinsic::is_constant) { - // We know we have a "Constant" argument. But we want to only - // return true for manifest constants, not those that depend on - // constants with unknowable values, e.g. GlobalValue or BlockAddress. - if (isManifestConstant(Operands[0])) - return ConstantInt::getTrue(Ty->getContext()); - return nullptr; - } - if (isa(Operands[0])) { - // cosine(arg) is between -1 and 1. cosine(invalid arg) is NaN. - // ctpop() is between 0 and bitwidth, pick 0 for undef. - if (IntrinsicID == Intrinsic::cos || - IntrinsicID == Intrinsic::ctpop) - return Constant::getNullValue(Ty); - if (IntrinsicID == Intrinsic::bswap || - IntrinsicID == Intrinsic::bitreverse || - IntrinsicID == Intrinsic::launder_invariant_group || - IntrinsicID == Intrinsic::strip_invariant_group) - return Operands[0]; - } +static Constant *ConstantFoldScalarCall1(StringRef Name, + Intrinsic::ID IntrinsicID, + Type *Ty, + ArrayRef Operands, + const TargetLibraryInfo *TLI, + const CallBase *Call) { + assert(Operands.size() == 1 && "Wrong number of operands."); + + if (IntrinsicID == Intrinsic::is_constant) { + // We know we have a "Constant" argument. But we want to only + // return true for manifest constants, not those that depend on + // constants with unknowable values, e.g. GlobalValue or BlockAddress. + if (isManifestConstant(Operands[0])) + return ConstantInt::getTrue(Ty->getContext()); + return nullptr; + } + if (isa(Operands[0])) { + // cosine(arg) is between -1 and 1. cosine(invalid arg) is NaN. + // ctpop() is between 0 and bitwidth, pick 0 for undef. + if (IntrinsicID == Intrinsic::cos || + IntrinsicID == Intrinsic::ctpop) + return Constant::getNullValue(Ty); + if (IntrinsicID == Intrinsic::bswap || + IntrinsicID == Intrinsic::bitreverse || + IntrinsicID == Intrinsic::launder_invariant_group || + IntrinsicID == Intrinsic::strip_invariant_group) + return Operands[0]; + } - if (isa(Operands[0])) { - // launder(null) == null == strip(null) iff in addrspace 0 - if (IntrinsicID == Intrinsic::launder_invariant_group || - IntrinsicID == Intrinsic::strip_invariant_group) { - // If instruction is not yet put in a basic block (e.g. when cloning - // a function during inlining), CS caller may not be available. - // So check CS's BB first before querying CS.getCaller. - const Function *Caller = CS.getParent() ? CS.getCaller() : nullptr; - if (Caller && - !NullPointerIsDefined( - Caller, Operands[0]->getType()->getPointerAddressSpace())) { - return Operands[0]; - } - return nullptr; + if (isa(Operands[0])) { + // launder(null) == null == strip(null) iff in addrspace 0 + if (IntrinsicID == Intrinsic::launder_invariant_group || + IntrinsicID == Intrinsic::strip_invariant_group) { + // If instruction is not yet put in a basic block (e.g. when cloning + // a function during inlining), Call's caller may not be available. + // So check Call's BB first before querying Call->getCaller. + const Function *Caller = + Call->getParent() ? Call->getCaller() : nullptr; + if (Caller && + !NullPointerIsDefined( + Caller, Operands[0]->getType()->getPointerAddressSpace())) { + return Operands[0]; } + return nullptr; } + } - if (auto *Op = dyn_cast(Operands[0])) { - if (IntrinsicID == Intrinsic::convert_to_fp16) { - APFloat Val(Op->getValueAPF()); - - bool lost = false; - Val.convert(APFloat::IEEEhalf(), APFloat::rmNearestTiesToEven, &lost); + if (auto *Op = dyn_cast(Operands[0])) { + if (IntrinsicID == Intrinsic::convert_to_fp16) { + APFloat Val(Op->getValueAPF()); - return ConstantInt::get(Ty->getContext(), Val.bitcastToAPInt()); - } + bool lost = false; + Val.convert(APFloat::IEEEhalf(), APFloat::rmNearestTiesToEven, &lost); - if (!Ty->isHalfTy() && !Ty->isFloatTy() && !Ty->isDoubleTy()) - return nullptr; + return ConstantInt::get(Ty->getContext(), Val.bitcastToAPInt()); + } - if (IntrinsicID == Intrinsic::round) { - APFloat V = Op->getValueAPF(); - V.roundToIntegral(APFloat::rmNearestTiesToAway); - return ConstantFP::get(Ty->getContext(), V); - } + if (!Ty->isHalfTy() && !Ty->isFloatTy() && !Ty->isDoubleTy()) + return nullptr; - if (IntrinsicID == Intrinsic::floor) { - APFloat V = Op->getValueAPF(); - V.roundToIntegral(APFloat::rmTowardNegative); - return ConstantFP::get(Ty->getContext(), V); - } + if (IntrinsicID == Intrinsic::round) { + APFloat V = Op->getValueAPF(); + V.roundToIntegral(APFloat::rmNearestTiesToAway); + return ConstantFP::get(Ty->getContext(), V); + } - if (IntrinsicID == Intrinsic::ceil) { - APFloat V = Op->getValueAPF(); - V.roundToIntegral(APFloat::rmTowardPositive); - return ConstantFP::get(Ty->getContext(), V); - } + if (IntrinsicID == Intrinsic::floor) { + APFloat V = Op->getValueAPF(); + V.roundToIntegral(APFloat::rmTowardNegative); + return ConstantFP::get(Ty->getContext(), V); + } - if (IntrinsicID == Intrinsic::trunc) { - APFloat V = Op->getValueAPF(); - V.roundToIntegral(APFloat::rmTowardZero); - return ConstantFP::get(Ty->getContext(), V); - } + if (IntrinsicID == Intrinsic::ceil) { + APFloat V = Op->getValueAPF(); + V.roundToIntegral(APFloat::rmTowardPositive); + return ConstantFP::get(Ty->getContext(), V); + } - if (IntrinsicID == Intrinsic::rint) { - APFloat V = Op->getValueAPF(); - V.roundToIntegral(APFloat::rmNearestTiesToEven); - return ConstantFP::get(Ty->getContext(), V); - } + if (IntrinsicID == Intrinsic::trunc) { + APFloat V = Op->getValueAPF(); + V.roundToIntegral(APFloat::rmTowardZero); + return ConstantFP::get(Ty->getContext(), V); + } - if (IntrinsicID == Intrinsic::nearbyint) { - APFloat V = Op->getValueAPF(); - V.roundToIntegral(APFloat::rmNearestTiesToEven); - return ConstantFP::get(Ty->getContext(), V); - } + if (IntrinsicID == Intrinsic::rint) { + APFloat V = Op->getValueAPF(); + V.roundToIntegral(APFloat::rmNearestTiesToEven); + return ConstantFP::get(Ty->getContext(), V); + } - /// We only fold functions with finite arguments. Folding NaN and inf is - /// likely to be aborted with an exception anyway, and some host libms - /// have known errors raising exceptions. - if (Op->getValueAPF().isNaN() || Op->getValueAPF().isInfinity()) - return nullptr; + if (IntrinsicID == Intrinsic::nearbyint) { + APFloat V = Op->getValueAPF(); + V.roundToIntegral(APFloat::rmNearestTiesToEven); + return ConstantFP::get(Ty->getContext(), V); + } - /// Currently APFloat versions of these functions do not exist, so we use - /// the host native double versions. Float versions are not called - /// directly but for all these it is true (float)(f((double)arg)) == - /// f(arg). Long double not supported yet. - double V = getValueAsDouble(Op); + /// We only fold functions with finite arguments. Folding NaN and inf is + /// likely to be aborted with an exception anyway, and some host libms + /// have known errors raising exceptions. + if (Op->getValueAPF().isNaN() || Op->getValueAPF().isInfinity()) + return nullptr; - switch (IntrinsicID) { - default: break; - case Intrinsic::fabs: - return ConstantFoldFP(fabs, V, Ty); - case Intrinsic::log2: - return ConstantFoldFP(Log2, V, Ty); - case Intrinsic::log: - return ConstantFoldFP(log, V, Ty); - case Intrinsic::log10: - return ConstantFoldFP(log10, V, Ty); - case Intrinsic::exp: - return ConstantFoldFP(exp, V, Ty); - case Intrinsic::exp2: - return ConstantFoldFP(exp2, V, Ty); - case Intrinsic::sin: - return ConstantFoldFP(sin, V, Ty); - case Intrinsic::cos: - return ConstantFoldFP(cos, V, Ty); - case Intrinsic::sqrt: - return ConstantFoldFP(sqrt, V, Ty); - } + /// Currently APFloat versions of these functions do not exist, so we use + /// the host native double versions. Float versions are not called + /// directly but for all these it is true (float)(f((double)arg)) == + /// f(arg). Long double not supported yet. + double V = getValueAsDouble(Op); - if (!TLI) - return nullptr; + switch (IntrinsicID) { + default: break; + case Intrinsic::fabs: + return ConstantFoldFP(fabs, V, Ty); + case Intrinsic::log2: + return ConstantFoldFP(Log2, V, Ty); + case Intrinsic::log: + return ConstantFoldFP(log, V, Ty); + case Intrinsic::log10: + return ConstantFoldFP(log10, V, Ty); + case Intrinsic::exp: + return ConstantFoldFP(exp, V, Ty); + case Intrinsic::exp2: + return ConstantFoldFP(exp2, V, Ty); + case Intrinsic::sin: + return ConstantFoldFP(sin, V, Ty); + case Intrinsic::cos: + return ConstantFoldFP(cos, V, Ty); + case Intrinsic::sqrt: + return ConstantFoldFP(sqrt, V, Ty); + } - char NameKeyChar = Name[0]; - if (Name[0] == '_' && Name.size() > 2 && Name[1] == '_') - NameKeyChar = Name[2]; - - switch (NameKeyChar) { - case 'a': - if ((Name == "acos" && TLI->has(LibFunc_acos)) || - (Name == "acosf" && TLI->has(LibFunc_acosf)) || - (Name == "__acos_finite" && TLI->has(LibFunc_acos_finite)) || - (Name == "__acosf_finite" && TLI->has(LibFunc_acosf_finite))) - return ConstantFoldFP(acos, V, Ty); - else if ((Name == "asin" && TLI->has(LibFunc_asin)) || - (Name == "asinf" && TLI->has(LibFunc_asinf)) || - (Name == "__asin_finite" && TLI->has(LibFunc_asin_finite)) || - (Name == "__asinf_finite" && TLI->has(LibFunc_asinf_finite))) - return ConstantFoldFP(asin, V, Ty); - else if ((Name == "atan" && TLI->has(LibFunc_atan)) || - (Name == "atanf" && TLI->has(LibFunc_atanf))) - return ConstantFoldFP(atan, V, Ty); - break; - case 'c': - if ((Name == "ceil" && TLI->has(LibFunc_ceil)) || - (Name == "ceilf" && TLI->has(LibFunc_ceilf))) - return ConstantFoldFP(ceil, V, Ty); - else if ((Name == "cos" && TLI->has(LibFunc_cos)) || - (Name == "cosf" && TLI->has(LibFunc_cosf))) - return ConstantFoldFP(cos, V, Ty); - else if ((Name == "cosh" && TLI->has(LibFunc_cosh)) || - (Name == "coshf" && TLI->has(LibFunc_coshf)) || - (Name == "__cosh_finite" && TLI->has(LibFunc_cosh_finite)) || - (Name == "__coshf_finite" && TLI->has(LibFunc_coshf_finite))) - return ConstantFoldFP(cosh, V, Ty); - break; - case 'e': - if ((Name == "exp" && TLI->has(LibFunc_exp)) || - (Name == "expf" && TLI->has(LibFunc_expf)) || - (Name == "__exp_finite" && TLI->has(LibFunc_exp_finite)) || - (Name == "__expf_finite" && TLI->has(LibFunc_expf_finite))) - return ConstantFoldFP(exp, V, Ty); - if ((Name == "exp2" && TLI->has(LibFunc_exp2)) || - (Name == "exp2f" && TLI->has(LibFunc_exp2f)) || - (Name == "__exp2_finite" && TLI->has(LibFunc_exp2_finite)) || - (Name == "__exp2f_finite" && TLI->has(LibFunc_exp2f_finite))) - // Constant fold exp2(x) as pow(2,x) in case the host doesn't have a - // C99 library. - return ConstantFoldBinaryFP(pow, 2.0, V, Ty); - break; - case 'f': - if ((Name == "fabs" && TLI->has(LibFunc_fabs)) || - (Name == "fabsf" && TLI->has(LibFunc_fabsf))) - return ConstantFoldFP(fabs, V, Ty); - else if ((Name == "floor" && TLI->has(LibFunc_floor)) || - (Name == "floorf" && TLI->has(LibFunc_floorf))) - return ConstantFoldFP(floor, V, Ty); - break; - case 'l': - if ((Name == "log" && V > 0 && TLI->has(LibFunc_log)) || - (Name == "logf" && V > 0 && TLI->has(LibFunc_logf)) || - (Name == "__log_finite" && V > 0 && - TLI->has(LibFunc_log_finite)) || - (Name == "__logf_finite" && V > 0 && - TLI->has(LibFunc_logf_finite))) - return ConstantFoldFP(log, V, Ty); - else if ((Name == "log10" && V > 0 && TLI->has(LibFunc_log10)) || - (Name == "log10f" && V > 0 && TLI->has(LibFunc_log10f)) || - (Name == "__log10_finite" && V > 0 && - TLI->has(LibFunc_log10_finite)) || - (Name == "__log10f_finite" && V > 0 && - TLI->has(LibFunc_log10f_finite))) - return ConstantFoldFP(log10, V, Ty); - break; - case 'r': - if ((Name == "round" && TLI->has(LibFunc_round)) || - (Name == "roundf" && TLI->has(LibFunc_roundf))) - return ConstantFoldFP(round, V, Ty); - break; - case 's': - if ((Name == "sin" && TLI->has(LibFunc_sin)) || - (Name == "sinf" && TLI->has(LibFunc_sinf))) - return ConstantFoldFP(sin, V, Ty); - else if ((Name == "sinh" && TLI->has(LibFunc_sinh)) || - (Name == "sinhf" && TLI->has(LibFunc_sinhf)) || - (Name == "__sinh_finite" && TLI->has(LibFunc_sinh_finite)) || - (Name == "__sinhf_finite" && TLI->has(LibFunc_sinhf_finite))) - return ConstantFoldFP(sinh, V, Ty); - else if ((Name == "sqrt" && V >= 0 && TLI->has(LibFunc_sqrt)) || - (Name == "sqrtf" && V >= 0 && TLI->has(LibFunc_sqrtf))) - return ConstantFoldFP(sqrt, V, Ty); - break; - case 't': - if ((Name == "tan" && TLI->has(LibFunc_tan)) || - (Name == "tanf" && TLI->has(LibFunc_tanf))) - return ConstantFoldFP(tan, V, Ty); - else if ((Name == "tanh" && TLI->has(LibFunc_tanh)) || - (Name == "tanhf" && TLI->has(LibFunc_tanhf))) - return ConstantFoldFP(tanh, V, Ty); - break; - default: - break; - } + if (!TLI) return nullptr; - } - if (auto *Op = dyn_cast(Operands[0])) { - switch (IntrinsicID) { - case Intrinsic::bswap: - return ConstantInt::get(Ty->getContext(), Op->getValue().byteSwap()); - case Intrinsic::ctpop: - return ConstantInt::get(Ty, Op->getValue().countPopulation()); - case Intrinsic::bitreverse: - return ConstantInt::get(Ty->getContext(), Op->getValue().reverseBits()); - case Intrinsic::convert_from_fp16: { - APFloat Val(APFloat::IEEEhalf(), Op->getValue()); - - bool lost = false; - APFloat::opStatus status = Val.convert( - Ty->getFltSemantics(), APFloat::rmNearestTiesToEven, &lost); - - // Conversion is always precise. - (void)status; - assert(status == APFloat::opOK && !lost && - "Precision lost during fp16 constfolding"); - - return ConstantFP::get(Ty->getContext(), Val); - } - default: - return nullptr; - } - } + char NameKeyChar = Name[0]; + if (Name[0] == '_' && Name.size() > 2 && Name[1] == '_') + NameKeyChar = Name[2]; - // Support ConstantVector in case we have an Undef in the top. - if (isa(Operands[0]) || - isa(Operands[0])) { - auto *Op = cast(Operands[0]); - switch (IntrinsicID) { - default: break; - case Intrinsic::x86_sse_cvtss2si: - case Intrinsic::x86_sse_cvtss2si64: - case Intrinsic::x86_sse2_cvtsd2si: - case Intrinsic::x86_sse2_cvtsd2si64: - if (ConstantFP *FPOp = - dyn_cast_or_null(Op->getAggregateElement(0U))) - return ConstantFoldSSEConvertToInt(FPOp->getValueAPF(), - /*roundTowardZero=*/false, Ty, - /*IsSigned*/true); - break; - case Intrinsic::x86_sse_cvttss2si: - case Intrinsic::x86_sse_cvttss2si64: - case Intrinsic::x86_sse2_cvttsd2si: - case Intrinsic::x86_sse2_cvttsd2si64: - if (ConstantFP *FPOp = - dyn_cast_or_null(Op->getAggregateElement(0U))) - return ConstantFoldSSEConvertToInt(FPOp->getValueAPF(), - /*roundTowardZero=*/true, Ty, - /*IsSigned*/true); - break; - } + switch (NameKeyChar) { + case 'a': + if ((Name == "acos" && TLI->has(LibFunc_acos)) || + (Name == "acosf" && TLI->has(LibFunc_acosf)) || + (Name == "__acos_finite" && TLI->has(LibFunc_acos_finite)) || + (Name == "__acosf_finite" && TLI->has(LibFunc_acosf_finite))) + return ConstantFoldFP(acos, V, Ty); + else if ((Name == "asin" && TLI->has(LibFunc_asin)) || + (Name == "asinf" && TLI->has(LibFunc_asinf)) || + (Name == "__asin_finite" && TLI->has(LibFunc_asin_finite)) || + (Name == "__asinf_finite" && TLI->has(LibFunc_asinf_finite))) + return ConstantFoldFP(asin, V, Ty); + else if ((Name == "atan" && TLI->has(LibFunc_atan)) || + (Name == "atanf" && TLI->has(LibFunc_atanf))) + return ConstantFoldFP(atan, V, Ty); + break; + case 'c': + if ((Name == "ceil" && TLI->has(LibFunc_ceil)) || + (Name == "ceilf" && TLI->has(LibFunc_ceilf))) + return ConstantFoldFP(ceil, V, Ty); + else if ((Name == "cos" && TLI->has(LibFunc_cos)) || + (Name == "cosf" && TLI->has(LibFunc_cosf))) + return ConstantFoldFP(cos, V, Ty); + else if ((Name == "cosh" && TLI->has(LibFunc_cosh)) || + (Name == "coshf" && TLI->has(LibFunc_coshf)) || + (Name == "__cosh_finite" && TLI->has(LibFunc_cosh_finite)) || + (Name == "__coshf_finite" && TLI->has(LibFunc_coshf_finite))) + return ConstantFoldFP(cosh, V, Ty); + break; + case 'e': + if ((Name == "exp" && TLI->has(LibFunc_exp)) || + (Name == "expf" && TLI->has(LibFunc_expf)) || + (Name == "__exp_finite" && TLI->has(LibFunc_exp_finite)) || + (Name == "__expf_finite" && TLI->has(LibFunc_expf_finite))) + return ConstantFoldFP(exp, V, Ty); + if ((Name == "exp2" && TLI->has(LibFunc_exp2)) || + (Name == "exp2f" && TLI->has(LibFunc_exp2f)) || + (Name == "__exp2_finite" && TLI->has(LibFunc_exp2_finite)) || + (Name == "__exp2f_finite" && TLI->has(LibFunc_exp2f_finite))) + // Constant fold exp2(x) as pow(2,x) in case the host doesn't have a + // C99 library. + return ConstantFoldBinaryFP(pow, 2.0, V, Ty); + break; + case 'f': + if ((Name == "fabs" && TLI->has(LibFunc_fabs)) || + (Name == "fabsf" && TLI->has(LibFunc_fabsf))) + return ConstantFoldFP(fabs, V, Ty); + else if ((Name == "floor" && TLI->has(LibFunc_floor)) || + (Name == "floorf" && TLI->has(LibFunc_floorf))) + return ConstantFoldFP(floor, V, Ty); + break; + case 'l': + if ((Name == "log" && V > 0 && TLI->has(LibFunc_log)) || + (Name == "logf" && V > 0 && TLI->has(LibFunc_logf)) || + (Name == "__log_finite" && V > 0 && + TLI->has(LibFunc_log_finite)) || + (Name == "__logf_finite" && V > 0 && + TLI->has(LibFunc_logf_finite))) + return ConstantFoldFP(log, V, Ty); + else if ((Name == "log10" && V > 0 && TLI->has(LibFunc_log10)) || + (Name == "log10f" && V > 0 && TLI->has(LibFunc_log10f)) || + (Name == "__log10_finite" && V > 0 && + TLI->has(LibFunc_log10_finite)) || + (Name == "__log10f_finite" && V > 0 && + TLI->has(LibFunc_log10f_finite))) + return ConstantFoldFP(log10, V, Ty); + break; + case 'r': + if ((Name == "round" && TLI->has(LibFunc_round)) || + (Name == "roundf" && TLI->has(LibFunc_roundf))) + return ConstantFoldFP(round, V, Ty); + break; + case 's': + if ((Name == "sin" && TLI->has(LibFunc_sin)) || + (Name == "sinf" && TLI->has(LibFunc_sinf))) + return ConstantFoldFP(sin, V, Ty); + else if ((Name == "sinh" && TLI->has(LibFunc_sinh)) || + (Name == "sinhf" && TLI->has(LibFunc_sinhf)) || + (Name == "__sinh_finite" && TLI->has(LibFunc_sinh_finite)) || + (Name == "__sinhf_finite" && TLI->has(LibFunc_sinhf_finite))) + return ConstantFoldFP(sinh, V, Ty); + else if ((Name == "sqrt" && V >= 0 && TLI->has(LibFunc_sqrt)) || + (Name == "sqrtf" && V >= 0 && TLI->has(LibFunc_sqrtf))) + return ConstantFoldFP(sqrt, V, Ty); + break; + case 't': + if ((Name == "tan" && TLI->has(LibFunc_tan)) || + (Name == "tanf" && TLI->has(LibFunc_tanf))) + return ConstantFoldFP(tan, V, Ty); + else if ((Name == "tanh" && TLI->has(LibFunc_tanh)) || + (Name == "tanhf" && TLI->has(LibFunc_tanhf))) + return ConstantFoldFP(tanh, V, Ty); + break; + default: + break; } - return nullptr; } - if (Operands.size() == 2) { - if (auto *Op1 = dyn_cast(Operands[0])) { - if (!Ty->isHalfTy() && !Ty->isFloatTy() && !Ty->isDoubleTy()) - return nullptr; - double Op1V = getValueAsDouble(Op1); - - if (auto *Op2 = dyn_cast(Operands[1])) { - if (Op2->getType() != Op1->getType()) - return nullptr; + if (auto *Op = dyn_cast(Operands[0])) { + switch (IntrinsicID) { + case Intrinsic::bswap: + return ConstantInt::get(Ty->getContext(), Op->getValue().byteSwap()); + case Intrinsic::ctpop: + return ConstantInt::get(Ty, Op->getValue().countPopulation()); + case Intrinsic::bitreverse: + return ConstantInt::get(Ty->getContext(), Op->getValue().reverseBits()); + case Intrinsic::convert_from_fp16: { + APFloat Val(APFloat::IEEEhalf(), Op->getValue()); + + bool lost = false; + APFloat::opStatus status = Val.convert( + Ty->getFltSemantics(), APFloat::rmNearestTiesToEven, &lost); + + // Conversion is always precise. + (void)status; + assert(status == APFloat::opOK && !lost && + "Precision lost during fp16 constfolding"); + + return ConstantFP::get(Ty->getContext(), Val); + } + default: + return nullptr; + } + } - double Op2V = getValueAsDouble(Op2); - if (IntrinsicID == Intrinsic::pow) { - return ConstantFoldBinaryFP(pow, Op1V, Op2V, Ty); - } - if (IntrinsicID == Intrinsic::copysign) { - APFloat V1 = Op1->getValueAPF(); - const APFloat &V2 = Op2->getValueAPF(); - V1.copySign(V2); - return ConstantFP::get(Ty->getContext(), V1); - } + // Support ConstantVector in case we have an Undef in the top. + if (isa(Operands[0]) || + isa(Operands[0])) { + auto *Op = cast(Operands[0]); + switch (IntrinsicID) { + default: break; + case Intrinsic::x86_sse_cvtss2si: + case Intrinsic::x86_sse_cvtss2si64: + case Intrinsic::x86_sse2_cvtsd2si: + case Intrinsic::x86_sse2_cvtsd2si64: + if (ConstantFP *FPOp = + dyn_cast_or_null(Op->getAggregateElement(0U))) + return ConstantFoldSSEConvertToInt(FPOp->getValueAPF(), + /*roundTowardZero=*/false, Ty, + /*IsSigned*/true); + break; + case Intrinsic::x86_sse_cvttss2si: + case Intrinsic::x86_sse_cvttss2si64: + case Intrinsic::x86_sse2_cvttsd2si: + case Intrinsic::x86_sse2_cvttsd2si64: + if (ConstantFP *FPOp = + dyn_cast_or_null(Op->getAggregateElement(0U))) + return ConstantFoldSSEConvertToInt(FPOp->getValueAPF(), + /*roundTowardZero=*/true, Ty, + /*IsSigned*/true); + break; + } + } - if (IntrinsicID == Intrinsic::minnum) { - const APFloat &C1 = Op1->getValueAPF(); - const APFloat &C2 = Op2->getValueAPF(); - return ConstantFP::get(Ty->getContext(), minnum(C1, C2)); - } + return nullptr; +} - if (IntrinsicID == Intrinsic::maxnum) { - const APFloat &C1 = Op1->getValueAPF(); - const APFloat &C2 = Op2->getValueAPF(); - return ConstantFP::get(Ty->getContext(), maxnum(C1, C2)); - } +static Constant *ConstantFoldScalarCall2(StringRef Name, + Intrinsic::ID IntrinsicID, + Type *Ty, + ArrayRef Operands, + const TargetLibraryInfo *TLI, + const CallBase *Call) { + assert(Operands.size() == 2 && "Wrong number of operands."); - if (IntrinsicID == Intrinsic::minimum) { - const APFloat &C1 = Op1->getValueAPF(); - const APFloat &C2 = Op2->getValueAPF(); - return ConstantFP::get(Ty->getContext(), minimum(C1, C2)); - } + if (auto *Op1 = dyn_cast(Operands[0])) { + if (!Ty->isHalfTy() && !Ty->isFloatTy() && !Ty->isDoubleTy()) + return nullptr; + double Op1V = getValueAsDouble(Op1); - if (IntrinsicID == Intrinsic::maximum) { - const APFloat &C1 = Op1->getValueAPF(); - const APFloat &C2 = Op2->getValueAPF(); - return ConstantFP::get(Ty->getContext(), maximum(C1, C2)); - } + if (auto *Op2 = dyn_cast(Operands[1])) { + if (Op2->getType() != Op1->getType()) + return nullptr; - if (!TLI) - return nullptr; - if ((Name == "pow" && TLI->has(LibFunc_pow)) || - (Name == "powf" && TLI->has(LibFunc_powf)) || - (Name == "__pow_finite" && TLI->has(LibFunc_pow_finite)) || - (Name == "__powf_finite" && TLI->has(LibFunc_powf_finite))) - return ConstantFoldBinaryFP(pow, Op1V, Op2V, Ty); - if ((Name == "fmod" && TLI->has(LibFunc_fmod)) || - (Name == "fmodf" && TLI->has(LibFunc_fmodf))) - return ConstantFoldBinaryFP(fmod, Op1V, Op2V, Ty); - if ((Name == "atan2" && TLI->has(LibFunc_atan2)) || - (Name == "atan2f" && TLI->has(LibFunc_atan2f)) || - (Name == "__atan2_finite" && TLI->has(LibFunc_atan2_finite)) || - (Name == "__atan2f_finite" && TLI->has(LibFunc_atan2f_finite))) - return ConstantFoldBinaryFP(atan2, Op1V, Op2V, Ty); - } else if (auto *Op2C = dyn_cast(Operands[1])) { - if (IntrinsicID == Intrinsic::powi && Ty->isHalfTy()) - return ConstantFP::get(Ty->getContext(), - APFloat((float)std::pow((float)Op1V, - (int)Op2C->getZExtValue()))); - if (IntrinsicID == Intrinsic::powi && Ty->isFloatTy()) - return ConstantFP::get(Ty->getContext(), - APFloat((float)std::pow((float)Op1V, - (int)Op2C->getZExtValue()))); - if (IntrinsicID == Intrinsic::powi && Ty->isDoubleTy()) - return ConstantFP::get(Ty->getContext(), - APFloat((double)std::pow((double)Op1V, - (int)Op2C->getZExtValue()))); + double Op2V = getValueAsDouble(Op2); + if (IntrinsicID == Intrinsic::pow) { + return ConstantFoldBinaryFP(pow, Op1V, Op2V, Ty); + } + if (IntrinsicID == Intrinsic::copysign) { + APFloat V1 = Op1->getValueAPF(); + const APFloat &V2 = Op2->getValueAPF(); + V1.copySign(V2); + return ConstantFP::get(Ty->getContext(), V1); } - return nullptr; - } - if (Operands[0]->getType()->isIntegerTy() && - Operands[1]->getType()->isIntegerTy()) { - const APInt *C0, *C1; - if (!getConstIntOrUndef(Operands[0], C0) || - !getConstIntOrUndef(Operands[1], C1)) - return nullptr; + if (IntrinsicID == Intrinsic::minnum) { + const APFloat &C1 = Op1->getValueAPF(); + const APFloat &C2 = Op2->getValueAPF(); + return ConstantFP::get(Ty->getContext(), minnum(C1, C2)); + } - switch (IntrinsicID) { - default: break; - case Intrinsic::smul_with_overflow: - case Intrinsic::umul_with_overflow: - // Even if both operands are undef, we cannot fold muls to undef - // in the general case. For example, on i2 there are no inputs - // that would produce { i2 -1, i1 true } as the result. - if (!C0 || !C1) - return Constant::getNullValue(Ty); - LLVM_FALLTHROUGH; - case Intrinsic::sadd_with_overflow: - case Intrinsic::uadd_with_overflow: - case Intrinsic::ssub_with_overflow: - case Intrinsic::usub_with_overflow: { - if (!C0 || !C1) - return UndefValue::get(Ty); + if (IntrinsicID == Intrinsic::maxnum) { + const APFloat &C1 = Op1->getValueAPF(); + const APFloat &C2 = Op2->getValueAPF(); + return ConstantFP::get(Ty->getContext(), maxnum(C1, C2)); + } - APInt Res; - bool Overflow; - switch (IntrinsicID) { - default: llvm_unreachable("Invalid case"); - case Intrinsic::sadd_with_overflow: - Res = C0->sadd_ov(*C1, Overflow); - break; - case Intrinsic::uadd_with_overflow: - Res = C0->uadd_ov(*C1, Overflow); - break; - case Intrinsic::ssub_with_overflow: - Res = C0->ssub_ov(*C1, Overflow); - break; - case Intrinsic::usub_with_overflow: - Res = C0->usub_ov(*C1, Overflow); - break; - case Intrinsic::smul_with_overflow: - Res = C0->smul_ov(*C1, Overflow); - break; - case Intrinsic::umul_with_overflow: - Res = C0->umul_ov(*C1, Overflow); - break; - } - Constant *Ops[] = { - ConstantInt::get(Ty->getContext(), Res), - ConstantInt::get(Type::getInt1Ty(Ty->getContext()), Overflow) - }; - return ConstantStruct::get(cast(Ty), Ops); + if (IntrinsicID == Intrinsic::minimum) { + const APFloat &C1 = Op1->getValueAPF(); + const APFloat &C2 = Op2->getValueAPF(); + return ConstantFP::get(Ty->getContext(), minimum(C1, C2)); } - case Intrinsic::uadd_sat: - case Intrinsic::sadd_sat: - if (!C0 && !C1) - return UndefValue::get(Ty); - if (!C0 || !C1) - return Constant::getAllOnesValue(Ty); - if (IntrinsicID == Intrinsic::uadd_sat) - return ConstantInt::get(Ty, C0->uadd_sat(*C1)); - else - return ConstantInt::get(Ty, C0->sadd_sat(*C1)); - case Intrinsic::usub_sat: - case Intrinsic::ssub_sat: - if (!C0 && !C1) - return UndefValue::get(Ty); - if (!C0 || !C1) - return Constant::getNullValue(Ty); - if (IntrinsicID == Intrinsic::usub_sat) - return ConstantInt::get(Ty, C0->usub_sat(*C1)); - else - return ConstantInt::get(Ty, C0->ssub_sat(*C1)); - case Intrinsic::cttz: - case Intrinsic::ctlz: - assert(C1 && "Must be constant int"); - - // cttz(0, 1) and ctlz(0, 1) are undef. - if (C1->isOneValue() && (!C0 || C0->isNullValue())) - return UndefValue::get(Ty); - if (!C0) - return Constant::getNullValue(Ty); - if (IntrinsicID == Intrinsic::cttz) - return ConstantInt::get(Ty, C0->countTrailingZeros()); - else - return ConstantInt::get(Ty, C0->countLeadingZeros()); + + if (IntrinsicID == Intrinsic::maximum) { + const APFloat &C1 = Op1->getValueAPF(); + const APFloat &C2 = Op2->getValueAPF(); + return ConstantFP::get(Ty->getContext(), maximum(C1, C2)); } - return nullptr; + if (!TLI) + return nullptr; + if ((Name == "pow" && TLI->has(LibFunc_pow)) || + (Name == "powf" && TLI->has(LibFunc_powf)) || + (Name == "__pow_finite" && TLI->has(LibFunc_pow_finite)) || + (Name == "__powf_finite" && TLI->has(LibFunc_powf_finite))) + return ConstantFoldBinaryFP(pow, Op1V, Op2V, Ty); + if ((Name == "fmod" && TLI->has(LibFunc_fmod)) || + (Name == "fmodf" && TLI->has(LibFunc_fmodf))) + return ConstantFoldBinaryFP(fmod, Op1V, Op2V, Ty); + if ((Name == "atan2" && TLI->has(LibFunc_atan2)) || + (Name == "atan2f" && TLI->has(LibFunc_atan2f)) || + (Name == "__atan2_finite" && TLI->has(LibFunc_atan2_finite)) || + (Name == "__atan2f_finite" && TLI->has(LibFunc_atan2f_finite))) + return ConstantFoldBinaryFP(atan2, Op1V, Op2V, Ty); + } else if (auto *Op2C = dyn_cast(Operands[1])) { + if (IntrinsicID == Intrinsic::powi && Ty->isHalfTy()) + return ConstantFP::get(Ty->getContext(), + APFloat((float)std::pow((float)Op1V, + (int)Op2C->getZExtValue()))); + if (IntrinsicID == Intrinsic::powi && Ty->isFloatTy()) + return ConstantFP::get(Ty->getContext(), + APFloat((float)std::pow((float)Op1V, + (int)Op2C->getZExtValue()))); + if (IntrinsicID == Intrinsic::powi && Ty->isDoubleTy()) + return ConstantFP::get(Ty->getContext(), + APFloat((double)std::pow((double)Op1V, + (int)Op2C->getZExtValue()))); } + return nullptr; + } - // Support ConstantVector in case we have an Undef in the top. - if ((isa(Operands[0]) || - isa(Operands[0])) && - // Check for default rounding mode. - // FIXME: Support other rounding modes? - isa(Operands[1]) && - cast(Operands[1])->getValue() == 4) { - auto *Op = cast(Operands[0]); + if (Operands[0]->getType()->isIntegerTy() && + Operands[1]->getType()->isIntegerTy()) { + const APInt *C0, *C1; + if (!getConstIntOrUndef(Operands[0], C0) || + !getConstIntOrUndef(Operands[1], C1)) + return nullptr; + + switch (IntrinsicID) { + default: break; + case Intrinsic::smul_with_overflow: + case Intrinsic::umul_with_overflow: + // Even if both operands are undef, we cannot fold muls to undef + // in the general case. For example, on i2 there are no inputs + // that would produce { i2 -1, i1 true } as the result. + if (!C0 || !C1) + return Constant::getNullValue(Ty); + LLVM_FALLTHROUGH; + case Intrinsic::sadd_with_overflow: + case Intrinsic::uadd_with_overflow: + case Intrinsic::ssub_with_overflow: + case Intrinsic::usub_with_overflow: { + if (!C0 || !C1) + return UndefValue::get(Ty); + + APInt Res; + bool Overflow; switch (IntrinsicID) { - default: break; - case Intrinsic::x86_avx512_vcvtss2si32: - case Intrinsic::x86_avx512_vcvtss2si64: - case Intrinsic::x86_avx512_vcvtsd2si32: - case Intrinsic::x86_avx512_vcvtsd2si64: - if (ConstantFP *FPOp = - dyn_cast_or_null(Op->getAggregateElement(0U))) - return ConstantFoldSSEConvertToInt(FPOp->getValueAPF(), - /*roundTowardZero=*/false, Ty, - /*IsSigned*/true); + default: llvm_unreachable("Invalid case"); + case Intrinsic::sadd_with_overflow: + Res = C0->sadd_ov(*C1, Overflow); + break; + case Intrinsic::uadd_with_overflow: + Res = C0->uadd_ov(*C1, Overflow); break; - case Intrinsic::x86_avx512_vcvtss2usi32: - case Intrinsic::x86_avx512_vcvtss2usi64: - case Intrinsic::x86_avx512_vcvtsd2usi32: - case Intrinsic::x86_avx512_vcvtsd2usi64: - if (ConstantFP *FPOp = - dyn_cast_or_null(Op->getAggregateElement(0U))) - return ConstantFoldSSEConvertToInt(FPOp->getValueAPF(), - /*roundTowardZero=*/false, Ty, - /*IsSigned*/false); + case Intrinsic::ssub_with_overflow: + Res = C0->ssub_ov(*C1, Overflow); + break; + case Intrinsic::usub_with_overflow: + Res = C0->usub_ov(*C1, Overflow); break; - case Intrinsic::x86_avx512_cvttss2si: - case Intrinsic::x86_avx512_cvttss2si64: - case Intrinsic::x86_avx512_cvttsd2si: - case Intrinsic::x86_avx512_cvttsd2si64: - if (ConstantFP *FPOp = - dyn_cast_or_null(Op->getAggregateElement(0U))) - return ConstantFoldSSEConvertToInt(FPOp->getValueAPF(), - /*roundTowardZero=*/true, Ty, - /*IsSigned*/true); + case Intrinsic::smul_with_overflow: + Res = C0->smul_ov(*C1, Overflow); break; - case Intrinsic::x86_avx512_cvttss2usi: - case Intrinsic::x86_avx512_cvttss2usi64: - case Intrinsic::x86_avx512_cvttsd2usi: - case Intrinsic::x86_avx512_cvttsd2usi64: - if (ConstantFP *FPOp = - dyn_cast_or_null(Op->getAggregateElement(0U))) - return ConstantFoldSSEConvertToInt(FPOp->getValueAPF(), - /*roundTowardZero=*/true, Ty, - /*IsSigned*/false); + case Intrinsic::umul_with_overflow: + Res = C0->umul_ov(*C1, Overflow); break; } + Constant *Ops[] = { + ConstantInt::get(Ty->getContext(), Res), + ConstantInt::get(Type::getInt1Ty(Ty->getContext()), Overflow) + }; + return ConstantStruct::get(cast(Ty), Ops); + } + case Intrinsic::uadd_sat: + case Intrinsic::sadd_sat: + if (!C0 && !C1) + return UndefValue::get(Ty); + if (!C0 || !C1) + return Constant::getAllOnesValue(Ty); + if (IntrinsicID == Intrinsic::uadd_sat) + return ConstantInt::get(Ty, C0->uadd_sat(*C1)); + else + return ConstantInt::get(Ty, C0->sadd_sat(*C1)); + case Intrinsic::usub_sat: + case Intrinsic::ssub_sat: + if (!C0 && !C1) + return UndefValue::get(Ty); + if (!C0 || !C1) + return Constant::getNullValue(Ty); + if (IntrinsicID == Intrinsic::usub_sat) + return ConstantInt::get(Ty, C0->usub_sat(*C1)); + else + return ConstantInt::get(Ty, C0->ssub_sat(*C1)); + case Intrinsic::cttz: + case Intrinsic::ctlz: + assert(C1 && "Must be constant int"); + + // cttz(0, 1) and ctlz(0, 1) are undef. + if (C1->isOneValue() && (!C0 || C0->isNullValue())) + return UndefValue::get(Ty); + if (!C0) + return Constant::getNullValue(Ty); + if (IntrinsicID == Intrinsic::cttz) + return ConstantInt::get(Ty, C0->countTrailingZeros()); + else + return ConstantInt::get(Ty, C0->countLeadingZeros()); } + return nullptr; } - if (Operands.size() != 3) - return nullptr; + // Support ConstantVector in case we have an Undef in the top. + if ((isa(Operands[0]) || + isa(Operands[0])) && + // Check for default rounding mode. + // FIXME: Support other rounding modes? + isa(Operands[1]) && + cast(Operands[1])->getValue() == 4) { + auto *Op = cast(Operands[0]); + switch (IntrinsicID) { + default: break; + case Intrinsic::x86_avx512_vcvtss2si32: + case Intrinsic::x86_avx512_vcvtss2si64: + case Intrinsic::x86_avx512_vcvtsd2si32: + case Intrinsic::x86_avx512_vcvtsd2si64: + if (ConstantFP *FPOp = + dyn_cast_or_null(Op->getAggregateElement(0U))) + return ConstantFoldSSEConvertToInt(FPOp->getValueAPF(), + /*roundTowardZero=*/false, Ty, + /*IsSigned*/true); + break; + case Intrinsic::x86_avx512_vcvtss2usi32: + case Intrinsic::x86_avx512_vcvtss2usi64: + case Intrinsic::x86_avx512_vcvtsd2usi32: + case Intrinsic::x86_avx512_vcvtsd2usi64: + if (ConstantFP *FPOp = + dyn_cast_or_null(Op->getAggregateElement(0U))) + return ConstantFoldSSEConvertToInt(FPOp->getValueAPF(), + /*roundTowardZero=*/false, Ty, + /*IsSigned*/false); + break; + case Intrinsic::x86_avx512_cvttss2si: + case Intrinsic::x86_avx512_cvttss2si64: + case Intrinsic::x86_avx512_cvttsd2si: + case Intrinsic::x86_avx512_cvttsd2si64: + if (ConstantFP *FPOp = + dyn_cast_or_null(Op->getAggregateElement(0U))) + return ConstantFoldSSEConvertToInt(FPOp->getValueAPF(), + /*roundTowardZero=*/true, Ty, + /*IsSigned*/true); + break; + case Intrinsic::x86_avx512_cvttss2usi: + case Intrinsic::x86_avx512_cvttss2usi64: + case Intrinsic::x86_avx512_cvttsd2usi: + case Intrinsic::x86_avx512_cvttsd2usi64: + if (ConstantFP *FPOp = + dyn_cast_or_null(Op->getAggregateElement(0U))) + return ConstantFoldSSEConvertToInt(FPOp->getValueAPF(), + /*roundTowardZero=*/true, Ty, + /*IsSigned*/false); + break; + } + } + return nullptr; +} + +static Constant *ConstantFoldScalarCall3(StringRef Name, + Intrinsic::ID IntrinsicID, + Type *Ty, + ArrayRef Operands, + const TargetLibraryInfo *TLI, + const CallBase *Call) { + assert(Operands.size() == 3 && "Wrong number of operands."); if (const auto *Op1 = dyn_cast(Operands[0])) { if (const auto *Op2 = dyn_cast(Operands[1])) { @@ -2179,6 +2207,43 @@ Constant *ConstantFoldScalarCall(StringRef Name, unsigned IntrinsicID, Type *Ty, } } + if (const auto *Op1 = dyn_cast(Operands[0])) { + if (const auto *Op2 = dyn_cast(Operands[1])) { + if (const auto *Op3 = dyn_cast(Operands[2])) { + switch (IntrinsicID) { + default: break; + case Intrinsic::smul_fix: + case Intrinsic::smul_fix_sat: { + // This code performs rounding towards negative infinity in case the + // result cannot be represented exactly for the given scale. Targets + // that do care about rounding should use a target hook for specifying + // how rounding should be done, and provide their own folding to be + // consistent with rounding. This is the same approach as used by + // DAGTypeLegalizer::ExpandIntRes_MULFIX. + APInt Lhs = Op1->getValue(); + APInt Rhs = Op2->getValue(); + unsigned Scale = Op3->getValue().getZExtValue(); + unsigned Width = Lhs.getBitWidth(); + assert(Scale < Width && "Illegal scale."); + unsigned ExtendedWidth = Width * 2; + APInt Product = (Lhs.sextOrSelf(ExtendedWidth) * + Rhs.sextOrSelf(ExtendedWidth)).ashr(Scale); + if (IntrinsicID == Intrinsic::smul_fix_sat) { + APInt MaxValue = + APInt::getSignedMaxValue(Width).sextOrSelf(ExtendedWidth); + APInt MinValue = + APInt::getSignedMinValue(Width).sextOrSelf(ExtendedWidth); + Product = APIntOps::smin(Product, MaxValue); + Product = APIntOps::smax(Product, MinValue); + } + return ConstantInt::get(Ty->getContext(), + Product.sextOrTrunc(Width)); + } + } + } + } + } + if (IntrinsicID == Intrinsic::fshl || IntrinsicID == Intrinsic::fshr) { const APInt *C0, *C1, *C2; if (!getConstIntOrUndef(Operands[0], C0) || @@ -2212,11 +2277,31 @@ Constant *ConstantFoldScalarCall(StringRef Name, unsigned IntrinsicID, Type *Ty, return nullptr; } -Constant *ConstantFoldVectorCall(StringRef Name, unsigned IntrinsicID, - VectorType *VTy, ArrayRef Operands, - const DataLayout &DL, - const TargetLibraryInfo *TLI, - ImmutableCallSite CS) { +static Constant *ConstantFoldScalarCall(StringRef Name, + Intrinsic::ID IntrinsicID, + Type *Ty, + ArrayRef Operands, + const TargetLibraryInfo *TLI, + const CallBase *Call) { + if (Operands.size() == 1) + return ConstantFoldScalarCall1(Name, IntrinsicID, Ty, Operands, TLI, Call); + + if (Operands.size() == 2) + return ConstantFoldScalarCall2(Name, IntrinsicID, Ty, Operands, TLI, Call); + + if (Operands.size() == 3) + return ConstantFoldScalarCall3(Name, IntrinsicID, Ty, Operands, TLI, Call); + + return nullptr; +} + +static Constant *ConstantFoldVectorCall(StringRef Name, + Intrinsic::ID IntrinsicID, + VectorType *VTy, + ArrayRef Operands, + const DataLayout &DL, + const TargetLibraryInfo *TLI, + const CallBase *Call) { SmallVector Result(VTy->getNumElements()); SmallVector Lane(Operands.size()); Type *Ty = VTy->getElementType(); @@ -2263,10 +2348,8 @@ Constant *ConstantFoldVectorCall(StringRef Name, unsigned IntrinsicID, for (unsigned I = 0, E = VTy->getNumElements(); I != E; ++I) { // Gather a column of constants. for (unsigned J = 0, JE = Operands.size(); J != JE; ++J) { - // These intrinsics use a scalar type for their second argument. - if (J == 1 && - (IntrinsicID == Intrinsic::cttz || IntrinsicID == Intrinsic::ctlz || - IntrinsicID == Intrinsic::powi)) { + // Some intrinsics use a scalar type for certain arguments. + if (hasVectorInstrinsicScalarOpd(IntrinsicID, J)) { Lane[J] = Operands[J]; continue; } @@ -2279,7 +2362,8 @@ Constant *ConstantFoldVectorCall(StringRef Name, unsigned IntrinsicID, } // Use the regular scalar folding to simplify this column. - Constant *Folded = ConstantFoldScalarCall(Name, IntrinsicID, Ty, Lane, TLI, CS); + Constant *Folded = + ConstantFoldScalarCall(Name, IntrinsicID, Ty, Lane, TLI, Call); if (!Folded) return nullptr; Result[I] = Folded; @@ -2290,11 +2374,10 @@ Constant *ConstantFoldVectorCall(StringRef Name, unsigned IntrinsicID, } // end anonymous namespace -Constant * -llvm::ConstantFoldCall(ImmutableCallSite CS, Function *F, - ArrayRef Operands, - const TargetLibraryInfo *TLI) { - if (CS.isNoBuiltin() || CS.isStrictFP()) +Constant *llvm::ConstantFoldCall(const CallBase *Call, Function *F, + ArrayRef Operands, + const TargetLibraryInfo *TLI) { + if (Call->isNoBuiltin() || Call->isStrictFP()) return nullptr; if (!F->hasName()) return nullptr; @@ -2304,17 +2387,19 @@ llvm::ConstantFoldCall(ImmutableCallSite CS, Function *F, if (auto *VTy = dyn_cast(Ty)) return ConstantFoldVectorCall(Name, F->getIntrinsicID(), VTy, Operands, - F->getParent()->getDataLayout(), TLI, CS); + F->getParent()->getDataLayout(), TLI, Call); - return ConstantFoldScalarCall(Name, F->getIntrinsicID(), Ty, Operands, TLI, CS); + return ConstantFoldScalarCall(Name, F->getIntrinsicID(), Ty, Operands, TLI, + Call); } -bool llvm::isMathLibCallNoop(CallSite CS, const TargetLibraryInfo *TLI) { +bool llvm::isMathLibCallNoop(const CallBase *Call, + const TargetLibraryInfo *TLI) { // FIXME: Refactor this code; this duplicates logic in LibCallsShrinkWrap // (and to some extent ConstantFoldScalarCall). - if (CS.isNoBuiltin() || CS.isStrictFP()) + if (Call->isNoBuiltin() || Call->isStrictFP()) return false; - Function *F = CS.getCalledFunction(); + Function *F = Call->getCalledFunction(); if (!F) return false; @@ -2322,8 +2407,8 @@ bool llvm::isMathLibCallNoop(CallSite CS, const TargetLibraryInfo *TLI) { if (!TLI || !TLI->getLibFunc(*F, Func)) return false; - if (CS.getNumArgOperands() == 1) { - if (ConstantFP *OpC = dyn_cast(CS.getArgOperand(0))) { + if (Call->getNumArgOperands() == 1) { + if (ConstantFP *OpC = dyn_cast(Call->getArgOperand(0))) { const APFloat &Op = OpC->getValueAPF(); switch (Func) { case LibFunc_logl: @@ -2421,9 +2506,9 @@ bool llvm::isMathLibCallNoop(CallSite CS, const TargetLibraryInfo *TLI) { } } - if (CS.getNumArgOperands() == 2) { - ConstantFP *Op0C = dyn_cast(CS.getArgOperand(0)); - ConstantFP *Op1C = dyn_cast(CS.getArgOperand(1)); + if (Call->getNumArgOperands() == 2) { + ConstantFP *Op0C = dyn_cast(Call->getArgOperand(0)); + ConstantFP *Op1C = dyn_cast(Call->getArgOperand(1)); if (Op0C && Op1C) { const APFloat &Op0 = Op0C->getValueAPF(); const APFloat &Op1 = Op1C->getValueAPF(); diff --git a/lib/Analysis/CostModel.cpp b/lib/Analysis/CostModel.cpp index 3d55bf20bb40..bf0cdbfd0c8b 100644 --- a/lib/Analysis/CostModel.cpp +++ b/lib/Analysis/CostModel.cpp @@ -1,9 +1,8 @@ //===- CostModel.cpp ------ Cost Model Analysis ---------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/Delinearization.cpp b/lib/Analysis/Delinearization.cpp index 4cafb7da16d3..c1043e446beb 100644 --- a/lib/Analysis/Delinearization.cpp +++ b/lib/Analysis/Delinearization.cpp @@ -1,9 +1,8 @@ //===---- Delinearization.cpp - MultiDimensional Index Delinearization ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/DemandedBits.cpp b/lib/Analysis/DemandedBits.cpp index 34f785fb02be..01b8ff10d355 100644 --- a/lib/Analysis/DemandedBits.cpp +++ b/lib/Analysis/DemandedBits.cpp @@ -1,9 +1,8 @@ //===- DemandedBits.cpp - Determine demanded bits -------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -340,6 +339,8 @@ void DemandedBits::performAnalysis() { Type *T = J->getType(); if (T->isIntOrIntVectorTy()) AliveBits[J] = APInt::getAllOnesValue(T->getScalarSizeInBits()); + else + Visited.insert(J); Worklist.insert(J); } } @@ -355,16 +356,18 @@ void DemandedBits::performAnalysis() { LLVM_DEBUG(dbgs() << "DemandedBits: Visiting: " << *UserI); APInt AOut; + bool InputIsKnownDead = false; if (UserI->getType()->isIntOrIntVectorTy()) { AOut = AliveBits[UserI]; LLVM_DEBUG(dbgs() << " Alive Out: 0x" << Twine::utohexstr(AOut.getLimitedValue())); + + // If all bits of the output are dead, then all bits of the input + // are also dead. + InputIsKnownDead = !AOut && !isAlwaysLive(UserI); } LLVM_DEBUG(dbgs() << "\n"); - if (!UserI->getType()->isIntOrIntVectorTy()) - Visited.insert(UserI); - KnownBits Known, Known2; bool KnownBitsComputed = false; // Compute the set of alive bits for each operand. These are anded into the @@ -381,10 +384,7 @@ void DemandedBits::performAnalysis() { if (T->isIntOrIntVectorTy()) { unsigned BitWidth = T->getScalarSizeInBits(); APInt AB = APInt::getAllOnesValue(BitWidth); - if (UserI->getType()->isIntOrIntVectorTy() && !AOut && - !isAlwaysLive(UserI)) { - // If all bits of the output are dead, then all bits of the input - // are also dead. + if (InputIsKnownDead) { AB = APInt(BitWidth, 0); } else { // Bits of each operand that are used to compute alive bits of the @@ -403,18 +403,13 @@ void DemandedBits::performAnalysis() { // If we've added to the set of alive bits (or the operand has not // been previously visited), then re-queue the operand to be visited // again. - APInt ABPrev(BitWidth, 0); - auto ABI = AliveBits.find(I); - if (ABI != AliveBits.end()) - ABPrev = ABI->second; - - APInt ABNew = AB | ABPrev; - if (ABNew != ABPrev || ABI == AliveBits.end()) { - AliveBits[I] = std::move(ABNew); + auto Res = AliveBits.try_emplace(I); + if (Res.second || (AB |= Res.first->second) != Res.first->second) { + Res.first->second = std::move(AB); Worklist.insert(I); } } - } else if (I && !Visited.count(I)) { + } else if (I && Visited.insert(I).second) { Worklist.insert(I); } } diff --git a/lib/Analysis/DependenceAnalysis.cpp b/lib/Analysis/DependenceAnalysis.cpp index 3f4dfa52e1da..75f269e84f9d 100644 --- a/lib/Analysis/DependenceAnalysis.cpp +++ b/lib/Analysis/DependenceAnalysis.cpp @@ -1,9 +1,8 @@ //===-- DependenceAnalysis.cpp - DA Implementation --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -110,6 +109,14 @@ STATISTIC(BanerjeeSuccesses, "Banerjee successes"); static cl::opt Delinearize("da-delinearize", cl::init(true), cl::Hidden, cl::ZeroOrMore, cl::desc("Try to delinearize array references.")); +static cl::opt DisableDelinearizationChecks( + "da-disable-delinearization-checks", cl::init(false), cl::Hidden, + cl::ZeroOrMore, + cl::desc( + "Disable checks that try to statically verify validity of " + "delinearized subscripts. Enabling this option may result in incorrect " + "dependence vectors for languages that allow the subscript of one " + "dimension to underflow or overflow into another dimension.")); //===----------------------------------------------------------------------===// // basics @@ -3317,19 +3324,20 @@ bool DependenceInfo::tryDelinearize(Instruction *Src, Instruction *Dst, // and dst. // FIXME: It may be better to record these sizes and add them as constraints // to the dependency checks. - for (int i = 1; i < size; ++i) { - if (!isKnownNonNegative(SrcSubscripts[i], SrcPtr)) - return false; + if (!DisableDelinearizationChecks) + for (int i = 1; i < size; ++i) { + if (!isKnownNonNegative(SrcSubscripts[i], SrcPtr)) + return false; - if (!isKnownLessThan(SrcSubscripts[i], Sizes[i - 1])) - return false; + if (!isKnownLessThan(SrcSubscripts[i], Sizes[i - 1])) + return false; - if (!isKnownNonNegative(DstSubscripts[i], DstPtr)) - return false; + if (!isKnownNonNegative(DstSubscripts[i], DstPtr)) + return false; - if (!isKnownLessThan(DstSubscripts[i], Sizes[i - 1])) - return false; - } + if (!isKnownLessThan(DstSubscripts[i], Sizes[i - 1])) + return false; + } LLVM_DEBUG({ dbgs() << "\nSrcSubscripts: "; @@ -3369,6 +3377,19 @@ static void dumpSmallBitVector(SmallBitVector &BV) { } #endif +bool DependenceInfo::invalidate(Function &F, const PreservedAnalyses &PA, + FunctionAnalysisManager::Invalidator &Inv) { + // Check if the analysis itself has been invalidated. + auto PAC = PA.getChecker(); + if (!PAC.preserved() && !PAC.preservedSet>()) + return true; + + // Check transitive dependencies. + return Inv.invalidate(F, PA) || + Inv.invalidate(F, PA) || + Inv.invalidate(F, PA); +} + // depends - // Returns NULL if there is no dependence. // Otherwise, return a Dependence with as many details as possible. @@ -3510,7 +3531,7 @@ DependenceInfo::depends(Instruction *Src, Instruction *Dst, // to either Separable or Coupled). // // Next, we consider 1 and 2. The intersection of the GroupLoops is empty. - // Next, 1 and 3. The intersectionof their GroupLoops = {2}, not empty, + // Next, 1 and 3. The intersection of their GroupLoops = {2}, not empty, // so Pair[3].Group = {0, 1, 3} and Done = false. // // Next, we compare 2 against 3. The intersection of the GroupLoops is empty. diff --git a/lib/Analysis/DivergenceAnalysis.cpp b/lib/Analysis/DivergenceAnalysis.cpp index 7ba23854a3cc..0ccd59ef2bfd 100644 --- a/lib/Analysis/DivergenceAnalysis.cpp +++ b/lib/Analysis/DivergenceAnalysis.cpp @@ -1,9 +1,8 @@ //===- DivergenceAnalysis.cpp --------- Divergence Analysis Implementation -==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/DomPrinter.cpp b/lib/Analysis/DomPrinter.cpp index 8abc0e7d0df9..d9f43dd746ef 100644 --- a/lib/Analysis/DomPrinter.cpp +++ b/lib/Analysis/DomPrinter.cpp @@ -1,9 +1,8 @@ //===- DomPrinter.cpp - DOT printer for the dominance trees ------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/DomTreeUpdater.cpp b/lib/Analysis/DomTreeUpdater.cpp new file mode 100644 index 000000000000..49215889cfd6 --- /dev/null +++ b/lib/Analysis/DomTreeUpdater.cpp @@ -0,0 +1,533 @@ +//===- DomTreeUpdater.cpp - DomTree/Post DomTree Updater --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements the DomTreeUpdater class, which provides a uniform way +// to update dominator tree related data structures. +// +//===----------------------------------------------------------------------===// + +#include "llvm/Analysis/DomTreeUpdater.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/Analysis/PostDominators.h" +#include "llvm/IR/Dominators.h" +#include "llvm/Support/GenericDomTree.h" +#include +#include +#include + +namespace llvm { + +bool DomTreeUpdater::isUpdateValid( + const DominatorTree::UpdateType Update) const { + const auto *From = Update.getFrom(); + const auto *To = Update.getTo(); + const auto Kind = Update.getKind(); + + // Discard updates by inspecting the current state of successors of From. + // Since isUpdateValid() must be called *after* the Terminator of From is + // altered we can determine if the update is unnecessary for batch updates + // or invalid for a single update. + const bool HasEdge = llvm::any_of( + successors(From), [To](const BasicBlock *B) { return B == To; }); + + // If the IR does not match the update, + // 1. In batch updates, this update is unnecessary. + // 2. When called by insertEdge*()/deleteEdge*(), this update is invalid. + // Edge does not exist in IR. + if (Kind == DominatorTree::Insert && !HasEdge) + return false; + + // Edge exists in IR. + if (Kind == DominatorTree::Delete && HasEdge) + return false; + + return true; +} + +bool DomTreeUpdater::isSelfDominance( + const DominatorTree::UpdateType Update) const { + // Won't affect DomTree and PostDomTree. + return Update.getFrom() == Update.getTo(); +} + +void DomTreeUpdater::applyDomTreeUpdates() { + // No pending DomTreeUpdates. + if (Strategy != UpdateStrategy::Lazy || !DT) + return; + + // Only apply updates not are applied by DomTree. + if (hasPendingDomTreeUpdates()) { + const auto I = PendUpdates.begin() + PendDTUpdateIndex; + const auto E = PendUpdates.end(); + assert(I < E && "Iterator range invalid; there should be DomTree updates."); + DT->applyUpdates(ArrayRef(I, E)); + PendDTUpdateIndex = PendUpdates.size(); + } +} + +void DomTreeUpdater::flush() { + applyDomTreeUpdates(); + applyPostDomTreeUpdates(); + dropOutOfDateUpdates(); +} + +void DomTreeUpdater::applyPostDomTreeUpdates() { + // No pending PostDomTreeUpdates. + if (Strategy != UpdateStrategy::Lazy || !PDT) + return; + + // Only apply updates not are applied by PostDomTree. + if (hasPendingPostDomTreeUpdates()) { + const auto I = PendUpdates.begin() + PendPDTUpdateIndex; + const auto E = PendUpdates.end(); + assert(I < E && + "Iterator range invalid; there should be PostDomTree updates."); + PDT->applyUpdates(ArrayRef(I, E)); + PendPDTUpdateIndex = PendUpdates.size(); + } +} + +void DomTreeUpdater::tryFlushDeletedBB() { + if (!hasPendingUpdates()) + forceFlushDeletedBB(); +} + +bool DomTreeUpdater::forceFlushDeletedBB() { + if (DeletedBBs.empty()) + return false; + + for (auto *BB : DeletedBBs) { + // After calling deleteBB or callbackDeleteBB under Lazy UpdateStrategy, + // validateDeleteBB() removes all instructions of DelBB and adds an + // UnreachableInst as its terminator. So we check whether the BasicBlock to + // delete only has an UnreachableInst inside. + assert(BB->getInstList().size() == 1 && + isa(BB->getTerminator()) && + "DelBB has been modified while awaiting deletion."); + BB->removeFromParent(); + eraseDelBBNode(BB); + delete BB; + } + DeletedBBs.clear(); + Callbacks.clear(); + return true; +} + +void DomTreeUpdater::recalculate(Function &F) { + + if (Strategy == UpdateStrategy::Eager) { + if (DT) + DT->recalculate(F); + if (PDT) + PDT->recalculate(F); + return; + } + + // There is little performance gain if we pend the recalculation under + // Lazy UpdateStrategy so we recalculate available trees immediately. + + // Prevent forceFlushDeletedBB() from erasing DomTree or PostDomTree nodes. + IsRecalculatingDomTree = IsRecalculatingPostDomTree = true; + + // Because all trees are going to be up-to-date after recalculation, + // flush awaiting deleted BasicBlocks. + forceFlushDeletedBB(); + if (DT) + DT->recalculate(F); + if (PDT) + PDT->recalculate(F); + + // Resume forceFlushDeletedBB() to erase DomTree or PostDomTree nodes. + IsRecalculatingDomTree = IsRecalculatingPostDomTree = false; + PendDTUpdateIndex = PendPDTUpdateIndex = PendUpdates.size(); + dropOutOfDateUpdates(); +} + +bool DomTreeUpdater::hasPendingUpdates() const { + return hasPendingDomTreeUpdates() || hasPendingPostDomTreeUpdates(); +} + +bool DomTreeUpdater::hasPendingDomTreeUpdates() const { + if (!DT) + return false; + return PendUpdates.size() != PendDTUpdateIndex; +} + +bool DomTreeUpdater::hasPendingPostDomTreeUpdates() const { + if (!PDT) + return false; + return PendUpdates.size() != PendPDTUpdateIndex; +} + +bool DomTreeUpdater::isBBPendingDeletion(llvm::BasicBlock *DelBB) const { + if (Strategy == UpdateStrategy::Eager || DeletedBBs.empty()) + return false; + return DeletedBBs.count(DelBB) != 0; +} + +// The DT and PDT require the nodes related to updates +// are not deleted when update functions are called. +// So BasicBlock deletions must be pended when the +// UpdateStrategy is Lazy. When the UpdateStrategy is +// Eager, the BasicBlock will be deleted immediately. +void DomTreeUpdater::deleteBB(BasicBlock *DelBB) { + validateDeleteBB(DelBB); + if (Strategy == UpdateStrategy::Lazy) { + DeletedBBs.insert(DelBB); + return; + } + + DelBB->removeFromParent(); + eraseDelBBNode(DelBB); + delete DelBB; +} + +void DomTreeUpdater::callbackDeleteBB( + BasicBlock *DelBB, std::function Callback) { + validateDeleteBB(DelBB); + if (Strategy == UpdateStrategy::Lazy) { + Callbacks.push_back(CallBackOnDeletion(DelBB, Callback)); + DeletedBBs.insert(DelBB); + return; + } + + DelBB->removeFromParent(); + eraseDelBBNode(DelBB); + Callback(DelBB); + delete DelBB; +} + +void DomTreeUpdater::eraseDelBBNode(BasicBlock *DelBB) { + if (DT && !IsRecalculatingDomTree) + if (DT->getNode(DelBB)) + DT->eraseNode(DelBB); + + if (PDT && !IsRecalculatingPostDomTree) + if (PDT->getNode(DelBB)) + PDT->eraseNode(DelBB); +} + +void DomTreeUpdater::validateDeleteBB(BasicBlock *DelBB) { + assert(DelBB && "Invalid push_back of nullptr DelBB."); + assert(pred_empty(DelBB) && "DelBB has one or more predecessors."); + // DelBB is unreachable and all its instructions are dead. + while (!DelBB->empty()) { + Instruction &I = DelBB->back(); + // Replace used instructions with an arbitrary value (undef). + if (!I.use_empty()) + I.replaceAllUsesWith(llvm::UndefValue::get(I.getType())); + DelBB->getInstList().pop_back(); + } + // Make sure DelBB has a valid terminator instruction. As long as DelBB is a + // Child of Function F it must contain valid IR. + new UnreachableInst(DelBB->getContext(), DelBB); +} + +void DomTreeUpdater::applyUpdates(ArrayRef Updates) { + if (!DT && !PDT) + return; + + if (Strategy == UpdateStrategy::Lazy) { + for (const auto U : Updates) + if (!isSelfDominance(U)) + PendUpdates.push_back(U); + + return; + } + + if (DT) + DT->applyUpdates(Updates); + if (PDT) + PDT->applyUpdates(Updates); +} + +void DomTreeUpdater::applyUpdatesPermissive( + ArrayRef Updates) { + if (!DT && !PDT) + return; + + SmallSet, 8> Seen; + SmallVector DeduplicatedUpdates; + for (const auto U : Updates) { + auto Edge = std::make_pair(U.getFrom(), U.getTo()); + // Because it is illegal to submit updates that have already been applied + // and updates to an edge need to be strictly ordered, + // it is safe to infer the existence of an edge from the first update + // to this edge. + // If the first update to an edge is "Delete", it means that the edge + // existed before. If the first update to an edge is "Insert", it means + // that the edge didn't exist before. + // + // For example, if the user submits {{Delete, A, B}, {Insert, A, B}}, + // because + // 1. it is illegal to submit updates that have already been applied, + // i.e., user cannot delete an nonexistent edge, + // 2. updates to an edge need to be strictly ordered, + // So, initially edge A -> B existed. + // We can then safely ignore future updates to this edge and directly + // inspect the current CFG: + // a. If the edge still exists, because the user cannot insert an existent + // edge, so both {Delete, A, B}, {Insert, A, B} actually happened and + // resulted in a no-op. DTU won't submit any update in this case. + // b. If the edge doesn't exist, we can then infer that {Delete, A, B} + // actually happened but {Insert, A, B} was an invalid update which never + // happened. DTU will submit {Delete, A, B} in this case. + if (!isSelfDominance(U) && Seen.count(Edge) == 0) { + Seen.insert(Edge); + // If the update doesn't appear in the CFG, it means that + // either the change isn't made or relevant operations + // result in a no-op. + if (isUpdateValid(U)) { + if (isLazy()) + PendUpdates.push_back(U); + else + DeduplicatedUpdates.push_back(U); + } + } + } + + if (Strategy == UpdateStrategy::Lazy) + return; + + if (DT) + DT->applyUpdates(DeduplicatedUpdates); + if (PDT) + PDT->applyUpdates(DeduplicatedUpdates); +} + +DominatorTree &DomTreeUpdater::getDomTree() { + assert(DT && "Invalid acquisition of a null DomTree"); + applyDomTreeUpdates(); + dropOutOfDateUpdates(); + return *DT; +} + +PostDominatorTree &DomTreeUpdater::getPostDomTree() { + assert(PDT && "Invalid acquisition of a null PostDomTree"); + applyPostDomTreeUpdates(); + dropOutOfDateUpdates(); + return *PDT; +} + +void DomTreeUpdater::insertEdge(BasicBlock *From, BasicBlock *To) { + +#ifndef NDEBUG + assert(isUpdateValid({DominatorTree::Insert, From, To}) && + "Inserted edge does not appear in the CFG"); +#endif + + if (!DT && !PDT) + return; + + // Won't affect DomTree and PostDomTree; discard update. + if (From == To) + return; + + if (Strategy == UpdateStrategy::Eager) { + if (DT) + DT->insertEdge(From, To); + if (PDT) + PDT->insertEdge(From, To); + return; + } + + PendUpdates.push_back({DominatorTree::Insert, From, To}); +} + +void DomTreeUpdater::insertEdgeRelaxed(BasicBlock *From, BasicBlock *To) { + if (From == To) + return; + + if (!DT && !PDT) + return; + + if (!isUpdateValid({DominatorTree::Insert, From, To})) + return; + + if (Strategy == UpdateStrategy::Eager) { + if (DT) + DT->insertEdge(From, To); + if (PDT) + PDT->insertEdge(From, To); + return; + } + + PendUpdates.push_back({DominatorTree::Insert, From, To}); +} + +void DomTreeUpdater::deleteEdge(BasicBlock *From, BasicBlock *To) { + +#ifndef NDEBUG + assert(isUpdateValid({DominatorTree::Delete, From, To}) && + "Deleted edge still exists in the CFG!"); +#endif + + if (!DT && !PDT) + return; + + // Won't affect DomTree and PostDomTree; discard update. + if (From == To) + return; + + if (Strategy == UpdateStrategy::Eager) { + if (DT) + DT->deleteEdge(From, To); + if (PDT) + PDT->deleteEdge(From, To); + return; + } + + PendUpdates.push_back({DominatorTree::Delete, From, To}); +} + +void DomTreeUpdater::deleteEdgeRelaxed(BasicBlock *From, BasicBlock *To) { + if (From == To) + return; + + if (!DT && !PDT) + return; + + if (!isUpdateValid({DominatorTree::Delete, From, To})) + return; + + if (Strategy == UpdateStrategy::Eager) { + if (DT) + DT->deleteEdge(From, To); + if (PDT) + PDT->deleteEdge(From, To); + return; + } + + PendUpdates.push_back({DominatorTree::Delete, From, To}); +} + +void DomTreeUpdater::dropOutOfDateUpdates() { + if (Strategy == DomTreeUpdater::UpdateStrategy::Eager) + return; + + tryFlushDeletedBB(); + + // Drop all updates applied by both trees. + if (!DT) + PendDTUpdateIndex = PendUpdates.size(); + if (!PDT) + PendPDTUpdateIndex = PendUpdates.size(); + + const size_t dropIndex = std::min(PendDTUpdateIndex, PendPDTUpdateIndex); + const auto B = PendUpdates.begin(); + const auto E = PendUpdates.begin() + dropIndex; + assert(B <= E && "Iterator out of range."); + PendUpdates.erase(B, E); + // Calculate current index. + PendDTUpdateIndex -= dropIndex; + PendPDTUpdateIndex -= dropIndex; +} + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +LLVM_DUMP_METHOD void DomTreeUpdater::dump() const { + raw_ostream &OS = llvm::dbgs(); + + OS << "Available Trees: "; + if (DT || PDT) { + if (DT) + OS << "DomTree "; + if (PDT) + OS << "PostDomTree "; + OS << "\n"; + } else + OS << "None\n"; + + OS << "UpdateStrategy: "; + if (Strategy == UpdateStrategy::Eager) { + OS << "Eager\n"; + return; + } else + OS << "Lazy\n"; + int Index = 0; + + auto printUpdates = + [&](ArrayRef::const_iterator begin, + ArrayRef::const_iterator end) { + if (begin == end) + OS << " None\n"; + Index = 0; + for (auto It = begin, ItEnd = end; It != ItEnd; ++It) { + auto U = *It; + OS << " " << Index << " : "; + ++Index; + if (U.getKind() == DominatorTree::Insert) + OS << "Insert, "; + else + OS << "Delete, "; + BasicBlock *From = U.getFrom(); + if (From) { + auto S = From->getName(); + if (!From->hasName()) + S = "(no name)"; + OS << S << "(" << From << "), "; + } else { + OS << "(badref), "; + } + BasicBlock *To = U.getTo(); + if (To) { + auto S = To->getName(); + if (!To->hasName()) + S = "(no_name)"; + OS << S << "(" << To << ")\n"; + } else { + OS << "(badref)\n"; + } + } + }; + + if (DT) { + const auto I = PendUpdates.begin() + PendDTUpdateIndex; + assert(PendUpdates.begin() <= I && I <= PendUpdates.end() && + "Iterator out of range."); + OS << "Applied but not cleared DomTreeUpdates:\n"; + printUpdates(PendUpdates.begin(), I); + OS << "Pending DomTreeUpdates:\n"; + printUpdates(I, PendUpdates.end()); + } + + if (PDT) { + const auto I = PendUpdates.begin() + PendPDTUpdateIndex; + assert(PendUpdates.begin() <= I && I <= PendUpdates.end() && + "Iterator out of range."); + OS << "Applied but not cleared PostDomTreeUpdates:\n"; + printUpdates(PendUpdates.begin(), I); + OS << "Pending PostDomTreeUpdates:\n"; + printUpdates(I, PendUpdates.end()); + } + + OS << "Pending DeletedBBs:\n"; + Index = 0; + for (auto BB : DeletedBBs) { + OS << " " << Index << " : "; + ++Index; + if (BB->hasName()) + OS << BB->getName() << "("; + else + OS << "(no_name)("; + OS << BB << ")\n"; + } + + OS << "Pending Callbacks:\n"; + Index = 0; + for (auto BB : Callbacks) { + OS << " " << Index << " : "; + ++Index; + if (BB->hasName()) + OS << BB->getName() << "("; + else + OS << "(no_name)("; + OS << BB << ")\n"; + } +} +#endif +} // namespace llvm diff --git a/lib/Analysis/DominanceFrontier.cpp b/lib/Analysis/DominanceFrontier.cpp index de7f62cf4ecd..f9a554acb7ea 100644 --- a/lib/Analysis/DominanceFrontier.cpp +++ b/lib/Analysis/DominanceFrontier.cpp @@ -1,9 +1,8 @@ //===- DominanceFrontier.cpp - Dominance Frontier Calculation -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/Analysis/EHPersonalities.cpp b/lib/Analysis/EHPersonalities.cpp index 0df73aeebbdc..2242541696a4 100644 --- a/lib/Analysis/EHPersonalities.cpp +++ b/lib/Analysis/EHPersonalities.cpp @@ -1,9 +1,8 @@ //===- EHPersonalities.cpp - Compute EH-related information ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/Analysis/GlobalsModRef.cpp b/lib/Analysis/GlobalsModRef.cpp index b28abcadca4a..0d6c0ffb18a8 100644 --- a/lib/Analysis/GlobalsModRef.cpp +++ b/lib/Analysis/GlobalsModRef.cpp @@ -1,9 +1,8 @@ //===- GlobalsModRef.cpp - Simple Mod/Ref Analysis for Globals ------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -514,7 +513,7 @@ void GlobalsAAResult::AnalyzeCallGraph(CallGraph &CG, Module &M) { break; } - if (F->isDeclaration() || F->hasFnAttribute(Attribute::OptimizeNone)) { + if (F->isDeclaration() || F->hasOptNone()) { // Try to get mod/ref behaviour from function attributes. if (F->doesNotAccessMemory()) { // Can't do better than that! @@ -567,7 +566,7 @@ void GlobalsAAResult::AnalyzeCallGraph(CallGraph &CG, Module &M) { // Don't prove any properties based on the implementation of an optnone // function. Function attributes were already used as a best approximation // above. - if (Node->getFunction()->hasFnAttribute(Attribute::OptimizeNone)) + if (Node->getFunction()->hasOptNone()) continue; for (Instruction &I : instructions(Node->getFunction())) { @@ -597,7 +596,7 @@ void GlobalsAAResult::AnalyzeCallGraph(CallGraph &CG, Module &M) { } // All non-call instructions we use the primary predicates for whether - // thay read or write memory. + // they read or write memory. if (I.mayReadFromMemory()) FI.addModRefInfo(ModRefInfo::Ref); if (I.mayWriteToMemory()) @@ -791,10 +790,10 @@ bool GlobalsAAResult::isNonEscapingGlobalNoAlias(const GlobalValue *GV, } // FIXME: It would be good to handle other obvious no-alias cases here, but - // it isn't clear how to do so reasonbly without building a small version + // it isn't clear how to do so reasonably without building a small version // of BasicAA into this code. We could recurse into AAResultBase::alias // here but that seems likely to go poorly as we're inside the - // implementation of such a query. Until then, just conservatievly retun + // implementation of such a query. Until then, just conservatively return // false. return false; } while (!Inputs.empty()); @@ -807,7 +806,8 @@ bool GlobalsAAResult::isNonEscapingGlobalNoAlias(const GlobalValue *GV, /// other is some random pointer, we know there cannot be an alias, because the /// address of the global isn't taken. AliasResult GlobalsAAResult::alias(const MemoryLocation &LocA, - const MemoryLocation &LocB) { + const MemoryLocation &LocB, + AAQueryInfo &AAQI) { // Get the base object these pointers point to. const Value *UV1 = GetUnderlyingObject(LocA.Ptr, DL); const Value *UV2 = GetUnderlyingObject(LocB.Ptr, DL); @@ -882,11 +882,12 @@ AliasResult GlobalsAAResult::alias(const MemoryLocation &LocA, if ((GV1 || GV2) && GV1 != GV2) return NoAlias; - return AAResultBase::alias(LocA, LocB); + return AAResultBase::alias(LocA, LocB, AAQI); } ModRefInfo GlobalsAAResult::getModRefInfoForArgument(const CallBase *Call, - const GlobalValue *GV) { + const GlobalValue *GV, + AAQueryInfo &AAQI) { if (Call->doesNotAccessMemory()) return ModRefInfo::NoModRef; ModRefInfo ConservativeResult = @@ -895,14 +896,15 @@ ModRefInfo GlobalsAAResult::getModRefInfoForArgument(const CallBase *Call, // Iterate through all the arguments to the called function. If any argument // is based on GV, return the conservative result. for (auto &A : Call->args()) { - SmallVector Objects; + SmallVector Objects; GetUnderlyingObjects(A, Objects, DL); // All objects must be identified. if (!all_of(Objects, isIdentifiedObject) && // Try ::alias to see if all objects are known not to alias GV. - !all_of(Objects, [&](Value *V) { - return this->alias(MemoryLocation(V), MemoryLocation(GV)) == NoAlias; + !all_of(Objects, [&](const Value *V) { + return this->alias(MemoryLocation(V), MemoryLocation(GV), AAQI) == + NoAlias; })) return ConservativeResult; @@ -915,7 +917,8 @@ ModRefInfo GlobalsAAResult::getModRefInfoForArgument(const CallBase *Call, } ModRefInfo GlobalsAAResult::getModRefInfo(const CallBase *Call, - const MemoryLocation &Loc) { + const MemoryLocation &Loc, + AAQueryInfo &AAQI) { ModRefInfo Known = ModRefInfo::ModRef; // If we are asking for mod/ref info of a direct call with a pointer to a @@ -927,11 +930,11 @@ ModRefInfo GlobalsAAResult::getModRefInfo(const CallBase *Call, if (NonAddressTakenGlobals.count(GV)) if (const FunctionInfo *FI = getFunctionInfo(F)) Known = unionModRef(FI->getModRefInfoForGlobal(*GV), - getModRefInfoForArgument(Call, GV)); + getModRefInfoForArgument(Call, GV, AAQI)); if (!isModOrRefSet(Known)) return ModRefInfo::NoModRef; // No need to query other mod/ref analyses - return intersectModRef(Known, AAResultBase::getModRefInfo(Call, Loc)); + return intersectModRef(Known, AAResultBase::getModRefInfo(Call, Loc, AAQI)); } GlobalsAAResult::GlobalsAAResult(const DataLayout &DL, diff --git a/lib/Analysis/GuardUtils.cpp b/lib/Analysis/GuardUtils.cpp index 08fa6abeafb5..cad92f6e56bb 100644 --- a/lib/Analysis/GuardUtils.cpp +++ b/lib/Analysis/GuardUtils.cpp @@ -1,9 +1,8 @@ //===-- GuardUtils.cpp - Utils for work with guards -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Utils that are used to perform analyzes related to guards and their @@ -19,3 +18,32 @@ bool llvm::isGuard(const User *U) { using namespace llvm::PatternMatch; return match(U, m_Intrinsic()); } + +bool llvm::isGuardAsWidenableBranch(const User *U) { + Value *Condition, *WidenableCondition; + BasicBlock *GuardedBB, *DeoptBB; + if (!parseWidenableBranch(U, Condition, WidenableCondition, GuardedBB, + DeoptBB)) + return false; + using namespace llvm::PatternMatch; + for (auto &Insn : *DeoptBB) { + if (match(&Insn, m_Intrinsic())) + return true; + if (Insn.mayHaveSideEffects()) + return false; + } + return false; +} + +bool llvm::parseWidenableBranch(const User *U, Value *&Condition, + Value *&WidenableCondition, + BasicBlock *&IfTrueBB, BasicBlock *&IfFalseBB) { + using namespace llvm::PatternMatch; + if (!match(U, m_Br(m_And(m_Value(Condition), m_Value(WidenableCondition)), + IfTrueBB, IfFalseBB))) + return false; + // TODO: At the moment, we only recognize the branch if the WC call in this + // specific position. We should generalize! + return match(WidenableCondition, + m_Intrinsic()); +} diff --git a/lib/Analysis/IVDescriptors.cpp b/lib/Analysis/IVDescriptors.cpp index aaebc4a481ec..ce285f82f720 100644 --- a/lib/Analysis/IVDescriptors.cpp +++ b/lib/Analysis/IVDescriptors.cpp @@ -1,9 +1,8 @@ //===- llvm/Analysis/IVDescriptors.cpp - IndVar Descriptors -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,6 +14,7 @@ #include "llvm/ADT/ScopeExit.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/Analysis/BasicAliasAnalysis.h" +#include "llvm/Analysis/DomTreeUpdater.h" #include "llvm/Analysis/GlobalsModRef.h" #include "llvm/Analysis/InstructionSimplify.h" #include "llvm/Analysis/LoopInfo.h" @@ -26,7 +26,6 @@ #include "llvm/Analysis/ScalarEvolutionExpressions.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/ValueTracking.h" -#include "llvm/IR/DomTreeUpdater.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Module.h" @@ -252,6 +251,10 @@ bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurrenceKind Kind, Worklist.push_back(Start); VisitedInsts.insert(Start); + // Start with all flags set because we will intersect this with the reduction + // flags from all the reduction operations. + FastMathFlags FMF = FastMathFlags::getFast(); + // A value in the reduction can be used: // - By the reduction: // - Reduction operation: @@ -297,6 +300,8 @@ bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurrenceKind Kind, ReduxDesc = isRecurrenceInstr(Cur, Kind, ReduxDesc, HasFunNoNaNAttr); if (!ReduxDesc.isRecurrence()) return false; + if (isa(ReduxDesc.getPatternInst())) + FMF &= ReduxDesc.getPatternInst()->getFastMathFlags(); } bool IsASelect = isa(Cur); @@ -442,7 +447,7 @@ bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurrenceKind Kind, // Save the description of this reduction variable. RecurrenceDescriptor RD( - RdxStart, ExitInstruction, Kind, ReduxDesc.getMinMaxKind(), + RdxStart, ExitInstruction, Kind, FMF, ReduxDesc.getMinMaxKind(), ReduxDesc.getUnsafeAlgebraInst(), RecurrenceType, IsSigned, CastInsts); RedDes = RD; @@ -550,9 +555,8 @@ RecurrenceDescriptor::isConditionalRdxPattern( RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(Instruction *I, RecurrenceKind Kind, InstDesc &Prev, bool HasFunNoNaNAttr) { - bool FP = I->getType()->isFloatingPointTy(); Instruction *UAI = Prev.getUnsafeAlgebraInst(); - if (!UAI && FP && !I->isFast()) + if (!UAI && isa(I) && !I->hasAllowReassoc()) UAI = I; // Found an unsafe (unvectorizable) algebra instruction. switch (I->getOpcode()) { @@ -1010,7 +1014,7 @@ bool InductionDescriptor::isInductionPHI(PHINode *Phi, const Loop *TheLoop, // If we started from an UnknownSCEV, and managed to build an addRecurrence // only after enabling Assume with PSCEV, this means we may have encountered // cast instructions that required adding a runtime check in order to - // guarantee the correctness of the AddRecurence respresentation of the + // guarantee the correctness of the AddRecurrence respresentation of the // induction. if (PhiScev != AR && SymbolicPhi) { SmallVector Casts; @@ -1049,6 +1053,13 @@ bool InductionDescriptor::isInductionPHI( Value *StartValue = Phi->getIncomingValueForBlock(AR->getLoop()->getLoopPreheader()); + + BasicBlock *Latch = AR->getLoop()->getLoopLatch(); + if (!Latch) + return false; + BinaryOperator *BOp = + dyn_cast(Phi->getIncomingValueForBlock(Latch)); + const SCEV *Step = AR->getStepRecurrence(*SE); // Calculate the pointer stride and check if it is consecutive. // The stride may be a constant or a loop invariant integer value. @@ -1057,7 +1068,7 @@ bool InductionDescriptor::isInductionPHI( return false; if (PhiTy->isIntegerTy()) { - D = InductionDescriptor(StartValue, IK_IntInduction, Step, /*BOp=*/nullptr, + D = InductionDescriptor(StartValue, IK_IntInduction, Step, BOp, CastsToIgnore); return true; } @@ -1084,6 +1095,6 @@ bool InductionDescriptor::isInductionPHI( return false; auto *StepValue = SE->getConstant(CV->getType(), CVSize / Size, true /* signed */); - D = InductionDescriptor(StartValue, IK_PtrInduction, StepValue); + D = InductionDescriptor(StartValue, IK_PtrInduction, StepValue, BOp); return true; } diff --git a/lib/Analysis/IVUsers.cpp b/lib/Analysis/IVUsers.cpp index 609e5e3a1448..681a0cf7e981 100644 --- a/lib/Analysis/IVUsers.cpp +++ b/lib/Analysis/IVUsers.cpp @@ -1,9 +1,8 @@ //===- IVUsers.cpp - Induction Variable Users -------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/IndirectCallPromotionAnalysis.cpp b/lib/Analysis/IndirectCallPromotionAnalysis.cpp index d6e6e76af03c..6ff840efcb64 100644 --- a/lib/Analysis/IndirectCallPromotionAnalysis.cpp +++ b/lib/Analysis/IndirectCallPromotionAnalysis.cpp @@ -1,9 +1,8 @@ //===-- IndirectCallPromotionAnalysis.cpp - Find promotion candidates ===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/InlineCost.cpp b/lib/Analysis/InlineCost.cpp index 6ddb3cbc01a3..0dec146e0465 100644 --- a/lib/Analysis/InlineCost.cpp +++ b/lib/Analysis/InlineCost.cpp @@ -1,9 +1,8 @@ //===- InlineCost.cpp - Cost analysis for inliner -------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -28,7 +27,6 @@ #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/Config/llvm-config.h" -#include "llvm/IR/CallSite.h" #include "llvm/IR/CallingConv.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/Dominators.h" @@ -37,6 +35,7 @@ #include "llvm/IR/InstVisitor.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Operator.h" +#include "llvm/IR/PatternMatch.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" @@ -51,19 +50,19 @@ static cl::opt InlineThreshold( cl::desc("Control the amount of inlining to perform (default = 225)")); static cl::opt HintThreshold( - "inlinehint-threshold", cl::Hidden, cl::init(325), + "inlinehint-threshold", cl::Hidden, cl::init(325), cl::ZeroOrMore, cl::desc("Threshold for inlining functions with inline hint")); static cl::opt ColdCallSiteThreshold("inline-cold-callsite-threshold", cl::Hidden, - cl::init(45), + cl::init(45), cl::ZeroOrMore, cl::desc("Threshold for inlining cold callsites")); // We introduce this threshold to help performance of instrumentation based // PGO before we actually hook up inliner with analysis passes such as BPI and // BFI. static cl::opt ColdThreshold( - "inlinecold-threshold", cl::Hidden, cl::init(45), + "inlinecold-threshold", cl::Hidden, cl::init(45), cl::ZeroOrMore, cl::desc("Threshold for inlining functions with cold attribute")); static cl::opt @@ -77,7 +76,7 @@ static cl::opt LocallyHotCallSiteThreshold( static cl::opt ColdCallSiteRelFreq( "cold-callsite-rel-freq", cl::Hidden, cl::init(2), cl::ZeroOrMore, - cl::desc("Maxmimum block frequency, expressed as a percentage of caller's " + cl::desc("Maximum block frequency, expressed as a percentage of caller's " "entry frequency, for a callsite to be cold in the absence of " "profile information.")); @@ -88,7 +87,7 @@ static cl::opt HotCallSiteRelFreq( "profile information.")); static cl::opt OptComputeFullInlineCost( - "inline-cost-full", cl::Hidden, cl::init(false), + "inline-cost-full", cl::Hidden, cl::init(false), cl::ZeroOrMore, cl::desc("Compute the full inline cost of a call site even when the cost " "exceeds the threshold.")); @@ -122,31 +121,43 @@ class CallAnalyzer : public InstVisitor { /// The candidate callsite being analyzed. Please do not use this to do /// analysis in the caller function; we want the inline cost query to be /// easily cacheable. Instead, use the cover function paramHasAttr. - CallSite CandidateCS; + CallBase &CandidateCall; /// Tunable parameters that control the analysis. const InlineParams &Params; + /// Upper bound for the inlining cost. Bonuses are being applied to account + /// for speculative "expected profit" of the inlining decision. int Threshold; - int Cost; + + /// Inlining cost measured in abstract units, accounts for all the + /// instructions expected to be executed for a given function invocation. + /// Instructions that are statically proven to be dead based on call-site + /// arguments are not counted here. + int Cost = 0; + bool ComputeFullInlineCost; - bool IsCallerRecursive; - bool IsRecursiveCall; - bool ExposesReturnsTwice; - bool HasDynamicAlloca; - bool ContainsNoDuplicateCall; - bool HasReturn; - bool HasIndirectBr; - bool HasUninlineableIntrinsic; - bool InitsVargArgs; + bool IsCallerRecursive = false; + bool IsRecursiveCall = false; + bool ExposesReturnsTwice = false; + bool HasDynamicAlloca = false; + bool ContainsNoDuplicateCall = false; + bool HasReturn = false; + bool HasIndirectBr = false; + bool HasUninlineableIntrinsic = false; + bool InitsVargArgs = false; /// Number of bytes allocated statically by the callee. - uint64_t AllocatedSize; - unsigned NumInstructions, NumVectorInstructions; - int VectorBonus, TenPercentVectorBonus; - // Bonus to be applied when the callee has only one reachable basic block. - int SingleBBBonus; + uint64_t AllocatedSize = 0; + unsigned NumInstructions = 0; + unsigned NumVectorInstructions = 0; + + /// Bonus to be applied when percentage of vector instructions in callee is + /// high (see more details in updateThreshold). + int VectorBonus = 0; + /// Bonus to be applied when the callee has only one reachable basic block. + int SingleBBBonus = 0; /// While we walk the potentially-inlined instructions, we build up and /// maintain a mapping of simplified values specific to this callsite. The @@ -181,7 +192,7 @@ class CallAnalyzer : public InstVisitor { /// loads. bool EnableLoadElimination; SmallPtrSet LoadAddrSet; - int LoadEliminationCost; + int LoadEliminationCost = 0; // Custom simplification helper routines. bool isAllocaDerivedArg(Value *V); @@ -196,7 +207,7 @@ class CallAnalyzer : public InstVisitor { bool isGEPFree(GetElementPtrInst &GEP); bool canFoldInboundsGEP(GetElementPtrInst &I); bool accumulateGEPOffset(GEPOperator &GEP, APInt &Offset); - bool simplifyCallSite(Function *F, CallSite CS); + bool simplifyCallSite(Function *F, CallBase &Call); template bool simplifyInstruction(Instruction &I, Callable Evaluate); ConstantInt *stripAndComputeInBoundsConstantOffsets(Value *&V); @@ -216,22 +227,28 @@ class CallAnalyzer : public InstVisitor { /// attributes and callee hotness for PGO builds. The Callee is explicitly /// passed to support analyzing indirect calls whose target is inferred by /// analysis. - void updateThreshold(CallSite CS, Function &Callee); + void updateThreshold(CallBase &Call, Function &Callee); - /// Return true if size growth is allowed when inlining the callee at CS. - bool allowSizeGrowth(CallSite CS); + /// Return true if size growth is allowed when inlining the callee at \p Call. + bool allowSizeGrowth(CallBase &Call); - /// Return true if \p CS is a cold callsite. - bool isColdCallSite(CallSite CS, BlockFrequencyInfo *CallerBFI); + /// Return true if \p Call is a cold callsite. + bool isColdCallSite(CallBase &Call, BlockFrequencyInfo *CallerBFI); - /// Return a higher threshold if \p CS is a hot callsite. - Optional getHotCallSiteThreshold(CallSite CS, + /// Return a higher threshold if \p Call is a hot callsite. + Optional getHotCallSiteThreshold(CallBase &Call, BlockFrequencyInfo *CallerBFI); // Custom analysis routines. InlineResult analyzeBlock(BasicBlock *BB, SmallPtrSetImpl &EphValues); + /// Handle a capped 'int' increment for Cost. + void addCost(int64_t Inc, int64_t UpperBound = INT_MAX) { + assert(UpperBound > 0 && UpperBound <= INT_MAX && "invalid upper bound"); + Cost = (int)std::min(UpperBound, Cost + Inc); + } + // Disable several entry points to the visitor so we don't accidentally use // them by declaring but not defining them here. void visit(Module *); @@ -256,11 +273,12 @@ class CallAnalyzer : public InstVisitor { bool visitCmpInst(CmpInst &I); bool visitSub(BinaryOperator &I); bool visitBinaryOperator(BinaryOperator &I); + bool visitFNeg(UnaryOperator &I); bool visitLoad(LoadInst &I); bool visitStore(StoreInst &I); bool visitExtractValue(ExtractValueInst &I); bool visitInsertValue(InsertValueInst &I); - bool visitCallSite(CallSite CS); + bool visitCallBase(CallBase &Call); bool visitReturnInst(ReturnInst &RI); bool visitBranchInst(BranchInst &BI); bool visitSelectInst(SelectInst &SI); @@ -276,38 +294,29 @@ public: std::function &GetAssumptionCache, Optional> &GetBFI, ProfileSummaryInfo *PSI, OptimizationRemarkEmitter *ORE, - Function &Callee, CallSite CSArg, const InlineParams &Params) + Function &Callee, CallBase &Call, const InlineParams &Params) : TTI(TTI), GetAssumptionCache(GetAssumptionCache), GetBFI(GetBFI), PSI(PSI), F(Callee), DL(F.getParent()->getDataLayout()), ORE(ORE), - CandidateCS(CSArg), Params(Params), Threshold(Params.DefaultThreshold), - Cost(0), ComputeFullInlineCost(OptComputeFullInlineCost || - Params.ComputeFullInlineCost || ORE), - IsCallerRecursive(false), IsRecursiveCall(false), - ExposesReturnsTwice(false), HasDynamicAlloca(false), - ContainsNoDuplicateCall(false), HasReturn(false), HasIndirectBr(false), - HasUninlineableIntrinsic(false), InitsVargArgs(false), AllocatedSize(0), - NumInstructions(0), NumVectorInstructions(0), VectorBonus(0), - SingleBBBonus(0), EnableLoadElimination(true), LoadEliminationCost(0), - NumConstantArgs(0), NumConstantOffsetPtrArgs(0), NumAllocaArgs(0), - NumConstantPtrCmps(0), NumConstantPtrDiffs(0), - NumInstructionsSimplified(0), SROACostSavings(0), - SROACostSavingsLost(0) {} - - InlineResult analyzeCall(CallSite CS); + CandidateCall(Call), Params(Params), Threshold(Params.DefaultThreshold), + ComputeFullInlineCost(OptComputeFullInlineCost || + Params.ComputeFullInlineCost || ORE), + EnableLoadElimination(true) {} + + InlineResult analyzeCall(CallBase &Call); int getThreshold() { return Threshold; } int getCost() { return Cost; } // Keep a bunch of stats about the cost savings found so we can print them // out when debugging. - unsigned NumConstantArgs; - unsigned NumConstantOffsetPtrArgs; - unsigned NumAllocaArgs; - unsigned NumConstantPtrCmps; - unsigned NumConstantPtrDiffs; - unsigned NumInstructionsSimplified; - unsigned SROACostSavings; - unsigned SROACostSavingsLost; + unsigned NumConstantArgs = 0; + unsigned NumConstantOffsetPtrArgs = 0; + unsigned NumAllocaArgs = 0; + unsigned NumConstantPtrCmps = 0; + unsigned NumConstantPtrDiffs = 0; + unsigned NumInstructionsSimplified = 0; + unsigned SROACostSavings = 0; + unsigned SROACostSavingsLost = 0; void dump(); }; @@ -342,7 +351,7 @@ bool CallAnalyzer::lookupSROAArgAndCost( void CallAnalyzer::disableSROA(DenseMap::iterator CostIt) { // If we're no longer able to perform SROA we need to undo its cost savings // and prevent subsequent analysis. - Cost += CostIt->second; + addCost(CostIt->second); SROACostSavings -= CostIt->second; SROACostSavingsLost += CostIt->second; SROAArgCosts.erase(CostIt); @@ -366,7 +375,7 @@ void CallAnalyzer::accumulateSROACost(DenseMap::iterator CostIt, void CallAnalyzer::disableLoadElimination() { if (EnableLoadElimination) { - Cost += LoadEliminationCost; + addCost(LoadEliminationCost); LoadEliminationCost = 0; EnableLoadElimination = false; } @@ -701,7 +710,7 @@ bool CallAnalyzer::visitIntToPtr(IntToPtrInst &I) { } bool CallAnalyzer::visitCastInst(CastInst &I) { - // Propagate constants through ptrtoint. + // Propagate constants through casts. if (simplifyInstruction(I, [&](SmallVectorImpl &COps) { return ConstantExpr::getCast(I.getOpcode(), COps[0], I.getType()); })) @@ -721,7 +730,7 @@ bool CallAnalyzer::visitCastInst(CastInst &I) { case Instruction::FPToUI: case Instruction::FPToSI: if (TTI.getFPOpCost(I.getType()) == TargetTransformInfo::TCC_Expensive) - Cost += InlineConstants::CallPenalty; + addCost(InlineConstants::CallPenalty); break; default: break; @@ -737,14 +746,14 @@ bool CallAnalyzer::visitUnaryInstruction(UnaryInstruction &I) { })) return true; - // Disable any SROA on the argument to arbitrary unary operators. + // Disable any SROA on the argument to arbitrary unary instructions. disableSROA(Operand); return false; } bool CallAnalyzer::paramHasAttr(Argument *A, Attribute::AttrKind Attr) { - return CandidateCS.paramHasAttr(A->getArgNo(), Attr); + return CandidateCall.paramHasAttr(A->getArgNo(), Attr); } bool CallAnalyzer::isKnownNonNullInCallee(Value *V) { @@ -769,7 +778,7 @@ bool CallAnalyzer::isKnownNonNullInCallee(Value *V) { return false; } -bool CallAnalyzer::allowSizeGrowth(CallSite CS) { +bool CallAnalyzer::allowSizeGrowth(CallBase &Call) { // If the normal destination of the invoke or the parent block of the call // site is unreachable-terminated, there is little point in inlining this // unless there is literally zero cost. @@ -785,21 +794,21 @@ bool CallAnalyzer::allowSizeGrowth(CallSite CS) { // For now, we are not handling this corner case here as it is rare in real // code. In future, we should elaborate this based on BPI and BFI in more // general threshold adjusting heuristics in updateThreshold(). - Instruction *Instr = CS.getInstruction(); - if (InvokeInst *II = dyn_cast(Instr)) { + if (InvokeInst *II = dyn_cast(&Call)) { if (isa(II->getNormalDest()->getTerminator())) return false; - } else if (isa(Instr->getParent()->getTerminator())) + } else if (isa(Call.getParent()->getTerminator())) return false; return true; } -bool CallAnalyzer::isColdCallSite(CallSite CS, BlockFrequencyInfo *CallerBFI) { +bool CallAnalyzer::isColdCallSite(CallBase &Call, + BlockFrequencyInfo *CallerBFI) { // If global profile summary is available, then callsite's coldness is // determined based on that. if (PSI && PSI->hasProfileSummary()) - return PSI->isColdCallSite(CS, CallerBFI); + return PSI->isColdCallSite(CallSite(&Call), CallerBFI); // Otherwise we need BFI to be available. if (!CallerBFI) @@ -810,20 +819,21 @@ bool CallAnalyzer::isColdCallSite(CallSite CS, BlockFrequencyInfo *CallerBFI) { // complexity is not worth it unless this scaling shows up high in the // profiles. const BranchProbability ColdProb(ColdCallSiteRelFreq, 100); - auto CallSiteBB = CS.getInstruction()->getParent(); + auto CallSiteBB = Call.getParent(); auto CallSiteFreq = CallerBFI->getBlockFreq(CallSiteBB); auto CallerEntryFreq = - CallerBFI->getBlockFreq(&(CS.getCaller()->getEntryBlock())); + CallerBFI->getBlockFreq(&(Call.getCaller()->getEntryBlock())); return CallSiteFreq < CallerEntryFreq * ColdProb; } Optional -CallAnalyzer::getHotCallSiteThreshold(CallSite CS, +CallAnalyzer::getHotCallSiteThreshold(CallBase &Call, BlockFrequencyInfo *CallerBFI) { // If global profile summary is available, then callsite's hotness is // determined based on that. - if (PSI && PSI->hasProfileSummary() && PSI->isHotCallSite(CS, CallerBFI)) + if (PSI && PSI->hasProfileSummary() && + PSI->isHotCallSite(CallSite(&Call), CallerBFI)) return Params.HotCallSiteThreshold; // Otherwise we need BFI to be available and to have a locally hot callsite @@ -835,7 +845,7 @@ CallAnalyzer::getHotCallSiteThreshold(CallSite CS, // potentially cache the computation of scaled entry frequency, but the added // complexity is not worth it unless this scaling shows up high in the // profiles. - auto CallSiteBB = CS.getInstruction()->getParent(); + auto CallSiteBB = Call.getParent(); auto CallSiteFreq = CallerBFI->getBlockFreq(CallSiteBB).getFrequency(); auto CallerEntryFreq = CallerBFI->getEntryFreq(); if (CallSiteFreq >= CallerEntryFreq * HotCallSiteRelFreq) @@ -845,14 +855,14 @@ CallAnalyzer::getHotCallSiteThreshold(CallSite CS, return None; } -void CallAnalyzer::updateThreshold(CallSite CS, Function &Callee) { +void CallAnalyzer::updateThreshold(CallBase &Call, Function &Callee) { // If no size growth is allowed for this inlining, set Threshold to 0. - if (!allowSizeGrowth(CS)) { + if (!allowSizeGrowth(Call)) { Threshold = 0; return; } - Function *Caller = CS.getCaller(); + Function *Caller = Call.getCaller(); // return min(A, B) if B is valid. auto MinIfValid = [](int A, Optional B) { @@ -870,15 +880,6 @@ void CallAnalyzer::updateThreshold(CallSite CS, Function &Callee) { // basic block at the given callsite context. This is speculatively applied // and withdrawn if more than one basic block is seen. // - // Vector bonuses: We want to more aggressively inline vector-dense kernels - // and apply this bonus based on the percentage of vector instructions. A - // bonus is applied if the vector instructions exceed 50% and half that amount - // is applied if it exceeds 10%. Note that these bonuses are some what - // arbitrary and evolved over time by accident as much as because they are - // principled bonuses. - // FIXME: It would be nice to base the bonus values on something more - // scientific. - // // LstCallToStaticBonus: This large bonus is applied to ensure the inlining // of the last call to a static function as inlining such functions is // guaranteed to reduce code size. @@ -886,7 +887,7 @@ void CallAnalyzer::updateThreshold(CallSite CS, Function &Callee) { // These bonus percentages may be set to 0 based on properties of the caller // and the callsite. int SingleBBBonusPercent = 50; - int VectorBonusPercent = 150; + int VectorBonusPercent = TTI.getInlinerVectorBonusPercent(); int LastCallToStaticBonus = InlineConstants::LastCallToStaticBonus; // Lambda to set all the above bonus and bonus percentages to 0. @@ -898,7 +899,7 @@ void CallAnalyzer::updateThreshold(CallSite CS, Function &Callee) { // Use the OptMinSizeThreshold or OptSizeThreshold knob if they are available // and reduce the threshold if the caller has the necessary attribute. - if (Caller->optForMinSize()) { + if (Caller->hasMinSize()) { Threshold = MinIfValid(Threshold, Params.OptMinSizeThreshold); // For minsize, we want to disable the single BB bonus and the vector // bonuses, but not the last-call-to-static bonus. Inlining the last call to @@ -906,12 +907,12 @@ void CallAnalyzer::updateThreshold(CallSite CS, Function &Callee) { // call/return instructions. SingleBBBonusPercent = 0; VectorBonusPercent = 0; - } else if (Caller->optForSize()) + } else if (Caller->hasOptSize()) Threshold = MinIfValid(Threshold, Params.OptSizeThreshold); // Adjust the threshold based on inlinehint attribute and profile based // hotness information if the caller does not have MinSize attribute. - if (!Caller->optForMinSize()) { + if (!Caller->hasMinSize()) { if (Callee.hasFnAttribute(Attribute::InlineHint)) Threshold = MaxIfValid(Threshold, Params.HintThreshold); @@ -923,15 +924,15 @@ void CallAnalyzer::updateThreshold(CallSite CS, Function &Callee) { // used (which adds hotness metadata to calls) or if caller's // BlockFrequencyInfo is available. BlockFrequencyInfo *CallerBFI = GetBFI ? &((*GetBFI)(*Caller)) : nullptr; - auto HotCallSiteThreshold = getHotCallSiteThreshold(CS, CallerBFI); - if (!Caller->optForSize() && HotCallSiteThreshold) { + auto HotCallSiteThreshold = getHotCallSiteThreshold(Call, CallerBFI); + if (!Caller->hasOptSize() && HotCallSiteThreshold) { LLVM_DEBUG(dbgs() << "Hot callsite.\n"); // FIXME: This should update the threshold only if it exceeds the // current threshold, but AutoFDO + ThinLTO currently relies on this // behavior to prevent inlining of hot callsites during ThinLTO // compile phase. Threshold = HotCallSiteThreshold.getValue(); - } else if (isColdCallSite(CS, CallerBFI)) { + } else if (isColdCallSite(Call, CallerBFI)) { LLVM_DEBUG(dbgs() << "Cold callsite.\n"); // Do not apply bonuses for a cold callsite including the // LastCallToStatic bonus. While this bonus might result in code size @@ -968,7 +969,7 @@ void CallAnalyzer::updateThreshold(CallSite CS, Function &Callee) { VectorBonus = Threshold * VectorBonusPercent / 100; bool OnlyOneCallAndLocalLinkage = - F.hasLocalLinkage() && F.hasOneUse() && &F == CS.getCalledFunction(); + F.hasLocalLinkage() && F.hasOneUse() && &F == Call.getCalledFunction(); // If there is only one call of the function, and it has internal linkage, // the cost of inlining it drops dramatically. It may seem odd to update // Cost in updateThreshold, but the bonus depends on the logic in this method. @@ -1087,10 +1088,34 @@ bool CallAnalyzer::visitBinaryOperator(BinaryOperator &I) { // If the instruction is floating point, and the target says this operation // is expensive, this may eventually become a library call. Treat the cost - // as such. + // as such. Unless it's fneg which can be implemented with an xor. + using namespace llvm::PatternMatch; if (I.getType()->isFloatingPointTy() && - TTI.getFPOpCost(I.getType()) == TargetTransformInfo::TCC_Expensive) - Cost += InlineConstants::CallPenalty; + TTI.getFPOpCost(I.getType()) == TargetTransformInfo::TCC_Expensive && + !match(&I, m_FNeg(m_Value()))) + addCost(InlineConstants::CallPenalty); + + return false; +} + +bool CallAnalyzer::visitFNeg(UnaryOperator &I) { + Value *Op = I.getOperand(0); + Constant *COp = dyn_cast(Op); + if (!COp) + COp = SimplifiedValues.lookup(Op); + + Value *SimpleV = SimplifyFNegInst(COp ? COp : Op, + cast(I).getFastMathFlags(), + DL); + + if (Constant *C = dyn_cast_or_null(SimpleV)) + SimplifiedValues[&I] = C; + + if (SimpleV) + return true; + + // Disable any SROA on arguments to arbitrary, unsimplified fneg. + disableSROA(Op); return false; } @@ -1173,62 +1198,61 @@ bool CallAnalyzer::visitInsertValue(InsertValueInst &I) { /// analyzing the arguments and call itself with instsimplify. Returns true if /// it has simplified the callsite to some other entity (a constant), making it /// free. -bool CallAnalyzer::simplifyCallSite(Function *F, CallSite CS) { +bool CallAnalyzer::simplifyCallSite(Function *F, CallBase &Call) { // FIXME: Using the instsimplify logic directly for this is inefficient // because we have to continually rebuild the argument list even when no // simplifications can be performed. Until that is fixed with remapping // inside of instsimplify, directly constant fold calls here. - if (!canConstantFoldCallTo(CS, F)) + if (!canConstantFoldCallTo(&Call, F)) return false; // Try to re-map the arguments to constants. SmallVector ConstantArgs; - ConstantArgs.reserve(CS.arg_size()); - for (CallSite::arg_iterator I = CS.arg_begin(), E = CS.arg_end(); I != E; - ++I) { - Constant *C = dyn_cast(*I); + ConstantArgs.reserve(Call.arg_size()); + for (Value *I : Call.args()) { + Constant *C = dyn_cast(I); if (!C) - C = dyn_cast_or_null(SimplifiedValues.lookup(*I)); + C = dyn_cast_or_null(SimplifiedValues.lookup(I)); if (!C) return false; // This argument doesn't map to a constant. ConstantArgs.push_back(C); } - if (Constant *C = ConstantFoldCall(CS, F, ConstantArgs)) { - SimplifiedValues[CS.getInstruction()] = C; + if (Constant *C = ConstantFoldCall(&Call, F, ConstantArgs)) { + SimplifiedValues[&Call] = C; return true; } return false; } -bool CallAnalyzer::visitCallSite(CallSite CS) { - if (CS.hasFnAttr(Attribute::ReturnsTwice) && +bool CallAnalyzer::visitCallBase(CallBase &Call) { + if (Call.hasFnAttr(Attribute::ReturnsTwice) && !F.hasFnAttribute(Attribute::ReturnsTwice)) { // This aborts the entire analysis. ExposesReturnsTwice = true; return false; } - if (CS.isCall() && cast(CS.getInstruction())->cannotDuplicate()) + if (isa(Call) && cast(Call).cannotDuplicate()) ContainsNoDuplicateCall = true; - if (Function *F = CS.getCalledFunction()) { + if (Function *F = Call.getCalledFunction()) { // When we have a concrete function, first try to simplify it directly. - if (simplifyCallSite(F, CS)) + if (simplifyCallSite(F, Call)) return true; // Next check if it is an intrinsic we know about. // FIXME: Lift this into part of the InstVisitor. - if (IntrinsicInst *II = dyn_cast(CS.getInstruction())) { + if (IntrinsicInst *II = dyn_cast(&Call)) { switch (II->getIntrinsicID()) { default: - if (!CS.onlyReadsMemory() && !isAssumeLikeIntrinsic(II)) + if (!Call.onlyReadsMemory() && !isAssumeLikeIntrinsic(II)) disableLoadElimination(); - return Base::visitCallSite(CS); + return Base::visitCallBase(Call); case Intrinsic::load_relative: // This is normally lowered to 4 LLVM instructions. - Cost += 3 * InlineConstants::InstrCost; + addCost(3 * InlineConstants::InstrCost); return false; case Intrinsic::memset: @@ -1247,7 +1271,7 @@ bool CallAnalyzer::visitCallSite(CallSite CS) { } } - if (F == CS.getInstruction()->getFunction()) { + if (F == Call.getFunction()) { // This flag will fully abort the analysis, so don't bother with anything // else. IsRecursiveCall = true; @@ -1257,34 +1281,34 @@ bool CallAnalyzer::visitCallSite(CallSite CS) { if (TTI.isLoweredToCall(F)) { // We account for the average 1 instruction per call argument setup // here. - Cost += CS.arg_size() * InlineConstants::InstrCost; + addCost(Call.arg_size() * InlineConstants::InstrCost); // Everything other than inline ASM will also have a significant cost // merely from making the call. - if (!isa(CS.getCalledValue())) - Cost += InlineConstants::CallPenalty; + if (!isa(Call.getCalledValue())) + addCost(InlineConstants::CallPenalty); } - if (!CS.onlyReadsMemory()) + if (!Call.onlyReadsMemory()) disableLoadElimination(); - return Base::visitCallSite(CS); + return Base::visitCallBase(Call); } // Otherwise we're in a very special case -- an indirect function call. See // if we can be particularly clever about this. - Value *Callee = CS.getCalledValue(); + Value *Callee = Call.getCalledValue(); // First, pay the price of the argument setup. We account for the average // 1 instruction per call argument setup here. - Cost += CS.arg_size() * InlineConstants::InstrCost; + addCost(Call.arg_size() * InlineConstants::InstrCost); // Next, check if this happens to be an indirect function call to a known // function in this inline context. If not, we've done all we can. Function *F = dyn_cast_or_null(SimplifiedValues.lookup(Callee)); if (!F) { - if (!CS.onlyReadsMemory()) + if (!Call.onlyReadsMemory()) disableLoadElimination(); - return Base::visitCallSite(CS); + return Base::visitCallBase(Call); } // If we have a constant that we are calling as a function, we can peer @@ -1294,9 +1318,9 @@ bool CallAnalyzer::visitCallSite(CallSite CS) { // out. Pretend to inline the function, with a custom threshold. auto IndirectCallParams = Params; IndirectCallParams.DefaultThreshold = InlineConstants::IndirectCallThreshold; - CallAnalyzer CA(TTI, GetAssumptionCache, GetBFI, PSI, ORE, *F, CS, + CallAnalyzer CA(TTI, GetAssumptionCache, GetBFI, PSI, ORE, *F, Call, IndirectCallParams); - if (CA.analyzeCall(CS)) { + if (CA.analyzeCall(Call)) { // We were able to inline the indirect call! Subtract the cost from the // threshold to get the bonus we want to apply, but don't go below zero. Cost -= std::max(0, CA.getThreshold() - CA.getCost()); @@ -1304,7 +1328,7 @@ bool CallAnalyzer::visitCallSite(CallSite CS) { if (!F->onlyReadsMemory()) disableLoadElimination(); - return Base::visitCallSite(CS); + return Base::visitCallBase(Call); } bool CallAnalyzer::visitReturnInst(ReturnInst &RI) { @@ -1438,7 +1462,7 @@ bool CallAnalyzer::visitSwitchInst(SwitchInst &SI) { (int64_t)SI.getNumCases() * InlineConstants::InstrCost + Cost); if (CostLowerBound > Threshold && !ComputeFullInlineCost) { - Cost = CostLowerBound; + addCost((int64_t)SI.getNumCases() * InlineConstants::InstrCost); return false; } @@ -1452,7 +1476,7 @@ bool CallAnalyzer::visitSwitchInst(SwitchInst &SI) { int64_t JTCost = (int64_t)JumpTableSize * InlineConstants::InstrCost + 4 * InlineConstants::InstrCost; - Cost = std::min((int64_t)CostUpperBound, JTCost + Cost); + addCost(JTCost, (int64_t)CostUpperBound); return false; } @@ -1473,7 +1497,7 @@ bool CallAnalyzer::visitSwitchInst(SwitchInst &SI) { // n + n / 2 - 1 = n * 3 / 2 - 1 if (NumCaseCluster <= 3) { // Suppose a comparison includes one compare and one conditional branch. - Cost += NumCaseCluster * 2 * InlineConstants::InstrCost; + addCost(NumCaseCluster * 2 * InlineConstants::InstrCost); return false; } @@ -1481,7 +1505,7 @@ bool CallAnalyzer::visitSwitchInst(SwitchInst &SI) { int64_t SwitchCost = ExpectedNumberOfCompare * 2 * InlineConstants::InstrCost; - Cost = std::min((int64_t)CostUpperBound, SwitchCost + Cost); + addCost(SwitchCost, (int64_t)CostUpperBound); return false; } @@ -1574,7 +1598,7 @@ CallAnalyzer::analyzeBlock(BasicBlock *BB, if (Base::visit(&*I)) ++NumInstructionsSimplified; else - Cost += InlineConstants::InstrCost; + addCost(InlineConstants::InstrCost); using namespace ore; // If the visit this instruction detected an uninlinable pattern, abort. @@ -1595,7 +1619,7 @@ CallAnalyzer::analyzeBlock(BasicBlock *BB, if (ORE) ORE->emit([&]() { return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline", - CandidateCS.getInstruction()) + &CandidateCall) << NV("Callee", &F) << " has uninlinable pattern (" << NV("InlineResult", IR.message) << ") and cost is not fully computed"; @@ -1612,14 +1636,14 @@ CallAnalyzer::analyzeBlock(BasicBlock *BB, if (ORE) ORE->emit([&]() { return OptimizationRemarkMissed(DEBUG_TYPE, "NeverInline", - CandidateCS.getInstruction()) + &CandidateCall) << NV("Callee", &F) << " is " << NV("InlineResult", IR.message) << ". Cost is not fully computed"; }); return IR; } - // Check if we've past the maximum possible threshold so we don't spin in + // Check if we've passed the maximum possible threshold so we don't spin in // huge basic blocks that will never inline. if (Cost >= Threshold && !ComputeFullInlineCost) return false; @@ -1676,7 +1700,7 @@ ConstantInt *CallAnalyzer::stripAndComputeInBoundsConstantOffsets(Value *&V) { /// blocks to see if all their incoming edges are dead or not. void CallAnalyzer::findDeadBlocks(BasicBlock *CurrBB, BasicBlock *NextBB) { auto IsEdgeDead = [&](BasicBlock *Pred, BasicBlock *Succ) { - // A CFG edge is dead if the predecessor is dead or the predessor has a + // A CFG edge is dead if the predecessor is dead or the predecessor has a // known successor which is not the one under exam. return (DeadBlocks.count(Pred) || (KnownSuccessors[Pred] && KnownSuccessors[Pred] != Succ)); @@ -1712,7 +1736,7 @@ void CallAnalyzer::findDeadBlocks(BasicBlock *CurrBB, BasicBlock *NextBB) { /// factors and heuristics. If this method returns false but the computed cost /// is below the computed threshold, then inlining was forcibly disabled by /// some artifact of the routine. -InlineResult CallAnalyzer::analyzeCall(CallSite CS) { +InlineResult CallAnalyzer::analyzeCall(CallBase &Call) { ++NumCallsAnalyzed; // Perform some tweaks to the cost and threshold based on the direct @@ -1729,7 +1753,7 @@ InlineResult CallAnalyzer::analyzeCall(CallSite CS) { assert(NumVectorInstructions == 0); // Update the threshold based on callsite properties - updateThreshold(CS, F); + updateThreshold(Call, F); // While Threshold depends on commandline options that can take negative // values, we want to enforce the invariant that the computed threshold and @@ -1745,7 +1769,7 @@ InlineResult CallAnalyzer::analyzeCall(CallSite CS) { // Give out bonuses for the callsite, as the instructions setting them up // will be gone after inlining. - Cost -= getCallsiteCost(CS, DL); + addCost(-getCallsiteCost(Call, DL)); // If this function uses the coldcc calling convention, prefer not to inline // it. @@ -1759,14 +1783,11 @@ InlineResult CallAnalyzer::analyzeCall(CallSite CS) { if (F.empty()) return true; - Function *Caller = CS.getInstruction()->getFunction(); + Function *Caller = Call.getFunction(); // Check if the caller function is recursive itself. for (User *U : Caller->users()) { - CallSite Site(U); - if (!Site) - continue; - Instruction *I = Site.getInstruction(); - if (I->getFunction() == Caller) { + CallBase *Call = dyn_cast(U); + if (Call && Call->getFunction() == Caller) { IsCallerRecursive = true; break; } @@ -1774,10 +1795,10 @@ InlineResult CallAnalyzer::analyzeCall(CallSite CS) { // Populate our simplified values by mapping from function arguments to call // arguments with known important simplifications. - CallSite::arg_iterator CAI = CS.arg_begin(); + auto CAI = Call.arg_begin(); for (Function::arg_iterator FAI = F.arg_begin(), FAE = F.arg_end(); FAI != FAE; ++FAI, ++CAI) { - assert(CAI != CS.arg_end()); + assert(CAI != Call.arg_end()); if (Constant *C = dyn_cast(CAI)) SimplifiedValues[&*FAI] = C; @@ -1826,14 +1847,18 @@ InlineResult CallAnalyzer::analyzeCall(CallSite CS) { if (BB->empty()) continue; - // Disallow inlining a blockaddress. A blockaddress only has defined - // behavior for an indirect branch in the same function, and we do not - // currently support inlining indirect branches. But, the inliner may not - // see an indirect branch that ends up being dead code at a particular call - // site. If the blockaddress escapes the function, e.g., via a global - // variable, inlining may lead to an invalid cross-function reference. + // Disallow inlining a blockaddress with uses other than strictly callbr. + // A blockaddress only has defined behavior for an indirect branch in the + // same function, and we do not currently support inlining indirect + // branches. But, the inliner may not see an indirect branch that ends up + // being dead code at a particular call site. If the blockaddress escapes + // the function, e.g., via a global variable, inlining may lead to an + // invalid cross-function reference. + // FIXME: pr/39560: continue relaxing this overt restriction. if (BB->hasAddressTaken()) - return "blockaddress"; + for (User *U : BlockAddress::get(&*BB)->users()) + if (!isa(*U)) + return "blockaddress used outside of callbr"; // Analyze the cost of this block. If we blow through the threshold, this // returns false, and we can bail on out. @@ -1887,7 +1912,7 @@ InlineResult CallAnalyzer::analyzeCall(CallSite CS) { } bool OnlyOneCallAndLocalLinkage = - F.hasLocalLinkage() && F.hasOneUse() && &F == CS.getCalledFunction(); + F.hasLocalLinkage() && F.hasOneUse() && &F == Call.getCalledFunction(); // If this is a noduplicate call, we can still inline as long as // inlining this would cause the removal of the caller (so the instruction // is not actually duplicated, just moved). @@ -1899,7 +1924,7 @@ InlineResult CallAnalyzer::analyzeCall(CallSite CS) { // size, we penalise any call sites that perform loops. We do this after all // other costs here, so will likely only be dealing with relatively small // functions (and hence DT and LI will hopefully be cheap). - if (Caller->optForMinSize()) { + if (Caller->hasMinSize()) { DominatorTree DT(F); LoopInfo LI(DT); int NumLoops = 0; @@ -1909,7 +1934,7 @@ InlineResult CallAnalyzer::analyzeCall(CallSite CS) { continue; NumLoops++; } - Cost += NumLoops * InlineConstants::CallPenalty; + addCost(NumLoops * InlineConstants::CallPenalty); } // We applied the maximum possible vector bonus at the beginning. Now, @@ -1953,13 +1978,13 @@ static bool functionsHaveCompatibleAttributes(Function *Caller, AttributeFuncs::areInlineCompatible(*Caller, *Callee); } -int llvm::getCallsiteCost(CallSite CS, const DataLayout &DL) { +int llvm::getCallsiteCost(CallBase &Call, const DataLayout &DL) { int Cost = 0; - for (unsigned I = 0, E = CS.arg_size(); I != E; ++I) { - if (CS.isByValArgument(I)) { + for (unsigned I = 0, E = Call.arg_size(); I != E; ++I) { + if (Call.isByValArgument(I)) { // We approximate the number of loads and stores needed by dividing the // size of the byval type by the target's pointer size. - PointerType *PTy = cast(CS.getArgument(I)->getType()); + PointerType *PTy = cast(Call.getArgOperand(I)->getType()); unsigned TypeSize = DL.getTypeSizeInBits(PTy->getElementType()); unsigned AS = PTy->getAddressSpace(); unsigned PointerSize = DL.getPointerSizeInBits(AS); @@ -1987,16 +2012,16 @@ int llvm::getCallsiteCost(CallSite CS, const DataLayout &DL) { } InlineCost llvm::getInlineCost( - CallSite CS, const InlineParams &Params, TargetTransformInfo &CalleeTTI, + CallBase &Call, const InlineParams &Params, TargetTransformInfo &CalleeTTI, std::function &GetAssumptionCache, Optional> GetBFI, ProfileSummaryInfo *PSI, OptimizationRemarkEmitter *ORE) { - return getInlineCost(CS, CS.getCalledFunction(), Params, CalleeTTI, + return getInlineCost(Call, Call.getCalledFunction(), Params, CalleeTTI, GetAssumptionCache, GetBFI, PSI, ORE); } InlineCost llvm::getInlineCost( - CallSite CS, Function *Callee, const InlineParams &Params, + CallBase &Call, Function *Callee, const InlineParams &Params, TargetTransformInfo &CalleeTTI, std::function &GetAssumptionCache, Optional> GetBFI, @@ -2012,9 +2037,9 @@ InlineCost llvm::getInlineCost( // argument is in the alloca address space (so it is a little bit complicated // to solve). unsigned AllocaAS = Callee->getParent()->getDataLayout().getAllocaAddrSpace(); - for (unsigned I = 0, E = CS.arg_size(); I != E; ++I) - if (CS.isByValArgument(I)) { - PointerType *PTy = cast(CS.getArgument(I)->getType()); + for (unsigned I = 0, E = Call.arg_size(); I != E; ++I) + if (Call.isByValArgument(I)) { + PointerType *PTy = cast(Call.getArgOperand(I)->getType()); if (PTy->getAddressSpace() != AllocaAS) return llvm::InlineCost::getNever("byval arguments without alloca" " address space"); @@ -2022,20 +2047,21 @@ InlineCost llvm::getInlineCost( // Calls to functions with always-inline attributes should be inlined // whenever possible. - if (CS.hasFnAttr(Attribute::AlwaysInline)) { - if (isInlineViable(*Callee)) + if (Call.hasFnAttr(Attribute::AlwaysInline)) { + auto IsViable = isInlineViable(*Callee); + if (IsViable) return llvm::InlineCost::getAlways("always inline attribute"); - return llvm::InlineCost::getNever("inapplicable always inline attribute"); + return llvm::InlineCost::getNever(IsViable.message); } // Never inline functions with conflicting attributes (unless callee has // always-inline attribute). - Function *Caller = CS.getCaller(); + Function *Caller = Call.getCaller(); if (!functionsHaveCompatibleAttributes(Caller, Callee, CalleeTTI)) return llvm::InlineCost::getNever("conflicting attributes"); // Don't inline this call if the caller has the optnone attribute. - if (Caller->hasFnAttribute(Attribute::OptimizeNone)) + if (Caller->hasOptNone()) return llvm::InlineCost::getNever("optnone attribute"); // Don't inline a function that treats null pointer as valid into a caller @@ -2052,15 +2078,15 @@ InlineCost llvm::getInlineCost( return llvm::InlineCost::getNever("noinline function attribute"); // Don't inline call sites marked noinline. - if (CS.isNoInline()) + if (Call.isNoInline()) return llvm::InlineCost::getNever("noinline call site attribute"); LLVM_DEBUG(llvm::dbgs() << " Analyzing call of " << Callee->getName() << "... (caller:" << Caller->getName() << ")\n"); - CallAnalyzer CA(CalleeTTI, GetAssumptionCache, GetBFI, PSI, ORE, *Callee, CS, - Params); - InlineResult ShouldInline = CA.analyzeCall(CS); + CallAnalyzer CA(CalleeTTI, GetAssumptionCache, GetBFI, PSI, ORE, *Callee, + Call, Params); + InlineResult ShouldInline = CA.analyzeCall(Call); LLVM_DEBUG(CA.dump()); @@ -2073,42 +2099,50 @@ InlineCost llvm::getInlineCost( return llvm::InlineCost::get(CA.getCost(), CA.getThreshold()); } -bool llvm::isInlineViable(Function &F) { +InlineResult llvm::isInlineViable(Function &F) { bool ReturnsTwice = F.hasFnAttribute(Attribute::ReturnsTwice); for (Function::iterator BI = F.begin(), BE = F.end(); BI != BE; ++BI) { - // Disallow inlining of functions which contain indirect branches or - // blockaddresses. - if (isa(BI->getTerminator()) || BI->hasAddressTaken()) - return false; + // Disallow inlining of functions which contain indirect branches. + if (isa(BI->getTerminator())) + return "contains indirect branches"; + + // Disallow inlining of blockaddresses which are used by non-callbr + // instructions. + if (BI->hasAddressTaken()) + for (User *U : BlockAddress::get(&*BI)->users()) + if (!isa(*U)) + return "blockaddress used outside of callbr"; for (auto &II : *BI) { - CallSite CS(&II); - if (!CS) + CallBase *Call = dyn_cast(&II); + if (!Call) continue; // Disallow recursive calls. - if (&F == CS.getCalledFunction()) - return false; + if (&F == Call->getCalledFunction()) + return "recursive call"; // Disallow calls which expose returns-twice to a function not previously // attributed as such. - if (!ReturnsTwice && CS.isCall() && - cast(CS.getInstruction())->canReturnTwice()) - return false; + if (!ReturnsTwice && isa(Call) && + cast(Call)->canReturnTwice()) + return "exposes returns-twice attribute"; - if (CS.getCalledFunction()) - switch (CS.getCalledFunction()->getIntrinsicID()) { + if (Call->getCalledFunction()) + switch (Call->getCalledFunction()->getIntrinsicID()) { default: break; // Disallow inlining of @llvm.icall.branch.funnel because current // backend can't separate call targets from call arguments. case llvm::Intrinsic::icall_branch_funnel: + return "disallowed inlining of @llvm.icall.branch.funnel"; // Disallow inlining functions that call @llvm.localescape. Doing this // correctly would require major changes to the inliner. case llvm::Intrinsic::localescape: + return "disallowed inlining of @llvm.localescape"; // Disallow inlining of functions that initialize VarArgs with va_start. case llvm::Intrinsic::vastart: - return false; + return "contains VarArgs initialized with va_start"; } } } diff --git a/lib/Analysis/InstCount.cpp b/lib/Analysis/InstCount.cpp index 95ab6ee3db5b..943a99a5f46d 100644 --- a/lib/Analysis/InstCount.cpp +++ b/lib/Analysis/InstCount.cpp @@ -1,9 +1,8 @@ //===-- InstCount.cpp - Collects the count of all instructions ------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/InstructionPrecedenceTracking.cpp b/lib/Analysis/InstructionPrecedenceTracking.cpp index 816126f407ca..35190ce3e11a 100644 --- a/lib/Analysis/InstructionPrecedenceTracking.cpp +++ b/lib/Analysis/InstructionPrecedenceTracking.cpp @@ -1,9 +1,8 @@ //===-- InstructionPrecedenceTracking.cpp -----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Implements a class that is able to define some instructions as "special" @@ -20,6 +19,7 @@ #include "llvm/Analysis/InstructionPrecedenceTracking.h" #include "llvm/Analysis/ValueTracking.h" +#include "llvm/IR/PatternMatch.h" using namespace llvm; @@ -153,5 +153,8 @@ bool ImplicitControlFlowTracking::isSpecialInstruction( bool MemoryWriteTracking::isSpecialInstruction( const Instruction *Insn) const { + using namespace PatternMatch; + if (match(Insn, m_Intrinsic())) + return false; return Insn->mayWriteToMemory(); } diff --git a/lib/Analysis/InstructionSimplify.cpp b/lib/Analysis/InstructionSimplify.cpp index ccf907c144f0..e34bf6f4e43f 100644 --- a/lib/Analysis/InstructionSimplify.cpp +++ b/lib/Analysis/InstructionSimplify.cpp @@ -1,9 +1,8 @@ //===- InstructionSimplify.cpp - Fold instruction operands ----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -34,6 +33,8 @@ #include "llvm/IR/Dominators.h" #include "llvm/IR/GetElementPtrTypeIterator.h" #include "llvm/IR/GlobalAlias.h" +#include "llvm/IR/InstrTypes.h" +#include "llvm/IR/Instructions.h" #include "llvm/IR/Operator.h" #include "llvm/IR/PatternMatch.h" #include "llvm/IR/ValueHandle.h" @@ -50,6 +51,9 @@ STATISTIC(NumExpand, "Number of expansions"); STATISTIC(NumReassoc, "Number of reassociations"); static Value *SimplifyAndInst(Value *, Value *, const SimplifyQuery &, unsigned); +static Value *simplifyUnOp(unsigned, Value *, const SimplifyQuery &, unsigned); +static Value *simplifyFPUnOp(unsigned, Value *, const FastMathFlags &, + const SimplifyQuery &, unsigned); static Value *SimplifyBinOp(unsigned, Value *, Value *, const SimplifyQuery &, unsigned); static Value *SimplifyFPBinOp(unsigned, Value *, Value *, const FastMathFlags &, @@ -655,32 +659,11 @@ static Constant *stripAndComputeConstantOffsets(const DataLayout &DL, Value *&V, Type *IntPtrTy = DL.getIntPtrType(V->getType())->getScalarType(); APInt Offset = APInt::getNullValue(IntPtrTy->getIntegerBitWidth()); - // Even though we don't look through PHI nodes, we could be called on an - // instruction in an unreachable block, which may be on a cycle. - SmallPtrSet Visited; - Visited.insert(V); - do { - if (GEPOperator *GEP = dyn_cast(V)) { - if ((!AllowNonInbounds && !GEP->isInBounds()) || - !GEP->accumulateConstantOffset(DL, Offset)) - break; - V = GEP->getPointerOperand(); - } else if (Operator::getOpcode(V) == Instruction::BitCast) { - V = cast(V)->getOperand(0); - } else if (GlobalAlias *GA = dyn_cast(V)) { - if (GA->isInterposable()) - break; - V = GA->getAliasee(); - } else { - if (auto CS = CallSite(V)) - if (Value *RV = CS.getReturnedArgOperand()) { - V = RV; - continue; - } - break; - } - assert(V->getType()->isPtrOrPtrVectorTy() && "Unexpected operand type!"); - } while (Visited.insert(V).second); + V = V->stripAndAccumulateConstantOffsets(DL, Offset, AllowNonInbounds); + // As that strip may trace through `addrspacecast`, need to sext or trunc + // the offset calculated. + IntPtrTy = DL.getIntPtrType(V->getType())->getScalarType(); + Offset = Offset.sextOrTrunc(IntPtrTy->getIntegerBitWidth()); Constant *OffsetIntPtr = ConstantInt::get(IntPtrTy, Offset); if (V->getType()->isVectorTy()) @@ -1841,6 +1824,16 @@ static Value *SimplifyAndInst(Value *Op0, Value *Op1, const SimplifyQuery &Q, return Op1; } + // This is a similar pattern used for checking if a value is a power-of-2: + // (A - 1) & A --> 0 (if A is a power-of-2 or 0) + // A & (A - 1) --> 0 (if A is a power-of-2 or 0) + if (match(Op0, m_Add(m_Specific(Op1), m_AllOnes())) && + isKnownToBeAPowerOfTwo(Op1, Q.DL, /*OrZero*/ true, 0, Q.AC, Q.CxtI, Q.DT)) + return Constant::getNullValue(Op1->getType()); + if (match(Op1, m_Add(m_Specific(Op0), m_AllOnes())) && + isKnownToBeAPowerOfTwo(Op0, Q.DL, /*OrZero*/ true, 0, Q.AC, Q.CxtI, Q.DT)) + return Constant::getNullValue(Op0->getType()); + if (Value *V = simplifyAndOrOfCmps(Q, Op0, Op1, true)) return V; @@ -2280,12 +2273,12 @@ computePointerICmp(const DataLayout &DL, const TargetLibraryInfo *TLI, // come from a pointer that cannot overlap with dynamically-allocated // memory within the lifetime of the current function (allocas, byval // arguments, globals), then determine the comparison result here. - SmallVector LHSUObjs, RHSUObjs; + SmallVector LHSUObjs, RHSUObjs; GetUnderlyingObjects(LHS, LHSUObjs, DL); GetUnderlyingObjects(RHS, RHSUObjs, DL); // Is the set of underlying objects all noalias calls? - auto IsNAC = [](ArrayRef Objects) { + auto IsNAC = [](ArrayRef Objects) { return all_of(Objects, isNoAliasCall); }; @@ -2295,8 +2288,8 @@ computePointerICmp(const DataLayout &DL, const TargetLibraryInfo *TLI, // live with the compared-to allocation). For globals, we exclude symbols // that might be resolve lazily to symbols in another dynamically-loaded // library (and, thus, could be malloc'ed by the implementation). - auto IsAllocDisjoint = [](ArrayRef Objects) { - return all_of(Objects, [](Value *V) { + auto IsAllocDisjoint = [](ArrayRef Objects) { + return all_of(Objects, [](const Value *V) { if (const AllocaInst *AI = dyn_cast(V)) return AI->getParent() && AI->getFunction() && AI->isStaticAlloca(); if (const GlobalValue *GV = dyn_cast(V)) @@ -2472,228 +2465,6 @@ static Value *simplifyICmpWithZero(CmpInst::Predicate Pred, Value *LHS, return nullptr; } -/// Many binary operators with a constant operand have an easy-to-compute -/// range of outputs. This can be used to fold a comparison to always true or -/// always false. -static void setLimitsForBinOp(BinaryOperator &BO, APInt &Lower, APInt &Upper, - const InstrInfoQuery &IIQ) { - unsigned Width = Lower.getBitWidth(); - const APInt *C; - switch (BO.getOpcode()) { - case Instruction::Add: - if (match(BO.getOperand(1), m_APInt(C)) && !C->isNullValue()) { - // FIXME: If we have both nuw and nsw, we should reduce the range further. - if (IIQ.hasNoUnsignedWrap(cast(&BO))) { - // 'add nuw x, C' produces [C, UINT_MAX]. - Lower = *C; - } else if (IIQ.hasNoSignedWrap(cast(&BO))) { - if (C->isNegative()) { - // 'add nsw x, -C' produces [SINT_MIN, SINT_MAX - C]. - Lower = APInt::getSignedMinValue(Width); - Upper = APInt::getSignedMaxValue(Width) + *C + 1; - } else { - // 'add nsw x, +C' produces [SINT_MIN + C, SINT_MAX]. - Lower = APInt::getSignedMinValue(Width) + *C; - Upper = APInt::getSignedMaxValue(Width) + 1; - } - } - } - break; - - case Instruction::And: - if (match(BO.getOperand(1), m_APInt(C))) - // 'and x, C' produces [0, C]. - Upper = *C + 1; - break; - - case Instruction::Or: - if (match(BO.getOperand(1), m_APInt(C))) - // 'or x, C' produces [C, UINT_MAX]. - Lower = *C; - break; - - case Instruction::AShr: - if (match(BO.getOperand(1), m_APInt(C)) && C->ult(Width)) { - // 'ashr x, C' produces [INT_MIN >> C, INT_MAX >> C]. - Lower = APInt::getSignedMinValue(Width).ashr(*C); - Upper = APInt::getSignedMaxValue(Width).ashr(*C) + 1; - } else if (match(BO.getOperand(0), m_APInt(C))) { - unsigned ShiftAmount = Width - 1; - if (!C->isNullValue() && IIQ.isExact(&BO)) - ShiftAmount = C->countTrailingZeros(); - if (C->isNegative()) { - // 'ashr C, x' produces [C, C >> (Width-1)] - Lower = *C; - Upper = C->ashr(ShiftAmount) + 1; - } else { - // 'ashr C, x' produces [C >> (Width-1), C] - Lower = C->ashr(ShiftAmount); - Upper = *C + 1; - } - } - break; - - case Instruction::LShr: - if (match(BO.getOperand(1), m_APInt(C)) && C->ult(Width)) { - // 'lshr x, C' produces [0, UINT_MAX >> C]. - Upper = APInt::getAllOnesValue(Width).lshr(*C) + 1; - } else if (match(BO.getOperand(0), m_APInt(C))) { - // 'lshr C, x' produces [C >> (Width-1), C]. - unsigned ShiftAmount = Width - 1; - if (!C->isNullValue() && IIQ.isExact(&BO)) - ShiftAmount = C->countTrailingZeros(); - Lower = C->lshr(ShiftAmount); - Upper = *C + 1; - } - break; - - case Instruction::Shl: - if (match(BO.getOperand(0), m_APInt(C))) { - if (IIQ.hasNoUnsignedWrap(&BO)) { - // 'shl nuw C, x' produces [C, C << CLZ(C)] - Lower = *C; - Upper = Lower.shl(Lower.countLeadingZeros()) + 1; - } else if (BO.hasNoSignedWrap()) { // TODO: What if both nuw+nsw? - if (C->isNegative()) { - // 'shl nsw C, x' produces [C << CLO(C)-1, C] - unsigned ShiftAmount = C->countLeadingOnes() - 1; - Lower = C->shl(ShiftAmount); - Upper = *C + 1; - } else { - // 'shl nsw C, x' produces [C, C << CLZ(C)-1] - unsigned ShiftAmount = C->countLeadingZeros() - 1; - Lower = *C; - Upper = C->shl(ShiftAmount) + 1; - } - } - } - break; - - case Instruction::SDiv: - if (match(BO.getOperand(1), m_APInt(C))) { - APInt IntMin = APInt::getSignedMinValue(Width); - APInt IntMax = APInt::getSignedMaxValue(Width); - if (C->isAllOnesValue()) { - // 'sdiv x, -1' produces [INT_MIN + 1, INT_MAX] - // where C != -1 and C != 0 and C != 1 - Lower = IntMin + 1; - Upper = IntMax + 1; - } else if (C->countLeadingZeros() < Width - 1) { - // 'sdiv x, C' produces [INT_MIN / C, INT_MAX / C] - // where C != -1 and C != 0 and C != 1 - Lower = IntMin.sdiv(*C); - Upper = IntMax.sdiv(*C); - if (Lower.sgt(Upper)) - std::swap(Lower, Upper); - Upper = Upper + 1; - assert(Upper != Lower && "Upper part of range has wrapped!"); - } - } else if (match(BO.getOperand(0), m_APInt(C))) { - if (C->isMinSignedValue()) { - // 'sdiv INT_MIN, x' produces [INT_MIN, INT_MIN / -2]. - Lower = *C; - Upper = Lower.lshr(1) + 1; - } else { - // 'sdiv C, x' produces [-|C|, |C|]. - Upper = C->abs() + 1; - Lower = (-Upper) + 1; - } - } - break; - - case Instruction::UDiv: - if (match(BO.getOperand(1), m_APInt(C)) && !C->isNullValue()) { - // 'udiv x, C' produces [0, UINT_MAX / C]. - Upper = APInt::getMaxValue(Width).udiv(*C) + 1; - } else if (match(BO.getOperand(0), m_APInt(C))) { - // 'udiv C, x' produces [0, C]. - Upper = *C + 1; - } - break; - - case Instruction::SRem: - if (match(BO.getOperand(1), m_APInt(C))) { - // 'srem x, C' produces (-|C|, |C|). - Upper = C->abs(); - Lower = (-Upper) + 1; - } - break; - - case Instruction::URem: - if (match(BO.getOperand(1), m_APInt(C))) - // 'urem x, C' produces [0, C). - Upper = *C; - break; - - default: - break; - } -} - -/// Some intrinsics with a constant operand have an easy-to-compute range of -/// outputs. This can be used to fold a comparison to always true or always -/// false. -static void setLimitsForIntrinsic(IntrinsicInst &II, APInt &Lower, - APInt &Upper) { - unsigned Width = Lower.getBitWidth(); - const APInt *C; - switch (II.getIntrinsicID()) { - case Intrinsic::uadd_sat: - // uadd.sat(x, C) produces [C, UINT_MAX]. - if (match(II.getOperand(0), m_APInt(C)) || - match(II.getOperand(1), m_APInt(C))) - Lower = *C; - break; - case Intrinsic::sadd_sat: - if (match(II.getOperand(0), m_APInt(C)) || - match(II.getOperand(1), m_APInt(C))) { - if (C->isNegative()) { - // sadd.sat(x, -C) produces [SINT_MIN, SINT_MAX + (-C)]. - Lower = APInt::getSignedMinValue(Width); - Upper = APInt::getSignedMaxValue(Width) + *C + 1; - } else { - // sadd.sat(x, +C) produces [SINT_MIN + C, SINT_MAX]. - Lower = APInt::getSignedMinValue(Width) + *C; - Upper = APInt::getSignedMaxValue(Width) + 1; - } - } - break; - case Intrinsic::usub_sat: - // usub.sat(C, x) produces [0, C]. - if (match(II.getOperand(0), m_APInt(C))) - Upper = *C + 1; - // usub.sat(x, C) produces [0, UINT_MAX - C]. - else if (match(II.getOperand(1), m_APInt(C))) - Upper = APInt::getMaxValue(Width) - *C + 1; - break; - case Intrinsic::ssub_sat: - if (match(II.getOperand(0), m_APInt(C))) { - if (C->isNegative()) { - // ssub.sat(-C, x) produces [SINT_MIN, -SINT_MIN + (-C)]. - Lower = APInt::getSignedMinValue(Width); - Upper = *C - APInt::getSignedMinValue(Width) + 1; - } else { - // ssub.sat(+C, x) produces [-SINT_MAX + C, SINT_MAX]. - Lower = *C - APInt::getSignedMaxValue(Width); - Upper = APInt::getSignedMaxValue(Width) + 1; - } - } else if (match(II.getOperand(1), m_APInt(C))) { - if (C->isNegative()) { - // ssub.sat(x, -C) produces [SINT_MIN - (-C), SINT_MAX]: - Lower = APInt::getSignedMinValue(Width) - *C; - Upper = APInt::getSignedMaxValue(Width) + 1; - } else { - // ssub.sat(x, +C) produces [SINT_MIN, SINT_MAX - C]. - Lower = APInt::getSignedMinValue(Width); - Upper = APInt::getSignedMaxValue(Width) - *C + 1; - } - } - break; - default: - break; - } -} - static Value *simplifyICmpWithConstant(CmpInst::Predicate Pred, Value *LHS, Value *RHS, const InstrInfoQuery &IIQ) { Type *ITy = GetCompareTy(RHS); // The return type. @@ -2721,22 +2492,7 @@ static Value *simplifyICmpWithConstant(CmpInst::Predicate Pred, Value *LHS, if (RHS_CR.isFullSet()) return ConstantInt::getTrue(ITy); - // Find the range of possible values for binary operators. - unsigned Width = C->getBitWidth(); - APInt Lower = APInt(Width, 0); - APInt Upper = APInt(Width, 0); - if (auto *BO = dyn_cast(LHS)) - setLimitsForBinOp(*BO, Lower, Upper, IIQ); - else if (auto *II = dyn_cast(LHS)) - setLimitsForIntrinsic(*II, Lower, Upper); - - ConstantRange LHS_CR = - Lower != Upper ? ConstantRange(Lower, Upper) : ConstantRange(Width, true); - - if (auto *I = dyn_cast(LHS)) - if (auto *Ranges = IIQ.getMetadata(I, LLVMContext::MD_range)) - LHS_CR = LHS_CR.intersectWith(getConstantRangeFromMetadata(*Ranges)); - + ConstantRange LHS_CR = computeConstantRange(LHS, IIQ.UseInstrInfo); if (!LHS_CR.isFullSet()) { if (RHS_CR.contains(LHS_CR)) return ConstantInt::getTrue(ITy); @@ -3062,44 +2818,6 @@ static Value *simplifyICmpWithBinOp(CmpInst::Predicate Pred, Value *LHS, return nullptr; } -static Value *simplifyICmpWithAbsNabs(CmpInst::Predicate Pred, Value *Op0, - Value *Op1) { - // We need a comparison with a constant. - const APInt *C; - if (!match(Op1, m_APInt(C))) - return nullptr; - - // matchSelectPattern returns the negation part of an abs pattern in SP1. - // If the negate has an NSW flag, abs(INT_MIN) is undefined. Without that - // constraint, we can't make a contiguous range for the result of abs. - ICmpInst::Predicate AbsPred = ICmpInst::BAD_ICMP_PREDICATE; - Value *SP0, *SP1; - SelectPatternFlavor SPF = matchSelectPattern(Op0, SP0, SP1).Flavor; - if (SPF == SelectPatternFlavor::SPF_ABS && - cast(SP1)->hasNoSignedWrap()) - // The result of abs(X) is >= 0 (with nsw). - AbsPred = ICmpInst::ICMP_SGE; - if (SPF == SelectPatternFlavor::SPF_NABS) - // The result of -abs(X) is <= 0. - AbsPred = ICmpInst::ICMP_SLE; - - if (AbsPred == ICmpInst::BAD_ICMP_PREDICATE) - return nullptr; - - // If there is no intersection between abs/nabs and the range of this icmp, - // the icmp must be false. If the abs/nabs range is a subset of the icmp - // range, the icmp must be true. - APInt Zero = APInt::getNullValue(C->getBitWidth()); - ConstantRange AbsRange = ConstantRange::makeExactICmpRegion(AbsPred, Zero); - ConstantRange CmpRange = ConstantRange::makeExactICmpRegion(Pred, *C); - if (AbsRange.intersectWith(CmpRange).isEmptySet()) - return getFalse(GetCompareTy(Op0)); - if (CmpRange.contains(AbsRange)) - return getTrue(GetCompareTy(Op0)); - - return nullptr; -} - /// Simplify integer comparisons where at least one operand of the compare /// matches an integer min/max idiom. static Value *simplifyICmpWithMinMax(CmpInst::Predicate Pred, Value *LHS, @@ -3319,9 +3037,16 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS, std::swap(LHS, RHS); Pred = CmpInst::getSwappedPredicate(Pred); } + assert(!isa(LHS) && "Unexpected icmp undef,%X"); Type *ITy = GetCompareTy(LHS); // The return type. + // For EQ and NE, we can always pick a value for the undef to make the + // predicate pass or fail, so we can return undef. + // Matches behavior in llvm::ConstantFoldCompareInstruction. + if (isa(RHS) && ICmpInst::isEquality(Pred)) + return UndefValue::get(ITy); + // icmp X, X -> true/false // icmp X, undef -> true/false because undef could be X. if (LHS == RHS || isa(RHS)) @@ -3531,9 +3256,6 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS, if (Value *V = simplifyICmpWithMinMax(Pred, LHS, RHS, Q, MaxRecurse)) return V; - if (Value *V = simplifyICmpWithAbsNabs(Pred, LHS, RHS)) - return V; - // Simplify comparisons of related pointers using a powerful, recursive // GEP-walk when we have target data available.. if (LHS->getType()->isPointerTy()) @@ -3647,6 +3369,8 @@ static Value *SimplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS, } // Handle fcmp with constant RHS. + // TODO: Use match with a specific FP value, so these work with vectors with + // undef lanes. const APFloat *C; if (match(RHS, m_APFloat(C))) { // Check whether the constant is an infinity. @@ -3675,28 +3399,7 @@ static Value *SimplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS, } } } - if (C->isZero()) { - switch (Pred) { - case FCmpInst::FCMP_OGE: - if (FMF.noNaNs() && CannotBeOrderedLessThanZero(LHS, Q.TLI)) - return getTrue(RetTy); - break; - case FCmpInst::FCMP_UGE: - if (CannotBeOrderedLessThanZero(LHS, Q.TLI)) - return getTrue(RetTy); - break; - case FCmpInst::FCMP_ULT: - if (FMF.noNaNs() && CannotBeOrderedLessThanZero(LHS, Q.TLI)) - return getFalse(RetTy); - break; - case FCmpInst::FCMP_OLT: - if (CannotBeOrderedLessThanZero(LHS, Q.TLI)) - return getFalse(RetTy); - break; - default: - break; - } - } else if (C->isNegative()) { + if (C->isNegative() && !C->isNegZero()) { assert(!C->isNaN() && "Unexpected NaN constant!"); // TODO: We can catch more cases by using a range check rather than // relying on CannotBeOrderedLessThanZero. @@ -3719,6 +3422,67 @@ static Value *SimplifyFCmpInst(unsigned Predicate, Value *LHS, Value *RHS, break; } } + + // Check comparison of [minnum/maxnum with constant] with other constant. + const APFloat *C2; + if ((match(LHS, m_Intrinsic(m_Value(), m_APFloat(C2))) && + C2->compare(*C) == APFloat::cmpLessThan) || + (match(LHS, m_Intrinsic(m_Value(), m_APFloat(C2))) && + C2->compare(*C) == APFloat::cmpGreaterThan)) { + bool IsMaxNum = + cast(LHS)->getIntrinsicID() == Intrinsic::maxnum; + // The ordered relationship and minnum/maxnum guarantee that we do not + // have NaN constants, so ordered/unordered preds are handled the same. + switch (Pred) { + case FCmpInst::FCMP_OEQ: case FCmpInst::FCMP_UEQ: + // minnum(X, LesserC) == C --> false + // maxnum(X, GreaterC) == C --> false + return getFalse(RetTy); + case FCmpInst::FCMP_ONE: case FCmpInst::FCMP_UNE: + // minnum(X, LesserC) != C --> true + // maxnum(X, GreaterC) != C --> true + return getTrue(RetTy); + case FCmpInst::FCMP_OGE: case FCmpInst::FCMP_UGE: + case FCmpInst::FCMP_OGT: case FCmpInst::FCMP_UGT: + // minnum(X, LesserC) >= C --> false + // minnum(X, LesserC) > C --> false + // maxnum(X, GreaterC) >= C --> true + // maxnum(X, GreaterC) > C --> true + return ConstantInt::get(RetTy, IsMaxNum); + case FCmpInst::FCMP_OLE: case FCmpInst::FCMP_ULE: + case FCmpInst::FCMP_OLT: case FCmpInst::FCMP_ULT: + // minnum(X, LesserC) <= C --> true + // minnum(X, LesserC) < C --> true + // maxnum(X, GreaterC) <= C --> false + // maxnum(X, GreaterC) < C --> false + return ConstantInt::get(RetTy, !IsMaxNum); + default: + // TRUE/FALSE/ORD/UNO should be handled before this. + llvm_unreachable("Unexpected fcmp predicate"); + } + } + } + + if (match(RHS, m_AnyZeroFP())) { + switch (Pred) { + case FCmpInst::FCMP_OGE: + case FCmpInst::FCMP_ULT: + // Positive or zero X >= 0.0 --> true + // Positive or zero X < 0.0 --> false + if ((FMF.noNaNs() || isKnownNeverNaN(LHS, Q.TLI)) && + CannotBeOrderedLessThanZero(LHS, Q.TLI)) + return Pred == FCmpInst::FCMP_OGE ? getTrue(RetTy) : getFalse(RetTy); + break; + case FCmpInst::FCMP_UGE: + case FCmpInst::FCMP_OLT: + // Positive or zero or nan X >= 0.0 --> true + // Positive or zero or nan X < 0.0 --> false + if (CannotBeOrderedLessThanZero(LHS, Q.TLI)) + return Pred == FCmpInst::FCMP_UGE ? getTrue(RetTy) : getFalse(RetTy); + break; + default: + break; + } } // If the comparison is with the result of a select instruction, check whether @@ -3904,27 +3668,44 @@ static Value *simplifySelectWithICmpCond(Value *CondVal, Value *TrueVal, Pred == ICmpInst::ICMP_EQ)) return V; - // Test for zero-shift-guard-ops around funnel shifts. These are used to - // avoid UB from oversized shifts in raw IR rotate patterns, but the - // intrinsics do not have that problem. + // Test for a bogus zero-shift-guard-op around funnel-shift or rotate. Value *ShAmt; auto isFsh = m_CombineOr(m_Intrinsic(m_Value(X), m_Value(), m_Value(ShAmt)), m_Intrinsic(m_Value(), m_Value(X), m_Value(ShAmt))); - // (ShAmt != 0) ? fshl(X, *, ShAmt) : X --> fshl(X, *, ShAmt) - // (ShAmt != 0) ? fshr(*, X, ShAmt) : X --> fshr(*, X, ShAmt) // (ShAmt == 0) ? fshl(X, *, ShAmt) : X --> X // (ShAmt == 0) ? fshr(*, X, ShAmt) : X --> X - if (match(TrueVal, isFsh) && FalseVal == X && CmpLHS == ShAmt) - return Pred == ICmpInst::ICMP_NE ? TrueVal : X; - - // (ShAmt == 0) ? X : fshl(X, *, ShAmt) --> fshl(X, *, ShAmt) - // (ShAmt == 0) ? X : fshr(*, X, ShAmt) --> fshr(*, X, ShAmt) + if (match(TrueVal, isFsh) && FalseVal == X && CmpLHS == ShAmt && + Pred == ICmpInst::ICMP_EQ) + return X; // (ShAmt != 0) ? X : fshl(X, *, ShAmt) --> X // (ShAmt != 0) ? X : fshr(*, X, ShAmt) --> X - if (match(FalseVal, isFsh) && TrueVal == X && CmpLHS == ShAmt) - return Pred == ICmpInst::ICMP_EQ ? FalseVal : X; + if (match(FalseVal, isFsh) && TrueVal == X && CmpLHS == ShAmt && + Pred == ICmpInst::ICMP_NE) + return X; + + // Test for a zero-shift-guard-op around rotates. These are used to + // avoid UB from oversized shifts in raw IR rotate patterns, but the + // intrinsics do not have that problem. + // We do not allow this transform for the general funnel shift case because + // that would not preserve the poison safety of the original code. + auto isRotate = m_CombineOr(m_Intrinsic(m_Value(X), + m_Deferred(X), + m_Value(ShAmt)), + m_Intrinsic(m_Value(X), + m_Deferred(X), + m_Value(ShAmt))); + // (ShAmt != 0) ? fshl(X, X, ShAmt) : X --> fshl(X, X, ShAmt) + // (ShAmt != 0) ? fshr(X, X, ShAmt) : X --> fshr(X, X, ShAmt) + if (match(TrueVal, isRotate) && FalseVal == X && CmpLHS == ShAmt && + Pred == ICmpInst::ICMP_NE) + return TrueVal; + // (ShAmt == 0) ? X : fshl(X, X, ShAmt) --> fshl(X, X, ShAmt) + // (ShAmt == 0) ? X : fshr(X, X, ShAmt) --> fshr(X, X, ShAmt) + if (match(FalseVal, isRotate) && TrueVal == X && CmpLHS == ShAmt && + Pred == ICmpInst::ICMP_EQ) + return FalseVal; } // Check for other compares that behave like bit test. @@ -4218,6 +3999,17 @@ Value *llvm::SimplifyInsertElementInst(Value *Vec, Value *Val, Value *Idx, if (isa(Idx)) return UndefValue::get(Vec->getType()); + // Inserting an undef scalar? Assume it is the same value as the existing + // vector element. + if (isa(Val)) + return Vec; + + // If we are extracting a value from a vector, then inserting it into the same + // place, that's the input vector: + // insertelt Vec, (extractelt Vec, Idx), Idx --> Vec + if (match(Val, m_ExtractElement(m_Specific(Vec), m_Specific(Idx)))) + return Vec; + return nullptr; } @@ -4495,6 +4287,33 @@ Value *llvm::SimplifyShuffleVectorInst(Value *Op0, Value *Op1, Constant *Mask, return ::SimplifyShuffleVectorInst(Op0, Op1, Mask, RetTy, Q, RecursionLimit); } +static Constant *foldConstant(Instruction::UnaryOps Opcode, + Value *&Op, const SimplifyQuery &Q) { + if (auto *C = dyn_cast(Op)) + return ConstantFoldUnaryOpOperand(Opcode, C, Q.DL); + return nullptr; +} + +/// Given the operand for an FNeg, see if we can fold the result. If not, this +/// returns null. +static Value *simplifyFNegInst(Value *Op, FastMathFlags FMF, + const SimplifyQuery &Q, unsigned MaxRecurse) { + if (Constant *C = foldConstant(Instruction::FNeg, Op, Q)) + return C; + + Value *X; + // fneg (fneg X) ==> X + if (match(Op, m_FNeg(m_Value(X)))) + return X; + + return nullptr; +} + +Value *llvm::SimplifyFNegInst(Value *Op, FastMathFlags FMF, + const SimplifyQuery &Q) { + return ::simplifyFNegInst(Op, FMF, Q, RecursionLimit); +} + static Constant *propagateNaN(Constant *In) { // If the input is a vector with undef elements, just return a default NaN. if (!In->isNaN()) @@ -4536,16 +4355,22 @@ static Value *SimplifyFAddInst(Value *Op0, Value *Op1, FastMathFlags FMF, (FMF.noSignedZeros() || CannotBeNegativeZero(Op0, Q.TLI))) return Op0; - // With nnan: (+/-0.0 - X) + X --> 0.0 (and commuted variant) + // With nnan: -X + X --> 0.0 (and commuted variant) // We don't have to explicitly exclude infinities (ninf): INF + -INF == NaN. // Negative zeros are allowed because we always end up with positive zero: // X = -0.0: (-0.0 - (-0.0)) + (-0.0) == ( 0.0) + (-0.0) == 0.0 // X = -0.0: ( 0.0 - (-0.0)) + (-0.0) == ( 0.0) + (-0.0) == 0.0 // X = 0.0: (-0.0 - ( 0.0)) + ( 0.0) == (-0.0) + ( 0.0) == 0.0 // X = 0.0: ( 0.0 - ( 0.0)) + ( 0.0) == ( 0.0) + ( 0.0) == 0.0 - if (FMF.noNaNs() && (match(Op0, m_FSub(m_AnyZeroFP(), m_Specific(Op1))) || - match(Op1, m_FSub(m_AnyZeroFP(), m_Specific(Op0))))) - return ConstantFP::getNullValue(Op0->getType()); + if (FMF.noNaNs()) { + if (match(Op0, m_FSub(m_AnyZeroFP(), m_Specific(Op1))) || + match(Op1, m_FSub(m_AnyZeroFP(), m_Specific(Op0)))) + return ConstantFP::getNullValue(Op0->getType()); + + if (match(Op0, m_FNeg(m_Specific(Op1))) || + match(Op1, m_FNeg(m_Specific(Op0)))) + return ConstantFP::getNullValue(Op0->getType()); + } // (X - Y) + Y --> X // Y + (X - Y) --> X @@ -4578,14 +4403,17 @@ static Value *SimplifyFSubInst(Value *Op0, Value *Op1, FastMathFlags FMF, return Op0; // fsub -0.0, (fsub -0.0, X) ==> X + // fsub -0.0, (fneg X) ==> X Value *X; if (match(Op0, m_NegZeroFP()) && - match(Op1, m_FSub(m_NegZeroFP(), m_Value(X)))) + match(Op1, m_FNeg(m_Value(X)))) return X; // fsub 0.0, (fsub 0.0, X) ==> X if signed zeros are ignored. + // fsub 0.0, (fneg X) ==> X if signed zeros are ignored. if (FMF.noSignedZeros() && match(Op0, m_AnyZeroFP()) && - match(Op1, m_FSub(m_AnyZeroFP(), m_Value(X)))) + (match(Op1, m_FSub(m_AnyZeroFP(), m_Value(X))) || + match(Op1, m_FNeg(m_Value(X))))) return X; // fsub nnan x, x ==> 0.0 @@ -4722,6 +4550,42 @@ Value *llvm::SimplifyFRemInst(Value *Op0, Value *Op1, FastMathFlags FMF, //=== Helper functions for higher up the class hierarchy. +/// Given the operand for a UnaryOperator, see if we can fold the result. +/// If not, this returns null. +static Value *simplifyUnOp(unsigned Opcode, Value *Op, const SimplifyQuery &Q, + unsigned MaxRecurse) { + switch (Opcode) { + case Instruction::FNeg: + return simplifyFNegInst(Op, FastMathFlags(), Q, MaxRecurse); + default: + llvm_unreachable("Unexpected opcode"); + } +} + +/// Given the operand for a UnaryOperator, see if we can fold the result. +/// If not, this returns null. +/// In contrast to SimplifyUnOp, try to use FastMathFlag when folding the +/// result. In case we don't need FastMathFlags, simply fall to SimplifyUnOp. +static Value *simplifyFPUnOp(unsigned Opcode, Value *Op, + const FastMathFlags &FMF, + const SimplifyQuery &Q, unsigned MaxRecurse) { + switch (Opcode) { + case Instruction::FNeg: + return simplifyFNegInst(Op, FMF, Q, MaxRecurse); + default: + return simplifyUnOp(Opcode, Op, Q, MaxRecurse); + } +} + +Value *llvm::SimplifyUnOp(unsigned Opcode, Value *Op, const SimplifyQuery &Q) { + return ::simplifyUnOp(Opcode, Op, Q, RecursionLimit); +} + +Value *llvm::SimplifyFPUnOp(unsigned Opcode, Value *Op, FastMathFlags FMF, + const SimplifyQuery &Q) { + return ::simplifyFPUnOp(Opcode, Op, FMF, Q, RecursionLimit); +} + /// Given operands for a BinaryOperator, see if we can fold the result. /// If not, this returns null. static Value *SimplifyBinOp(unsigned Opcode, Value *LHS, Value *RHS, @@ -4885,22 +4749,6 @@ static Value *SimplifyRelativeLoad(Constant *Ptr, Constant *Offset, return ConstantExpr::getBitCast(LoadedLHSPtr, Int8PtrTy); } -static bool maskIsAllZeroOrUndef(Value *Mask) { - auto *ConstMask = dyn_cast(Mask); - if (!ConstMask) - return false; - if (ConstMask->isNullValue() || isa(ConstMask)) - return true; - for (unsigned I = 0, E = ConstMask->getType()->getVectorNumElements(); I != E; - ++I) { - if (auto *MaskElt = ConstMask->getAggregateElement(I)) - if (MaskElt->isNullValue() || isa(MaskElt)) - continue; - return false; - } - return true; -} - static Value *simplifyUnaryIntrinsic(Function *F, Value *Op0, const SimplifyQuery &Q) { // Idempotent functions return the same result when called repeatedly. @@ -4941,8 +4789,32 @@ static Value *simplifyUnaryIntrinsic(Function *F, Value *Op0, case Intrinsic::log2: // log2(exp2(x)) -> x if (Q.CxtI->hasAllowReassoc() && - match(Op0, m_Intrinsic(m_Value(X)))) return X; + (match(Op0, m_Intrinsic(m_Value(X))) || + match(Op0, m_Intrinsic(m_SpecificFP(2.0), + m_Value(X))))) return X; + break; + case Intrinsic::log10: + // log10(pow(10.0, x)) -> x + if (Q.CxtI->hasAllowReassoc() && + match(Op0, m_Intrinsic(m_SpecificFP(10.0), + m_Value(X)))) return X; break; + case Intrinsic::floor: + case Intrinsic::trunc: + case Intrinsic::ceil: + case Intrinsic::round: + case Intrinsic::nearbyint: + case Intrinsic::rint: { + // floor (sitofp x) -> sitofp x + // floor (uitofp x) -> uitofp x + // + // Converting from int always results in a finite integral number or + // infinity. For either of those inputs, these rounding functions always + // return the same value, so the rounding can be eliminated. + if (match(Op0, m_SIToFP(m_Value())) || match(Op0, m_UIToFP(m_Value()))) + return Op0; + break; + } default: break; } @@ -4960,16 +4832,19 @@ static Value *simplifyBinaryIntrinsic(Function *F, Value *Op0, Value *Op1, // X - X -> { 0, false } if (Op0 == Op1) return Constant::getNullValue(ReturnType); - // X - undef -> undef - // undef - X -> undef - if (isa(Op0) || isa(Op1)) - return UndefValue::get(ReturnType); - break; + LLVM_FALLTHROUGH; case Intrinsic::uadd_with_overflow: case Intrinsic::sadd_with_overflow: - // X + undef -> undef - if (isa(Op0) || isa(Op1)) - return UndefValue::get(ReturnType); + // X - undef -> { undef, false } + // undef - X -> { undef, false } + // X + undef -> { undef, false } + // undef + x -> { undef, false } + if (isa(Op0) || isa(Op1)) { + return ConstantStruct::get( + cast(ReturnType), + {UndefValue::get(ReturnType->getStructElementType(0)), + Constant::getNullValue(ReturnType->getStructElementType(1))}); + } break; case Intrinsic::umul_with_overflow: case Intrinsic::smul_with_overflow: @@ -5085,26 +4960,28 @@ static Value *simplifyBinaryIntrinsic(Function *F, Value *Op0, Value *Op1, return nullptr; } -template -static Value *simplifyIntrinsic(Function *F, IterTy ArgBegin, IterTy ArgEnd, - const SimplifyQuery &Q) { +static Value *simplifyIntrinsic(CallBase *Call, const SimplifyQuery &Q) { + // Intrinsics with no operands have some kind of side effect. Don't simplify. - unsigned NumOperands = std::distance(ArgBegin, ArgEnd); - if (NumOperands == 0) + unsigned NumOperands = Call->getNumArgOperands(); + if (!NumOperands) return nullptr; + Function *F = cast(Call->getCalledFunction()); Intrinsic::ID IID = F->getIntrinsicID(); if (NumOperands == 1) - return simplifyUnaryIntrinsic(F, ArgBegin[0], Q); + return simplifyUnaryIntrinsic(F, Call->getArgOperand(0), Q); if (NumOperands == 2) - return simplifyBinaryIntrinsic(F, ArgBegin[0], ArgBegin[1], Q); + return simplifyBinaryIntrinsic(F, Call->getArgOperand(0), + Call->getArgOperand(1), Q); // Handle intrinsics with 3 or more arguments. switch (IID) { - case Intrinsic::masked_load: { - Value *MaskArg = ArgBegin[2]; - Value *PassthruArg = ArgBegin[3]; + case Intrinsic::masked_load: + case Intrinsic::masked_gather: { + Value *MaskArg = Call->getArgOperand(2); + Value *PassthruArg = Call->getArgOperand(3); // If the mask is all zeros or undef, the "passthru" argument is the result. if (maskIsAllZeroOrUndef(MaskArg)) return PassthruArg; @@ -5112,7 +4989,8 @@ static Value *simplifyIntrinsic(Function *F, IterTy ArgBegin, IterTy ArgEnd, } case Intrinsic::fshl: case Intrinsic::fshr: { - Value *Op0 = ArgBegin[0], *Op1 = ArgBegin[1], *ShAmtArg = ArgBegin[2]; + Value *Op0 = Call->getArgOperand(0), *Op1 = Call->getArgOperand(1), + *ShAmtArg = Call->getArgOperand(2); // If both operands are undef, the result is undef. if (match(Op0, m_Undef()) && match(Op1, m_Undef())) @@ -5120,15 +4998,14 @@ static Value *simplifyIntrinsic(Function *F, IterTy ArgBegin, IterTy ArgEnd, // If shift amount is undef, assume it is zero. if (match(ShAmtArg, m_Undef())) - return ArgBegin[IID == Intrinsic::fshl ? 0 : 1]; + return Call->getArgOperand(IID == Intrinsic::fshl ? 0 : 1); const APInt *ShAmtC; if (match(ShAmtArg, m_APInt(ShAmtC))) { // If there's effectively no shift, return the 1st arg or 2nd arg. - // TODO: For vectors, we could check each element of a non-splat constant. APInt BitWidth = APInt(ShAmtC->getBitWidth(), ShAmtC->getBitWidth()); if (ShAmtC->urem(BitWidth).isNullValue()) - return ArgBegin[IID == Intrinsic::fshl ? 0 : 1]; + return Call->getArgOperand(IID == Intrinsic::fshl ? 0 : 1); } return nullptr; } @@ -5137,58 +5014,36 @@ static Value *simplifyIntrinsic(Function *F, IterTy ArgBegin, IterTy ArgEnd, } } -template -static Value *SimplifyCall(ImmutableCallSite CS, Value *V, IterTy ArgBegin, - IterTy ArgEnd, const SimplifyQuery &Q, - unsigned MaxRecurse) { - Type *Ty = V->getType(); - if (PointerType *PTy = dyn_cast(Ty)) - Ty = PTy->getElementType(); - FunctionType *FTy = cast(Ty); +Value *llvm::SimplifyCall(CallBase *Call, const SimplifyQuery &Q) { + Value *Callee = Call->getCalledValue(); // call undef -> undef // call null -> undef - if (isa(V) || isa(V)) - return UndefValue::get(FTy->getReturnType()); + if (isa(Callee) || isa(Callee)) + return UndefValue::get(Call->getType()); - Function *F = dyn_cast(V); + Function *F = dyn_cast(Callee); if (!F) return nullptr; if (F->isIntrinsic()) - if (Value *Ret = simplifyIntrinsic(F, ArgBegin, ArgEnd, Q)) + if (Value *Ret = simplifyIntrinsic(Call, Q)) return Ret; - if (!canConstantFoldCallTo(CS, F)) + if (!canConstantFoldCallTo(Call, F)) return nullptr; SmallVector ConstantArgs; - ConstantArgs.reserve(ArgEnd - ArgBegin); - for (IterTy I = ArgBegin, E = ArgEnd; I != E; ++I) { - Constant *C = dyn_cast(*I); + unsigned NumArgs = Call->getNumArgOperands(); + ConstantArgs.reserve(NumArgs); + for (auto &Arg : Call->args()) { + Constant *C = dyn_cast(&Arg); if (!C) return nullptr; ConstantArgs.push_back(C); } - return ConstantFoldCall(CS, F, ConstantArgs, Q.TLI); -} - -Value *llvm::SimplifyCall(ImmutableCallSite CS, Value *V, - User::op_iterator ArgBegin, User::op_iterator ArgEnd, - const SimplifyQuery &Q) { - return ::SimplifyCall(CS, V, ArgBegin, ArgEnd, Q, RecursionLimit); -} - -Value *llvm::SimplifyCall(ImmutableCallSite CS, Value *V, - ArrayRef Args, const SimplifyQuery &Q) { - return ::SimplifyCall(CS, V, Args.begin(), Args.end(), Q, RecursionLimit); -} - -Value *llvm::SimplifyCall(ImmutableCallSite ICS, const SimplifyQuery &Q) { - CallSite CS(const_cast(ICS.getInstruction())); - return ::SimplifyCall(CS, CS.getCalledValue(), CS.arg_begin(), CS.arg_end(), - Q, RecursionLimit); + return ConstantFoldCall(Call, F, ConstantArgs, Q.TLI); } /// See if we can compute a simplified version of this instruction. @@ -5203,6 +5058,9 @@ Value *llvm::SimplifyInstruction(Instruction *I, const SimplifyQuery &SQ, default: Result = ConstantFoldInstruction(I, Q.DL, Q.TLI); break; + case Instruction::FNeg: + Result = SimplifyFNegInst(I->getOperand(0), I->getFastMathFlags(), Q); + break; case Instruction::FAdd: Result = SimplifyFAddInst(I->getOperand(0), I->getOperand(1), I->getFastMathFlags(), Q); @@ -5327,8 +5185,7 @@ Value *llvm::SimplifyInstruction(Instruction *I, const SimplifyQuery &SQ, Result = SimplifyPHINode(cast(I), Q); break; case Instruction::Call: { - CallSite CS(cast(I)); - Result = SimplifyCall(CS, Q); + Result = SimplifyCall(cast(I), Q); break; } #define HANDLE_CAST_INST(num, opc, clas) case Instruction::opc: diff --git a/lib/Analysis/Interval.cpp b/lib/Analysis/Interval.cpp index 6d5de22cb93f..07d6e27c13be 100644 --- a/lib/Analysis/Interval.cpp +++ b/lib/Analysis/Interval.cpp @@ -1,9 +1,8 @@ //===- Interval.cpp - Interval class code ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/IntervalPartition.cpp b/lib/Analysis/IntervalPartition.cpp index c777d91b67c6..d12db010db6a 100644 --- a/lib/Analysis/IntervalPartition.cpp +++ b/lib/Analysis/IntervalPartition.cpp @@ -1,9 +1,8 @@ //===- IntervalPartition.cpp - Interval Partition module code -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/IteratedDominanceFrontier.cpp b/lib/Analysis/IteratedDominanceFrontier.cpp deleted file mode 100644 index 000fe5ddad54..000000000000 --- a/lib/Analysis/IteratedDominanceFrontier.cpp +++ /dev/null @@ -1,110 +0,0 @@ -//===- IteratedDominanceFrontier.cpp - Compute IDF ------------------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// Compute iterated dominance frontiers using a linear time algorithm. -// -//===----------------------------------------------------------------------===// - -#include "llvm/Analysis/IteratedDominanceFrontier.h" -#include "llvm/IR/CFG.h" -#include "llvm/IR/Dominators.h" -#include - -namespace llvm { - -template -void IDFCalculator::calculate( - SmallVectorImpl &PHIBlocks) { - // Use a priority queue keyed on dominator tree level so that inserted nodes - // are handled from the bottom of the dominator tree upwards. We also augment - // the level with a DFS number to ensure that the blocks are ordered in a - // deterministic way. - typedef std::pair> - DomTreeNodePair; - typedef std::priority_queue, - less_second> IDFPriorityQueue; - IDFPriorityQueue PQ; - - DT.updateDFSNumbers(); - - for (BasicBlock *BB : *DefBlocks) { - if (DomTreeNode *Node = DT.getNode(BB)) - PQ.push({Node, std::make_pair(Node->getLevel(), Node->getDFSNumIn())}); - } - - SmallVector Worklist; - SmallPtrSet VisitedPQ; - SmallPtrSet VisitedWorklist; - - while (!PQ.empty()) { - DomTreeNodePair RootPair = PQ.top(); - PQ.pop(); - DomTreeNode *Root = RootPair.first; - unsigned RootLevel = RootPair.second.first; - - // Walk all dominator tree children of Root, inspecting their CFG edges with - // targets elsewhere on the dominator tree. Only targets whose level is at - // most Root's level are added to the iterated dominance frontier of the - // definition set. - - Worklist.clear(); - Worklist.push_back(Root); - VisitedWorklist.insert(Root); - - while (!Worklist.empty()) { - DomTreeNode *Node = Worklist.pop_back_val(); - BasicBlock *BB = Node->getBlock(); - // Succ is the successor in the direction we are calculating IDF, so it is - // successor for IDF, and predecessor for Reverse IDF. - auto DoWork = [&](BasicBlock *Succ) { - DomTreeNode *SuccNode = DT.getNode(Succ); - - // Quickly skip all CFG edges that are also dominator tree edges instead - // of catching them below. - if (SuccNode->getIDom() == Node) - return; - - const unsigned SuccLevel = SuccNode->getLevel(); - if (SuccLevel > RootLevel) - return; - - if (!VisitedPQ.insert(SuccNode).second) - return; - - BasicBlock *SuccBB = SuccNode->getBlock(); - if (useLiveIn && !LiveInBlocks->count(SuccBB)) - return; - - PHIBlocks.emplace_back(SuccBB); - if (!DefBlocks->count(SuccBB)) - PQ.push(std::make_pair( - SuccNode, std::make_pair(SuccLevel, SuccNode->getDFSNumIn()))); - }; - - if (GD) { - for (auto Pair : children< - std::pair *, NodeTy>>( - {GD, BB})) - DoWork(Pair.second); - } else { - for (auto *Succ : children(BB)) - DoWork(Succ); - } - - for (auto DomChild : *Node) { - if (VisitedWorklist.insert(DomChild).second) - Worklist.push_back(DomChild); - } - } - } -} - -template class IDFCalculator; -template class IDFCalculator, true>; -} diff --git a/lib/Analysis/LazyBlockFrequencyInfo.cpp b/lib/Analysis/LazyBlockFrequencyInfo.cpp index 93c23bca96af..439758560284 100644 --- a/lib/Analysis/LazyBlockFrequencyInfo.cpp +++ b/lib/Analysis/LazyBlockFrequencyInfo.cpp @@ -1,9 +1,8 @@ //===- LazyBlockFrequencyInfo.cpp - Lazy Block Frequency Analysis ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/LazyBranchProbabilityInfo.cpp b/lib/Analysis/LazyBranchProbabilityInfo.cpp index 429b78c3a47e..f2592c26b373 100644 --- a/lib/Analysis/LazyBranchProbabilityInfo.cpp +++ b/lib/Analysis/LazyBranchProbabilityInfo.cpp @@ -1,9 +1,8 @@ //===- LazyBranchProbabilityInfo.cpp - Lazy Branch Probability Analysis ---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/LazyCallGraph.cpp b/lib/Analysis/LazyCallGraph.cpp index 3f22ada803c9..797fcf516429 100644 --- a/lib/Analysis/LazyCallGraph.cpp +++ b/lib/Analysis/LazyCallGraph.cpp @@ -1,9 +1,8 @@ //===- LazyCallGraph.cpp - Analysis of a Module's call graph --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -173,6 +172,19 @@ LazyCallGraph::LazyCallGraph(Module &M, TargetLibraryInfo &TLI) { addEdge(EntryEdges.Edges, EntryEdges.EdgeIndexMap, get(F), Edge::Ref); } + // Externally visible aliases of internal functions are also viable entry + // edges to the module. + for (auto &A : M.aliases()) { + if (A.hasLocalLinkage()) + continue; + if (Function* F = dyn_cast(A.getAliasee())) { + LLVM_DEBUG(dbgs() << " Adding '" << F->getName() + << "' with alias '" << A.getName() + << "' to entry set of the graph.\n"); + addEdge(EntryEdges.Edges, EntryEdges.EdgeIndexMap, get(*F), Edge::Ref); + } + } + // Now add entry nodes for functions reachable via initializers to globals. SmallVector Worklist; SmallPtrSet Visited; diff --git a/lib/Analysis/LazyValueInfo.cpp b/lib/Analysis/LazyValueInfo.cpp index 110c085d3f35..542ff709d475 100644 --- a/lib/Analysis/LazyValueInfo.cpp +++ b/lib/Analysis/LazyValueInfo.cpp @@ -1,9 +1,8 @@ //===- LazyValueInfo.cpp - Value constraint analysis ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -423,10 +422,18 @@ namespace { BasicBlock *BB); Optional getRangeForOperand(unsigned Op, Instruction *I, BasicBlock *BB); + bool solveBlockValueBinaryOpImpl( + ValueLatticeElement &BBLV, Instruction *I, BasicBlock *BB, + std::function OpFn); bool solveBlockValueBinaryOp(ValueLatticeElement &BBLV, BinaryOperator *BBI, BasicBlock *BB); bool solveBlockValueCast(ValueLatticeElement &BBLV, CastInst *CI, BasicBlock *BB); + bool solveBlockValueOverflowIntrinsic( + ValueLatticeElement &BBLV, WithOverflowInst *WO, BasicBlock *BB); + bool solveBlockValueIntrinsic(ValueLatticeElement &BBLV, IntrinsicInst *II, + BasicBlock *BB); void intersectAssumeOrGuardBlockValueConstantRange(Value *Val, ValueLatticeElement &BBLV, Instruction *BBI); @@ -625,7 +632,7 @@ bool LazyValueInfoImpl::solveBlockValueImpl(ValueLatticeElement &Res, // and the like to prove non-nullness, but it's not clear that's worth it // compile time wise. The context-insensitive value walk done inside // isKnownNonZero gets most of the profitable cases at much less expense. - // This does mean that we have a sensativity to where the defining + // This does mean that we have a sensitivity to where the defining // instruction is placed, even if it could legally be hoisted much higher. // That is unfortunate. PointerType *PT = dyn_cast(BBI->getType()); @@ -639,6 +646,14 @@ bool LazyValueInfoImpl::solveBlockValueImpl(ValueLatticeElement &Res, if (BinaryOperator *BO = dyn_cast(BBI)) return solveBlockValueBinaryOp(Res, BO, BB); + + if (auto *EVI = dyn_cast(BBI)) + if (auto *WO = dyn_cast(EVI->getAggregateOperand())) + if (EVI->getNumIndices() == 1 && *EVI->idx_begin() == 0) + return solveBlockValueOverflowIntrinsic(Res, WO, BB); + + if (auto *II = dyn_cast(BBI)) + return solveBlockValueIntrinsic(Res, II, BB); } LLVM_DEBUG(dbgs() << " compute BB '" << BB->getName() @@ -824,7 +839,9 @@ void LazyValueInfoImpl::intersectAssumeOrGuardBlockValueConstantRange( if (!GuardDecl || GuardDecl->use_empty()) return; - for (Instruction &I : make_range(BBI->getIterator().getReverse(), + if (BBI->getIterator() == BBI->getParent()->begin()) + return; + for (Instruction &I : make_range(std::next(BBI->getIterator().getReverse()), BBI->getParent()->rend())) { Value *Cond = nullptr; if (match(&I, m_Intrinsic(m_Value(Cond)))) @@ -892,7 +909,28 @@ bool LazyValueInfoImpl::solveBlockValueSelect(ValueLatticeElement &BBLV, return true; } - // TODO: ABS, NABS from the SelectPatternResult + if (SPR.Flavor == SPF_ABS) { + if (LHS == SI->getTrueValue()) { + BBLV = ValueLatticeElement::getRange(TrueCR.abs()); + return true; + } + if (LHS == SI->getFalseValue()) { + BBLV = ValueLatticeElement::getRange(FalseCR.abs()); + return true; + } + } + + if (SPR.Flavor == SPF_NABS) { + ConstantRange Zero(APInt::getNullValue(TrueCR.getBitWidth())); + if (LHS == SI->getTrueValue()) { + BBLV = ValueLatticeElement::getRange(Zero.sub(TrueCR.abs())); + return true; + } + if (LHS == SI->getFalseValue()) { + BBLV = ValueLatticeElement::getRange(Zero.sub(FalseCR.abs())); + return true; + } + } } // Can we constrain the facts about the true and false values by using the @@ -962,7 +1000,7 @@ Optional LazyValueInfoImpl::getRangeForOperand(unsigned Op, const unsigned OperandBitWidth = DL.getTypeSizeInBits(I->getOperand(Op)->getType()); - ConstantRange Range = ConstantRange(OperandBitWidth); + ConstantRange Range = ConstantRange::getFull(OperandBitWidth); if (hasBlockValue(I->getOperand(Op), BB)) { ValueLatticeElement Val = getBlockValue(I->getOperand(Op), BB); intersectAssumeOrGuardBlockValueConstantRange(I->getOperand(Op), Val, I); @@ -1018,56 +1056,83 @@ bool LazyValueInfoImpl::solveBlockValueCast(ValueLatticeElement &BBLV, return true; } +bool LazyValueInfoImpl::solveBlockValueBinaryOpImpl( + ValueLatticeElement &BBLV, Instruction *I, BasicBlock *BB, + std::function OpFn) { + // Figure out the ranges of the operands. If that fails, use a + // conservative range, but apply the transfer rule anyways. This + // lets us pick up facts from expressions like "and i32 (call i32 + // @foo()), 32" + Optional LHSRes = getRangeForOperand(0, I, BB); + Optional RHSRes = getRangeForOperand(1, I, BB); + if (!LHSRes.hasValue() || !RHSRes.hasValue()) + // More work to do before applying this transfer rule. + return false; + + ConstantRange LHSRange = LHSRes.getValue(); + ConstantRange RHSRange = RHSRes.getValue(); + BBLV = ValueLatticeElement::getRange(OpFn(LHSRange, RHSRange)); + return true; +} + bool LazyValueInfoImpl::solveBlockValueBinaryOp(ValueLatticeElement &BBLV, BinaryOperator *BO, BasicBlock *BB) { assert(BO->getOperand(0)->getType()->isSized() && "all operands to binary operators are sized"); - - // Filter out operators we don't know how to reason about before attempting to - // recurse on our operand(s). This can cut a long search short if we know - // we're not going to be able to get any useful information anyways. - switch (BO->getOpcode()) { - case Instruction::Add: - case Instruction::Sub: - case Instruction::Mul: - case Instruction::UDiv: - case Instruction::Shl: - case Instruction::LShr: - case Instruction::AShr: - case Instruction::And: - case Instruction::Or: - // continue into the code below - break; - default: - // Unhandled instructions are overdefined. + if (BO->getOpcode() == Instruction::Xor) { + // Xor is the only operation not supported by ConstantRange::binaryOp(). LLVM_DEBUG(dbgs() << " compute BB '" << BB->getName() << "' - overdefined (unknown binary operator).\n"); BBLV = ValueLatticeElement::getOverdefined(); return true; - }; - - // Figure out the ranges of the operands. If that fails, use a - // conservative range, but apply the transfer rule anyways. This - // lets us pick up facts from expressions like "and i32 (call i32 - // @foo()), 32" - Optional LHSRes = getRangeForOperand(0, BO, BB); - Optional RHSRes = getRangeForOperand(1, BO, BB); + } - if (!LHSRes.hasValue() || !RHSRes.hasValue()) - // More work to do before applying this transfer rule. - return false; + return solveBlockValueBinaryOpImpl(BBLV, BO, BB, + [BO](const ConstantRange &CR1, const ConstantRange &CR2) { + return CR1.binaryOp(BO->getOpcode(), CR2); + }); +} - ConstantRange LHSRange = LHSRes.getValue(); - ConstantRange RHSRange = RHSRes.getValue(); +bool LazyValueInfoImpl::solveBlockValueOverflowIntrinsic( + ValueLatticeElement &BBLV, WithOverflowInst *WO, BasicBlock *BB) { + return solveBlockValueBinaryOpImpl(BBLV, WO, BB, + [WO](const ConstantRange &CR1, const ConstantRange &CR2) { + return CR1.binaryOp(WO->getBinaryOp(), CR2); + }); +} - // NOTE: We're currently limited by the set of operations that ConstantRange - // can evaluate symbolically. Enhancing that set will allows us to analyze - // more definitions. - Instruction::BinaryOps BinOp = BO->getOpcode(); - BBLV = ValueLatticeElement::getRange(LHSRange.binaryOp(BinOp, RHSRange)); - return true; +bool LazyValueInfoImpl::solveBlockValueIntrinsic( + ValueLatticeElement &BBLV, IntrinsicInst *II, BasicBlock *BB) { + switch (II->getIntrinsicID()) { + case Intrinsic::uadd_sat: + return solveBlockValueBinaryOpImpl(BBLV, II, BB, + [](const ConstantRange &CR1, const ConstantRange &CR2) { + return CR1.uadd_sat(CR2); + }); + case Intrinsic::usub_sat: + return solveBlockValueBinaryOpImpl(BBLV, II, BB, + [](const ConstantRange &CR1, const ConstantRange &CR2) { + return CR1.usub_sat(CR2); + }); + case Intrinsic::sadd_sat: + return solveBlockValueBinaryOpImpl(BBLV, II, BB, + [](const ConstantRange &CR1, const ConstantRange &CR2) { + return CR1.sadd_sat(CR2); + }); + case Intrinsic::ssub_sat: + return solveBlockValueBinaryOpImpl(BBLV, II, BB, + [](const ConstantRange &CR1, const ConstantRange &CR2) { + return CR1.ssub_sat(CR2); + }); + default: + LLVM_DEBUG(dbgs() << " compute BB '" << BB->getName() + << "' - overdefined (unknown intrinsic).\n"); + BBLV = ValueLatticeElement::getOverdefined(); + return true; + } } static ValueLatticeElement getValueFromICmpCondition(Value *Val, ICmpInst *ICI, @@ -1133,6 +1198,28 @@ static ValueLatticeElement getValueFromICmpCondition(Value *Val, ICmpInst *ICI, return ValueLatticeElement::getOverdefined(); } +// Handle conditions of the form +// extractvalue(op.with.overflow(%x, C), 1). +static ValueLatticeElement getValueFromOverflowCondition( + Value *Val, WithOverflowInst *WO, bool IsTrueDest) { + // TODO: This only works with a constant RHS for now. We could also compute + // the range of the RHS, but this doesn't fit into the current structure of + // the edge value calculation. + const APInt *C; + if (WO->getLHS() != Val || !match(WO->getRHS(), m_APInt(C))) + return ValueLatticeElement::getOverdefined(); + + // Calculate the possible values of %x for which no overflow occurs. + ConstantRange NWR = ConstantRange::makeExactNoWrapRegion( + WO->getBinaryOp(), *C, WO->getNoWrapKind()); + + // If overflow is false, %x is constrained to NWR. If overflow is true, %x is + // constrained to it's inverse (all values that might cause overflow). + if (IsTrueDest) + NWR = NWR.inverse(); + return ValueLatticeElement::getRange(NWR); +} + static ValueLatticeElement getValueFromCondition(Value *Val, Value *Cond, bool isTrueDest, DenseMap &Visited); @@ -1143,6 +1230,11 @@ getValueFromConditionImpl(Value *Val, Value *Cond, bool isTrueDest, if (ICmpInst *ICI = dyn_cast(Cond)) return getValueFromICmpCondition(Val, ICI, isTrueDest); + if (auto *EVI = dyn_cast(Cond)) + if (auto *WO = dyn_cast(EVI->getAggregateOperand())) + if (EVI->getNumIndices() == 1 && *EVI->idx_begin() == 1) + return getValueFromOverflowCondition(Val, WO, isTrueDest); + // Handle conditions in the form of (cond1 && cond2), we know that on the // true dest path both of the conditions hold. Similarly for conditions of // the form (cond1 || cond2), we know that on the false dest path neither @@ -1575,14 +1667,14 @@ ConstantRange LazyValueInfo::getConstantRange(Value *V, BasicBlock *BB, ValueLatticeElement Result = getImpl(PImpl, AC, &DL, DT).getValueInBlock(V, BB, CxtI); if (Result.isUndefined()) - return ConstantRange(Width, /*isFullSet=*/false); + return ConstantRange::getEmpty(Width); if (Result.isConstantRange()) return Result.getConstantRange(); // We represent ConstantInt constants as constant ranges but other kinds // of integer constants, i.e. ConstantExpr will be tagged as constants assert(!(Result.isConstant() && isa(Result.getConstant())) && "ConstantInt value must be represented as constantrange"); - return ConstantRange(Width, /*isFullSet=*/true); + return ConstantRange::getFull(Width); } /// Determine whether the specified value is known to be a @@ -1614,14 +1706,14 @@ ConstantRange LazyValueInfo::getConstantRangeOnEdge(Value *V, getImpl(PImpl, AC, &DL, DT).getValueOnEdge(V, FromBB, ToBB, CxtI); if (Result.isUndefined()) - return ConstantRange(Width, /*isFullSet=*/false); + return ConstantRange::getEmpty(Width); if (Result.isConstantRange()) return Result.getConstantRange(); // We represent ConstantInt constants as constant ranges but other kinds // of integer constants, i.e. ConstantExpr will be tagged as constants assert(!(Result.isConstant() && isa(Result.getConstant())) && "ConstantInt value must be represented as constantrange"); - return ConstantRange(Width, /*isFullSet=*/true); + return ConstantRange::getFull(Width); } static LazyValueInfo::Tristate @@ -1711,7 +1803,7 @@ LazyValueInfo::getPredicateAt(unsigned Pred, Value *V, Constant *C, // through would still be correct. const DataLayout &DL = CxtI->getModule()->getDataLayout(); if (V->getType()->isPointerTy() && C->isNullValue() && - isKnownNonZero(V->stripPointerCasts(), DL)) { + isKnownNonZero(V->stripPointerCastsSameRepresentation(), DL)) { if (Pred == ICmpInst::ICMP_EQ) return LazyValueInfo::False; else if (Pred == ICmpInst::ICMP_NE) diff --git a/lib/Analysis/LegacyDivergenceAnalysis.cpp b/lib/Analysis/LegacyDivergenceAnalysis.cpp index 5540859ebdda..52212e1c42aa 100644 --- a/lib/Analysis/LegacyDivergenceAnalysis.cpp +++ b/lib/Analysis/LegacyDivergenceAnalysis.cpp @@ -1,10 +1,9 @@ //===- LegacyDivergenceAnalysis.cpp --------- Legacy Divergence Analysis //Implementation -==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/Lint.cpp b/lib/Analysis/Lint.cpp index 5d0a627f8426..d28b8a189d4b 100644 --- a/lib/Analysis/Lint.cpp +++ b/lib/Analysis/Lint.cpp @@ -1,9 +1,8 @@ //===-- Lint.cpp - Check for common errors in LLVM IR ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -268,10 +267,14 @@ void Lint::visitCallSite(CallSite CS) { if (Formal->hasNoAliasAttr() && Actual->getType()->isPointerTy()) { AttributeList PAL = CS.getAttributes(); unsigned ArgNo = 0; - for (CallSite::arg_iterator BI = CS.arg_begin(); BI != AE; ++BI) { + for (CallSite::arg_iterator BI = CS.arg_begin(); BI != AE; + ++BI, ++ArgNo) { // Skip ByVal arguments since they will be memcpy'd to the callee's // stack so we're not really passing the pointer anyway. - if (PAL.hasParamAttribute(ArgNo++, Attribute::ByVal)) + if (PAL.hasParamAttribute(ArgNo, Attribute::ByVal)) + continue; + // If both arguments are readonly, they have no dependence. + if (Formal->onlyReadsMemory() && CS.onlyReadsMemory(ArgNo)) continue; if (AI != BI && (*BI)->getType()->isPointerTy()) { AliasResult Result = AA->alias(*AI, *BI); diff --git a/lib/Analysis/Loads.cpp b/lib/Analysis/Loads.cpp index 8129795bc0c1..31da4e9ec783 100644 --- a/lib/Analysis/Loads.cpp +++ b/lib/Analysis/Loads.cpp @@ -1,9 +1,8 @@ //===- Loads.cpp - Local load analysis ------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -126,7 +125,8 @@ bool llvm::isDereferenceableAndAlignedPointer(const Value *V, unsigned Align, Visited); } -bool llvm::isDereferenceableAndAlignedPointer(const Value *V, unsigned Align, +bool llvm::isDereferenceableAndAlignedPointer(const Value *V, Type *Ty, + unsigned Align, const DataLayout &DL, const Instruction *CtxI, const DominatorTree *DT) { @@ -134,8 +134,6 @@ bool llvm::isDereferenceableAndAlignedPointer(const Value *V, unsigned Align, // attribute, we know exactly how many bytes are dereferenceable. If we can // determine the exact offset to the attributed variable, we can use that // information here. - Type *VTy = V->getType(); - Type *Ty = VTy->getPointerElementType(); // Require ABI alignment for loads without alignment specification if (Align == 0) @@ -146,14 +144,16 @@ bool llvm::isDereferenceableAndAlignedPointer(const Value *V, unsigned Align, SmallPtrSet Visited; return ::isDereferenceableAndAlignedPointer( - V, Align, APInt(DL.getIndexTypeSizeInBits(VTy), DL.getTypeStoreSize(Ty)), DL, - CtxI, DT, Visited); + V, Align, + APInt(DL.getIndexTypeSizeInBits(V->getType()), DL.getTypeStoreSize(Ty)), + DL, CtxI, DT, Visited); } -bool llvm::isDereferenceablePointer(const Value *V, const DataLayout &DL, +bool llvm::isDereferenceablePointer(const Value *V, Type *Ty, + const DataLayout &DL, const Instruction *CtxI, const DominatorTree *DT) { - return isDereferenceableAndAlignedPointer(V, 1, DL, CtxI, DT); + return isDereferenceableAndAlignedPointer(V, Ty, 1, DL, CtxI, DT); } /// Test if A and B will obviously have the same value. @@ -198,7 +198,7 @@ static bool AreEquivalentAddressValues(const Value *A, const Value *B) { /// /// This uses the pointee type to determine how many bytes need to be safe to /// load from the pointer. -bool llvm::isSafeToLoadUnconditionally(Value *V, unsigned Align, +bool llvm::isSafeToLoadUnconditionally(Value *V, unsigned Align, APInt &Size, const DataLayout &DL, Instruction *ScanFrom, const DominatorTree *DT) { @@ -209,7 +209,7 @@ bool llvm::isSafeToLoadUnconditionally(Value *V, unsigned Align, // If DT is not specified we can't make context-sensitive query const Instruction* CtxI = DT ? ScanFrom : nullptr; - if (isDereferenceableAndAlignedPointer(V, Align, DL, CtxI, DT)) + if (isDereferenceableAndAlignedPointer(V, Align, Size, DL, CtxI, DT)) return true; int64_t ByteOffset = 0; @@ -281,9 +281,17 @@ bool llvm::isSafeToLoadUnconditionally(Value *V, unsigned Align, Value *AccessedPtr; unsigned AccessedAlign; if (LoadInst *LI = dyn_cast(BBI)) { + // Ignore volatile loads. The execution of a volatile load cannot + // be used to prove an address is backed by regular memory; it can, + // for example, point to an MMIO register. + if (LI->isVolatile()) + continue; AccessedPtr = LI->getPointerOperand(); AccessedAlign = LI->getAlignment(); } else if (StoreInst *SI = dyn_cast(BBI)) { + // Ignore volatile stores (see comment for loads). + if (SI->isVolatile()) + continue; AccessedPtr = SI->getPointerOperand(); AccessedAlign = SI->getAlignment(); } else @@ -306,7 +314,15 @@ bool llvm::isSafeToLoadUnconditionally(Value *V, unsigned Align, return false; } -/// DefMaxInstsToScan - the default number of maximum instructions +bool llvm::isSafeToLoadUnconditionally(Value *V, Type *Ty, unsigned Align, + const DataLayout &DL, + Instruction *ScanFrom, + const DominatorTree *DT) { + APInt Size(DL.getIndexTypeSizeInBits(V->getType()), DL.getTypeStoreSize(Ty)); + return isSafeToLoadUnconditionally(V, Align, Size, DL, ScanFrom, DT); +} + + /// DefMaxInstsToScan - the default number of maximum instructions /// to scan in the block, used by FindAvailableLoadedValue(). /// FindAvailableLoadedValue() was introduced in r60148, to improve jump /// threading in part by eliminating partially redundant loads. diff --git a/lib/Analysis/LoopAccessAnalysis.cpp b/lib/Analysis/LoopAccessAnalysis.cpp index 7f3480f512ab..36bd9a8b7ea7 100644 --- a/lib/Analysis/LoopAccessAnalysis.cpp +++ b/lib/Analysis/LoopAccessAnalysis.cpp @@ -1,9 +1,8 @@ //===- LoopAccessAnalysis.cpp - Loop Access Analysis Implementation --------==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -843,7 +842,7 @@ void AccessAnalysis::processMemAccesses() { bool SetHasWrite = false; // Map of pointers to last access encountered. - typedef DenseMap UnderlyingObjToAccessMap; + typedef DenseMap UnderlyingObjToAccessMap; UnderlyingObjToAccessMap ObjToLastAccess; // Set of access to check after all writes have been processed. @@ -904,13 +903,13 @@ void AccessAnalysis::processMemAccesses() { // Create sets of pointers connected by a shared alias set and // underlying object. - typedef SmallVector ValueVector; + typedef SmallVector ValueVector; ValueVector TempObjects; GetUnderlyingObjects(Ptr, TempObjects, DL, LI); LLVM_DEBUG(dbgs() << "Underlying objects for pointer " << *Ptr << "\n"); - for (Value *UnderlyingObj : TempObjects) { + for (const Value *UnderlyingObj : TempObjects) { // nullptr never alias, don't join sets for pointer that have "null" // in their UnderlyingObjects list. if (isa(UnderlyingObj) && @@ -1014,7 +1013,7 @@ int64_t llvm::getPtrStride(PredicatedScalarEvolution &PSE, Value *Ptr, return 0; } - // The accesss function must stride over the innermost loop. + // The access function must stride over the innermost loop. if (Lp != AR->getLoop()) { LLVM_DEBUG(dbgs() << "LAA: Bad stride - Not striding over innermost loop " << *Ptr << " SCEV: " << *AR << "\n"); @@ -1086,7 +1085,7 @@ int64_t llvm::getPtrStride(PredicatedScalarEvolution &PSE, Value *Ptr, if (Assume) { // We can avoid this case by adding a run-time check. LLVM_DEBUG(dbgs() << "LAA: Non unit strided pointer which is not either " - << "inbouds or in address space 0 may wrap:\n" + << "inbounds or in address space 0 may wrap:\n" << "LAA: Pointer: " << *Ptr << "\n" << "LAA: SCEV: " << *AR << "\n" << "LAA: Added an overflow assumption\n"); @@ -1145,10 +1144,9 @@ bool llvm::sortPtrAccesses(ArrayRef VL, const DataLayout &DL, std::iota(SortedIndices.begin(), SortedIndices.end(), 0); // Sort the memory accesses and keep the order of their uses in UseOrder. - std::stable_sort(SortedIndices.begin(), SortedIndices.end(), - [&OffValPairs](unsigned Left, unsigned Right) { - return OffValPairs[Left].first < OffValPairs[Right].first; - }); + llvm::stable_sort(SortedIndices, [&](unsigned Left, unsigned Right) { + return OffValPairs[Left].first < OffValPairs[Right].first; + }); // Check if the order is consecutive already. if (llvm::all_of(SortedIndices, [&SortedIndices](const unsigned I) { @@ -1346,7 +1344,7 @@ static bool isSafeDependenceDistance(const DataLayout &DL, ScalarEvolution &SE, // where Step is the absolute stride of the memory accesses in bytes, // then there is no dependence. // - // Ratioanle: + // Rationale: // We basically want to check if the absolute distance (|Dist/Step|) // is >= the loop iteration count (or > BackedgeTakenCount). // This is equivalent to the Strong SIV Test (Practical Dependence Testing, @@ -1369,7 +1367,7 @@ static bool isSafeDependenceDistance(const DataLayout &DL, ScalarEvolution &SE, // The dependence distance can be positive/negative, so we sign extend Dist; // The multiplication of the absolute stride in bytes and the - // backdgeTakenCount is non-negative, so we zero extend Product. + // backedgeTakenCount is non-negative, so we zero extend Product. if (DistTypeSize > ProductTypeSize) CastedProduct = SE.getZeroExtendExpr(Product, Dist.getType()); else @@ -1780,6 +1778,11 @@ void LoopAccessInfo::analyzeLoop(AliasAnalysis *AA, LoopInfo *LI, unsigned NumReads = 0; unsigned NumReadWrites = 0; + bool HasComplexMemInst = false; + + // A runtime check is only legal to insert if there are no convergent calls. + HasConvergentOp = false; + PtrRtChecking->Pointers.clear(); PtrRtChecking->Need = false; @@ -1787,8 +1790,25 @@ void LoopAccessInfo::analyzeLoop(AliasAnalysis *AA, LoopInfo *LI, // For each block. for (BasicBlock *BB : TheLoop->blocks()) { - // Scan the BB and collect legal loads and stores. + // Scan the BB and collect legal loads and stores. Also detect any + // convergent instructions. for (Instruction &I : *BB) { + if (auto *Call = dyn_cast(&I)) { + if (Call->isConvergent()) + HasConvergentOp = true; + } + + // With both a non-vectorizable memory instruction and a convergent + // operation, found in this loop, no reason to continue the search. + if (HasComplexMemInst && HasConvergentOp) { + CanVecMem = false; + return; + } + + // Avoid hitting recordAnalysis multiple times. + if (HasComplexMemInst) + continue; + // If this is a load, save it. If this instruction can read from memory // but is not a load, then we quit. Notice that we don't handle function // calls that read or write. @@ -1807,12 +1827,18 @@ void LoopAccessInfo::analyzeLoop(AliasAnalysis *AA, LoopInfo *LI, continue; auto *Ld = dyn_cast(&I); - if (!Ld || (!Ld->isSimple() && !IsAnnotatedParallel)) { + if (!Ld) { + recordAnalysis("CantVectorizeInstruction", Ld) + << "instruction cannot be vectorized"; + HasComplexMemInst = true; + continue; + } + if (!Ld->isSimple() && !IsAnnotatedParallel) { recordAnalysis("NonSimpleLoad", Ld) << "read with atomic ordering or volatile read"; LLVM_DEBUG(dbgs() << "LAA: Found a non-simple load.\n"); - CanVecMem = false; - return; + HasComplexMemInst = true; + continue; } NumLoads++; Loads.push_back(Ld); @@ -1828,15 +1854,15 @@ void LoopAccessInfo::analyzeLoop(AliasAnalysis *AA, LoopInfo *LI, if (!St) { recordAnalysis("CantVectorizeInstruction", St) << "instruction cannot be vectorized"; - CanVecMem = false; - return; + HasComplexMemInst = true; + continue; } if (!St->isSimple() && !IsAnnotatedParallel) { recordAnalysis("NonSimpleStore", St) << "write with atomic ordering or volatile write"; LLVM_DEBUG(dbgs() << "LAA: Found a non-simple store.\n"); - CanVecMem = false; - return; + HasComplexMemInst = true; + continue; } NumStores++; Stores.push_back(St); @@ -1847,6 +1873,11 @@ void LoopAccessInfo::analyzeLoop(AliasAnalysis *AA, LoopInfo *LI, } // Next instr. } // Next block. + if (HasComplexMemInst) { + CanVecMem = false; + return; + } + // Now we have two lists that hold the loads and the stores. // Next, we find the pointers that they use. @@ -1964,7 +1995,7 @@ void LoopAccessInfo::analyzeLoop(AliasAnalysis *AA, LoopInfo *LI, } LLVM_DEBUG( - dbgs() << "LAA: We can perform a memory runtime check if needed.\n"); + dbgs() << "LAA: May be able to perform a memory runtime check if needed.\n"); CanVecMem = true; if (Accesses.isDependencyCheckNeeded()) { @@ -1999,6 +2030,15 @@ void LoopAccessInfo::analyzeLoop(AliasAnalysis *AA, LoopInfo *LI, } } + if (HasConvergentOp) { + recordAnalysis("CantInsertRuntimeCheckWithConvergent") + << "cannot add control dependency to convergent operation"; + LLVM_DEBUG(dbgs() << "LAA: We can't vectorize because a runtime check " + "would be needed with a convergent operation\n"); + CanVecMem = false; + return; + } + if (CanVecMem) LLVM_DEBUG( dbgs() << "LAA: No unsafe dependent memory operations in loop. We" @@ -2252,7 +2292,7 @@ void LoopAccessInfo::collectStridedAccess(Value *MemAccess) { // Match the types so we can compare the stride and the BETakenCount. // The Stride can be positive/negative, so we sign extend Stride; - // The backdgeTakenCount is non-negative, so we zero extend BETakenCount. + // The backedgeTakenCount is non-negative, so we zero extend BETakenCount. const DataLayout &DL = TheLoop->getHeader()->getModule()->getDataLayout(); uint64_t StrideTypeSize = DL.getTypeAllocSize(StrideExpr->getType()); uint64_t BETypeSize = DL.getTypeAllocSize(BETakenCount->getType()); @@ -2287,6 +2327,7 @@ LoopAccessInfo::LoopAccessInfo(Loop *L, ScalarEvolution *SE, PtrRtChecking(llvm::make_unique(SE)), DepChecker(llvm::make_unique(*PSE, L)), TheLoop(L), NumLoads(0), NumStores(0), MaxSafeDepDistBytes(-1), CanVecMem(false), + HasConvergentOp(false), HasDependenceInvolvingLoopInvariantAddress(false) { if (canAnalyzeLoop()) analyzeLoop(AA, LI, TLI, DT); @@ -2303,6 +2344,9 @@ void LoopAccessInfo::print(raw_ostream &OS, unsigned Depth) const { OS << "\n"; } + if (HasConvergentOp) + OS.indent(Depth) << "Has convergent operation in loop\n"; + if (Report) OS.indent(Depth) << "Report: " << Report->getMsg() << "\n"; diff --git a/lib/Analysis/LoopAnalysisManager.cpp b/lib/Analysis/LoopAnalysisManager.cpp index 2a3b29d7fbca..a10a87ce113b 100644 --- a/lib/Analysis/LoopAnalysisManager.cpp +++ b/lib/Analysis/LoopAnalysisManager.cpp @@ -1,9 +1,8 @@ //===- LoopAnalysisManager.cpp - Loop analysis management -----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -19,11 +18,6 @@ using namespace llvm; namespace llvm { -/// Enables memory ssa as a dependency for loop passes in legacy pass manager. -cl::opt EnableMSSALoopDependency( - "enable-mssa-loop-dependency", cl::Hidden, cl::init(false), - cl::desc("Enable MemorySSA dependency for loop pass manager")); - // Explicit template instantiations and specialization definitions for core // template typedefs. template class AllAnalysesOn; @@ -147,8 +141,6 @@ PreservedAnalyses llvm::getLoopPassPreservedAnalyses() { PA.preserve(); PA.preserve(); PA.preserve(); - if (EnableMSSALoopDependency) - PA.preserve(); // FIXME: What we really want to do here is preserve an AA category, but that // concept doesn't exist yet. PA.preserve(); diff --git a/lib/Analysis/LoopInfo.cpp b/lib/Analysis/LoopInfo.cpp index ef2b1257015c..aa5da0859805 100644 --- a/lib/Analysis/LoopInfo.cpp +++ b/lib/Analysis/LoopInfo.cpp @@ -1,9 +1,8 @@ //===- LoopInfo.cpp - Natural Loop Calculator -----------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,8 +17,12 @@ #include "llvm/ADT/DepthFirstIterator.h" #include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/SmallPtrSet.h" +#include "llvm/Analysis/IVDescriptors.h" #include "llvm/Analysis/LoopInfoImpl.h" #include "llvm/Analysis/LoopIterator.h" +#include "llvm/Analysis/MemorySSA.h" +#include "llvm/Analysis/MemorySSAUpdater.h" +#include "llvm/Analysis/ScalarEvolutionExpressions.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/Config/llvm-config.h" #include "llvm/IR/CFG.h" @@ -65,15 +68,16 @@ bool Loop::hasLoopInvariantOperands(const Instruction *I) const { return all_of(I->operands(), [this](Value *V) { return isLoopInvariant(V); }); } -bool Loop::makeLoopInvariant(Value *V, bool &Changed, - Instruction *InsertPt) const { +bool Loop::makeLoopInvariant(Value *V, bool &Changed, Instruction *InsertPt, + MemorySSAUpdater *MSSAU) const { if (Instruction *I = dyn_cast(V)) - return makeLoopInvariant(I, Changed, InsertPt); + return makeLoopInvariant(I, Changed, InsertPt, MSSAU); return true; // All non-instructions are loop-invariant. } bool Loop::makeLoopInvariant(Instruction *I, bool &Changed, - Instruction *InsertPt) const { + Instruction *InsertPt, + MemorySSAUpdater *MSSAU) const { // Test if the value is already loop-invariant. if (isLoopInvariant(I)) return true; @@ -94,11 +98,14 @@ bool Loop::makeLoopInvariant(Instruction *I, bool &Changed, } // Don't hoist instructions with loop-variant operands. for (Value *Operand : I->operands()) - if (!makeLoopInvariant(Operand, Changed, InsertPt)) + if (!makeLoopInvariant(Operand, Changed, InsertPt, MSSAU)) return false; // Hoist. I->moveBefore(InsertPt); + if (MSSAU) + if (auto *MUD = MSSAU->getMemorySSA()->getMemoryAccess(I)) + MSSAU->moveToPlace(MUD, InsertPt->getParent(), MemorySSA::End); // There is possibility of hoisting this instruction above some arbitrary // condition. Any metadata defined on it can be control dependent on this @@ -110,24 +117,37 @@ bool Loop::makeLoopInvariant(Instruction *I, bool &Changed, return true; } -PHINode *Loop::getCanonicalInductionVariable() const { +bool Loop::getIncomingAndBackEdge(BasicBlock *&Incoming, + BasicBlock *&Backedge) const { BasicBlock *H = getHeader(); - BasicBlock *Incoming = nullptr, *Backedge = nullptr; + Incoming = nullptr; + Backedge = nullptr; pred_iterator PI = pred_begin(H); assert(PI != pred_end(H) && "Loop must have at least one backedge!"); Backedge = *PI++; if (PI == pred_end(H)) - return nullptr; // dead loop + return false; // dead loop Incoming = *PI++; if (PI != pred_end(H)) - return nullptr; // multiple backedges? + return false; // multiple backedges? if (contains(Incoming)) { if (contains(Backedge)) - return nullptr; + return false; std::swap(Incoming, Backedge); } else if (!contains(Backedge)) + return false; + + assert(Incoming && Backedge && "expected non-null incoming and backedges"); + return true; +} + +PHINode *Loop::getCanonicalInductionVariable() const { + BasicBlock *H = getHeader(); + + BasicBlock *Incoming = nullptr, *Backedge = nullptr; + if (!getIncomingAndBackEdge(Incoming, Backedge)) return nullptr; // Loop over all of the PHI nodes, looking for a canonical indvar. @@ -146,6 +166,218 @@ PHINode *Loop::getCanonicalInductionVariable() const { return nullptr; } +/// Get the latch condition instruction. +static ICmpInst *getLatchCmpInst(const Loop &L) { + if (BasicBlock *Latch = L.getLoopLatch()) + if (BranchInst *BI = dyn_cast_or_null(Latch->getTerminator())) + if (BI->isConditional()) + return dyn_cast(BI->getCondition()); + + return nullptr; +} + +/// Return the final value of the loop induction variable if found. +static Value *findFinalIVValue(const Loop &L, const PHINode &IndVar, + const Instruction &StepInst) { + ICmpInst *LatchCmpInst = getLatchCmpInst(L); + if (!LatchCmpInst) + return nullptr; + + Value *Op0 = LatchCmpInst->getOperand(0); + Value *Op1 = LatchCmpInst->getOperand(1); + if (Op0 == &IndVar || Op0 == &StepInst) + return Op1; + + if (Op1 == &IndVar || Op1 == &StepInst) + return Op0; + + return nullptr; +} + +Optional Loop::LoopBounds::getBounds(const Loop &L, + PHINode &IndVar, + ScalarEvolution &SE) { + InductionDescriptor IndDesc; + if (!InductionDescriptor::isInductionPHI(&IndVar, &L, &SE, IndDesc)) + return None; + + Value *InitialIVValue = IndDesc.getStartValue(); + Instruction *StepInst = IndDesc.getInductionBinOp(); + if (!InitialIVValue || !StepInst) + return None; + + const SCEV *Step = IndDesc.getStep(); + Value *StepInstOp1 = StepInst->getOperand(1); + Value *StepInstOp0 = StepInst->getOperand(0); + Value *StepValue = nullptr; + if (SE.getSCEV(StepInstOp1) == Step) + StepValue = StepInstOp1; + else if (SE.getSCEV(StepInstOp0) == Step) + StepValue = StepInstOp0; + + Value *FinalIVValue = findFinalIVValue(L, IndVar, *StepInst); + if (!FinalIVValue) + return None; + + return LoopBounds(L, *InitialIVValue, *StepInst, StepValue, *FinalIVValue, + SE); +} + +using Direction = Loop::LoopBounds::Direction; + +ICmpInst::Predicate Loop::LoopBounds::getCanonicalPredicate() const { + BasicBlock *Latch = L.getLoopLatch(); + assert(Latch && "Expecting valid latch"); + + BranchInst *BI = dyn_cast_or_null(Latch->getTerminator()); + assert(BI && BI->isConditional() && "Expecting conditional latch branch"); + + ICmpInst *LatchCmpInst = dyn_cast(BI->getCondition()); + assert(LatchCmpInst && + "Expecting the latch compare instruction to be a CmpInst"); + + // Need to inverse the predicate when first successor is not the loop + // header + ICmpInst::Predicate Pred = (BI->getSuccessor(0) == L.getHeader()) + ? LatchCmpInst->getPredicate() + : LatchCmpInst->getInversePredicate(); + + if (LatchCmpInst->getOperand(0) == &getFinalIVValue()) + Pred = ICmpInst::getSwappedPredicate(Pred); + + // Need to flip strictness of the predicate when the latch compare instruction + // is not using StepInst + if (LatchCmpInst->getOperand(0) == &getStepInst() || + LatchCmpInst->getOperand(1) == &getStepInst()) + return Pred; + + // Cannot flip strictness of NE and EQ + if (Pred != ICmpInst::ICMP_NE && Pred != ICmpInst::ICMP_EQ) + return ICmpInst::getFlippedStrictnessPredicate(Pred); + + Direction D = getDirection(); + if (D == Direction::Increasing) + return ICmpInst::ICMP_SLT; + + if (D == Direction::Decreasing) + return ICmpInst::ICMP_SGT; + + // If cannot determine the direction, then unable to find the canonical + // predicate + return ICmpInst::BAD_ICMP_PREDICATE; +} + +Direction Loop::LoopBounds::getDirection() const { + if (const SCEVAddRecExpr *StepAddRecExpr = + dyn_cast(SE.getSCEV(&getStepInst()))) + if (const SCEV *StepRecur = StepAddRecExpr->getStepRecurrence(SE)) { + if (SE.isKnownPositive(StepRecur)) + return Direction::Increasing; + if (SE.isKnownNegative(StepRecur)) + return Direction::Decreasing; + } + + return Direction::Unknown; +} + +Optional Loop::getBounds(ScalarEvolution &SE) const { + if (PHINode *IndVar = getInductionVariable(SE)) + return LoopBounds::getBounds(*this, *IndVar, SE); + + return None; +} + +PHINode *Loop::getInductionVariable(ScalarEvolution &SE) const { + if (!isLoopSimplifyForm()) + return nullptr; + + BasicBlock *Header = getHeader(); + assert(Header && "Expected a valid loop header"); + ICmpInst *CmpInst = getLatchCmpInst(*this); + if (!CmpInst) + return nullptr; + + Instruction *LatchCmpOp0 = dyn_cast(CmpInst->getOperand(0)); + Instruction *LatchCmpOp1 = dyn_cast(CmpInst->getOperand(1)); + + for (PHINode &IndVar : Header->phis()) { + InductionDescriptor IndDesc; + if (!InductionDescriptor::isInductionPHI(&IndVar, this, &SE, IndDesc)) + continue; + + Instruction *StepInst = IndDesc.getInductionBinOp(); + + // case 1: + // IndVar = phi[{InitialValue, preheader}, {StepInst, latch}] + // StepInst = IndVar + step + // cmp = StepInst < FinalValue + if (StepInst == LatchCmpOp0 || StepInst == LatchCmpOp1) + return &IndVar; + + // case 2: + // IndVar = phi[{InitialValue, preheader}, {StepInst, latch}] + // StepInst = IndVar + step + // cmp = IndVar < FinalValue + if (&IndVar == LatchCmpOp0 || &IndVar == LatchCmpOp1) + return &IndVar; + } + + return nullptr; +} + +bool Loop::getInductionDescriptor(ScalarEvolution &SE, + InductionDescriptor &IndDesc) const { + if (PHINode *IndVar = getInductionVariable(SE)) + return InductionDescriptor::isInductionPHI(IndVar, this, &SE, IndDesc); + + return false; +} + +bool Loop::isAuxiliaryInductionVariable(PHINode &AuxIndVar, + ScalarEvolution &SE) const { + // Located in the loop header + BasicBlock *Header = getHeader(); + if (AuxIndVar.getParent() != Header) + return false; + + // No uses outside of the loop + for (User *U : AuxIndVar.users()) + if (const Instruction *I = dyn_cast(U)) + if (!contains(I)) + return false; + + InductionDescriptor IndDesc; + if (!InductionDescriptor::isInductionPHI(&AuxIndVar, this, &SE, IndDesc)) + return false; + + // The step instruction opcode should be add or sub. + if (IndDesc.getInductionOpcode() != Instruction::Add && + IndDesc.getInductionOpcode() != Instruction::Sub) + return false; + + // Incremented by a loop invariant step for each loop iteration + return SE.isLoopInvariant(IndDesc.getStep(), this); +} + +bool Loop::isCanonical(ScalarEvolution &SE) const { + InductionDescriptor IndDesc; + if (!getInductionDescriptor(SE, IndDesc)) + return false; + + ConstantInt *Init = dyn_cast_or_null(IndDesc.getStartValue()); + if (!Init || !Init->isZero()) + return false; + + if (IndDesc.getInductionOpcode() != Instruction::Add) + return false; + + ConstantInt *Step = IndDesc.getConstIntStepValue(); + if (!Step || !Step->isOne()) + return false; + + return true; +} + // Check that 'BB' doesn't have any uses outside of the 'L' static bool isBlockInLCSSAForm(const Loop &L, const BasicBlock &BB, DominatorTree &DT) { @@ -200,8 +432,11 @@ bool Loop::isLoopSimplifyForm() const { bool Loop::isSafeToClone() const { // Return false if any loop blocks contain indirectbrs, or there are any calls // to noduplicate functions. + // FIXME: it should be ok to clone CallBrInst's if we correctly update the + // operand list to reflect the newly cloned labels. for (BasicBlock *BB : this->blocks()) { - if (isa(BB->getTerminator())) + if (isa(BB->getTerminator()) || + isa(BB->getTerminator())) return false; for (Instruction &I : *BB) @@ -242,48 +477,20 @@ void Loop::setLoopID(MDNode *LoopID) const { assert((!LoopID || LoopID->getOperand(0) == LoopID) && "Loop ID should refer to itself"); - BasicBlock *H = getHeader(); - for (BasicBlock *BB : this->blocks()) { - Instruction *TI = BB->getTerminator(); - for (BasicBlock *Successor : successors(TI)) { - if (Successor == H) { - TI->setMetadata(LLVMContext::MD_loop, LoopID); - break; - } - } - } + SmallVector LoopLatches; + getLoopLatches(LoopLatches); + for (BasicBlock *BB : LoopLatches) + BB->getTerminator()->setMetadata(LLVMContext::MD_loop, LoopID); } void Loop::setLoopAlreadyUnrolled() { - MDNode *LoopID = getLoopID(); - // First remove any existing loop unrolling metadata. - SmallVector MDs; - // Reserve first location for self reference to the LoopID metadata node. - MDs.push_back(nullptr); - - if (LoopID) { - for (unsigned i = 1, ie = LoopID->getNumOperands(); i < ie; ++i) { - bool IsUnrollMetadata = false; - MDNode *MD = dyn_cast(LoopID->getOperand(i)); - if (MD) { - const MDString *S = dyn_cast(MD->getOperand(0)); - IsUnrollMetadata = S && S->getString().startswith("llvm.loop.unroll."); - } - if (!IsUnrollMetadata) - MDs.push_back(LoopID->getOperand(i)); - } - } - - // Add unroll(disable) metadata to disable future unrolling. LLVMContext &Context = getHeader()->getContext(); - SmallVector DisableOperands; - DisableOperands.push_back(MDString::get(Context, "llvm.loop.unroll.disable")); - MDNode *DisableNode = MDNode::get(Context, DisableOperands); - MDs.push_back(DisableNode); - MDNode *NewLoopID = MDNode::get(Context, MDs); - // Set operand 0 to refer to the loop id itself. - NewLoopID->replaceOperandWith(0, NewLoopID); + MDNode *DisableUnrollMD = + MDNode::get(Context, MDString::get(Context, "llvm.loop.unroll.disable")); + MDNode *LoopID = getLoopID(); + MDNode *NewLoopID = makePostTransformationMetadata( + Context, LoopID, {"llvm.loop.unroll."}, {DisableUnrollMD}); setLoopID(NewLoopID); } @@ -761,6 +968,46 @@ bool llvm::isValidAsAccessGroup(MDNode *Node) { return Node->getNumOperands() == 0 && Node->isDistinct(); } +MDNode *llvm::makePostTransformationMetadata(LLVMContext &Context, + MDNode *OrigLoopID, + ArrayRef RemovePrefixes, + ArrayRef AddAttrs) { + // First remove any existing loop metadata related to this transformation. + SmallVector MDs; + + // Reserve first location for self reference to the LoopID metadata node. + TempMDTuple TempNode = MDNode::getTemporary(Context, None); + MDs.push_back(TempNode.get()); + + // Remove metadata for the transformation that has been applied or that became + // outdated. + if (OrigLoopID) { + for (unsigned i = 1, ie = OrigLoopID->getNumOperands(); i < ie; ++i) { + bool IsVectorMetadata = false; + Metadata *Op = OrigLoopID->getOperand(i); + if (MDNode *MD = dyn_cast(Op)) { + const MDString *S = dyn_cast(MD->getOperand(0)); + if (S) + IsVectorMetadata = + llvm::any_of(RemovePrefixes, [S](StringRef Prefix) -> bool { + return S->getString().startswith(Prefix); + }); + } + if (!IsVectorMetadata) + MDs.push_back(Op); + } + } + + // Add metadata to avoid reapplying a transformation, such as + // llvm.loop.unroll.disable and llvm.loop.isvectorized. + MDs.append(AddAttrs.begin(), AddAttrs.end()); + + MDNode *NewLoopID = MDNode::getDistinct(Context, MDs); + // Replace the temporary node with a self-reference. + NewLoopID->replaceOperandWith(0, NewLoopID); + return NewLoopID; +} + //===----------------------------------------------------------------------===// // LoopInfo implementation // @@ -792,7 +1039,7 @@ void LoopInfoWrapperPass::verifyAnalysis() const { void LoopInfoWrapperPass::getAnalysisUsage(AnalysisUsage &AU) const { AU.setPreservesAll(); - AU.addRequired(); + AU.addRequiredTransitive(); } void LoopInfoWrapperPass::print(raw_ostream &OS, const Module *) const { diff --git a/lib/Analysis/LoopPass.cpp b/lib/Analysis/LoopPass.cpp index a68f114b83a0..4ab3798039d8 100644 --- a/lib/Analysis/LoopPass.cpp +++ b/lib/Analysis/LoopPass.cpp @@ -1,9 +1,8 @@ //===- LoopPass.cpp - Loop Pass and Loop Pass Manager ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -23,6 +22,7 @@ #include "llvm/IR/PassTimingInfo.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Timer.h" +#include "llvm/Support/TimeProfiler.h" #include "llvm/Support/raw_ostream.h" using namespace llvm; @@ -210,6 +210,8 @@ bool LPPassManager::runOnFunction(Function &F) { for (unsigned Index = 0; Index < getNumContainedPasses(); ++Index) { LoopPass *P = getContainedPass(Index); + llvm::TimeTraceScope LoopPassScope("RunLoopPass", P->getPassName()); + dumpPassInfo(P, EXECUTION_MSG, ON_LOOP_MSG, CurrentLoop->getHeader()->getName()); dumpRequiredSet(P); @@ -384,16 +386,20 @@ void LoopPass::assignPassManager(PMStack &PMS, LPPM->add(this); } +static std::string getDescription(const Loop &L) { + return "loop"; +} + bool LoopPass::skipLoop(const Loop *L) const { const Function *F = L->getHeader()->getParent(); if (!F) return false; // Check the opt bisect limit. - LLVMContext &Context = F->getContext(); - if (!Context.getOptPassGate().shouldRunPass(this, *L)) + OptPassGate &Gate = F->getContext().getOptPassGate(); + if (Gate.isEnabled() && !Gate.shouldRunPass(this, getDescription(*L))) return true; // Check for the OptimizeNone attribute. - if (F->hasFnAttribute(Attribute::OptimizeNone)) { + if (F->hasOptNone()) { // FIXME: Report this to dbgs() only once per function. LLVM_DEBUG(dbgs() << "Skipping pass '" << getPassName() << "' in function " << F->getName() << "\n"); diff --git a/lib/Analysis/LoopUnrollAnalyzer.cpp b/lib/Analysis/LoopUnrollAnalyzer.cpp index c8b91a7a1a51..1728b5e9f6d2 100644 --- a/lib/Analysis/LoopUnrollAnalyzer.cpp +++ b/lib/Analysis/LoopUnrollAnalyzer.cpp @@ -1,9 +1,8 @@ //===- LoopUnrollAnalyzer.cpp - Unrolling Effect Estimation -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/MemDepPrinter.cpp b/lib/Analysis/MemDepPrinter.cpp index 907b321b231a..6e1bb50e8893 100644 --- a/lib/Analysis/MemDepPrinter.cpp +++ b/lib/Analysis/MemDepPrinter.cpp @@ -1,9 +1,8 @@ //===- MemDepPrinter.cpp - Printer for MemoryDependenceAnalysis -----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/MemDerefPrinter.cpp b/lib/Analysis/MemDerefPrinter.cpp index 4a136c5a0c6d..77ebf89d9a08 100644 --- a/lib/Analysis/MemDerefPrinter.cpp +++ b/lib/Analysis/MemDerefPrinter.cpp @@ -1,9 +1,8 @@ //===- MemDerefPrinter.cpp - Printer for isDereferenceablePointer ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -54,9 +53,10 @@ bool MemDerefPrinter::runOnFunction(Function &F) { for (auto &I: instructions(F)) { if (LoadInst *LI = dyn_cast(&I)) { Value *PO = LI->getPointerOperand(); - if (isDereferenceablePointer(PO, DL)) + if (isDereferenceablePointer(PO, LI->getType(), DL)) Deref.push_back(PO); - if (isDereferenceableAndAlignedPointer(PO, LI->getAlignment(), DL)) + if (isDereferenceableAndAlignedPointer(PO, LI->getType(), + LI->getAlignment(), DL)) DerefAndAligned.insert(PO); } } diff --git a/lib/Analysis/MemoryBuiltins.cpp b/lib/Analysis/MemoryBuiltins.cpp index 686ad294378c..729dad463657 100644 --- a/lib/Analysis/MemoryBuiltins.cpp +++ b/lib/Analysis/MemoryBuiltins.cpp @@ -1,9 +1,8 @@ //===- MemoryBuiltins.cpp - Identify calls to memory builtins -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -264,6 +263,19 @@ bool llvm::isAllocLikeFn(const Value *V, const TargetLibraryInfo *TLI, return getAllocationData(V, AllocLike, TLI, LookThroughBitCast).hasValue(); } +/// Tests if a value is a call or invoke to a library function that +/// reallocates memory (e.g., realloc). +bool llvm::isReallocLikeFn(const Value *V, const TargetLibraryInfo *TLI, + bool LookThroughBitCast) { + return getAllocationData(V, ReallocLike, TLI, LookThroughBitCast).hasValue(); +} + +/// Tests if a functions is a call or invoke to a library function that +/// reallocates memory (e.g., realloc). +bool llvm::isReallocLikeFn(const Function *F, const TargetLibraryInfo *TLI) { + return getAllocationDataForFunction(F, ReallocLike, TLI).hasValue(); +} + /// extractMallocCall - Returns the corresponding CallInst if the instruction /// is a malloc call. Since CallInst::CreateMalloc() only creates calls, we /// ignore InvokeInst here. @@ -359,19 +371,8 @@ const CallInst *llvm::extractCallocCall(const Value *I, return isCallocLikeFn(I, TLI) ? cast(I) : nullptr; } -/// isFreeCall - Returns non-null if the value is a call to the builtin free() -const CallInst *llvm::isFreeCall(const Value *I, const TargetLibraryInfo *TLI) { - bool IsNoBuiltinCall; - const Function *Callee = - getCalledFunction(I, /*LookThroughBitCast=*/false, IsNoBuiltinCall); - if (Callee == nullptr || IsNoBuiltinCall) - return nullptr; - - StringRef FnName = Callee->getName(); - LibFunc TLIFn; - if (!TLI || !TLI->getLibFunc(FnName, TLIFn) || !TLI->has(TLIFn)) - return nullptr; - +/// isLibFreeFunction - Returns true if the function is a builtin free() +bool llvm::isLibFreeFunction(const Function *F, const LibFunc TLIFn) { unsigned ExpectedNumParams; if (TLIFn == LibFunc_free || TLIFn == LibFunc_ZdlPv || // operator delete(void*) @@ -402,22 +403,39 @@ const CallInst *llvm::isFreeCall(const Value *I, const TargetLibraryInfo *TLI) { TLIFn == LibFunc_ZdlPvSt11align_val_tRKSt9nothrow_t) // delete[](void*, align_val_t, nothrow) ExpectedNumParams = 3; else - return nullptr; + return false; // Check free prototype. // FIXME: workaround for PR5130, this will be obsolete when a nobuiltin // attribute will exist. - FunctionType *FTy = Callee->getFunctionType(); + FunctionType *FTy = F->getFunctionType(); if (!FTy->getReturnType()->isVoidTy()) - return nullptr; + return false; if (FTy->getNumParams() != ExpectedNumParams) + return false; + if (FTy->getParamType(0) != Type::getInt8PtrTy(F->getContext())) + return false; + + return true; +} + +/// isFreeCall - Returns non-null if the value is a call to the builtin free() +const CallInst *llvm::isFreeCall(const Value *I, const TargetLibraryInfo *TLI) { + bool IsNoBuiltinCall; + const Function *Callee = + getCalledFunction(I, /*LookThroughBitCast=*/false, IsNoBuiltinCall); + if (Callee == nullptr || IsNoBuiltinCall) return nullptr; - if (FTy->getParamType(0) != Type::getInt8PtrTy(Callee->getContext())) + + StringRef FnName = Callee->getName(); + LibFunc TLIFn; + if (!TLI || !TLI->getLibFunc(FnName, TLIFn) || !TLI->has(TLIFn)) return nullptr; - return dyn_cast(I); + return isLibFreeFunction(Callee, TLIFn) ? dyn_cast(I) : nullptr; } + //===----------------------------------------------------------------------===// // Utility functions to compute size of objects. // @@ -442,10 +460,10 @@ bool llvm::getObjectSize(const Value *Ptr, uint64_t &Size, const DataLayout &DL, return true; } -ConstantInt *llvm::lowerObjectSizeCall(IntrinsicInst *ObjectSize, - const DataLayout &DL, - const TargetLibraryInfo *TLI, - bool MustSucceed) { +Value *llvm::lowerObjectSizeCall(IntrinsicInst *ObjectSize, + const DataLayout &DL, + const TargetLibraryInfo *TLI, + bool MustSucceed) { assert(ObjectSize->getIntrinsicID() == Intrinsic::objectsize && "ObjectSize must be a call to llvm.objectsize!"); @@ -462,13 +480,35 @@ ConstantInt *llvm::lowerObjectSizeCall(IntrinsicInst *ObjectSize, EvalOptions.NullIsUnknownSize = cast(ObjectSize->getArgOperand(2))->isOne(); - // FIXME: Does it make sense to just return a failure value if the size won't - // fit in the output and `!MustSucceed`? - uint64_t Size; auto *ResultType = cast(ObjectSize->getType()); - if (getObjectSize(ObjectSize->getArgOperand(0), Size, DL, TLI, EvalOptions) && - isUIntN(ResultType->getBitWidth(), Size)) - return ConstantInt::get(ResultType, Size); + bool StaticOnly = cast(ObjectSize->getArgOperand(3))->isZero(); + if (StaticOnly) { + // FIXME: Does it make sense to just return a failure value if the size won't + // fit in the output and `!MustSucceed`? + uint64_t Size; + if (getObjectSize(ObjectSize->getArgOperand(0), Size, DL, TLI, EvalOptions) && + isUIntN(ResultType->getBitWidth(), Size)) + return ConstantInt::get(ResultType, Size); + } else { + LLVMContext &Ctx = ObjectSize->getFunction()->getContext(); + ObjectSizeOffsetEvaluator Eval(DL, TLI, Ctx, EvalOptions); + SizeOffsetEvalType SizeOffsetPair = + Eval.compute(ObjectSize->getArgOperand(0)); + + if (SizeOffsetPair != ObjectSizeOffsetEvaluator::unknown()) { + IRBuilder Builder(Ctx, TargetFolder(DL)); + Builder.SetInsertPoint(ObjectSize); + + // If we've outside the end of the object, then we can always access + // exactly 0 bytes. + Value *ResultSize = + Builder.CreateSub(SizeOffsetPair.first, SizeOffsetPair.second); + Value *UseZero = + Builder.CreateICmpULT(SizeOffsetPair.first, SizeOffsetPair.second); + return Builder.CreateSelect(UseZero, ConstantInt::get(ResultType, 0), + ResultSize); + } + } if (!MustSucceed) return nullptr; @@ -684,7 +724,7 @@ SizeOffsetType ObjectSizeOffsetVisitor::visitGlobalVariable(GlobalVariable &GV){ if (!GV.hasDefinitiveInitializer()) return unknown(); - APInt Size(IntTyBits, DL.getTypeAllocSize(GV.getType()->getElementType())); + APInt Size(IntTyBits, DL.getTypeAllocSize(GV.getValueType())); return std::make_pair(align(Size, GV.getAlignment()), Zero); } @@ -743,9 +783,12 @@ SizeOffsetType ObjectSizeOffsetVisitor::visitInstruction(Instruction &I) { ObjectSizeOffsetEvaluator::ObjectSizeOffsetEvaluator( const DataLayout &DL, const TargetLibraryInfo *TLI, LLVMContext &Context, - bool RoundToAlign) - : DL(DL), TLI(TLI), Context(Context), Builder(Context, TargetFolder(DL)), - RoundToAlign(RoundToAlign) { + ObjectSizeOpts EvalOpts) + : DL(DL), TLI(TLI), Context(Context), + Builder(Context, TargetFolder(DL), + IRBuilderCallbackInserter( + [&](Instruction *I) { InsertedInstructions.insert(I); })), + EvalOpts(EvalOpts) { // IntTy and Zero must be set for each compute() since the address space may // be different for later objects. } @@ -767,17 +810,21 @@ SizeOffsetEvalType ObjectSizeOffsetEvaluator::compute(Value *V) { if (CacheIt != CacheMap.end() && anyKnown(CacheIt->second)) CacheMap.erase(CacheIt); } + + // Erase any instructions we inserted as part of the traversal. + for (Instruction *I : InsertedInstructions) { + I->replaceAllUsesWith(UndefValue::get(I->getType())); + I->eraseFromParent(); + } } SeenVals.clear(); + InsertedInstructions.clear(); return Result; } SizeOffsetEvalType ObjectSizeOffsetEvaluator::compute_(Value *V) { - ObjectSizeOpts ObjSizeOptions; - ObjSizeOptions.RoundToAlign = RoundToAlign; - - ObjectSizeOffsetVisitor Visitor(DL, TLI, Context, ObjSizeOptions); + ObjectSizeOffsetVisitor Visitor(DL, TLI, Context, EvalOpts); SizeOffsetType Const = Visitor.compute(V); if (Visitor.bothKnown(Const)) return std::make_pair(ConstantInt::get(Context, Const.first), @@ -916,24 +963,28 @@ SizeOffsetEvalType ObjectSizeOffsetEvaluator::visitPHINode(PHINode &PHI) { if (!bothKnown(EdgeData)) { OffsetPHI->replaceAllUsesWith(UndefValue::get(IntTy)); OffsetPHI->eraseFromParent(); + InsertedInstructions.erase(OffsetPHI); SizePHI->replaceAllUsesWith(UndefValue::get(IntTy)); SizePHI->eraseFromParent(); + InsertedInstructions.erase(SizePHI); return unknown(); } SizePHI->addIncoming(EdgeData.first, PHI.getIncomingBlock(i)); OffsetPHI->addIncoming(EdgeData.second, PHI.getIncomingBlock(i)); } - Value *Size = SizePHI, *Offset = OffsetPHI, *Tmp; - if ((Tmp = SizePHI->hasConstantValue())) { + Value *Size = SizePHI, *Offset = OffsetPHI; + if (Value *Tmp = SizePHI->hasConstantValue()) { Size = Tmp; SizePHI->replaceAllUsesWith(Size); SizePHI->eraseFromParent(); + InsertedInstructions.erase(SizePHI); } - if ((Tmp = OffsetPHI->hasConstantValue())) { + if (Value *Tmp = OffsetPHI->hasConstantValue()) { Offset = Tmp; OffsetPHI->replaceAllUsesWith(Offset); OffsetPHI->eraseFromParent(); + InsertedInstructions.erase(OffsetPHI); } return std::make_pair(Size, Offset); } diff --git a/lib/Analysis/MemoryDependenceAnalysis.cpp b/lib/Analysis/MemoryDependenceAnalysis.cpp index e22182b99e11..b25b655165d7 100644 --- a/lib/Analysis/MemoryDependenceAnalysis.cpp +++ b/lib/Analysis/MemoryDependenceAnalysis.cpp @@ -1,9 +1,8 @@ //===- MemoryDependenceAnalysis.cpp - Mem Deps Implementation -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -327,7 +326,8 @@ static bool isVolatile(Instruction *Inst) { MemDepResult MemoryDependenceResults::getPointerDependencyFrom( const MemoryLocation &MemLoc, bool isLoad, BasicBlock::iterator ScanIt, - BasicBlock *BB, Instruction *QueryInst, unsigned *Limit) { + BasicBlock *BB, Instruction *QueryInst, unsigned *Limit, + OrderedBasicBlock *OBB) { MemDepResult InvariantGroupDependency = MemDepResult::getUnknown(); if (QueryInst != nullptr) { if (auto *LI = dyn_cast(QueryInst)) { @@ -338,7 +338,7 @@ MemDepResult MemoryDependenceResults::getPointerDependencyFrom( } } MemDepResult SimpleDep = getSimplePointerDependencyFrom( - MemLoc, isLoad, ScanIt, BB, QueryInst, Limit); + MemLoc, isLoad, ScanIt, BB, QueryInst, Limit, OBB); if (SimpleDep.isDef()) return SimpleDep; // Non-local invariant group dependency indicates there is non local Def @@ -439,14 +439,13 @@ MemoryDependenceResults::getInvariantGroupPointerDependency(LoadInst *LI, MemDepResult MemoryDependenceResults::getSimplePointerDependencyFrom( const MemoryLocation &MemLoc, bool isLoad, BasicBlock::iterator ScanIt, - BasicBlock *BB, Instruction *QueryInst, unsigned *Limit) { + BasicBlock *BB, Instruction *QueryInst, unsigned *Limit, + OrderedBasicBlock *OBB) { bool isInvariantLoad = false; - if (!Limit) { - unsigned DefaultLimit = BlockScanLimit; - return getSimplePointerDependencyFrom(MemLoc, isLoad, ScanIt, BB, QueryInst, - &DefaultLimit); - } + unsigned DefaultLimit = BlockScanLimit; + if (!Limit) + Limit = &DefaultLimit; // We must be careful with atomic accesses, as they may allow another thread // to touch this location, clobbering it. We are conservative: if the @@ -488,11 +487,14 @@ MemDepResult MemoryDependenceResults::getSimplePointerDependencyFrom( const DataLayout &DL = BB->getModule()->getDataLayout(); - // Create a numbered basic block to lazily compute and cache instruction + // If the caller did not provide an ordered basic block, + // create one to lazily compute and cache instruction // positions inside a BB. This is used to provide fast queries for relative // position between two instructions in a BB and can be used by // AliasAnalysis::callCapturesBefore. - OrderedBasicBlock OBB(BB); + OrderedBasicBlock OBBTmp(BB); + if (!OBB) + OBB = &OBBTmp; // Return "true" if and only if the instruction I is either a non-simple // load or a non-simple store. @@ -673,7 +675,7 @@ MemDepResult MemoryDependenceResults::getSimplePointerDependencyFrom( // A release fence requires that all stores complete before it, but does // not prevent the reordering of following loads or stores 'before' the // fence. As a result, we look past it when finding a dependency for - // loads. DSE uses this to find preceeding stores to delete and thus we + // loads. DSE uses this to find preceding stores to delete and thus we // can't bypass the fence if the query instruction is a store. if (FenceInst *FI = dyn_cast(Inst)) if (isLoad && FI->getOrdering() == AtomicOrdering::Release) @@ -683,7 +685,7 @@ MemDepResult MemoryDependenceResults::getSimplePointerDependencyFrom( ModRefInfo MR = AA.getModRefInfo(Inst, MemLoc); // If necessary, perform additional analysis. if (isModAndRefSet(MR)) - MR = AA.callCapturesBefore(Inst, MemLoc, &DT, &OBB); + MR = AA.callCapturesBefore(Inst, MemLoc, &DT, OBB); switch (clearMust(MR)) { case ModRefInfo::NoModRef: // If the call has no effect on the queried pointer, just ignore it. @@ -709,7 +711,8 @@ MemDepResult MemoryDependenceResults::getSimplePointerDependencyFrom( return MemDepResult::getNonFuncLocal(); } -MemDepResult MemoryDependenceResults::getDependency(Instruction *QueryInst) { +MemDepResult MemoryDependenceResults::getDependency(Instruction *QueryInst, + OrderedBasicBlock *OBB) { Instruction *ScanPos = QueryInst; // Check for a cached result @@ -747,8 +750,9 @@ MemDepResult MemoryDependenceResults::getDependency(Instruction *QueryInst) { if (auto *II = dyn_cast(QueryInst)) isLoad |= II->getIntrinsicID() == Intrinsic::lifetime_start; - LocalCache = getPointerDependencyFrom( - MemLoc, isLoad, ScanPos->getIterator(), QueryParent, QueryInst); + LocalCache = + getPointerDependencyFrom(MemLoc, isLoad, ScanPos->getIterator(), + QueryParent, QueryInst, nullptr, OBB); } else if (auto *QueryCall = dyn_cast(QueryInst)) { bool isReadOnly = AA.onlyReadsMemory(QueryCall); LocalCache = getCallDependencyFrom(QueryCall, isReadOnly, diff --git a/lib/Analysis/MemoryLocation.cpp b/lib/Analysis/MemoryLocation.cpp index 27e8d72b8e89..163830eee797 100644 --- a/lib/Analysis/MemoryLocation.cpp +++ b/lib/Analysis/MemoryLocation.cpp @@ -1,9 +1,8 @@ //===- MemoryLocation.cpp - Memory location descriptions -------------------==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/Analysis/MemorySSA.cpp b/lib/Analysis/MemorySSA.cpp index 6a5567ed765b..17f5d9b9f0ad 100644 --- a/lib/Analysis/MemorySSA.cpp +++ b/lib/Analysis/MemorySSA.cpp @@ -1,9 +1,8 @@ //===- MemorySSA.cpp - Memory SSA Builder ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -82,6 +81,11 @@ bool llvm::VerifyMemorySSA = true; #else bool llvm::VerifyMemorySSA = false; #endif +/// Enables memory ssa as a dependency for loop passes in legacy pass manager. +cl::opt llvm::EnableMSSALoopDependency( + "enable-mssa-loop-dependency", cl::Hidden, cl::init(false), + cl::desc("Enable MemorySSA dependency for loop pass manager")); + static cl::opt VerifyMemorySSAX("verify-memoryssa", cl::location(VerifyMemorySSA), cl::Hidden, cl::desc("Enable verification of MemorySSA.")); @@ -252,10 +256,10 @@ struct ClobberAlias { // Return a pair of {IsClobber (bool), AR (AliasResult)}. It relies on AR being // ignored if IsClobber = false. -static ClobberAlias instructionClobbersQuery(const MemoryDef *MD, - const MemoryLocation &UseLoc, - const Instruction *UseInst, - AliasAnalysis &AA) { +template +static ClobberAlias +instructionClobbersQuery(const MemoryDef *MD, const MemoryLocation &UseLoc, + const Instruction *UseInst, AliasAnalysisType &AA) { Instruction *DefInst = MD->getMemoryInst(); assert(DefInst && "Defining instruction not actually an instruction"); const auto *UseCall = dyn_cast(UseInst); @@ -300,10 +304,11 @@ static ClobberAlias instructionClobbersQuery(const MemoryDef *MD, return {isModSet(I), AR}; } +template static ClobberAlias instructionClobbersQuery(MemoryDef *MD, const MemoryUseOrDef *MU, const MemoryLocOrCall &UseMLOC, - AliasAnalysis &AA) { + AliasAnalysisType &AA) { // FIXME: This is a temporary hack to allow a single instructionClobbersQuery // to exist while MemoryLocOrCall is pushed through places. if (UseMLOC.IsCall) @@ -346,12 +351,12 @@ struct UpwardsMemoryQuery { } // end anonymous namespace static bool lifetimeEndsAt(MemoryDef *MD, const MemoryLocation &Loc, - AliasAnalysis &AA) { + BatchAAResults &AA) { Instruction *Inst = MD->getMemoryInst(); if (IntrinsicInst *II = dyn_cast(Inst)) { switch (II->getIntrinsicID()) { case Intrinsic::lifetime_end: - return AA.isMustAlias(MemoryLocation(II->getArgOperand(1)), Loc); + return AA.alias(MemoryLocation(II->getArgOperand(1)), Loc) == MustAlias; default: return false; } @@ -359,13 +364,14 @@ static bool lifetimeEndsAt(MemoryDef *MD, const MemoryLocation &Loc, return false; } -static bool isUseTriviallyOptimizableToLiveOnEntry(AliasAnalysis &AA, +template +static bool isUseTriviallyOptimizableToLiveOnEntry(AliasAnalysisType &AA, const Instruction *I) { // If the memory can't be changed, then loads of the memory can't be // clobbered. return isa(I) && (I->getMetadata(LLVMContext::MD_invariant_load) || - AA.pointsToConstantMemory(cast(I)-> - getPointerOperand())); + AA.pointsToConstantMemory(MemoryLocation( + cast(I)->getPointerOperand()))); } /// Verifies that `Start` is clobbered by `ClobberAt`, and that nothing @@ -381,10 +387,12 @@ static bool isUseTriviallyOptimizableToLiveOnEntry(AliasAnalysis &AA, /// \param Query The UpwardsMemoryQuery we used for our search. /// \param AA The AliasAnalysis we used for our search. /// \param AllowImpreciseClobber Always false, unless we do relaxed verify. -static void + +template +LLVM_ATTRIBUTE_UNUSED static void checkClobberSanity(const MemoryAccess *Start, MemoryAccess *ClobberAt, const MemoryLocation &StartLoc, const MemorySSA &MSSA, - const UpwardsMemoryQuery &Query, AliasAnalysis &AA, + const UpwardsMemoryQuery &Query, AliasAnalysisType &AA, bool AllowImpreciseClobber = false) { assert(MSSA.dominates(ClobberAt, Start) && "Clobber doesn't dominate start?"); @@ -474,7 +482,7 @@ namespace { /// Our algorithm for walking (and trying to optimize) clobbers, all wrapped up /// in one class. -class ClobberWalker { +template class ClobberWalker { /// Save a few bytes by using unsigned instead of size_t. using ListIndex = unsigned; @@ -498,9 +506,10 @@ class ClobberWalker { }; const MemorySSA &MSSA; - AliasAnalysis &AA; + AliasAnalysisType &AA; DominatorTree &DT; UpwardsMemoryQuery *Query; + unsigned *UpwardWalkLimit; // Phi optimization bookkeeping SmallVector Paths; @@ -539,6 +548,16 @@ class ClobberWalker { walkToPhiOrClobber(DefPath &Desc, const MemoryAccess *StopAt = nullptr, const MemoryAccess *SkipStopAt = nullptr) const { assert(!isa(Desc.Last) && "Uses don't exist in my world"); + assert(UpwardWalkLimit && "Need a valid walk limit"); + bool LimitAlreadyReached = false; + // (*UpwardWalkLimit) may be 0 here, due to the loop in tryOptimizePhi. Set + // it to 1. This will not do any alias() calls. It either returns in the + // first iteration in the loop below, or is set back to 0 if all def chains + // are free of MemoryDefs. + if (!*UpwardWalkLimit) { + *UpwardWalkLimit = 1; + LimitAlreadyReached = true; + } for (MemoryAccess *Current : def_chain(Desc.Last)) { Desc.Last = Current; @@ -548,6 +567,10 @@ class ClobberWalker { if (auto *MD = dyn_cast(Current)) { if (MSSA.isLiveOnEntryDef(MD)) return {MD, true, MustAlias}; + + if (!--*UpwardWalkLimit) + return {Current, true, MayAlias}; + ClobberAlias CA = instructionClobbersQuery(MD, Desc.Loc, Query->Inst, AA); if (CA.IsClobber) @@ -555,6 +578,9 @@ class ClobberWalker { } } + if (LimitAlreadyReached) + *UpwardWalkLimit = 0; + assert(isa(Desc.Last) && "Ended at a non-clobber that's not a phi?"); return {Desc.Last, false, MayAlias}; @@ -626,10 +652,12 @@ class ClobberWalker { SkipStopWhere = Query->OriginalAccess; } - UpwardsWalkResult Res = walkToPhiOrClobber(Node, /*StopAt=*/StopWhere, + UpwardsWalkResult Res = walkToPhiOrClobber(Node, + /*StopAt=*/StopWhere, /*SkipStopAt=*/SkipStopWhere); if (Res.IsKnownClobber) { assert(Res.Result != StopWhere && Res.Result != SkipStopWhere); + // If this wasn't a cache hit, we hit a clobber when walking. That's a // failure. TerminatedPath Term{Res.Result, PathIndex}; @@ -662,7 +690,7 @@ class ClobberWalker { struct generic_def_path_iterator : public iterator_facade_base, std::forward_iterator_tag, T *> { - generic_def_path_iterator() = default; + generic_def_path_iterator() {} generic_def_path_iterator(Walker *W, ListIndex N) : W(W), N(N) {} T &operator*() const { return curNode(); } @@ -887,13 +915,19 @@ class ClobberWalker { } public: - ClobberWalker(const MemorySSA &MSSA, AliasAnalysis &AA, DominatorTree &DT) + ClobberWalker(const MemorySSA &MSSA, AliasAnalysisType &AA, DominatorTree &DT) : MSSA(MSSA), AA(AA), DT(DT) {} + AliasAnalysisType *getAA() { return &AA; } /// Finds the nearest clobber for the given query, optimizing phis if /// possible. - MemoryAccess *findClobber(MemoryAccess *Start, UpwardsMemoryQuery &Q) { + MemoryAccess *findClobber(MemoryAccess *Start, UpwardsMemoryQuery &Q, + unsigned &UpWalkLimit) { Query = &Q; + UpwardWalkLimit = &UpWalkLimit; + // Starting limit must be > 0. + if (!UpWalkLimit) + UpWalkLimit++; MemoryAccess *Current = Start; // This walker pretends uses don't exist. If we're handed one, silently grab @@ -918,13 +952,11 @@ public: } #ifdef EXPENSIVE_CHECKS - if (!Q.SkipSelfAccess) + if (!Q.SkipSelfAccess && *UpwardWalkLimit > 0) checkClobberSanity(Current, Result, Q.StartingLoc, MSSA, Q, AA); #endif return Result; } - - void verify(const MemorySSA *MSSA) { assert(MSSA == &this->MSSA); } }; struct RenamePassData { @@ -947,77 +979,99 @@ struct RenamePassData { namespace llvm { -class MemorySSA::ClobberWalkerBase { - ClobberWalker Walker; +template class MemorySSA::ClobberWalkerBase { + ClobberWalker Walker; MemorySSA *MSSA; public: - ClobberWalkerBase(MemorySSA *M, AliasAnalysis *A, DominatorTree *D) + ClobberWalkerBase(MemorySSA *M, AliasAnalysisType *A, DominatorTree *D) : Walker(*M, *A, *D), MSSA(M) {} MemoryAccess *getClobberingMemoryAccessBase(MemoryAccess *, - const MemoryLocation &); - // Second argument (bool), defines whether the clobber search should skip the + const MemoryLocation &, + unsigned &); + // Third argument (bool), defines whether the clobber search should skip the // original queried access. If true, there will be a follow-up query searching // for a clobber access past "self". Note that the Optimized access is not // updated if a new clobber is found by this SkipSelf search. If this // additional query becomes heavily used we may decide to cache the result. // Walker instantiations will decide how to set the SkipSelf bool. - MemoryAccess *getClobberingMemoryAccessBase(MemoryAccess *, bool); - void verify(const MemorySSA *MSSA) { Walker.verify(MSSA); } + MemoryAccess *getClobberingMemoryAccessBase(MemoryAccess *, unsigned &, bool); }; /// A MemorySSAWalker that does AA walks to disambiguate accesses. It no /// longer does caching on its own, but the name has been retained for the /// moment. +template class MemorySSA::CachingWalker final : public MemorySSAWalker { - ClobberWalkerBase *Walker; + ClobberWalkerBase *Walker; public: - CachingWalker(MemorySSA *M, ClobberWalkerBase *W) + CachingWalker(MemorySSA *M, ClobberWalkerBase *W) : MemorySSAWalker(M), Walker(W) {} ~CachingWalker() override = default; using MemorySSAWalker::getClobberingMemoryAccess; - MemoryAccess *getClobberingMemoryAccess(MemoryAccess *MA) override; + MemoryAccess *getClobberingMemoryAccess(MemoryAccess *MA, unsigned &UWL) { + return Walker->getClobberingMemoryAccessBase(MA, UWL, false); + } MemoryAccess *getClobberingMemoryAccess(MemoryAccess *MA, - const MemoryLocation &Loc) override; + const MemoryLocation &Loc, + unsigned &UWL) { + return Walker->getClobberingMemoryAccessBase(MA, Loc, UWL); + } + + MemoryAccess *getClobberingMemoryAccess(MemoryAccess *MA) override { + unsigned UpwardWalkLimit = MaxCheckLimit; + return getClobberingMemoryAccess(MA, UpwardWalkLimit); + } + MemoryAccess *getClobberingMemoryAccess(MemoryAccess *MA, + const MemoryLocation &Loc) override { + unsigned UpwardWalkLimit = MaxCheckLimit; + return getClobberingMemoryAccess(MA, Loc, UpwardWalkLimit); + } void invalidateInfo(MemoryAccess *MA) override { if (auto *MUD = dyn_cast(MA)) MUD->resetOptimized(); } - - void verify(const MemorySSA *MSSA) override { - MemorySSAWalker::verify(MSSA); - Walker->verify(MSSA); - } }; +template class MemorySSA::SkipSelfWalker final : public MemorySSAWalker { - ClobberWalkerBase *Walker; + ClobberWalkerBase *Walker; public: - SkipSelfWalker(MemorySSA *M, ClobberWalkerBase *W) + SkipSelfWalker(MemorySSA *M, ClobberWalkerBase *W) : MemorySSAWalker(M), Walker(W) {} ~SkipSelfWalker() override = default; using MemorySSAWalker::getClobberingMemoryAccess; - MemoryAccess *getClobberingMemoryAccess(MemoryAccess *MA) override; + MemoryAccess *getClobberingMemoryAccess(MemoryAccess *MA, unsigned &UWL) { + return Walker->getClobberingMemoryAccessBase(MA, UWL, true); + } MemoryAccess *getClobberingMemoryAccess(MemoryAccess *MA, - const MemoryLocation &Loc) override; + const MemoryLocation &Loc, + unsigned &UWL) { + return Walker->getClobberingMemoryAccessBase(MA, Loc, UWL); + } + + MemoryAccess *getClobberingMemoryAccess(MemoryAccess *MA) override { + unsigned UpwardWalkLimit = MaxCheckLimit; + return getClobberingMemoryAccess(MA, UpwardWalkLimit); + } + MemoryAccess *getClobberingMemoryAccess(MemoryAccess *MA, + const MemoryLocation &Loc) override { + unsigned UpwardWalkLimit = MaxCheckLimit; + return getClobberingMemoryAccess(MA, Loc, UpwardWalkLimit); + } void invalidateInfo(MemoryAccess *MA) override { if (auto *MUD = dyn_cast(MA)) MUD->resetOptimized(); } - - void verify(const MemorySSA *MSSA) override { - MemorySSAWalker::verify(MSSA); - Walker->verify(MSSA); - } }; } // end namespace llvm @@ -1071,6 +1125,8 @@ MemoryAccess *MemorySSA::renameBlock(BasicBlock *BB, MemoryAccess *IncomingVal, void MemorySSA::renamePass(DomTreeNode *Root, MemoryAccess *IncomingVal, SmallPtrSetImpl &Visited, bool SkipVisited, bool RenameAllUses) { + assert(Root && "Trying to rename accesses in an unreachable block"); + SmallVector WorkStack; // Skip everything if we already renamed this block and we are skipping. // Note: You can't sink this into the if, because we need it to occur @@ -1154,9 +1210,20 @@ void MemorySSA::markUnreachableAsLiveOnEntry(BasicBlock *BB) { } MemorySSA::MemorySSA(Function &Func, AliasAnalysis *AA, DominatorTree *DT) - : AA(AA), DT(DT), F(Func), LiveOnEntryDef(nullptr), Walker(nullptr), + : AA(nullptr), DT(DT), F(Func), LiveOnEntryDef(nullptr), Walker(nullptr), SkipWalker(nullptr), NextID(0) { - buildMemorySSA(); + // Build MemorySSA using a batch alias analysis. This reuses the internal + // state that AA collects during an alias()/getModRefInfo() call. This is + // safe because there are no CFG changes while building MemorySSA and can + // significantly reduce the time spent by the compiler in AA, because we will + // make queries about all the instructions in the Function. + BatchAAResults BatchAA(*AA); + buildMemorySSA(BatchAA); + // Intentionally leave AA to nullptr while building so we don't accidently + // use non-batch AliasAnalysis. + this->AA = AA; + // Also create the walker here. + getWalker(); } MemorySSA::~MemorySSA() { @@ -1193,11 +1260,9 @@ namespace llvm { /// which is walking bottom-up. class MemorySSA::OptimizeUses { public: - OptimizeUses(MemorySSA *MSSA, MemorySSAWalker *Walker, AliasAnalysis *AA, - DominatorTree *DT) - : MSSA(MSSA), Walker(Walker), AA(AA), DT(DT) { - Walker = MSSA->getWalker(); - } + OptimizeUses(MemorySSA *MSSA, CachingWalker *Walker, + BatchAAResults *BAA, DominatorTree *DT) + : MSSA(MSSA), Walker(Walker), AA(BAA), DT(DT) {} void optimizeUses(); @@ -1225,8 +1290,8 @@ private: DenseMap &); MemorySSA *MSSA; - MemorySSAWalker *Walker; - AliasAnalysis *AA; + CachingWalker *Walker; + BatchAAResults *AA; DominatorTree *DT; }; @@ -1343,11 +1408,12 @@ void MemorySSA::OptimizeUses::optimizeUsesInBlock( continue; } bool FoundClobberResult = false; + unsigned UpwardWalkLimit = MaxCheckLimit; while (UpperBound > LocInfo.LowerBound) { if (isa(VersionStack[UpperBound])) { // For phis, use the walker, see where we ended up, go there - Instruction *UseInst = MU->getMemoryInst(); - MemoryAccess *Result = Walker->getClobberingMemoryAccess(UseInst); + MemoryAccess *Result = + Walker->getClobberingMemoryAccess(MU, UpwardWalkLimit); // We are guaranteed to find it or something is wrong while (VersionStack[UpperBound] != Result) { assert(UpperBound != 0); @@ -1423,7 +1489,7 @@ void MemorySSA::placePHINodes( createMemoryPhi(BB); } -void MemorySSA::buildMemorySSA() { +void MemorySSA::buildMemorySSA(BatchAAResults &BAA) { // We create an access to represent "live on entry", for things like // arguments or users of globals, where the memory they use is defined before // the beginning of the function. We do not actually insert it into the IR. @@ -1445,7 +1511,7 @@ void MemorySSA::buildMemorySSA() { AccessList *Accesses = nullptr; DefsList *Defs = nullptr; for (Instruction &I : B) { - MemoryUseOrDef *MUD = createNewAccess(&I); + MemoryUseOrDef *MUD = createNewAccess(&I, &BAA); if (!MUD) continue; @@ -1469,9 +1535,9 @@ void MemorySSA::buildMemorySSA() { SmallPtrSet Visited; renamePass(DT->getRootNode(), LiveOnEntryDef.get(), Visited); - CachingWalker *Walker = getWalkerImpl(); - - OptimizeUses(this, Walker, AA, DT).optimizeUses(); + ClobberWalkerBase WalkerBase(this, &BAA, DT); + CachingWalker WalkerLocal(this, &WalkerBase); + OptimizeUses(this, &WalkerLocal, &BAA, DT).optimizeUses(); // Mark the uses in unreachable blocks as live on entry, so that they go // somewhere. @@ -1482,14 +1548,16 @@ void MemorySSA::buildMemorySSA() { MemorySSAWalker *MemorySSA::getWalker() { return getWalkerImpl(); } -MemorySSA::CachingWalker *MemorySSA::getWalkerImpl() { +MemorySSA::CachingWalker *MemorySSA::getWalkerImpl() { if (Walker) return Walker.get(); if (!WalkerBase) - WalkerBase = llvm::make_unique(this, AA, DT); + WalkerBase = + llvm::make_unique>(this, AA, DT); - Walker = llvm::make_unique(this, WalkerBase.get()); + Walker = + llvm::make_unique>(this, WalkerBase.get()); return Walker.get(); } @@ -1498,9 +1566,11 @@ MemorySSAWalker *MemorySSA::getSkipSelfWalker() { return SkipWalker.get(); if (!WalkerBase) - WalkerBase = llvm::make_unique(this, AA, DT); + WalkerBase = + llvm::make_unique>(this, AA, DT); - SkipWalker = llvm::make_unique(this, WalkerBase.get()); + SkipWalker = + llvm::make_unique>(this, WalkerBase.get()); return SkipWalker.get(); } @@ -1619,7 +1689,7 @@ MemoryUseOrDef *MemorySSA::createDefinedAccess(Instruction *I, MemoryAccess *Definition, const MemoryUseOrDef *Template) { assert(!isa(I) && "Cannot create a defined access for a PHI"); - MemoryUseOrDef *NewAccess = createNewAccess(I, Template); + MemoryUseOrDef *NewAccess = createNewAccess(I, AA, Template); assert( NewAccess != nullptr && "Tried to create a memory access for a non-memory touching instruction"); @@ -1642,7 +1712,9 @@ static inline bool isOrdered(const Instruction *I) { } /// Helper function to create new memory accesses +template MemoryUseOrDef *MemorySSA::createNewAccess(Instruction *I, + AliasAnalysisType *AAP, const MemoryUseOrDef *Template) { // The assume intrinsic has a control dependency which we model by claiming // that it writes arbitrarily. Ignore that fake memory dependency here. @@ -1657,7 +1729,7 @@ MemoryUseOrDef *MemorySSA::createNewAccess(Instruction *I, Def = dyn_cast_or_null(Template) != nullptr; Use = dyn_cast_or_null(Template) != nullptr; #if !defined(NDEBUG) - ModRefInfo ModRef = AA->getModRefInfo(I, None); + ModRefInfo ModRef = AAP->getModRefInfo(I, None); bool DefCheck, UseCheck; DefCheck = isModSet(ModRef) || isOrdered(I); UseCheck = isRefSet(ModRef); @@ -1665,7 +1737,7 @@ MemoryUseOrDef *MemorySSA::createNewAccess(Instruction *I, #endif } else { // Find out what affect this instruction has on memory. - ModRefInfo ModRef = AA->getModRefInfo(I, None); + ModRefInfo ModRef = AAP->getModRefInfo(I, None); // The isOrdered check is used to ensure that volatiles end up as defs // (atomics end up as ModRef right now anyway). Until we separate the // ordering chain from the memory chain, this enables people to see at least @@ -1718,7 +1790,7 @@ void MemorySSA::removeFromLookups(MemoryAccess *MA) { MUD->setDefiningAccess(nullptr); // Invalidate our walker's cache if necessary if (!isa(MA)) - Walker->invalidateInfo(MA); + getWalker()->invalidateInfo(MA); Value *MemoryInst; if (const auto *MUD = dyn_cast(MA)) @@ -1778,35 +1850,16 @@ void MemorySSA::verifyMemorySSA() const { verifyDomination(F); verifyOrdering(F); verifyDominationNumbers(F); - Walker->verify(this); - verifyClobberSanity(F); -} - -/// Check sanity of the clobbering instruction for access MA. -void MemorySSA::checkClobberSanityAccess(const MemoryAccess *MA) const { - if (const auto *MUD = dyn_cast(MA)) { - if (!MUD->isOptimized()) - return; - auto *I = MUD->getMemoryInst(); - auto Loc = MemoryLocation::getOrNone(I); - if (Loc == None) - return; - auto *Clobber = MUD->getOptimized(); - UpwardsMemoryQuery Q(I, MUD); - checkClobberSanity(MUD, Clobber, *Loc, *this, Q, *AA, true); - } -} - -void MemorySSA::verifyClobberSanity(const Function &F) const { -#if !defined(NDEBUG) && defined(EXPENSIVE_CHECKS) - for (const BasicBlock &BB : F) { - const AccessList *Accesses = getBlockAccesses(&BB); - if (!Accesses) - continue; - for (const MemoryAccess &MA : *Accesses) - checkClobberSanityAccess(&MA); - } -#endif + // Previously, the verification used to also verify that the clobberingAccess + // cached by MemorySSA is the same as the clobberingAccess found at a later + // query to AA. This does not hold true in general due to the current fragility + // of BasicAA which has arbitrary caps on the things it analyzes before giving + // up. As a result, transformations that are correct, will lead to BasicAA + // returning different Alias answers before and after that transformation. + // Invalidating MemorySSA is not an option, as the results in BasicAA can be so + // random, in the worst case we'd need to rebuild MemorySSA from scratch after + // every transformation, which defeats the purpose of using it. For such an + // example, see test4 added in D51960. } /// Verify that all of the blocks we believe to have valid domination numbers @@ -2162,6 +2215,15 @@ MemorySSAAnalysis::Result MemorySSAAnalysis::run(Function &F, return MemorySSAAnalysis::Result(llvm::make_unique(F, &AA, &DT)); } +bool MemorySSAAnalysis::Result::invalidate( + Function &F, const PreservedAnalyses &PA, + FunctionAnalysisManager::Invalidator &Inv) { + auto PAC = PA.getChecker(); + return !(PAC.preserved() || PAC.preservedSet>()) || + Inv.invalidate(F, PA) || + Inv.invalidate(F, PA); +} + PreservedAnalyses MemorySSAPrinterPass::run(Function &F, FunctionAnalysisManager &AM) { OS << "MemorySSA for function: " << F.getName() << "\n"; @@ -2210,8 +2272,11 @@ MemorySSAWalker::MemorySSAWalker(MemorySSA *M) : MSSA(M) {} /// the MemoryAccess that actually clobbers Loc. /// /// \returns our clobbering memory access -MemoryAccess *MemorySSA::ClobberWalkerBase::getClobberingMemoryAccessBase( - MemoryAccess *StartingAccess, const MemoryLocation &Loc) { +template +MemoryAccess * +MemorySSA::ClobberWalkerBase::getClobberingMemoryAccessBase( + MemoryAccess *StartingAccess, const MemoryLocation &Loc, + unsigned &UpwardWalkLimit) { if (isa(StartingAccess)) return StartingAccess; @@ -2239,7 +2304,8 @@ MemoryAccess *MemorySSA::ClobberWalkerBase::getClobberingMemoryAccessBase( ? StartingUseOrDef->getDefiningAccess() : StartingUseOrDef; - MemoryAccess *Clobber = Walker.findClobber(DefiningAccess, Q); + MemoryAccess *Clobber = + Walker.findClobber(DefiningAccess, Q, UpwardWalkLimit); LLVM_DEBUG(dbgs() << "Starting Memory SSA clobber for " << *I << " is "); LLVM_DEBUG(dbgs() << *StartingUseOrDef << "\n"); LLVM_DEBUG(dbgs() << "Final Memory SSA clobber for " << *I << " is "); @@ -2247,9 +2313,10 @@ MemoryAccess *MemorySSA::ClobberWalkerBase::getClobberingMemoryAccessBase( return Clobber; } +template MemoryAccess * -MemorySSA::ClobberWalkerBase::getClobberingMemoryAccessBase(MemoryAccess *MA, - bool SkipSelf) { +MemorySSA::ClobberWalkerBase::getClobberingMemoryAccessBase( + MemoryAccess *MA, unsigned &UpwardWalkLimit, bool SkipSelf) { auto *StartingAccess = dyn_cast(MA); // If this is a MemoryPhi, we can't do anything. if (!StartingAccess) @@ -2275,7 +2342,7 @@ MemorySSA::ClobberWalkerBase::getClobberingMemoryAccessBase(MemoryAccess *MA, UpwardsMemoryQuery Q(I, StartingAccess); - if (isUseTriviallyOptimizableToLiveOnEntry(*MSSA->AA, I)) { + if (isUseTriviallyOptimizableToLiveOnEntry(*Walker.getAA(), I)) { MemoryAccess *LiveOnEntry = MSSA->getLiveOnEntryDef(); StartingAccess->setOptimized(LiveOnEntry); StartingAccess->setOptimizedAccessType(None); @@ -2295,7 +2362,7 @@ MemorySSA::ClobberWalkerBase::getClobberingMemoryAccessBase(MemoryAccess *MA, return DefiningAccess; } - OptimizedAccess = Walker.findClobber(DefiningAccess, Q); + OptimizedAccess = Walker.findClobber(DefiningAccess, Q, UpwardWalkLimit); StartingAccess->setOptimized(OptimizedAccess); if (MSSA->isLiveOnEntryDef(OptimizedAccess)) StartingAccess->setOptimizedAccessType(None); @@ -2311,10 +2378,10 @@ MemorySSA::ClobberWalkerBase::getClobberingMemoryAccessBase(MemoryAccess *MA, MemoryAccess *Result; if (SkipSelf && isa(OptimizedAccess) && - isa(StartingAccess)) { + isa(StartingAccess) && UpwardWalkLimit) { assert(isa(Q.OriginalAccess)); Q.SkipSelfAccess = true; - Result = Walker.findClobber(OptimizedAccess, Q); + Result = Walker.findClobber(OptimizedAccess, Q, UpwardWalkLimit); } else Result = OptimizedAccess; @@ -2324,28 +2391,6 @@ MemorySSA::ClobberWalkerBase::getClobberingMemoryAccessBase(MemoryAccess *MA, return Result; } -MemoryAccess * -MemorySSA::CachingWalker::getClobberingMemoryAccess(MemoryAccess *MA) { - return Walker->getClobberingMemoryAccessBase(MA, false); -} - -MemoryAccess * -MemorySSA::CachingWalker::getClobberingMemoryAccess(MemoryAccess *MA, - const MemoryLocation &Loc) { - return Walker->getClobberingMemoryAccessBase(MA, Loc); -} - -MemoryAccess * -MemorySSA::SkipSelfWalker::getClobberingMemoryAccess(MemoryAccess *MA) { - return Walker->getClobberingMemoryAccessBase(MA, true); -} - -MemoryAccess * -MemorySSA::SkipSelfWalker::getClobberingMemoryAccess(MemoryAccess *MA, - const MemoryLocation &Loc) { - return Walker->getClobberingMemoryAccessBase(MA, Loc); -} - MemoryAccess * DoNothingMemorySSAWalker::getClobberingMemoryAccess(MemoryAccess *MA) { if (auto *Use = dyn_cast(MA)) diff --git a/lib/Analysis/MemorySSAUpdater.cpp b/lib/Analysis/MemorySSAUpdater.cpp index 6c817d203684..4c1feee7fd9a 100644 --- a/lib/Analysis/MemorySSAUpdater.cpp +++ b/lib/Analysis/MemorySSAUpdater.cpp @@ -1,9 +1,8 @@ //===-- MemorySSAUpdater.cpp - Memory SSA Updater--------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------===// // @@ -73,7 +72,10 @@ MemoryAccess *MemorySSAUpdater::getPreviousDefRecursive( // potential phi node. This will insert phi nodes if we cycle in order to // break the cycle and have an operand. for (auto *Pred : predecessors(BB)) - PhiOps.push_back(getPreviousDefFromEnd(Pred, CachedPreviousDef)); + if (MSSA->DT->isReachableFromEntry(Pred)) + PhiOps.push_back(getPreviousDefFromEnd(Pred, CachedPreviousDef)); + else + PhiOps.push_back(MSSA->getLiveOnEntryDef()); // Now try to simplify the ops to avoid placing a phi. // This may return null if we never created a phi yet, that's okay @@ -157,8 +159,10 @@ MemoryAccess *MemorySSAUpdater::getPreviousDefFromEnd( DenseMap> &CachedPreviousDef) { auto *Defs = MSSA->getWritableBlockDefs(BB); - if (Defs) + if (Defs) { + CachedPreviousDef.insert({BB, &*Defs->rbegin()}); return &*Defs->rbegin(); + } return getPreviousDefRecursive(BB, CachedPreviousDef); } @@ -270,6 +274,8 @@ void MemorySSAUpdater::insertDef(MemoryDef *MD, bool RenameUses) { // Also make sure we skip ourselves to avoid self references. if (isa(U.getUser()) || U.getUser() == MD) continue; + // Defs are automatically unoptimized when the user is set to MD below, + // because the isOptimized() call will fail to find the same ID. U.set(MD); } } @@ -277,6 +283,9 @@ void MemorySSAUpdater::insertDef(MemoryDef *MD, bool RenameUses) { // and that def is now our defining access. MD->setDefiningAccess(DefBefore); + // Remember the index where we may insert new phis below. + unsigned NewPhiIndex = InsertedPHIs.size(); + SmallVector FixupList(InsertedPHIs.begin(), InsertedPHIs.end()); if (!DefBeforeSameBlock) { // If there was a local def before us, we must have the same effect it @@ -290,9 +299,56 @@ void MemorySSAUpdater::insertDef(MemoryDef *MD, bool RenameUses) { // backwards to find the def. To make that work, we'd have to track whether // getDefRecursive only ever used the single predecessor case. These types // of paths also only exist in between CFG simplifications. + + // If this is the first def in the block and this insert is in an arbitrary + // place, compute IDF and place phis. + auto Iter = MD->getDefsIterator(); + ++Iter; + auto IterEnd = MSSA->getBlockDefs(MD->getBlock())->end(); + if (Iter == IterEnd) { + ForwardIDFCalculator IDFs(*MSSA->DT); + SmallVector IDFBlocks; + SmallPtrSet DefiningBlocks; + DefiningBlocks.insert(MD->getBlock()); + IDFs.setDefiningBlocks(DefiningBlocks); + IDFs.calculate(IDFBlocks); + SmallVector, 4> NewInsertedPHIs; + for (auto *BBIDF : IDFBlocks) + if (!MSSA->getMemoryAccess(BBIDF)) { + auto *MPhi = MSSA->createMemoryPhi(BBIDF); + NewInsertedPHIs.push_back(MPhi); + // Add the phis created into the IDF blocks to NonOptPhis, so they are + // not optimized out as trivial by the call to getPreviousDefFromEnd + // below. Once they are complete, all these Phis are added to the + // FixupList, and removed from NonOptPhis inside fixupDefs(). + NonOptPhis.insert(MPhi); + } + + for (auto &MPhi : NewInsertedPHIs) { + auto *BBIDF = MPhi->getBlock(); + for (auto *Pred : predecessors(BBIDF)) { + DenseMap> CachedPreviousDef; + MPhi->addIncoming(getPreviousDefFromEnd(Pred, CachedPreviousDef), + Pred); + } + } + + // Re-take the index where we're adding the new phis, because the above + // call to getPreviousDefFromEnd, may have inserted into InsertedPHIs. + NewPhiIndex = InsertedPHIs.size(); + for (auto &MPhi : NewInsertedPHIs) { + InsertedPHIs.push_back(&*MPhi); + FixupList.push_back(&*MPhi); + } + } + FixupList.push_back(MD); } + // Remember the index where we stopped inserting new phis above, since the + // fixupDefs call in the loop below may insert more, that are already minimal. + unsigned NewPhiIndexEnd = InsertedPHIs.size(); + while (!FixupList.empty()) { unsigned StartingPHISize = InsertedPHIs.size(); fixupDefs(FixupList); @@ -300,6 +356,12 @@ void MemorySSAUpdater::insertDef(MemoryDef *MD, bool RenameUses) { // Put any new phis on the fixup list, and process them FixupList.append(InsertedPHIs.begin() + StartingPHISize, InsertedPHIs.end()); } + + // Optimize potentially non-minimal phis added in this method. + unsigned NewPhiSize = NewPhiIndexEnd - NewPhiIndex; + if (NewPhiSize) + tryRemoveTrivialPhis(ArrayRef(&InsertedPHIs[NewPhiIndex], NewPhiSize)); + // Now that all fixups are done, rename all uses if we are asked. if (RenameUses) { SmallPtrSet Visited; @@ -401,8 +463,8 @@ void MemorySSAUpdater::removeEdge(BasicBlock *From, BasicBlock *To) { } } -void MemorySSAUpdater::removeDuplicatePhiEdgesBetween(BasicBlock *From, - BasicBlock *To) { +void MemorySSAUpdater::removeDuplicatePhiEdgesBetween(const BasicBlock *From, + const BasicBlock *To) { if (MemoryPhi *MPhi = MSSA->getMemoryAccess(To)) { bool Found = false; MPhi->unorderedDeleteIncomingIf([&](const MemoryAccess *, BasicBlock *B) { @@ -420,7 +482,8 @@ void MemorySSAUpdater::removeDuplicatePhiEdgesBetween(BasicBlock *From, void MemorySSAUpdater::cloneUsesAndDefs(BasicBlock *BB, BasicBlock *NewBB, const ValueToValueMapTy &VMap, - PhiToDefMap &MPhiMap) { + PhiToDefMap &MPhiMap, + bool CloneWasSimplified) { auto GetNewDefiningAccess = [&](MemoryAccess *MA) -> MemoryAccess * { MemoryAccess *InsnDefining = MA; if (MemoryUseOrDef *DefMUD = dyn_cast(InsnDefining)) { @@ -450,16 +513,60 @@ void MemorySSAUpdater::cloneUsesAndDefs(BasicBlock *BB, BasicBlock *NewBB, // instructions. This occurs in LoopRotate when cloning instructions // from the old header to the old preheader. The cloned instruction may // also be a simplified Value, not an Instruction (see LoopRotate). + // Also in LoopRotate, even when it's an instruction, due to it being + // simplified, it may be a Use rather than a Def, so we cannot use MUD as + // template. Calls coming from updateForClonedBlockIntoPred, ensure this. if (Instruction *NewInsn = dyn_cast_or_null(VMap.lookup(Insn))) { MemoryAccess *NewUseOrDef = MSSA->createDefinedAccess( - NewInsn, GetNewDefiningAccess(MUD->getDefiningAccess()), MUD); + NewInsn, GetNewDefiningAccess(MUD->getDefiningAccess()), + CloneWasSimplified ? nullptr : MUD); MSSA->insertIntoListsForBlock(NewUseOrDef, NewBB, MemorySSA::End); } } } } +void MemorySSAUpdater::updatePhisWhenInsertingUniqueBackedgeBlock( + BasicBlock *Header, BasicBlock *Preheader, BasicBlock *BEBlock) { + auto *MPhi = MSSA->getMemoryAccess(Header); + if (!MPhi) + return; + + // Create phi node in the backedge block and populate it with the same + // incoming values as MPhi. Skip incoming values coming from Preheader. + auto *NewMPhi = MSSA->createMemoryPhi(BEBlock); + bool HasUniqueIncomingValue = true; + MemoryAccess *UniqueValue = nullptr; + for (unsigned I = 0, E = MPhi->getNumIncomingValues(); I != E; ++I) { + BasicBlock *IBB = MPhi->getIncomingBlock(I); + MemoryAccess *IV = MPhi->getIncomingValue(I); + if (IBB != Preheader) { + NewMPhi->addIncoming(IV, IBB); + if (HasUniqueIncomingValue) { + if (!UniqueValue) + UniqueValue = IV; + else if (UniqueValue != IV) + HasUniqueIncomingValue = false; + } + } + } + + // Update incoming edges into MPhi. Remove all but the incoming edge from + // Preheader. Add an edge from NewMPhi + auto *AccFromPreheader = MPhi->getIncomingValueForBlock(Preheader); + MPhi->setIncomingValue(0, AccFromPreheader); + MPhi->setIncomingBlock(0, Preheader); + for (unsigned I = MPhi->getNumIncomingValues() - 1; I >= 1; --I) + MPhi->unorderedDeleteIncoming(I); + MPhi->addIncoming(NewMPhi, BEBlock); + + // If NewMPhi is a trivial phi, remove it. Its use in the header MPhi will be + // replaced with the unique value. + if (HasUniqueIncomingValue) + removeMemoryAccess(NewMPhi); +} + void MemorySSAUpdater::updateForClonedLoop(const LoopBlocksRPO &LoopBlocks, ArrayRef ExitBlocks, const ValueToValueMapTy &VMap, @@ -543,10 +650,13 @@ void MemorySSAUpdater::updateForClonedBlockIntoPred( // Defs from BB being used in BB will be replaced with the cloned defs from // VM. The uses of BB's Phi (if it exists) in BB will be replaced by the // incoming def into the Phi from P1. + // Instructions cloned into the predecessor are in practice sometimes + // simplified, so disable the use of the template, and create an access from + // scratch. PhiToDefMap MPhiMap; if (MemoryPhi *MPhi = MSSA->getMemoryAccess(BB)) MPhiMap[MPhi] = MPhi->getIncomingValueForBlock(P1); - cloneUsesAndDefs(BB, P1, VM, MPhiMap); + cloneUsesAndDefs(BB, P1, VM, MPhiMap, /*CloneWasSimplified=*/true); } template @@ -599,7 +709,7 @@ void MemorySSAUpdater::applyUpdates(ArrayRef Updates, if (!RevDeleteUpdates.empty()) { // Update for inserted edges: use newDT and snapshot CFG as if deletes had - // not occured. + // not occurred. // FIXME: This creates a new DT, so it's more expensive to do mix // delete/inserts vs just inserts. We can do an incremental update on the DT // to revert deletes, than re-delete the edges. Teaching DT to do this, is @@ -697,7 +807,7 @@ void MemorySSAUpdater::applyInsertUpdates(ArrayRef Updates, // Map a BB to its predecessors: added + previously existing. To get a // deterministic order, store predecessors as SetVectors. The order in each - // will be defined by teh order in Updates (fixed) and the order given by + // will be defined by the order in Updates (fixed) and the order given by // children<> (also fixed). Since we further iterate over these ordered sets, // we lose the information of multiple edges possibly existing between two // blocks, so we'll keep and EdgeCount map for that. @@ -756,15 +866,15 @@ void MemorySSAUpdater::applyInsertUpdates(ArrayRef Updates, for (auto *BB : NewBlocks) PredMap.erase(BB); - SmallVector BlocksToProcess; SmallVector BlocksWithDefsToReplace; + SmallVector InsertedPhis; // First create MemoryPhis in all blocks that don't have one. Create in the // order found in Updates, not in PredMap, to get deterministic numbering. for (auto &Edge : Updates) { BasicBlock *BB = Edge.getTo(); if (PredMap.count(BB) && !MSSA->getMemoryAccess(BB)) - MSSA->createMemoryPhi(BB); + InsertedPhis.push_back(MSSA->createMemoryPhi(BB)); } // Now we'll fill in the MemoryPhis with the right incoming values. @@ -831,10 +941,6 @@ void MemorySSAUpdater::applyInsertUpdates(ArrayRef Updates, for (auto *Pred : PrevBlockSet) for (int I = 0, E = EdgeCountMap[{Pred, BB}]; I < E; ++I) NewPhi->addIncoming(DefP1, Pred); - - // Insert BB in the set of blocks that now have definition. We'll use this - // to compute IDF and add Phis there next. - BlocksToProcess.push_back(BB); } // Get all blocks that used to dominate BB and no longer do after adding @@ -849,22 +955,41 @@ void MemorySSAUpdater::applyInsertUpdates(ArrayRef Updates, GetNoLongerDomBlocks(PrevIDom, NewIDom, BlocksWithDefsToReplace); } + tryRemoveTrivialPhis(InsertedPhis); + // Create the set of blocks that now have a definition. We'll use this to + // compute IDF and add Phis there next. + SmallVector BlocksToProcess; + for (auto &VH : InsertedPhis) + if (auto *MPhi = cast_or_null(VH)) + BlocksToProcess.push_back(MPhi->getBlock()); + // Compute IDF and add Phis in all IDF blocks that do not have one. SmallVector IDFBlocks; if (!BlocksToProcess.empty()) { - ForwardIDFCalculator IDFs(DT); + ForwardIDFCalculator IDFs(DT, GD); SmallPtrSet DefiningBlocks(BlocksToProcess.begin(), BlocksToProcess.end()); IDFs.setDefiningBlocks(DefiningBlocks); IDFs.calculate(IDFBlocks); + + SmallSetVector PhisToFill; + // First create all needed Phis. + for (auto *BBIDF : IDFBlocks) + if (!MSSA->getMemoryAccess(BBIDF)) { + auto *IDFPhi = MSSA->createMemoryPhi(BBIDF); + InsertedPhis.push_back(IDFPhi); + PhisToFill.insert(IDFPhi); + } + // Then update or insert their correct incoming values. for (auto *BBIDF : IDFBlocks) { - if (auto *IDFPhi = MSSA->getMemoryAccess(BBIDF)) { + auto *IDFPhi = MSSA->getMemoryAccess(BBIDF); + assert(IDFPhi && "Phi must exist"); + if (!PhisToFill.count(IDFPhi)) { // Update existing Phi. // FIXME: some updates may be redundant, try to optimize and skip some. for (unsigned I = 0, E = IDFPhi->getNumIncomingValues(); I < E; ++I) IDFPhi->setIncomingValue(I, GetLastDef(IDFPhi->getIncomingBlock(I))); } else { - IDFPhi = MSSA->createMemoryPhi(BBIDF); for (auto &Pair : children({GD, BBIDF})) { BasicBlock *Pi = Pair.second; IDFPhi->addIncoming(GetLastDef(Pi), Pi); @@ -907,6 +1032,7 @@ void MemorySSAUpdater::applyInsertUpdates(ArrayRef Updates, } } } + tryRemoveTrivialPhis(InsertedPhis); } // Move What before Where in the MemorySSA IR. @@ -1052,7 +1178,7 @@ void MemorySSAUpdater::wireOldPredecessorsToNewImmediatePredecessor( } } -void MemorySSAUpdater::removeMemoryAccess(MemoryAccess *MA) { +void MemorySSAUpdater::removeMemoryAccess(MemoryAccess *MA, bool OptimizePhis) { assert(!MSSA->isLiveOnEntryDef(MA) && "Trying to remove the live on entry def"); // We can only delete phi nodes if they have no uses, or we can replace all @@ -1071,6 +1197,8 @@ void MemorySSAUpdater::removeMemoryAccess(MemoryAccess *MA) { NewDefTarget = cast(MA)->getDefiningAccess(); } + SmallSetVector PhisToCheck; + // Re-point the uses at our defining access if (!isa(MA) && !MA->use_empty()) { // Reset optimized on users of this store, and reset the uses. @@ -1090,6 +1218,9 @@ void MemorySSAUpdater::removeMemoryAccess(MemoryAccess *MA) { Use &U = *MA->use_begin(); if (auto *MUD = dyn_cast(U.getUser())) MUD->resetOptimized(); + if (OptimizePhis) + if (MemoryPhi *MP = dyn_cast(U.getUser())) + PhisToCheck.insert(MP); U.set(NewDefTarget); } } @@ -1098,10 +1229,25 @@ void MemorySSAUpdater::removeMemoryAccess(MemoryAccess *MA) { // are doing things here MSSA->removeFromLookups(MA); MSSA->removeFromLists(MA); + + // Optionally optimize Phi uses. This will recursively remove trivial phis. + if (!PhisToCheck.empty()) { + SmallVector PhisToOptimize{PhisToCheck.begin(), + PhisToCheck.end()}; + PhisToCheck.clear(); + + unsigned PhisSize = PhisToOptimize.size(); + while (PhisSize-- > 0) + if (MemoryPhi *MP = + cast_or_null(PhisToOptimize.pop_back_val())) { + auto OperRange = MP->operands(); + tryRemoveTrivialPhi(MP, OperRange); + } + } } void MemorySSAUpdater::removeBlocks( - const SmallPtrSetImpl &DeadBlocks) { + const SmallSetVector &DeadBlocks) { // First delete all uses of BB in MemoryPhis. for (BasicBlock *BB : DeadBlocks) { Instruction *TI = BB->getTerminator(); @@ -1133,6 +1279,51 @@ void MemorySSAUpdater::removeBlocks( } } +void MemorySSAUpdater::tryRemoveTrivialPhis(ArrayRef UpdatedPHIs) { + for (auto &VH : UpdatedPHIs) + if (auto *MPhi = cast_or_null(VH)) { + auto OperRange = MPhi->operands(); + tryRemoveTrivialPhi(MPhi, OperRange); + } +} + +void MemorySSAUpdater::changeToUnreachable(const Instruction *I) { + const BasicBlock *BB = I->getParent(); + // Remove memory accesses in BB for I and all following instructions. + auto BBI = I->getIterator(), BBE = BB->end(); + // FIXME: If this becomes too expensive, iterate until the first instruction + // with a memory access, then iterate over MemoryAccesses. + while (BBI != BBE) + removeMemoryAccess(&*(BBI++)); + // Update phis in BB's successors to remove BB. + SmallVector UpdatedPHIs; + for (const BasicBlock *Successor : successors(BB)) { + removeDuplicatePhiEdgesBetween(BB, Successor); + if (MemoryPhi *MPhi = MSSA->getMemoryAccess(Successor)) { + MPhi->unorderedDeleteIncomingBlock(BB); + UpdatedPHIs.push_back(MPhi); + } + } + // Optimize trivial phis. + tryRemoveTrivialPhis(UpdatedPHIs); +} + +void MemorySSAUpdater::changeCondBranchToUnconditionalTo(const BranchInst *BI, + const BasicBlock *To) { + const BasicBlock *BB = BI->getParent(); + SmallVector UpdatedPHIs; + for (const BasicBlock *Succ : successors(BB)) { + removeDuplicatePhiEdgesBetween(BB, Succ); + if (Succ != To) + if (auto *MPhi = MSSA->getMemoryAccess(Succ)) { + MPhi->unorderedDeleteIncomingBlock(BB); + UpdatedPHIs.push_back(MPhi); + } + } + // Optimize trivial phis. + tryRemoveTrivialPhis(UpdatedPHIs); +} + MemoryAccess *MemorySSAUpdater::createMemoryAccessInBB( Instruction *I, MemoryAccess *Definition, const BasicBlock *BB, MemorySSA::InsertionPlace Point) { diff --git a/lib/Analysis/ModuleDebugInfoPrinter.cpp b/lib/Analysis/ModuleDebugInfoPrinter.cpp index 1e321f17d59f..519242759824 100644 --- a/lib/Analysis/ModuleDebugInfoPrinter.cpp +++ b/lib/Analysis/ModuleDebugInfoPrinter.cpp @@ -1,9 +1,8 @@ //===-- ModuleDebugInfoPrinter.cpp - Prints module debug info metadata ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/ModuleSummaryAnalysis.cpp b/lib/Analysis/ModuleSummaryAnalysis.cpp index 87f76d43bb1e..e25eb290a665 100644 --- a/lib/Analysis/ModuleSummaryAnalysis.cpp +++ b/lib/Analysis/ModuleSummaryAnalysis.cpp @@ -1,9 +1,8 @@ //===- ModuleSummaryAnalysis.cpp - Module summary index builder -----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -71,6 +70,11 @@ cl::opt FSEC( "all-non-critical", "All non-critical edges."), clEnumValN(FunctionSummary::FSHT_All, "all", "All edges."))); +cl::opt ModuleSummaryDotFile( + "module-summary-dot-file", cl::init(""), cl::Hidden, + cl::value_desc("filename"), + cl::desc("File to emit dot graph of new summary into.")); + // Walk through the operands of a given User via worklist iteration and populate // the set of GlobalValue references encountered. Invoked either on an // Instruction or a GlobalVariable (which walks its initializer). @@ -227,6 +231,13 @@ static bool isNonVolatileLoad(const Instruction *I) { return false; } +static bool isNonVolatileStore(const Instruction *I) { + if (const auto *SI = dyn_cast(I)) + return !SI->isVolatile(); + + return false; +} + static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M, const Function &F, BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, DominatorTree &DT, @@ -241,7 +252,7 @@ static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M, // Map from callee ValueId to profile count. Used to accumulate profile // counts for all static calls to a given callee. MapVector CallGraphEdges; - SetVector RefEdges; + SetVector RefEdges, LoadRefEdges, StoreRefEdges; SetVector TypeTests; SetVector TypeTestAssumeVCalls, TypeCheckedLoadVCalls; @@ -254,6 +265,7 @@ static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M, // list. findRefEdges(Index, &F, RefEdges, Visited); std::vector NonVolatileLoads; + std::vector NonVolatileStores; bool HasInlineAsmMaybeReferencingInternal = false; for (const BasicBlock &BB : F) @@ -261,12 +273,34 @@ static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M, if (isa(I)) continue; ++NumInsts; - if (isNonVolatileLoad(&I)) { - // Postpone processing of non-volatile load instructions - // See comments below - Visited.insert(&I); - NonVolatileLoads.push_back(&I); - continue; + // Regular LTO module doesn't participate in ThinLTO import, + // so no reference from it can be read/writeonly, since this + // would require importing variable as local copy + if (IsThinLTO) { + if (isNonVolatileLoad(&I)) { + // Postpone processing of non-volatile load instructions + // See comments below + Visited.insert(&I); + NonVolatileLoads.push_back(&I); + continue; + } else if (isNonVolatileStore(&I)) { + Visited.insert(&I); + NonVolatileStores.push_back(&I); + // All references from second operand of store (destination address) + // can be considered write-only if they're not referenced by any + // non-store instruction. References from first operand of store + // (stored value) can't be treated either as read- or as write-only + // so we add them to RefEdges as we do with all other instructions + // except non-volatile load. + Value *Stored = I.getOperand(0); + if (auto *GV = dyn_cast(Stored)) + // findRefEdges will try to examine GV operands, so instead + // of calling it we should add GV to RefEdges directly. + RefEdges.insert(Index.getOrInsertValueInfo(GV)); + else if (auto *U = dyn_cast(Stored)) + findRefEdges(Index, U, RefEdges, Visited); + continue; + } } findRefEdges(Index, &I, RefEdges, Visited); auto CS = ImmutableCallSite(&I); @@ -357,24 +391,61 @@ static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M, } } - // By now we processed all instructions in a function, except - // non-volatile loads. All new refs we add in a loop below - // are obviously constant. All constant refs are grouped in the - // end of RefEdges vector, so we can use a single integer value - // to identify them. - unsigned RefCnt = RefEdges.size(); - for (const Instruction *I : NonVolatileLoads) { - Visited.erase(I); - findRefEdges(Index, I, RefEdges, Visited); - } - std::vector Refs = RefEdges.takeVector(); - // Regular LTO module doesn't participate in ThinLTO import, - // so no reference from it can be readonly, since this would - // require importing variable as local copy - if (IsThinLTO) - for (; RefCnt < Refs.size(); ++RefCnt) + std::vector Refs; + if (IsThinLTO) { + auto AddRefEdges = [&](const std::vector &Instrs, + SetVector &Edges, + SmallPtrSet &Cache) { + for (const auto *I : Instrs) { + Cache.erase(I); + findRefEdges(Index, I, Edges, Cache); + } + }; + + // By now we processed all instructions in a function, except + // non-volatile loads and non-volatile value stores. Let's find + // ref edges for both of instruction sets + AddRefEdges(NonVolatileLoads, LoadRefEdges, Visited); + // We can add some values to the Visited set when processing load + // instructions which are also used by stores in NonVolatileStores. + // For example this can happen if we have following code: + // + // store %Derived* @foo, %Derived** bitcast (%Base** @bar to %Derived**) + // %42 = load %Derived*, %Derived** bitcast (%Base** @bar to %Derived**) + // + // After processing loads we'll add bitcast to the Visited set, and if + // we use the same set while processing stores, we'll never see store + // to @bar and @bar will be mistakenly treated as readonly. + SmallPtrSet StoreCache; + AddRefEdges(NonVolatileStores, StoreRefEdges, StoreCache); + + // If both load and store instruction reference the same variable + // we won't be able to optimize it. Add all such reference edges + // to RefEdges set. + for (auto &VI : StoreRefEdges) + if (LoadRefEdges.remove(VI)) + RefEdges.insert(VI); + + unsigned RefCnt = RefEdges.size(); + // All new reference edges inserted in two loops below are either + // read or write only. They will be grouped in the end of RefEdges + // vector, so we can use a single integer value to identify them. + for (auto &VI : LoadRefEdges) + RefEdges.insert(VI); + + unsigned FirstWORef = RefEdges.size(); + for (auto &VI : StoreRefEdges) + RefEdges.insert(VI); + + Refs = RefEdges.takeVector(); + for (; RefCnt < FirstWORef; ++RefCnt) Refs[RefCnt].setReadOnly(); + for (; RefCnt < Refs.size(); ++RefCnt) + Refs[RefCnt].setWriteOnly(); + } else { + Refs = RefEdges.takeVector(); + } // Explicit add hot edges to enforce importing for designated GUIDs for // sample PGO, to enable the same inlines as the profiled optimized binary. for (auto &I : F.getImportGUIDs()) @@ -387,7 +458,8 @@ static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M, bool NotEligibleForImport = NonRenamableLocal || HasInlineAsmMaybeReferencingInternal; GlobalValueSummary::GVFlags Flags(F.getLinkage(), NotEligibleForImport, - /* Live = */ false, F.isDSOLocal()); + /* Live = */ false, F.isDSOLocal(), + F.hasLinkOnceODRLinkage() && F.hasGlobalUnnamedAddr()); FunctionSummary::FFlags FunFlags{ F.hasFnAttribute(Attribute::ReadNone), F.hasFnAttribute(Attribute::ReadOnly), @@ -406,26 +478,134 @@ static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M, Index.addGlobalValueSummary(F, std::move(FuncSummary)); } +/// Find function pointers referenced within the given vtable initializer +/// (or subset of an initializer) \p I. The starting offset of \p I within +/// the vtable initializer is \p StartingOffset. Any discovered function +/// pointers are added to \p VTableFuncs along with their cumulative offset +/// within the initializer. +static void findFuncPointers(const Constant *I, uint64_t StartingOffset, + const Module &M, ModuleSummaryIndex &Index, + VTableFuncList &VTableFuncs) { + // First check if this is a function pointer. + if (I->getType()->isPointerTy()) { + auto Fn = dyn_cast(I->stripPointerCasts()); + // We can disregard __cxa_pure_virtual as a possible call target, as + // calls to pure virtuals are UB. + if (Fn && Fn->getName() != "__cxa_pure_virtual") + VTableFuncs.push_back({Index.getOrInsertValueInfo(Fn), StartingOffset}); + return; + } + + // Walk through the elements in the constant struct or array and recursively + // look for virtual function pointers. + const DataLayout &DL = M.getDataLayout(); + if (auto *C = dyn_cast(I)) { + StructType *STy = dyn_cast(C->getType()); + assert(STy); + const StructLayout *SL = DL.getStructLayout(C->getType()); + + for (StructType::element_iterator EB = STy->element_begin(), EI = EB, + EE = STy->element_end(); + EI != EE; ++EI) { + auto Offset = SL->getElementOffset(EI - EB); + unsigned Op = SL->getElementContainingOffset(Offset); + findFuncPointers(cast(I->getOperand(Op)), + StartingOffset + Offset, M, Index, VTableFuncs); + } + } else if (auto *C = dyn_cast(I)) { + ArrayType *ATy = C->getType(); + Type *EltTy = ATy->getElementType(); + uint64_t EltSize = DL.getTypeAllocSize(EltTy); + for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) { + findFuncPointers(cast(I->getOperand(i)), + StartingOffset + i * EltSize, M, Index, VTableFuncs); + } + } +} + +// Identify the function pointers referenced by vtable definition \p V. +static void computeVTableFuncs(ModuleSummaryIndex &Index, + const GlobalVariable &V, const Module &M, + VTableFuncList &VTableFuncs) { + if (!V.isConstant()) + return; + + findFuncPointers(V.getInitializer(), /*StartingOffset=*/0, M, Index, + VTableFuncs); + +#ifndef NDEBUG + // Validate that the VTableFuncs list is ordered by offset. + uint64_t PrevOffset = 0; + for (auto &P : VTableFuncs) { + // The findVFuncPointers traversal should have encountered the + // functions in offset order. We need to use ">=" since PrevOffset + // starts at 0. + assert(P.VTableOffset >= PrevOffset); + PrevOffset = P.VTableOffset; + } +#endif +} + +/// Record vtable definition \p V for each type metadata it references. static void -computeVariableSummary(ModuleSummaryIndex &Index, const GlobalVariable &V, - DenseSet &CantBePromoted) { +recordTypeIdCompatibleVtableReferences(ModuleSummaryIndex &Index, + const GlobalVariable &V, + SmallVectorImpl &Types) { + for (MDNode *Type : Types) { + auto TypeID = Type->getOperand(1).get(); + + uint64_t Offset = + cast( + cast(Type->getOperand(0))->getValue()) + ->getZExtValue(); + + if (auto *TypeId = dyn_cast(TypeID)) + Index.getOrInsertTypeIdCompatibleVtableSummary(TypeId->getString()) + .push_back({Offset, Index.getOrInsertValueInfo(&V)}); + } +} + +static void computeVariableSummary(ModuleSummaryIndex &Index, + const GlobalVariable &V, + DenseSet &CantBePromoted, + const Module &M, + SmallVectorImpl &Types) { SetVector RefEdges; SmallPtrSet Visited; bool HasBlockAddress = findRefEdges(Index, &V, RefEdges, Visited); bool NonRenamableLocal = isNonRenamableLocal(V); GlobalValueSummary::GVFlags Flags(V.getLinkage(), NonRenamableLocal, - /* Live = */ false, V.isDSOLocal()); + /* Live = */ false, V.isDSOLocal(), + V.hasLinkOnceODRLinkage() && V.hasGlobalUnnamedAddr()); + + VTableFuncList VTableFuncs; + // If splitting is not enabled, then we compute the summary information + // necessary for index-based whole program devirtualization. + if (!Index.enableSplitLTOUnit()) { + Types.clear(); + V.getMetadata(LLVMContext::MD_type, Types); + if (!Types.empty()) { + // Identify the function pointers referenced by this vtable definition. + computeVTableFuncs(Index, V, M, VTableFuncs); + + // Record this vtable definition for each type metadata it references. + recordTypeIdCompatibleVtableReferences(Index, V, Types); + } + } - // Don't mark variables we won't be able to internalize as read-only. - GlobalVarSummary::GVarFlags VarFlags( + // Don't mark variables we won't be able to internalize as read/write-only. + bool CanBeInternalized = !V.hasComdat() && !V.hasAppendingLinkage() && !V.isInterposable() && - !V.hasAvailableExternallyLinkage() && !V.hasDLLExportStorageClass()); + !V.hasAvailableExternallyLinkage() && !V.hasDLLExportStorageClass(); + GlobalVarSummary::GVarFlags VarFlags(CanBeInternalized, CanBeInternalized); auto GVarSummary = llvm::make_unique(Flags, VarFlags, RefEdges.takeVector()); if (NonRenamableLocal) CantBePromoted.insert(V.getGUID()); if (HasBlockAddress) GVarSummary->setNotEligibleToImport(); + if (!VTableFuncs.empty()) + GVarSummary->setVTableFuncs(VTableFuncs); Index.addGlobalValueSummary(V, std::move(GVarSummary)); } @@ -434,12 +614,15 @@ computeAliasSummary(ModuleSummaryIndex &Index, const GlobalAlias &A, DenseSet &CantBePromoted) { bool NonRenamableLocal = isNonRenamableLocal(A); GlobalValueSummary::GVFlags Flags(A.getLinkage(), NonRenamableLocal, - /* Live = */ false, A.isDSOLocal()); + /* Live = */ false, A.isDSOLocal(), + A.hasLinkOnceODRLinkage() && A.hasGlobalUnnamedAddr()); auto AS = llvm::make_unique(Flags); auto *Aliasee = A.getBaseObject(); - auto *AliaseeSummary = Index.getGlobalValueSummary(*Aliasee); - assert(AliaseeSummary && "Alias expects aliasee summary to be parsed"); - AS->setAliasee(AliaseeSummary); + auto AliaseeVI = Index.getValueInfo(Aliasee->getGUID()); + assert(AliaseeVI && "Alias expects aliasee summary to be available"); + assert(AliaseeVI.getSummaryList().size() == 1 && + "Expected a single entry per aliasee in per-module index"); + AS->setAliasee(AliaseeVI, AliaseeVI.getSummaryList()[0].get()); if (NonRenamableLocal) CantBePromoted.insert(A.getGUID()); Index.addGlobalValueSummary(A, std::move(AS)); @@ -507,7 +690,8 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex( GlobalValueSummary::GVFlags GVFlags(GlobalValue::InternalLinkage, /* NotEligibleToImport = */ true, /* Live = */ true, - /* Local */ GV->isDSOLocal()); + /* Local */ GV->isDSOLocal(), + GV->hasLinkOnceODRLinkage() && GV->hasGlobalUnnamedAddr()); CantBePromoted.insert(GV->getGUID()); // Create the appropriate summary type. if (Function *F = dyn_cast(GV)) { @@ -531,7 +715,7 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex( } else { std::unique_ptr Summary = llvm::make_unique( - GVFlags, GlobalVarSummary::GVarFlags(), + GVFlags, GlobalVarSummary::GVarFlags(false, false), ArrayRef{}); Index.addGlobalValueSummary(*GV, std::move(Summary)); } @@ -568,10 +752,11 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex( // Compute summaries for all variables defined in module, and save in the // index. + SmallVector Types; for (const GlobalVariable &G : M.globals()) { if (G.isDeclaration()) continue; - computeVariableSummary(Index, G, CantBePromoted); + computeVariableSummary(Index, G, CantBePromoted, M, Types); } // Compute summaries for all aliases defined in module, and save in the @@ -626,6 +811,15 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex( } } + if (!ModuleSummaryDotFile.empty()) { + std::error_code EC; + raw_fd_ostream OSDot(ModuleSummaryDotFile, EC, sys::fs::OpenFlags::F_None); + if (EC) + report_fatal_error(Twine("Failed to open dot file ") + + ModuleSummaryDotFile + ": " + EC.message() + "\n"); + Index.exportToDot(OSDot); + } + return Index; } diff --git a/lib/Analysis/MustExecute.cpp b/lib/Analysis/MustExecute.cpp index 180c38ddacc2..b616cd6f762b 100644 --- a/lib/Analysis/MustExecute.cpp +++ b/lib/Analysis/MustExecute.cpp @@ -1,9 +1,8 @@ //===- MustExecute.cpp - Printer for isGuaranteedToExecute ----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -194,7 +193,8 @@ bool LoopSafetyInfo::allLoopPathsLeadToBlock(const Loop *CurLoop, SmallPtrSet Predecessors; collectTransitivePredecessors(CurLoop, BB, Predecessors); - // Make sure that all successors of all predecessors of BB are either: + // Make sure that all successors of, all predecessors of BB which are not + // dominated by BB, are either: // 1) BB, // 2) Also predecessors of BB, // 3) Exit blocks which are not taken on 1st iteration. @@ -204,6 +204,12 @@ bool LoopSafetyInfo::allLoopPathsLeadToBlock(const Loop *CurLoop, // Predecessor block may throw, so it has a side exit. if (blockMayThrow(Pred)) return false; + + // BB dominates Pred, so if Pred runs, BB must run. + // This is true when Pred is a loop latch. + if (DT->dominates(BB, Pred)) + continue; + for (auto *Succ : successors(Pred)) if (CheckedSuccessors.insert(Succ).second && Succ != BB && !Predecessors.count(Succ)) diff --git a/lib/Analysis/ObjCARCAliasAnalysis.cpp b/lib/Analysis/ObjCARCAliasAnalysis.cpp index 95ae1a6e744f..811033e73147 100644 --- a/lib/Analysis/ObjCARCAliasAnalysis.cpp +++ b/lib/Analysis/ObjCARCAliasAnalysis.cpp @@ -1,9 +1,8 @@ //===- ObjCARCAliasAnalysis.cpp - ObjC ARC Optimization -------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -38,9 +37,10 @@ using namespace llvm; using namespace llvm::objcarc; AliasResult ObjCARCAAResult::alias(const MemoryLocation &LocA, - const MemoryLocation &LocB) { + const MemoryLocation &LocB, + AAQueryInfo &AAQI) { if (!EnableARCOpts) - return AAResultBase::alias(LocA, LocB); + return AAResultBase::alias(LocA, LocB, AAQI); // First, strip off no-ops, including ObjC-specific no-ops, and try making a // precise alias query. @@ -48,7 +48,7 @@ AliasResult ObjCARCAAResult::alias(const MemoryLocation &LocA, const Value *SB = GetRCIdentityRoot(LocB.Ptr); AliasResult Result = AAResultBase::alias(MemoryLocation(SA, LocA.Size, LocA.AATags), - MemoryLocation(SB, LocB.Size, LocB.AATags)); + MemoryLocation(SB, LocB.Size, LocB.AATags), AAQI); if (Result != MayAlias) return Result; @@ -57,7 +57,7 @@ AliasResult ObjCARCAAResult::alias(const MemoryLocation &LocA, const Value *UA = GetUnderlyingObjCPtr(SA, DL); const Value *UB = GetUnderlyingObjCPtr(SB, DL); if (UA != SA || UB != SB) { - Result = AAResultBase::alias(MemoryLocation(UA), MemoryLocation(UB)); + Result = AAResultBase::alias(MemoryLocation(UA), MemoryLocation(UB), AAQI); // We can't use MustAlias or PartialAlias results here because // GetUnderlyingObjCPtr may return an offsetted pointer value. if (Result == NoAlias) @@ -70,22 +70,23 @@ AliasResult ObjCARCAAResult::alias(const MemoryLocation &LocA, } bool ObjCARCAAResult::pointsToConstantMemory(const MemoryLocation &Loc, - bool OrLocal) { + AAQueryInfo &AAQI, bool OrLocal) { if (!EnableARCOpts) - return AAResultBase::pointsToConstantMemory(Loc, OrLocal); + return AAResultBase::pointsToConstantMemory(Loc, AAQI, OrLocal); // First, strip off no-ops, including ObjC-specific no-ops, and try making // a precise alias query. const Value *S = GetRCIdentityRoot(Loc.Ptr); if (AAResultBase::pointsToConstantMemory( - MemoryLocation(S, Loc.Size, Loc.AATags), OrLocal)) + MemoryLocation(S, Loc.Size, Loc.AATags), AAQI, OrLocal)) return true; // If that failed, climb to the underlying object, including climbing through // ObjC-specific no-ops, and try making an imprecise alias query. const Value *U = GetUnderlyingObjCPtr(S, DL); if (U != S) - return AAResultBase::pointsToConstantMemory(MemoryLocation(U), OrLocal); + return AAResultBase::pointsToConstantMemory(MemoryLocation(U), AAQI, + OrLocal); // If that failed, fail. We don't need to chain here, since that's covered // by the earlier precise query. @@ -107,9 +108,10 @@ FunctionModRefBehavior ObjCARCAAResult::getModRefBehavior(const Function *F) { } ModRefInfo ObjCARCAAResult::getModRefInfo(const CallBase *Call, - const MemoryLocation &Loc) { + const MemoryLocation &Loc, + AAQueryInfo &AAQI) { if (!EnableARCOpts) - return AAResultBase::getModRefInfo(Call, Loc); + return AAResultBase::getModRefInfo(Call, Loc, AAQI); switch (GetBasicARCInstKind(Call)) { case ARCInstKind::Retain: @@ -128,7 +130,7 @@ ModRefInfo ObjCARCAAResult::getModRefInfo(const CallBase *Call, break; } - return AAResultBase::getModRefInfo(Call, Loc); + return AAResultBase::getModRefInfo(Call, Loc, AAQI); } ObjCARCAAResult ObjCARCAA::run(Function &F, FunctionAnalysisManager &AM) { diff --git a/lib/Analysis/ObjCARCAnalysisUtils.cpp b/lib/Analysis/ObjCARCAnalysisUtils.cpp index d6db6386c38b..56d1cb421225 100644 --- a/lib/Analysis/ObjCARCAnalysisUtils.cpp +++ b/lib/Analysis/ObjCARCAnalysisUtils.cpp @@ -1,9 +1,8 @@ //===- ObjCARCAnalysisUtils.cpp -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/ObjCARCInstKind.cpp b/lib/Analysis/ObjCARCInstKind.cpp index 31c432711834..0e96c6e975c9 100644 --- a/lib/Analysis/ObjCARCInstKind.cpp +++ b/lib/Analysis/ObjCARCInstKind.cpp @@ -1,9 +1,8 @@ //===- ARCInstKind.cpp - ObjC ARC Optimization ----------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -482,6 +481,41 @@ bool llvm::objcarc::IsNoopOnNull(ARCInstKind Class) { llvm_unreachable("covered switch isn't covered?"); } +/// Test if the given class represents instructions which do nothing if +/// passed a global variable. +bool llvm::objcarc::IsNoopOnGlobal(ARCInstKind Class) { + switch (Class) { + case ARCInstKind::Retain: + case ARCInstKind::RetainRV: + case ARCInstKind::ClaimRV: + case ARCInstKind::Release: + case ARCInstKind::Autorelease: + case ARCInstKind::AutoreleaseRV: + case ARCInstKind::RetainBlock: + case ARCInstKind::FusedRetainAutorelease: + case ARCInstKind::FusedRetainAutoreleaseRV: + return true; + case ARCInstKind::AutoreleasepoolPush: + case ARCInstKind::AutoreleasepoolPop: + case ARCInstKind::LoadWeakRetained: + case ARCInstKind::StoreWeak: + case ARCInstKind::InitWeak: + case ARCInstKind::LoadWeak: + case ARCInstKind::MoveWeak: + case ARCInstKind::CopyWeak: + case ARCInstKind::DestroyWeak: + case ARCInstKind::StoreStrong: + case ARCInstKind::IntrinsicUser: + case ARCInstKind::CallOrUser: + case ARCInstKind::Call: + case ARCInstKind::User: + case ARCInstKind::None: + case ARCInstKind::NoopCast: + return false; + } + llvm_unreachable("covered switch isn't covered?"); +} + /// Test if the given class represents instructions which are always safe /// to mark with the "tail" keyword. bool llvm::objcarc::IsAlwaysTail(ARCInstKind Class) { diff --git a/lib/Analysis/OptimizationRemarkEmitter.cpp b/lib/Analysis/OptimizationRemarkEmitter.cpp index 8ece0a2a3ed3..72c40a0be232 100644 --- a/lib/Analysis/OptimizationRemarkEmitter.cpp +++ b/lib/Analysis/OptimizationRemarkEmitter.cpp @@ -1,9 +1,8 @@ //===- OptimizationRemarkEmitter.cpp - Optimization Diagnostic --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/OrderedBasicBlock.cpp b/lib/Analysis/OrderedBasicBlock.cpp index 5f4fe0f7dda2..48f2a4020c66 100644 --- a/lib/Analysis/OrderedBasicBlock.cpp +++ b/lib/Analysis/OrderedBasicBlock.cpp @@ -1,9 +1,8 @@ //===- OrderedBasicBlock.cpp --------------------------------- -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -86,3 +85,27 @@ bool OrderedBasicBlock::dominates(const Instruction *A, const Instruction *B) { return comesBefore(A, B); } + +void OrderedBasicBlock::eraseInstruction(const Instruction *I) { + if (LastInstFound != BB->end() && I == &*LastInstFound) { + if (LastInstFound == BB->begin()) { + LastInstFound = BB->end(); + NextInstPos = 0; + } else + LastInstFound--; + } + + NumberedInsts.erase(I); +} + +void OrderedBasicBlock::replaceInstruction(const Instruction *Old, + const Instruction *New) { + auto OI = NumberedInsts.find(Old); + if (OI == NumberedInsts.end()) + return; + + NumberedInsts.insert({New, OI->second}); + if (LastInstFound != BB->end() && Old == &*LastInstFound) + LastInstFound = New->getIterator(); + NumberedInsts.erase(Old); +} diff --git a/lib/Analysis/OrderedInstructions.cpp b/lib/Analysis/OrderedInstructions.cpp index 7b155208c02e..458c0a7de6c2 100644 --- a/lib/Analysis/OrderedInstructions.cpp +++ b/lib/Analysis/OrderedInstructions.cpp @@ -1,9 +1,8 @@ //===-- OrderedInstructions.cpp - Instruction dominance function ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/PHITransAddr.cpp b/lib/Analysis/PHITransAddr.cpp index 858f08f6537a..7f77ab146c4c 100644 --- a/lib/Analysis/PHITransAddr.cpp +++ b/lib/Analysis/PHITransAddr.cpp @@ -1,9 +1,8 @@ //===- PHITransAddr.cpp - PHI Translation for Addresses -------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/PhiValues.cpp b/lib/Analysis/PhiValues.cpp index 729227c86697..49749bc44746 100644 --- a/lib/Analysis/PhiValues.cpp +++ b/lib/Analysis/PhiValues.cpp @@ -1,9 +1,8 @@ //===- PhiValues.cpp - Phi Value Analysis ---------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/Analysis/PostDominators.cpp b/lib/Analysis/PostDominators.cpp index e6b660fe26d7..4afe22bd5342 100644 --- a/lib/Analysis/PostDominators.cpp +++ b/lib/Analysis/PostDominators.cpp @@ -1,9 +1,8 @@ //===- PostDominators.cpp - Post-Dominator Calculation --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/ProfileSummaryInfo.cpp b/lib/Analysis/ProfileSummaryInfo.cpp index 1d70c75f2e1c..dce19d6d546e 100644 --- a/lib/Analysis/ProfileSummaryInfo.cpp +++ b/lib/Analysis/ProfileSummaryInfo.cpp @@ -1,9 +1,8 @@ //===- ProfileSummaryInfo.cpp - Global profile summary information --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -61,10 +60,9 @@ static cl::opt ProfileSummaryColdCount( // Find the summary entry for a desired percentile of counts. static const ProfileSummaryEntry &getEntryForPercentile(SummaryEntryVector &DS, uint64_t Percentile) { - auto Compare = [](const ProfileSummaryEntry &Entry, uint64_t Percentile) { + auto It = partition_point(DS, [=](const ProfileSummaryEntry &Entry) { return Entry.Cutoff < Percentile; - }; - auto It = std::lower_bound(DS.begin(), DS.end(), Percentile, Compare); + }); // The required percentile has to be <= one of the percentiles in the // detailed summary. if (It == DS.end()) @@ -80,7 +78,14 @@ static const ProfileSummaryEntry &getEntryForPercentile(SummaryEntryVector &DS, bool ProfileSummaryInfo::computeSummary() { if (Summary) return true; - auto *SummaryMD = M.getProfileSummary(); + // First try to get context sensitive ProfileSummary. + auto *SummaryMD = M.getProfileSummary(/* IsCS */ true); + if (SummaryMD) { + Summary.reset(ProfileSummary::getFromMD(SummaryMD)); + return true; + } + // This will actually return PSK_Instr or PSK_Sample summary. + SummaryMD = M.getProfileSummary(/* IsCS */ false); if (!SummaryMD) return false; Summary.reset(ProfileSummary::getFromMD(SummaryMD)); @@ -89,7 +94,8 @@ bool ProfileSummaryInfo::computeSummary() { Optional ProfileSummaryInfo::getProfileCount(const Instruction *Inst, - BlockFrequencyInfo *BFI) { + BlockFrequencyInfo *BFI, + bool AllowSynthetic) { if (!Inst) return None; assert((isa(Inst) || isa(Inst)) && @@ -105,7 +111,7 @@ ProfileSummaryInfo::getProfileCount(const Instruction *Inst, return None; } if (BFI) - return BFI->getBlockProfileCount(Inst->getParent()); + return BFI->getBlockProfileCount(Inst->getParent(), AllowSynthetic); return None; } diff --git a/lib/Analysis/PtrUseVisitor.cpp b/lib/Analysis/PtrUseVisitor.cpp index 1fdaf4d55b59..9a834ba4866a 100644 --- a/lib/Analysis/PtrUseVisitor.cpp +++ b/lib/Analysis/PtrUseVisitor.cpp @@ -1,9 +1,8 @@ //===- PtrUseVisitor.cpp - InstVisitors over a pointers uses --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -35,5 +34,11 @@ bool detail::PtrUseVisitorBase::adjustOffsetForGEP(GetElementPtrInst &GEPI) { if (!IsOffsetKnown) return false; - return GEPI.accumulateConstantOffset(DL, Offset); + APInt TmpOffset(DL.getIndexTypeSizeInBits(GEPI.getType()), 0); + if (GEPI.accumulateConstantOffset(DL, TmpOffset)) { + Offset += TmpOffset.sextOrTrunc(Offset.getBitWidth()); + return true; + } + + return false; } diff --git a/lib/Analysis/RegionInfo.cpp b/lib/Analysis/RegionInfo.cpp index 2bd611350f46..8ba38adfb0d2 100644 --- a/lib/Analysis/RegionInfo.cpp +++ b/lib/Analysis/RegionInfo.cpp @@ -1,9 +1,8 @@ //===- RegionInfo.cpp - SESE region detection analysis --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Detects single entry single exit regions in the control flow graph. diff --git a/lib/Analysis/RegionPass.cpp b/lib/Analysis/RegionPass.cpp index a101ff109199..6c0d17b45c62 100644 --- a/lib/Analysis/RegionPass.cpp +++ b/lib/Analysis/RegionPass.cpp @@ -1,9 +1,8 @@ //===- RegionPass.cpp - Region Pass and Region Pass Manager ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -279,12 +278,17 @@ Pass *RegionPass::createPrinterPass(raw_ostream &O, return new PrintRegionPass(Banner, O); } +static std::string getDescription(const Region &R) { + return "region"; +} + bool RegionPass::skipRegion(Region &R) const { Function &F = *R.getEntry()->getParent(); - if (!F.getContext().getOptPassGate().shouldRunPass(this, R)) + OptPassGate &Gate = F.getContext().getOptPassGate(); + if (Gate.isEnabled() && !Gate.shouldRunPass(this, getDescription(R))) return true; - if (F.hasFnAttribute(Attribute::OptimizeNone)) { + if (F.hasOptNone()) { // Report this only once per function. if (R.getEntry() == &F.getEntryBlock()) LLVM_DEBUG(dbgs() << "Skipping pass '" << getPassName() diff --git a/lib/Analysis/RegionPrinter.cpp b/lib/Analysis/RegionPrinter.cpp index 5986b8c4e0c3..5bdcb31fbe99 100644 --- a/lib/Analysis/RegionPrinter.cpp +++ b/lib/Analysis/RegionPrinter.cpp @@ -1,9 +1,8 @@ //===- RegionPrinter.cpp - Print regions tree pass ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Print out the region tree of a function using dotty/graphviz. diff --git a/lib/Analysis/ScalarEvolution.cpp b/lib/Analysis/ScalarEvolution.cpp index e5134f2eeda9..bc2cfd6fcc42 100644 --- a/lib/Analysis/ScalarEvolution.cpp +++ b/lib/Analysis/ScalarEvolution.cpp @@ -1,9 +1,8 @@ //===- ScalarEvolution.cpp - Scalar Evolution Analysis --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -203,15 +202,20 @@ static cl::opt MaxConstantEvolvingDepth( cl::desc("Maximum depth of recursive constant evolving"), cl::init(32)); static cl::opt - MaxExtDepth("scalar-evolution-max-ext-depth", cl::Hidden, - cl::desc("Maximum depth of recursive SExt/ZExt"), - cl::init(8)); + MaxCastDepth("scalar-evolution-max-cast-depth", cl::Hidden, + cl::desc("Maximum depth of recursive SExt/ZExt/Trunc"), + cl::init(8)); static cl::opt MaxAddRecSize("scalar-evolution-max-add-rec-size", cl::Hidden, cl::desc("Max coefficients in AddRec during evolving"), cl::init(8)); +static cl::opt + HugeExprThreshold("scalar-evolution-huge-expr-threshold", cl::Hidden, + cl::desc("Size of the expression which is considered huge"), + cl::init(4096)); + //===----------------------------------------------------------------------===// // SCEV class definitions //===----------------------------------------------------------------------===// @@ -273,7 +277,9 @@ void SCEV::print(raw_ostream &OS) const { case scAddExpr: case scMulExpr: case scUMaxExpr: - case scSMaxExpr: { + case scSMaxExpr: + case scUMinExpr: + case scSMinExpr: { const SCEVNAryExpr *NAry = cast(this); const char *OpStr = nullptr; switch (NAry->getSCEVType()) { @@ -281,6 +287,12 @@ void SCEV::print(raw_ostream &OS) const { case scMulExpr: OpStr = " * "; break; case scUMaxExpr: OpStr = " umax "; break; case scSMaxExpr: OpStr = " smax "; break; + case scUMinExpr: + OpStr = " umin "; + break; + case scSMinExpr: + OpStr = " smin "; + break; } OS << "("; for (SCEVNAryExpr::op_iterator I = NAry->op_begin(), E = NAry->op_end(); @@ -349,6 +361,8 @@ Type *SCEV::getType() const { case scMulExpr: case scUMaxExpr: case scSMaxExpr: + case scUMinExpr: + case scSMinExpr: return cast(this)->getType(); case scAddExpr: return cast(this)->getType(); @@ -393,7 +407,7 @@ bool SCEV::isNonConstantNegative() const { } SCEVCouldNotCompute::SCEVCouldNotCompute() : - SCEV(FoldingSetNodeIDRef(), scCouldNotCompute) {} + SCEV(FoldingSetNodeIDRef(), scCouldNotCompute, 0) {} bool SCEVCouldNotCompute::classof(const SCEV *S) { return S->getSCEVType() == scCouldNotCompute; @@ -422,7 +436,7 @@ ScalarEvolution::getConstant(Type *Ty, uint64_t V, bool isSigned) { SCEVCastExpr::SCEVCastExpr(const FoldingSetNodeIDRef ID, unsigned SCEVTy, const SCEV *op, Type *ty) - : SCEV(ID, SCEVTy), Op(op), Ty(ty) {} + : SCEV(ID, SCEVTy, computeExpressionSize(op)), Op(op), Ty(ty) {} SCEVTruncateExpr::SCEVTruncateExpr(const FoldingSetNodeIDRef ID, const SCEV *op, Type *ty) @@ -713,7 +727,9 @@ static int CompareSCEVComplexity( case scAddExpr: case scMulExpr: case scSMaxExpr: - case scUMaxExpr: { + case scUMaxExpr: + case scSMinExpr: + case scUMinExpr: { const SCEVNAryExpr *LC = cast(LHS); const SCEVNAryExpr *RC = cast(RHS); @@ -795,11 +811,10 @@ static void GroupByComplexity(SmallVectorImpl &Ops, } // Do the rough sort by complexity. - std::stable_sort(Ops.begin(), Ops.end(), - [&](const SCEV *LHS, const SCEV *RHS) { - return CompareSCEVComplexity(EqCacheSCEV, EqCacheValue, LI, - LHS, RHS, DT) < 0; - }); + llvm::stable_sort(Ops, [&](const SCEV *LHS, const SCEV *RHS) { + return CompareSCEVComplexity(EqCacheSCEV, EqCacheValue, LI, LHS, RHS, DT) < + 0; + }); // Now that we are sorted by complexity, group elements of the same // complexity. Note that this is, at worst, N^2, but the vector is likely to @@ -846,6 +861,17 @@ static inline int sizeOfSCEV(const SCEV *S) { return F.Size; } +/// Returns true if the subtree of \p S contains at least HugeExprThreshold +/// nodes. +static bool isHugeExpression(const SCEV *S) { + return S->getExpressionSize() >= HugeExprThreshold; +} + +/// Returns true of \p Ops contains a huge SCEV (see definition above). +static bool hasHugeExpression(ArrayRef Ops) { + return any_of(Ops, isHugeExpression); +} + namespace { struct SCEVDivision : public SCEVVisitor { @@ -913,6 +939,8 @@ public: void visitUDivExpr(const SCEVUDivExpr *Numerator) {} void visitSMaxExpr(const SCEVSMaxExpr *Numerator) {} void visitUMaxExpr(const SCEVUMaxExpr *Numerator) {} + void visitSMinExpr(const SCEVSMinExpr *Numerator) {} + void visitUMinExpr(const SCEVUMinExpr *Numerator) {} void visitUnknown(const SCEVUnknown *Numerator) {} void visitCouldNotCompute(const SCEVCouldNotCompute *Numerator) {} @@ -1219,8 +1247,8 @@ const SCEV *SCEVAddRecExpr::evaluateAtIteration(const SCEV *It, // SCEV Expression folder implementations //===----------------------------------------------------------------------===// -const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op, - Type *Ty) { +const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op, Type *Ty, + unsigned Depth) { assert(getTypeSizeInBits(Op->getType()) > getTypeSizeInBits(Ty) && "This is not a truncating conversion!"); assert(isSCEVable(Ty) && @@ -1241,15 +1269,23 @@ const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op, // trunc(trunc(x)) --> trunc(x) if (const SCEVTruncateExpr *ST = dyn_cast(Op)) - return getTruncateExpr(ST->getOperand(), Ty); + return getTruncateExpr(ST->getOperand(), Ty, Depth + 1); // trunc(sext(x)) --> sext(x) if widening or trunc(x) if narrowing if (const SCEVSignExtendExpr *SS = dyn_cast(Op)) - return getTruncateOrSignExtend(SS->getOperand(), Ty); + return getTruncateOrSignExtend(SS->getOperand(), Ty, Depth + 1); // trunc(zext(x)) --> zext(x) if widening or trunc(x) if narrowing if (const SCEVZeroExtendExpr *SZ = dyn_cast(Op)) - return getTruncateOrZeroExtend(SZ->getOperand(), Ty); + return getTruncateOrZeroExtend(SZ->getOperand(), Ty, Depth + 1); + + if (Depth > MaxCastDepth) { + SCEV *S = + new (SCEVAllocator) SCEVTruncateExpr(ID.Intern(SCEVAllocator), Op, Ty); + UniqueSCEVs.InsertNode(S, IP); + addToLoopUseLists(S); + return S; + } // trunc(x1 + ... + xN) --> trunc(x1) + ... + trunc(xN) and // trunc(x1 * ... * xN) --> trunc(x1) * ... * trunc(xN), @@ -1261,7 +1297,7 @@ const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op, unsigned numTruncs = 0; for (unsigned i = 0, e = CommOp->getNumOperands(); i != e && numTruncs < 2; ++i) { - const SCEV *S = getTruncateExpr(CommOp->getOperand(i), Ty); + const SCEV *S = getTruncateExpr(CommOp->getOperand(i), Ty, Depth + 1); if (!isa(CommOp->getOperand(i)) && isa(S)) numTruncs++; Operands.push_back(S); @@ -1285,7 +1321,7 @@ const SCEV *ScalarEvolution::getTruncateExpr(const SCEV *Op, if (const SCEVAddRecExpr *AddRec = dyn_cast(Op)) { SmallVector Operands; for (const SCEV *Op : AddRec->operands()) - Operands.push_back(getTruncateExpr(Op, Ty)); + Operands.push_back(getTruncateExpr(Op, Ty, Depth + 1)); return getAddRecExpr(Operands, AddRec->getLoop(), SCEV::FlagAnyWrap); } @@ -1619,7 +1655,7 @@ ScalarEvolution::getZeroExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth) { ID.AddPointer(Ty); void *IP = nullptr; if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; - if (Depth > MaxExtDepth) { + if (Depth > MaxCastDepth) { SCEV *S = new (SCEVAllocator) SCEVZeroExtendExpr(ID.Intern(SCEVAllocator), Op, Ty); UniqueSCEVs.InsertNode(S, IP); @@ -1637,7 +1673,7 @@ ScalarEvolution::getZeroExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth) { unsigned NewBits = getTypeSizeInBits(Ty); if (CR.truncate(TruncBits).zeroExtend(NewBits).contains( CR.zextOrTrunc(NewBits))) - return getTruncateOrZeroExtend(X, Ty); + return getTruncateOrZeroExtend(X, Ty, Depth); } // If the input value is a chrec scev, and we can prove that the value @@ -1679,9 +1715,9 @@ ScalarEvolution::getZeroExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth) { // Check whether the backedge-taken count can be losslessly casted to // the addrec's type. The count is always unsigned. const SCEV *CastedMaxBECount = - getTruncateOrZeroExtend(MaxBECount, Start->getType()); - const SCEV *RecastedMaxBECount = - getTruncateOrZeroExtend(CastedMaxBECount, MaxBECount->getType()); + getTruncateOrZeroExtend(MaxBECount, Start->getType(), Depth); + const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend( + CastedMaxBECount, MaxBECount->getType(), Depth); if (MaxBECount == RecastedMaxBECount) { Type *WideTy = IntegerType::get(getContext(), BitWidth * 2); // Check whether Start+Step*MaxBECount has no unsigned overflow. @@ -1930,7 +1966,7 @@ ScalarEvolution::getSignExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth) { void *IP = nullptr; if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; // Limit recursion depth. - if (Depth > MaxExtDepth) { + if (Depth > MaxCastDepth) { SCEV *S = new (SCEVAllocator) SCEVSignExtendExpr(ID.Intern(SCEVAllocator), Op, Ty); UniqueSCEVs.InsertNode(S, IP); @@ -1948,7 +1984,7 @@ ScalarEvolution::getSignExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth) { unsigned NewBits = getTypeSizeInBits(Ty); if (CR.truncate(TruncBits).signExtend(NewBits).contains( CR.sextOrTrunc(NewBits))) - return getTruncateOrSignExtend(X, Ty); + return getTruncateOrSignExtend(X, Ty, Depth); } if (auto *SA = dyn_cast(Op)) { @@ -2023,9 +2059,9 @@ ScalarEvolution::getSignExtendExpr(const SCEV *Op, Type *Ty, unsigned Depth) { // Check whether the backedge-taken count can be losslessly casted to // the addrec's type. The count is always unsigned. const SCEV *CastedMaxBECount = - getTruncateOrZeroExtend(MaxBECount, Start->getType()); - const SCEV *RecastedMaxBECount = - getTruncateOrZeroExtend(CastedMaxBECount, MaxBECount->getType()); + getTruncateOrZeroExtend(MaxBECount, Start->getType(), Depth); + const SCEV *RecastedMaxBECount = getTruncateOrZeroExtend( + CastedMaxBECount, MaxBECount->getType(), Depth); if (MaxBECount == RecastedMaxBECount) { Type *WideTy = IntegerType::get(getContext(), BitWidth * 2); // Check whether Start+Step*MaxBECount has no signed overflow. @@ -2295,7 +2331,7 @@ CollectAddOperandsWithScales(DenseMap &M, // can't-overflow flags for the operation if possible. static SCEV::NoWrapFlags StrengthenNoWrapFlags(ScalarEvolution *SE, SCEVTypes Type, - const SmallVectorImpl &Ops, + const ArrayRef Ops, SCEV::NoWrapFlags Flags) { using namespace std::placeholders; @@ -2405,7 +2441,7 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl &Ops, } // Limit recursion calls depth. - if (Depth > MaxArithDepth) + if (Depth > MaxArithDepth || hasHugeExpression(Ops)) return getOrCreateAddExpr(Ops, Flags); // Okay, check to see if the same value occurs in the operand list more than @@ -2743,7 +2779,7 @@ const SCEV *ScalarEvolution::getAddExpr(SmallVectorImpl &Ops, } const SCEV * -ScalarEvolution::getOrCreateAddExpr(SmallVectorImpl &Ops, +ScalarEvolution::getOrCreateAddExpr(ArrayRef Ops, SCEV::NoWrapFlags Flags) { FoldingSetNodeID ID; ID.AddInteger(scAddExpr); @@ -2765,7 +2801,7 @@ ScalarEvolution::getOrCreateAddExpr(SmallVectorImpl &Ops, } const SCEV * -ScalarEvolution::getOrCreateAddRecExpr(SmallVectorImpl &Ops, +ScalarEvolution::getOrCreateAddRecExpr(ArrayRef Ops, const Loop *L, SCEV::NoWrapFlags Flags) { FoldingSetNodeID ID; ID.AddInteger(scAddRecExpr); @@ -2788,7 +2824,7 @@ ScalarEvolution::getOrCreateAddRecExpr(SmallVectorImpl &Ops, } const SCEV * -ScalarEvolution::getOrCreateMulExpr(SmallVectorImpl &Ops, +ScalarEvolution::getOrCreateMulExpr(ArrayRef Ops, SCEV::NoWrapFlags Flags) { FoldingSetNodeID ID; ID.AddInteger(scMulExpr); @@ -2884,7 +2920,7 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl &Ops, Flags = StrengthenNoWrapFlags(this, scMulExpr, Ops, Flags); // Limit recursion calls depth. - if (Depth > MaxArithDepth) + if (Depth > MaxArithDepth || hasHugeExpression(Ops)) return getOrCreateMulExpr(Ops, Flags); // If there are any constants, fold them together. @@ -3057,7 +3093,8 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl &Ops, // Limit max number of arguments to avoid creation of unreasonably big // SCEVAddRecs with very complex operands. if (AddRec->getNumOperands() + OtherAddRec->getNumOperands() - 1 > - MaxAddRecSize) + MaxAddRecSize || isHugeExpression(AddRec) || + isHugeExpression(OtherAddRec)) continue; bool Overflow = false; @@ -3090,7 +3127,7 @@ const SCEV *ScalarEvolution::getMulExpr(SmallVectorImpl &Ops, AddRecOps.push_back(getAddExpr(SumOps, SCEV::FlagAnyWrap, Depth + 1)); } if (!Overflow) { - const SCEV *NewAddRec = getAddRecExpr(AddRecOps, AddRec->getLoop(), + const SCEV *NewAddRec = getAddRecExpr(AddRecOps, AddRecLoop, SCEV::FlagAnyWrap); if (Ops.size() == 2) return NewAddRec; Ops[Idx] = NewAddRec; @@ -3493,209 +3530,166 @@ ScalarEvolution::getGEPExpr(GEPOperator *GEP, return getAddExpr(BaseExpr, TotalOffset, Wrap); } -const SCEV *ScalarEvolution::getSMaxExpr(const SCEV *LHS, - const SCEV *RHS) { - SmallVector Ops = {LHS, RHS}; - return getSMaxExpr(Ops); +std::tuple +ScalarEvolution::findExistingSCEVInCache(int SCEVType, + ArrayRef Ops) { + FoldingSetNodeID ID; + void *IP = nullptr; + ID.AddInteger(SCEVType); + for (unsigned i = 0, e = Ops.size(); i != e; ++i) + ID.AddPointer(Ops[i]); + return std::tuple( + UniqueSCEVs.FindNodeOrInsertPos(ID, IP), std::move(ID), IP); } -const SCEV * -ScalarEvolution::getSMaxExpr(SmallVectorImpl &Ops) { - assert(!Ops.empty() && "Cannot get empty smax!"); +const SCEV *ScalarEvolution::getMinMaxExpr(unsigned Kind, + SmallVectorImpl &Ops) { + assert(!Ops.empty() && "Cannot get empty (u|s)(min|max)!"); if (Ops.size() == 1) return Ops[0]; #ifndef NDEBUG Type *ETy = getEffectiveSCEVType(Ops[0]->getType()); for (unsigned i = 1, e = Ops.size(); i != e; ++i) assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy && - "SCEVSMaxExpr operand types don't match!"); + "Operand types don't match!"); #endif + bool IsSigned = Kind == scSMaxExpr || Kind == scSMinExpr; + bool IsMax = Kind == scSMaxExpr || Kind == scUMaxExpr; + // Sort by complexity, this groups all similar expression types together. GroupByComplexity(Ops, &LI, DT); + // Check if we have created the same expression before. + if (const SCEV *S = std::get<0>(findExistingSCEVInCache(Kind, Ops))) { + return S; + } + // If there are any constants, fold them together. unsigned Idx = 0; if (const SCEVConstant *LHSC = dyn_cast(Ops[0])) { ++Idx; assert(Idx < Ops.size()); + auto FoldOp = [&](const APInt &LHS, const APInt &RHS) { + if (Kind == scSMaxExpr) + return APIntOps::smax(LHS, RHS); + else if (Kind == scSMinExpr) + return APIntOps::smin(LHS, RHS); + else if (Kind == scUMaxExpr) + return APIntOps::umax(LHS, RHS); + else if (Kind == scUMinExpr) + return APIntOps::umin(LHS, RHS); + llvm_unreachable("Unknown SCEV min/max opcode"); + }; + while (const SCEVConstant *RHSC = dyn_cast(Ops[Idx])) { // We found two constants, fold them together! ConstantInt *Fold = ConstantInt::get( - getContext(), APIntOps::smax(LHSC->getAPInt(), RHSC->getAPInt())); + getContext(), FoldOp(LHSC->getAPInt(), RHSC->getAPInt())); Ops[0] = getConstant(Fold); Ops.erase(Ops.begin()+1); // Erase the folded element if (Ops.size() == 1) return Ops[0]; LHSC = cast(Ops[0]); } - // If we are left with a constant minimum-int, strip it off. - if (cast(Ops[0])->getValue()->isMinValue(true)) { + bool IsMinV = LHSC->getValue()->isMinValue(IsSigned); + bool IsMaxV = LHSC->getValue()->isMaxValue(IsSigned); + + if (IsMax ? IsMinV : IsMaxV) { + // If we are left with a constant minimum(/maximum)-int, strip it off. Ops.erase(Ops.begin()); --Idx; - } else if (cast(Ops[0])->getValue()->isMaxValue(true)) { - // If we have an smax with a constant maximum-int, it will always be - // maximum-int. - return Ops[0]; + } else if (IsMax ? IsMaxV : IsMinV) { + // If we have a max(/min) with a constant maximum(/minimum)-int, + // it will always be the extremum. + return LHSC; } if (Ops.size() == 1) return Ops[0]; } - // Find the first SMax - while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scSMaxExpr) + // Find the first operation of the same kind + while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < Kind) ++Idx; - // Check to see if one of the operands is an SMax. If so, expand its operands - // onto our operand list, and recurse to simplify. + // Check to see if one of the operands is of the same kind. If so, expand its + // operands onto our operand list, and recurse to simplify. if (Idx < Ops.size()) { - bool DeletedSMax = false; - while (const SCEVSMaxExpr *SMax = dyn_cast(Ops[Idx])) { + bool DeletedAny = false; + while (Ops[Idx]->getSCEVType() == Kind) { + const SCEVMinMaxExpr *SMME = cast(Ops[Idx]); Ops.erase(Ops.begin()+Idx); - Ops.append(SMax->op_begin(), SMax->op_end()); - DeletedSMax = true; + Ops.append(SMME->op_begin(), SMME->op_end()); + DeletedAny = true; } - if (DeletedSMax) - return getSMaxExpr(Ops); + if (DeletedAny) + return getMinMaxExpr(Kind, Ops); } // Okay, check to see if the same value occurs in the operand list twice. If // so, delete one. Since we sorted the list, these values are required to // be adjacent. - for (unsigned i = 0, e = Ops.size()-1; i != e; ++i) - // X smax Y smax Y --> X smax Y - // X smax Y --> X, if X is always greater than Y - if (Ops[i] == Ops[i+1] || - isKnownPredicate(ICmpInst::ICMP_SGE, Ops[i], Ops[i+1])) { - Ops.erase(Ops.begin()+i+1, Ops.begin()+i+2); - --i; --e; - } else if (isKnownPredicate(ICmpInst::ICMP_SLE, Ops[i], Ops[i+1])) { - Ops.erase(Ops.begin()+i, Ops.begin()+i+1); - --i; --e; + llvm::CmpInst::Predicate GEPred = + IsSigned ? ICmpInst::ICMP_SGE : ICmpInst::ICMP_UGE; + llvm::CmpInst::Predicate LEPred = + IsSigned ? ICmpInst::ICMP_SLE : ICmpInst::ICMP_ULE; + llvm::CmpInst::Predicate FirstPred = IsMax ? GEPred : LEPred; + llvm::CmpInst::Predicate SecondPred = IsMax ? LEPred : GEPred; + for (unsigned i = 0, e = Ops.size() - 1; i != e; ++i) { + if (Ops[i] == Ops[i + 1] || + isKnownViaNonRecursiveReasoning(FirstPred, Ops[i], Ops[i + 1])) { + // X op Y op Y --> X op Y + // X op Y --> X, if we know X, Y are ordered appropriately + Ops.erase(Ops.begin() + i + 1, Ops.begin() + i + 2); + --i; + --e; + } else if (isKnownViaNonRecursiveReasoning(SecondPred, Ops[i], + Ops[i + 1])) { + // X op Y --> Y, if we know X, Y are ordered appropriately + Ops.erase(Ops.begin() + i, Ops.begin() + i + 1); + --i; + --e; } + } if (Ops.size() == 1) return Ops[0]; assert(!Ops.empty() && "Reduced smax down to nothing!"); - // Okay, it looks like we really DO need an smax expr. Check to see if we + // Okay, it looks like we really DO need an expr. Check to see if we // already have one, otherwise create a new one. + const SCEV *ExistingSCEV; FoldingSetNodeID ID; - ID.AddInteger(scSMaxExpr); - for (unsigned i = 0, e = Ops.size(); i != e; ++i) - ID.AddPointer(Ops[i]); - void *IP = nullptr; - if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; + void *IP; + std::tie(ExistingSCEV, ID, IP) = findExistingSCEVInCache(Kind, Ops); + if (ExistingSCEV) + return ExistingSCEV; const SCEV **O = SCEVAllocator.Allocate(Ops.size()); std::uninitialized_copy(Ops.begin(), Ops.end(), O); - SCEV *S = new (SCEVAllocator) SCEVSMaxExpr(ID.Intern(SCEVAllocator), - O, Ops.size()); + SCEV *S = new (SCEVAllocator) SCEVMinMaxExpr( + ID.Intern(SCEVAllocator), static_cast(Kind), O, Ops.size()); + UniqueSCEVs.InsertNode(S, IP); addToLoopUseLists(S); return S; } -const SCEV *ScalarEvolution::getUMaxExpr(const SCEV *LHS, - const SCEV *RHS) { +const SCEV *ScalarEvolution::getSMaxExpr(const SCEV *LHS, const SCEV *RHS) { SmallVector Ops = {LHS, RHS}; - return getUMaxExpr(Ops); + return getSMaxExpr(Ops); } -const SCEV * -ScalarEvolution::getUMaxExpr(SmallVectorImpl &Ops) { - assert(!Ops.empty() && "Cannot get empty umax!"); - if (Ops.size() == 1) return Ops[0]; -#ifndef NDEBUG - Type *ETy = getEffectiveSCEVType(Ops[0]->getType()); - for (unsigned i = 1, e = Ops.size(); i != e; ++i) - assert(getEffectiveSCEVType(Ops[i]->getType()) == ETy && - "SCEVUMaxExpr operand types don't match!"); -#endif - - // Sort by complexity, this groups all similar expression types together. - GroupByComplexity(Ops, &LI, DT); - - // If there are any constants, fold them together. - unsigned Idx = 0; - if (const SCEVConstant *LHSC = dyn_cast(Ops[0])) { - ++Idx; - assert(Idx < Ops.size()); - while (const SCEVConstant *RHSC = dyn_cast(Ops[Idx])) { - // We found two constants, fold them together! - ConstantInt *Fold = ConstantInt::get( - getContext(), APIntOps::umax(LHSC->getAPInt(), RHSC->getAPInt())); - Ops[0] = getConstant(Fold); - Ops.erase(Ops.begin()+1); // Erase the folded element - if (Ops.size() == 1) return Ops[0]; - LHSC = cast(Ops[0]); - } - - // If we are left with a constant minimum-int, strip it off. - if (cast(Ops[0])->getValue()->isMinValue(false)) { - Ops.erase(Ops.begin()); - --Idx; - } else if (cast(Ops[0])->getValue()->isMaxValue(false)) { - // If we have an umax with a constant maximum-int, it will always be - // maximum-int. - return Ops[0]; - } - - if (Ops.size() == 1) return Ops[0]; - } - - // Find the first UMax - while (Idx < Ops.size() && Ops[Idx]->getSCEVType() < scUMaxExpr) - ++Idx; - - // Check to see if one of the operands is a UMax. If so, expand its operands - // onto our operand list, and recurse to simplify. - if (Idx < Ops.size()) { - bool DeletedUMax = false; - while (const SCEVUMaxExpr *UMax = dyn_cast(Ops[Idx])) { - Ops.erase(Ops.begin()+Idx); - Ops.append(UMax->op_begin(), UMax->op_end()); - DeletedUMax = true; - } - - if (DeletedUMax) - return getUMaxExpr(Ops); - } - - // Okay, check to see if the same value occurs in the operand list twice. If - // so, delete one. Since we sorted the list, these values are required to - // be adjacent. - for (unsigned i = 0, e = Ops.size()-1; i != e; ++i) - // X umax Y umax Y --> X umax Y - // X umax Y --> X, if X is always greater than Y - if (Ops[i] == Ops[i + 1] || isKnownViaNonRecursiveReasoning( - ICmpInst::ICMP_UGE, Ops[i], Ops[i + 1])) { - Ops.erase(Ops.begin() + i + 1, Ops.begin() + i + 2); - --i; --e; - } else if (isKnownViaNonRecursiveReasoning(ICmpInst::ICMP_ULE, Ops[i], - Ops[i + 1])) { - Ops.erase(Ops.begin() + i, Ops.begin() + i + 1); - --i; --e; - } - - if (Ops.size() == 1) return Ops[0]; +const SCEV *ScalarEvolution::getSMaxExpr(SmallVectorImpl &Ops) { + return getMinMaxExpr(scSMaxExpr, Ops); +} - assert(!Ops.empty() && "Reduced umax down to nothing!"); +const SCEV *ScalarEvolution::getUMaxExpr(const SCEV *LHS, const SCEV *RHS) { + SmallVector Ops = {LHS, RHS}; + return getUMaxExpr(Ops); +} - // Okay, it looks like we really DO need a umax expr. Check to see if we - // already have one, otherwise create a new one. - FoldingSetNodeID ID; - ID.AddInteger(scUMaxExpr); - for (unsigned i = 0, e = Ops.size(); i != e; ++i) - ID.AddPointer(Ops[i]); - void *IP = nullptr; - if (const SCEV *S = UniqueSCEVs.FindNodeOrInsertPos(ID, IP)) return S; - const SCEV **O = SCEVAllocator.Allocate(Ops.size()); - std::uninitialized_copy(Ops.begin(), Ops.end(), O); - SCEV *S = new (SCEVAllocator) SCEVUMaxExpr(ID.Intern(SCEVAllocator), - O, Ops.size()); - UniqueSCEVs.InsertNode(S, IP); - addToLoopUseLists(S); - return S; +const SCEV *ScalarEvolution::getUMaxExpr(SmallVectorImpl &Ops) { + return getMinMaxExpr(scUMaxExpr, Ops); } const SCEV *ScalarEvolution::getSMinExpr(const SCEV *LHS, @@ -3705,11 +3699,7 @@ const SCEV *ScalarEvolution::getSMinExpr(const SCEV *LHS, } const SCEV *ScalarEvolution::getSMinExpr(SmallVectorImpl &Ops) { - // ~smax(~x, ~y, ~z) == smin(x, y, z). - SmallVector NotOps; - for (auto *S : Ops) - NotOps.push_back(getNotSCEV(S)); - return getNotSCEV(getSMaxExpr(NotOps)); + return getMinMaxExpr(scSMinExpr, Ops); } const SCEV *ScalarEvolution::getUMinExpr(const SCEV *LHS, @@ -3719,16 +3709,7 @@ const SCEV *ScalarEvolution::getUMinExpr(const SCEV *LHS, } const SCEV *ScalarEvolution::getUMinExpr(SmallVectorImpl &Ops) { - assert(!Ops.empty() && "At least one operand must be!"); - // Trivial case. - if (Ops.size() == 1) - return Ops[0]; - - // ~umax(~x, ~y, ~z) == umin(x, y, z). - SmallVector NotOps; - for (auto *S : Ops) - NotOps.push_back(getNotSCEV(S)); - return getNotSCEV(getUMaxExpr(NotOps)); + return getMinMaxExpr(scUMinExpr, Ops); } const SCEV *ScalarEvolution::getSizeOfExpr(Type *IntTy, Type *AllocTy) { @@ -3892,7 +3873,7 @@ void ScalarEvolution::eraseValueFromMap(Value *V) { } /// Check whether value has nuw/nsw/exact set but SCEV does not. -/// TODO: In reality it is better to check the poison recursevely +/// TODO: In reality it is better to check the poison recursively /// but this is better than nothing. static bool SCEVLostPoisonFlags(const SCEV *S, const Value *V) { if (auto *I = dyn_cast(V)) { @@ -3970,12 +3951,45 @@ const SCEV *ScalarEvolution::getNegativeSCEV(const SCEV *V, V, getConstant(cast(Constant::getAllOnesValue(Ty))), Flags); } +/// If Expr computes ~A, return A else return nullptr +static const SCEV *MatchNotExpr(const SCEV *Expr) { + const SCEVAddExpr *Add = dyn_cast(Expr); + if (!Add || Add->getNumOperands() != 2 || + !Add->getOperand(0)->isAllOnesValue()) + return nullptr; + + const SCEVMulExpr *AddRHS = dyn_cast(Add->getOperand(1)); + if (!AddRHS || AddRHS->getNumOperands() != 2 || + !AddRHS->getOperand(0)->isAllOnesValue()) + return nullptr; + + return AddRHS->getOperand(1); +} + /// Return a SCEV corresponding to ~V = -1-V const SCEV *ScalarEvolution::getNotSCEV(const SCEV *V) { if (const SCEVConstant *VC = dyn_cast(V)) return getConstant( cast(ConstantExpr::getNot(VC->getValue()))); + // Fold ~(u|s)(min|max)(~x, ~y) to (u|s)(max|min)(x, y) + if (const SCEVMinMaxExpr *MME = dyn_cast(V)) { + auto MatchMinMaxNegation = [&](const SCEVMinMaxExpr *MME) { + SmallVector MatchedOperands; + for (const SCEV *Operand : MME->operands()) { + const SCEV *Matched = MatchNotExpr(Operand); + if (!Matched) + return (const SCEV *)nullptr; + MatchedOperands.push_back(Matched); + } + return getMinMaxExpr( + SCEVMinMaxExpr::negate(static_cast(MME->getSCEVType())), + MatchedOperands); + }; + if (const SCEV *Replaced = MatchMinMaxNegation(MME)) + return Replaced; + } + Type *Ty = V->getType(); Ty = getEffectiveSCEVType(Ty); const SCEV *AllOnes = @@ -4022,29 +4036,28 @@ const SCEV *ScalarEvolution::getMinusSCEV(const SCEV *LHS, const SCEV *RHS, return getAddExpr(LHS, getNegativeSCEV(RHS, NegFlags), AddFlags, Depth); } -const SCEV * -ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, Type *Ty) { +const SCEV *ScalarEvolution::getTruncateOrZeroExtend(const SCEV *V, Type *Ty, + unsigned Depth) { Type *SrcTy = V->getType(); assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() && "Cannot truncate or zero extend with non-integer arguments!"); if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) return V; // No conversion if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty)) - return getTruncateExpr(V, Ty); - return getZeroExtendExpr(V, Ty); + return getTruncateExpr(V, Ty, Depth); + return getZeroExtendExpr(V, Ty, Depth); } -const SCEV * -ScalarEvolution::getTruncateOrSignExtend(const SCEV *V, - Type *Ty) { +const SCEV *ScalarEvolution::getTruncateOrSignExtend(const SCEV *V, Type *Ty, + unsigned Depth) { Type *SrcTy = V->getType(); assert(SrcTy->isIntOrPtrTy() && Ty->isIntOrPtrTy() && "Cannot truncate or zero extend with non-integer arguments!"); if (getTypeSizeInBits(SrcTy) == getTypeSizeInBits(Ty)) return V; // No conversion if (getTypeSizeInBits(SrcTy) > getTypeSizeInBits(Ty)) - return getTruncateExpr(V, Ty); - return getSignExtendExpr(V, Ty); + return getTruncateExpr(V, Ty, Depth); + return getSignExtendExpr(V, Ty, Depth); } const SCEV * @@ -4530,52 +4543,21 @@ static Optional MatchBinaryOp(Value *V, DominatorTree &DT) { if (EVI->getNumIndices() != 1 || EVI->getIndices()[0] != 0) break; - auto *CI = dyn_cast(EVI->getAggregateOperand()); - if (!CI) + auto *WO = dyn_cast(EVI->getAggregateOperand()); + if (!WO) break; - if (auto *F = CI->getCalledFunction()) - switch (F->getIntrinsicID()) { - case Intrinsic::sadd_with_overflow: - case Intrinsic::uadd_with_overflow: - if (!isOverflowIntrinsicNoWrap(cast(CI), DT)) - return BinaryOp(Instruction::Add, CI->getArgOperand(0), - CI->getArgOperand(1)); - - // Now that we know that all uses of the arithmetic-result component of - // CI are guarded by the overflow check, we can go ahead and pretend - // that the arithmetic is non-overflowing. - if (F->getIntrinsicID() == Intrinsic::sadd_with_overflow) - return BinaryOp(Instruction::Add, CI->getArgOperand(0), - CI->getArgOperand(1), /* IsNSW = */ true, - /* IsNUW = */ false); - else - return BinaryOp(Instruction::Add, CI->getArgOperand(0), - CI->getArgOperand(1), /* IsNSW = */ false, - /* IsNUW*/ true); - case Intrinsic::ssub_with_overflow: - case Intrinsic::usub_with_overflow: - if (!isOverflowIntrinsicNoWrap(cast(CI), DT)) - return BinaryOp(Instruction::Sub, CI->getArgOperand(0), - CI->getArgOperand(1)); - - // The same reasoning as sadd/uadd above. - if (F->getIntrinsicID() == Intrinsic::ssub_with_overflow) - return BinaryOp(Instruction::Sub, CI->getArgOperand(0), - CI->getArgOperand(1), /* IsNSW = */ true, - /* IsNUW = */ false); - else - return BinaryOp(Instruction::Sub, CI->getArgOperand(0), - CI->getArgOperand(1), /* IsNSW = */ false, - /* IsNUW = */ true); - case Intrinsic::smul_with_overflow: - case Intrinsic::umul_with_overflow: - return BinaryOp(Instruction::Mul, CI->getArgOperand(0), - CI->getArgOperand(1)); - default: - break; - } - break; + Instruction::BinaryOps BinOp = WO->getBinaryOp(); + bool Signed = WO->isSigned(); + // TODO: Should add nuw/nsw flags for mul as well. + if (BinOp == Instruction::Mul || !isOverflowIntrinsicNoWrap(WO, DT)) + return BinaryOp(BinOp, WO->getLHS(), WO->getRHS()); + + // Now that we know that all uses of the arithmetic-result component of + // CI are guarded by the overflow check, we can go ahead and pretend + // that the arithmetic is non-overflowing. + return BinaryOp(BinOp, WO->getLHS(), WO->getRHS(), + /* IsNSW = */ Signed, /* IsNUW = */ !Signed); } default: @@ -5009,7 +4991,7 @@ const SCEV *ScalarEvolution::createSimpleAffineAddRec(PHINode *PN, // overflow. if (auto *BEInst = dyn_cast(BEValueV)) if (isLoopInvariant(Accum, L) && isAddRecNeverPoison(BEInst, L)) - (void)getAddRecExpr(getAddExpr(StartVal, Accum), Accum, L, Flags); + (void)getAddRecExpr(getAddExpr(StartVal, Accum, Flags), Accum, L, Flags); return PHISCEV; } @@ -5196,6 +5178,8 @@ static bool IsAvailableOnEntry(const Loop *L, DominatorTree &DT, const SCEV *S, switch (S->getSCEVType()) { case scConstant: case scTruncate: case scZeroExtend: case scSignExtend: case scAddExpr: case scMulExpr: case scUMaxExpr: case scSMaxExpr: + case scUMinExpr: + case scSMinExpr: // These expressions are available if their operand(s) is/are. return true; @@ -5551,6 +5535,9 @@ ScalarEvolution::getRangeRef(const SCEV *S, DenseMap &Cache = SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED ? UnsignedRanges : SignedRanges; + ConstantRange::PreferredRangeType RangeType = + SignHint == ScalarEvolution::HINT_RANGE_UNSIGNED + ? ConstantRange::Unsigned : ConstantRange::Signed; // See if we've computed this range already. DenseMap::iterator I = Cache.find(S); @@ -5581,53 +5568,60 @@ ScalarEvolution::getRangeRef(const SCEV *S, ConstantRange X = getRangeRef(Add->getOperand(0), SignHint); for (unsigned i = 1, e = Add->getNumOperands(); i != e; ++i) X = X.add(getRangeRef(Add->getOperand(i), SignHint)); - return setRange(Add, SignHint, ConservativeResult.intersectWith(X)); + return setRange(Add, SignHint, + ConservativeResult.intersectWith(X, RangeType)); } if (const SCEVMulExpr *Mul = dyn_cast(S)) { ConstantRange X = getRangeRef(Mul->getOperand(0), SignHint); for (unsigned i = 1, e = Mul->getNumOperands(); i != e; ++i) X = X.multiply(getRangeRef(Mul->getOperand(i), SignHint)); - return setRange(Mul, SignHint, ConservativeResult.intersectWith(X)); + return setRange(Mul, SignHint, + ConservativeResult.intersectWith(X, RangeType)); } if (const SCEVSMaxExpr *SMax = dyn_cast(S)) { ConstantRange X = getRangeRef(SMax->getOperand(0), SignHint); for (unsigned i = 1, e = SMax->getNumOperands(); i != e; ++i) X = X.smax(getRangeRef(SMax->getOperand(i), SignHint)); - return setRange(SMax, SignHint, ConservativeResult.intersectWith(X)); + return setRange(SMax, SignHint, + ConservativeResult.intersectWith(X, RangeType)); } if (const SCEVUMaxExpr *UMax = dyn_cast(S)) { ConstantRange X = getRangeRef(UMax->getOperand(0), SignHint); for (unsigned i = 1, e = UMax->getNumOperands(); i != e; ++i) X = X.umax(getRangeRef(UMax->getOperand(i), SignHint)); - return setRange(UMax, SignHint, ConservativeResult.intersectWith(X)); + return setRange(UMax, SignHint, + ConservativeResult.intersectWith(X, RangeType)); } if (const SCEVUDivExpr *UDiv = dyn_cast(S)) { ConstantRange X = getRangeRef(UDiv->getLHS(), SignHint); ConstantRange Y = getRangeRef(UDiv->getRHS(), SignHint); return setRange(UDiv, SignHint, - ConservativeResult.intersectWith(X.udiv(Y))); + ConservativeResult.intersectWith(X.udiv(Y), RangeType)); } if (const SCEVZeroExtendExpr *ZExt = dyn_cast(S)) { ConstantRange X = getRangeRef(ZExt->getOperand(), SignHint); return setRange(ZExt, SignHint, - ConservativeResult.intersectWith(X.zeroExtend(BitWidth))); + ConservativeResult.intersectWith(X.zeroExtend(BitWidth), + RangeType)); } if (const SCEVSignExtendExpr *SExt = dyn_cast(S)) { ConstantRange X = getRangeRef(SExt->getOperand(), SignHint); return setRange(SExt, SignHint, - ConservativeResult.intersectWith(X.signExtend(BitWidth))); + ConservativeResult.intersectWith(X.signExtend(BitWidth), + RangeType)); } if (const SCEVTruncateExpr *Trunc = dyn_cast(S)) { ConstantRange X = getRangeRef(Trunc->getOperand(), SignHint); return setRange(Trunc, SignHint, - ConservativeResult.intersectWith(X.truncate(BitWidth))); + ConservativeResult.intersectWith(X.truncate(BitWidth), + RangeType)); } if (const SCEVAddRecExpr *AddRec = dyn_cast(S)) { @@ -5637,7 +5631,7 @@ ScalarEvolution::getRangeRef(const SCEV *S, if (const SCEVConstant *C = dyn_cast(AddRec->getStart())) if (!C->getValue()->isZero()) ConservativeResult = ConservativeResult.intersectWith( - ConstantRange(C->getAPInt(), APInt(BitWidth, 0))); + ConstantRange(C->getAPInt(), APInt(BitWidth, 0)), RangeType); // If there's no signed wrap, and all the operands have the same sign or // zero, the value won't ever change sign. @@ -5651,11 +5645,11 @@ ScalarEvolution::getRangeRef(const SCEV *S, if (AllNonNeg) ConservativeResult = ConservativeResult.intersectWith( ConstantRange(APInt(BitWidth, 0), - APInt::getSignedMinValue(BitWidth))); + APInt::getSignedMinValue(BitWidth)), RangeType); else if (AllNonPos) ConservativeResult = ConservativeResult.intersectWith( ConstantRange(APInt::getSignedMinValue(BitWidth), - APInt(BitWidth, 1))); + APInt(BitWidth, 1)), RangeType); } // TODO: non-affine addrec @@ -5668,14 +5662,14 @@ ScalarEvolution::getRangeRef(const SCEV *S, BitWidth); if (!RangeFromAffine.isFullSet()) ConservativeResult = - ConservativeResult.intersectWith(RangeFromAffine); + ConservativeResult.intersectWith(RangeFromAffine, RangeType); auto RangeFromFactoring = getRangeViaFactoring( AddRec->getStart(), AddRec->getStepRecurrence(*this), MaxBECount, BitWidth); if (!RangeFromFactoring.isFullSet()) ConservativeResult = - ConservativeResult.intersectWith(RangeFromFactoring); + ConservativeResult.intersectWith(RangeFromFactoring, RangeType); } } @@ -5686,7 +5680,8 @@ ScalarEvolution::getRangeRef(const SCEV *S, // Check if the IR explicitly contains !range metadata. Optional MDRange = GetRangeFromMetadata(U->getValue()); if (MDRange.hasValue()) - ConservativeResult = ConservativeResult.intersectWith(MDRange.getValue()); + ConservativeResult = ConservativeResult.intersectWith(MDRange.getValue(), + RangeType); // Split here to avoid paying the compile-time cost of calling both // computeKnownBits and ComputeNumSignBits. This restriction can be lifted @@ -5697,8 +5692,8 @@ ScalarEvolution::getRangeRef(const SCEV *S, KnownBits Known = computeKnownBits(U->getValue(), DL, 0, &AC, nullptr, &DT); if (Known.One != ~Known.Zero + 1) ConservativeResult = - ConservativeResult.intersectWith(ConstantRange(Known.One, - ~Known.Zero + 1)); + ConservativeResult.intersectWith( + ConstantRange(Known.One, ~Known.Zero + 1), RangeType); } else { assert(SignHint == ScalarEvolution::HINT_RANGE_SIGNED && "generalize as needed!"); @@ -5706,7 +5701,8 @@ ScalarEvolution::getRangeRef(const SCEV *S, if (NS > 1) ConservativeResult = ConservativeResult.intersectWith( ConstantRange(APInt::getSignedMinValue(BitWidth).ashr(NS - 1), - APInt::getSignedMaxValue(BitWidth).ashr(NS - 1) + 1)); + APInt::getSignedMaxValue(BitWidth).ashr(NS - 1) + 1), + RangeType); } // A range of Phi is a subset of union of all ranges of its input. @@ -5721,7 +5717,8 @@ ScalarEvolution::getRangeRef(const SCEV *S, if (RangeFromOps.isFullSet()) break; } - ConservativeResult = ConservativeResult.intersectWith(RangeFromOps); + ConservativeResult = + ConservativeResult.intersectWith(RangeFromOps, RangeType); bool Erased = PendingPhiRanges.erase(Phi); assert(Erased && "Failed to erase Phi properly?"); (void) Erased; @@ -5751,7 +5748,7 @@ static ConstantRange getRangeForAffineARHelper(APInt Step, // FullRange), then we don't know anything about the final range either. // Return FullRange. if (StartRange.isFullSet()) - return ConstantRange(BitWidth, /* isFullSet = */ true); + return ConstantRange::getFull(BitWidth); // If Step is signed and negative, then we use its absolute value, but we also // note that we're moving in the opposite direction. @@ -5767,7 +5764,7 @@ static ConstantRange getRangeForAffineARHelper(APInt Step, // Check if Offset is more than full span of BitWidth. If it is, the // expression is guaranteed to overflow. if (APInt::getMaxValue(StartRange.getBitWidth()).udiv(Step).ult(MaxBECount)) - return ConstantRange(BitWidth, /* isFullSet = */ true); + return ConstantRange::getFull(BitWidth); // Offset is by how much the expression can change. Checks above guarantee no // overflow here. @@ -5786,7 +5783,7 @@ static ConstantRange getRangeForAffineARHelper(APInt Step, // range (due to wrap around). This means that the expression can take any // value in this bitwidth, and we have to return full range. if (StartRange.contains(MovedBoundary)) - return ConstantRange(BitWidth, /* isFullSet = */ true); + return ConstantRange::getFull(BitWidth); APInt NewLower = Descending ? std::move(MovedBoundary) : std::move(StartLower); @@ -5794,12 +5791,8 @@ static ConstantRange getRangeForAffineARHelper(APInt Step, Descending ? std::move(StartUpper) : std::move(MovedBoundary); NewUpper += 1; - // If we end up with full range, return a proper full range. - if (NewLower == NewUpper) - return ConstantRange(BitWidth, /* isFullSet = */ true); - // No overflow detected, return [StartLower, StartUpper + Offset + 1) range. - return ConstantRange(std::move(NewLower), std::move(NewUpper)); + return ConstantRange::getNonEmpty(std::move(NewLower), std::move(NewUpper)); } ConstantRange ScalarEvolution::getRangeForAffineAR(const SCEV *Start, @@ -5832,7 +5825,7 @@ ConstantRange ScalarEvolution::getRangeForAffineAR(const SCEV *Start, MaxBECountValue, BitWidth, /* Signed = */ false); // Finally, intersect signed and unsigned ranges. - return SR.intersectWith(UR); + return SR.intersectWith(UR, ConstantRange::Smallest); } ConstantRange ScalarEvolution::getRangeViaFactoring(const SCEV *Start, @@ -5916,17 +5909,17 @@ ConstantRange ScalarEvolution::getRangeViaFactoring(const SCEV *Start, SelectPattern StartPattern(*this, BitWidth, Start); if (!StartPattern.isRecognized()) - return ConstantRange(BitWidth, /* isFullSet = */ true); + return ConstantRange::getFull(BitWidth); SelectPattern StepPattern(*this, BitWidth, Step); if (!StepPattern.isRecognized()) - return ConstantRange(BitWidth, /* isFullSet = */ true); + return ConstantRange::getFull(BitWidth); if (StartPattern.Condition != StepPattern.Condition) { // We don't handle this case today; but we could, by considering four // possibilities below instead of two. I'm not sure if there are cases where // that will help over what getRange already does, though. - return ConstantRange(BitWidth, /* isFullSet = */ true); + return ConstantRange::getFull(BitWidth); } // NB! Calling ScalarEvolution::getConstant is fine, but we should not try to @@ -6128,7 +6121,7 @@ const SCEV *ScalarEvolution::createSCEV(Value *V) { // to obey basic rules for definitions dominating uses which this // analysis depends on. if (!DT.isReachableFromEntry(I->getParent())) - return getUnknown(V); + return getUnknown(UndefValue::get(V->getType())); } else if (ConstantInt *CI = dyn_cast(V)) return getConstant(CI); else if (isa(V)) @@ -6744,6 +6737,28 @@ ScalarEvolution::getBackedgeTakenInfo(const Loop *L) { return BackedgeTakenCounts.find(L)->second = std::move(Result); } +void ScalarEvolution::forgetAllLoops() { + // This method is intended to forget all info about loops. It should + // invalidate caches as if the following happened: + // - The trip counts of all loops have changed arbitrarily + // - Every llvm::Value has been updated in place to produce a different + // result. + BackedgeTakenCounts.clear(); + PredicatedBackedgeTakenCounts.clear(); + LoopPropertiesCache.clear(); + ConstantEvolutionLoopExitValue.clear(); + ValueExprMap.clear(); + ValuesAtScopes.clear(); + LoopDispositions.clear(); + BlockDispositions.clear(); + UnsignedRanges.clear(); + SignedRanges.clear(); + ExprValueMap.clear(); + HasRecMap.clear(); + MinTrailingZerosCache.clear(); + PredicatedSCEVRewrites.clear(); +} + void ScalarEvolution::forgetLoop(const Loop *L) { // Drop any stored trip count value. auto RemoveLoopFromBackedgeMap = @@ -6972,8 +6987,8 @@ ScalarEvolution::ExitLimit::ExitLimit(const SCEV *E, const SCEV *M, /// Allocate memory for BackedgeTakenInfo and copy the not-taken count of each /// computable exit into a persistent ExitNotTakenInfo array. ScalarEvolution::BackedgeTakenInfo::BackedgeTakenInfo( - SmallVectorImpl - &&ExitCounts, + ArrayRef + ExitCounts, bool Complete, const SCEV *MaxCount, bool MaxOrZero) : MaxAndComplete(MaxCount, Complete), MaxOrZero(MaxOrZero) { using EdgeExitInfo = ScalarEvolution::BackedgeTakenInfo::EdgeExitInfo; @@ -7256,6 +7271,14 @@ ScalarEvolution::ExitLimit ScalarEvolution::computeExitLimitFromCondImpl( if (EL0.ExactNotTaken == EL1.ExactNotTaken) BECount = EL0.ExactNotTaken; } + // There are cases (e.g. PR26207) where computeExitLimitFromCond is able + // to be more aggressive when computing BECount than when computing + // MaxBECount. In these cases it is possible for EL0.ExactNotTaken and + // EL1.ExactNotTaken to match, but for EL0.MaxNotTaken and EL1.MaxNotTaken + // to not. + if (isa(MaxBECount) && + !isa(BECount)) + MaxBECount = getConstant(getUnsignedRangeMax(BECount)); return ExitLimit(BECount, MaxBECount, false, {&EL0.Predicates, &EL1.Predicates}); @@ -7651,7 +7674,7 @@ ScalarEvolution::ExitLimit ScalarEvolution::computeShiftCompareExitLimit( static bool CanConstantFold(const Instruction *I) { if (isa(I) || isa(I) || isa(I) || isa(I) || isa(I) || - isa(I)) + isa(I) || isa(I)) return true; if (const CallInst *CI = dyn_cast(I)) @@ -8075,7 +8098,9 @@ static Constant *BuildConstantFromSCEV(const SCEV *V) { } case scSMaxExpr: case scUMaxExpr: - break; // TODO: smax, umax. + case scSMinExpr: + case scUMinExpr: + break; // TODO: smax, umax, smin, umax. } return nullptr; } @@ -8087,44 +8112,64 @@ const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) { // exit value from the loop without using SCEVs. if (const SCEVUnknown *SU = dyn_cast(V)) { if (Instruction *I = dyn_cast(SU->getValue())) { - const Loop *LI = this->LI[I->getParent()]; - if (LI && LI->getParentLoop() == L) // Looking for loop exit value. - if (PHINode *PN = dyn_cast(I)) - if (PN->getParent() == LI->getHeader()) { - // Okay, there is no closed form solution for the PHI node. Check - // to see if the loop that contains it has a known backedge-taken - // count. If so, we may be able to force computation of the exit - // value. - const SCEV *BackedgeTakenCount = getBackedgeTakenCount(LI); - if (const SCEVConstant *BTCC = - dyn_cast(BackedgeTakenCount)) { - - // This trivial case can show up in some degenerate cases where - // the incoming IR has not yet been fully simplified. - if (BTCC->getValue()->isZero()) { - Value *InitValue = nullptr; - bool MultipleInitValues = false; - for (unsigned i = 0; i < PN->getNumIncomingValues(); i++) { - if (!LI->contains(PN->getIncomingBlock(i))) { - if (!InitValue) - InitValue = PN->getIncomingValue(i); - else if (InitValue != PN->getIncomingValue(i)) { - MultipleInitValues = true; - break; - } - } - if (!MultipleInitValues && InitValue) - return getSCEV(InitValue); + if (PHINode *PN = dyn_cast(I)) { + const Loop *LI = this->LI[I->getParent()]; + // Looking for loop exit value. + if (LI && LI->getParentLoop() == L && + PN->getParent() == LI->getHeader()) { + // Okay, there is no closed form solution for the PHI node. Check + // to see if the loop that contains it has a known backedge-taken + // count. If so, we may be able to force computation of the exit + // value. + const SCEV *BackedgeTakenCount = getBackedgeTakenCount(LI); + // This trivial case can show up in some degenerate cases where + // the incoming IR has not yet been fully simplified. + if (BackedgeTakenCount->isZero()) { + Value *InitValue = nullptr; + bool MultipleInitValues = false; + for (unsigned i = 0; i < PN->getNumIncomingValues(); i++) { + if (!LI->contains(PN->getIncomingBlock(i))) { + if (!InitValue) + InitValue = PN->getIncomingValue(i); + else if (InitValue != PN->getIncomingValue(i)) { + MultipleInitValues = true; + break; } } - // Okay, we know how many times the containing loop executes. If - // this is a constant evolving PHI node, get the final value at - // the specified iteration number. - Constant *RV = - getConstantEvolutionLoopExitValue(PN, BTCC->getAPInt(), LI); - if (RV) return getSCEV(RV); } + if (!MultipleInitValues && InitValue) + return getSCEV(InitValue); } + // Do we have a loop invariant value flowing around the backedge + // for a loop which must execute the backedge? + if (!isa(BackedgeTakenCount) && + isKnownPositive(BackedgeTakenCount) && + PN->getNumIncomingValues() == 2) { + unsigned InLoopPred = LI->contains(PN->getIncomingBlock(0)) ? 0 : 1; + const SCEV *OnBackedge = getSCEV(PN->getIncomingValue(InLoopPred)); + if (IsAvailableOnEntry(LI, DT, OnBackedge, PN->getParent())) + return OnBackedge; + } + if (auto *BTCC = dyn_cast(BackedgeTakenCount)) { + // Okay, we know how many times the containing loop executes. If + // this is a constant evolving PHI node, get the final value at + // the specified iteration number. + Constant *RV = + getConstantEvolutionLoopExitValue(PN, BTCC->getAPInt(), LI); + if (RV) return getSCEV(RV); + } + } + + // If there is a single-input Phi, evaluate it at our scope. If we can + // prove that this replacement does not break LCSSA form, use new value. + if (PN->getNumOperands() == 1) { + const SCEV *Input = getSCEV(PN->getOperand(0)); + const SCEV *InputAtScope = getSCEVAtScope(Input, L); + // TODO: We can generalize it using LI.replacementPreservesLCSSAForm, + // for the simplest case just support constants. + if (isa(InputAtScope)) return InputAtScope; + } + } // Okay, this is an expression that we cannot symbolically evaluate // into a SCEV. Check to see if it's possible to symbolically evaluate @@ -8198,13 +8243,11 @@ const SCEV *ScalarEvolution::computeSCEVAtScope(const SCEV *V, const Loop *L) { NewOps.push_back(OpAtScope); } if (isa(Comm)) - return getAddExpr(NewOps); + return getAddExpr(NewOps, Comm->getNoWrapFlags()); if (isa(Comm)) - return getMulExpr(NewOps); - if (isa(Comm)) - return getSMaxExpr(NewOps); - if (isa(Comm)) - return getUMaxExpr(NewOps); + return getMulExpr(NewOps, Comm->getNoWrapFlags()); + if (isa(Comm)) + return getMinMaxExpr(Comm->getSCEVType(), NewOps); llvm_unreachable("Unknown commutative SCEV type!"); } } @@ -10045,41 +10088,15 @@ bool ScalarEvolution::isImpliedCondOperands(ICmpInst::Predicate Pred, getNotSCEV(FoundLHS)); } -/// If Expr computes ~A, return A else return nullptr -static const SCEV *MatchNotExpr(const SCEV *Expr) { - const SCEVAddExpr *Add = dyn_cast(Expr); - if (!Add || Add->getNumOperands() != 2 || - !Add->getOperand(0)->isAllOnesValue()) - return nullptr; - - const SCEVMulExpr *AddRHS = dyn_cast(Add->getOperand(1)); - if (!AddRHS || AddRHS->getNumOperands() != 2 || - !AddRHS->getOperand(0)->isAllOnesValue()) - return nullptr; - - return AddRHS->getOperand(1); -} - -/// Is MaybeMaxExpr an SMax or UMax of Candidate and some other values? -template -static bool IsMaxConsistingOf(const SCEV *MaybeMaxExpr, - const SCEV *Candidate) { - const MaxExprType *MaxExpr = dyn_cast(MaybeMaxExpr); - if (!MaxExpr) return false; - - return find(MaxExpr->operands(), Candidate) != MaxExpr->op_end(); -} - -/// Is MaybeMinExpr an SMin or UMin of Candidate and some other values? -template -static bool IsMinConsistingOf(ScalarEvolution &SE, - const SCEV *MaybeMinExpr, - const SCEV *Candidate) { - const SCEV *MaybeMaxExpr = MatchNotExpr(MaybeMinExpr); - if (!MaybeMaxExpr) +/// Is MaybeMinMaxExpr an (U|S)(Min|Max) of Candidate and some other values? +template +static bool IsMinMaxConsistingOf(const SCEV *MaybeMinMaxExpr, + const SCEV *Candidate) { + const MinMaxExprType *MinMaxExpr = dyn_cast(MaybeMinMaxExpr); + if (!MinMaxExpr) return false; - return IsMaxConsistingOf(MaybeMaxExpr, SE.getNotSCEV(Candidate)); + return find(MinMaxExpr->operands(), Candidate) != MinMaxExpr->op_end(); } static bool IsKnownPredicateViaAddRecStart(ScalarEvolution &SE, @@ -10128,20 +10145,20 @@ static bool IsKnownPredicateViaMinOrMax(ScalarEvolution &SE, LLVM_FALLTHROUGH; case ICmpInst::ICMP_SLE: return - // min(A, ...) <= A - IsMinConsistingOf(SE, LHS, RHS) || - // A <= max(A, ...) - IsMaxConsistingOf(RHS, LHS); + // min(A, ...) <= A + IsMinMaxConsistingOf(LHS, RHS) || + // A <= max(A, ...) + IsMinMaxConsistingOf(RHS, LHS); case ICmpInst::ICMP_UGE: std::swap(LHS, RHS); LLVM_FALLTHROUGH; case ICmpInst::ICMP_ULE: return - // min(A, ...) <= A - IsMinConsistingOf(SE, LHS, RHS) || - // A <= max(A, ...) - IsMaxConsistingOf(RHS, LHS); + // min(A, ...) <= A + IsMinMaxConsistingOf(LHS, RHS) || + // A <= max(A, ...) + IsMinMaxConsistingOf(RHS, LHS); } llvm_unreachable("covered switch fell through?!"); @@ -10691,13 +10708,10 @@ ScalarEvolution::howManyGreaterThans(const SCEV *LHS, const SCEV *RHS, IsSigned ? APIntOps::smax(getSignedRangeMin(RHS), Limit) : APIntOps::umax(getUnsignedRangeMin(RHS), Limit); - - const SCEV *MaxBECount = getCouldNotCompute(); - if (isa(BECount)) - MaxBECount = BECount; - else - MaxBECount = computeBECount(getConstant(MaxStart - MinEnd), - getConstant(MinStride), false); + const SCEV *MaxBECount = isa(BECount) + ? BECount + : computeBECount(getConstant(MaxStart - MinEnd), + getConstant(MinStride), false); if (isa(MaxBECount)) MaxBECount = BECount; @@ -10806,8 +10820,6 @@ static inline bool containsUndefs(const SCEV *S) { return SCEVExprContains(S, [](const SCEV *S) { if (const auto *SU = dyn_cast(S)) return isa(SU->getValue()); - else if (const auto *SC = dyn_cast(S)) - return isa(SC->getValue()); return false; }); } @@ -11402,19 +11414,23 @@ static void PrintLoopInfo(raw_ostream &OS, ScalarEvolution *SE, L->getHeader()->printAsOperand(OS, /*PrintType=*/false); OS << ": "; - SmallVector ExitBlocks; - L->getExitBlocks(ExitBlocks); - if (ExitBlocks.size() != 1) + SmallVector ExitingBlocks; + L->getExitingBlocks(ExitingBlocks); + if (ExitingBlocks.size() != 1) OS << " "; - if (SE->hasLoopInvariantBackedgeTakenCount(L)) { - OS << "backedge-taken count is " << *SE->getBackedgeTakenCount(L); - } else { - OS << "Unpredictable backedge-taken count. "; - } + if (SE->hasLoopInvariantBackedgeTakenCount(L)) + OS << "backedge-taken count is " << *SE->getBackedgeTakenCount(L) << "\n"; + else + OS << "Unpredictable backedge-taken count.\n"; - OS << "\n" - "Loop "; + if (ExitingBlocks.size() > 1) + for (BasicBlock *ExitingBlock : ExitingBlocks) { + OS << " exit count for " << ExitingBlock->getName() << ": " + << *SE->getExitCount(L, ExitingBlock) << "\n"; + } + + OS << "Loop "; L->getHeader()->printAsOperand(OS, /*PrintType=*/false); OS << ": "; @@ -11611,7 +11627,9 @@ ScalarEvolution::computeLoopDisposition(const SCEV *S, const Loop *L) { case scAddExpr: case scMulExpr: case scUMaxExpr: - case scSMaxExpr: { + case scSMaxExpr: + case scUMinExpr: + case scSMinExpr: { bool HasVarying = false; for (auto *Op : cast(S)->operands()) { LoopDisposition D = getLoopDisposition(Op, L); @@ -11698,7 +11716,9 @@ ScalarEvolution::computeBlockDisposition(const SCEV *S, const BasicBlock *BB) { case scAddExpr: case scMulExpr: case scUMaxExpr: - case scSMaxExpr: { + case scSMaxExpr: + case scUMinExpr: + case scSMinExpr: { const SCEVNAryExpr *NAry = cast(S); bool Proper = true; for (const SCEV *NAryOp : NAry->operands()) { diff --git a/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp b/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp index 289d4f8ae49a..96da0a24cddd 100644 --- a/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp +++ b/lib/Analysis/ScalarEvolutionAliasAnalysis.cpp @@ -1,9 +1,8 @@ //===- ScalarEvolutionAliasAnalysis.cpp - SCEV-based Alias Analysis -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -23,7 +22,7 @@ using namespace llvm; AliasResult SCEVAAResult::alias(const MemoryLocation &LocA, - const MemoryLocation &LocB) { + const MemoryLocation &LocB, AAQueryInfo &AAQI) { // If either of the memory references is empty, it doesn't matter what the // pointer values are. This allows the code below to ignore this special // case. @@ -86,11 +85,12 @@ AliasResult SCEVAAResult::alias(const MemoryLocation &LocA, AO ? AAMDNodes() : LocA.AATags), MemoryLocation(BO ? BO : LocB.Ptr, BO ? LocationSize::unknown() : LocB.Size, - BO ? AAMDNodes() : LocB.AATags)) == NoAlias) + BO ? AAMDNodes() : LocB.AATags), + AAQI) == NoAlias) return NoAlias; // Forward the query to the next analysis. - return AAResultBase::alias(LocA, LocB); + return AAResultBase::alias(LocA, LocB, AAQI); } /// Given an expression, try to find a base value. diff --git a/lib/Analysis/ScalarEvolutionExpander.cpp b/lib/Analysis/ScalarEvolutionExpander.cpp index ca5cf1663b83..e8a95d35482c 100644 --- a/lib/Analysis/ScalarEvolutionExpander.cpp +++ b/lib/Analysis/ScalarEvolutionExpander.cpp @@ -1,9 +1,8 @@ //===- ScalarEvolutionExpander.cpp - Scalar Evolution Analysis ------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -61,12 +60,10 @@ Value *SCEVExpander::ReuseOrCreateCast(Value *V, Type *Ty, // instructions that might be inserted before BIP. if (BasicBlock::iterator(CI) != IP || BIP == IP) { // Create a new cast, and leave the old cast in place in case - // it is being used as an insert point. Clear its operand - // so that it doesn't hold anything live. + // it is being used as an insert point. Ret = CastInst::Create(Op, V, Ty, "", &*IP); Ret->takeName(CI); CI->replaceAllUsesWith(Ret); - CI->setOperand(0, UndefValue::get(V->getType())); break; } Ret = CI; @@ -167,9 +164,11 @@ Value *SCEVExpander::InsertNoopCastOfTo(Value *V, Type *Ty) { } /// InsertBinop - Insert the specified binary operator, doing a small amount -/// of work to avoid inserting an obviously redundant operation. +/// of work to avoid inserting an obviously redundant operation, and hoisting +/// to an outer loop when the opportunity is there and it is safe. Value *SCEVExpander::InsertBinop(Instruction::BinaryOps Opcode, - Value *LHS, Value *RHS) { + Value *LHS, Value *RHS, + SCEV::NoWrapFlags Flags, bool IsSafeToHoist) { // Fold a binop with constant operands. if (Constant *CLHS = dyn_cast(LHS)) if (Constant *CRHS = dyn_cast(RHS)) @@ -188,20 +187,22 @@ Value *SCEVExpander::InsertBinop(Instruction::BinaryOps Opcode, if (isa(IP)) ScanLimit++; - // Conservatively, do not use any instruction which has any of wrap/exact - // flags installed. - // TODO: Instead of simply disable poison instructions we can be clever - // here and match SCEV to this instruction. - auto canGeneratePoison = [](Instruction *I) { - if (isa(I) && - (I->hasNoSignedWrap() || I->hasNoUnsignedWrap())) - return true; + auto canGenerateIncompatiblePoison = [&Flags](Instruction *I) { + // Ensure that no-wrap flags match. + if (isa(I)) { + if (I->hasNoSignedWrap() != (Flags & SCEV::FlagNSW)) + return true; + if (I->hasNoUnsignedWrap() != (Flags & SCEV::FlagNUW)) + return true; + } + // Conservatively, do not use any instruction which has any of exact + // flags installed. if (isa(I) && I->isExact()) return true; return false; }; if (IP->getOpcode() == (unsigned)Opcode && IP->getOperand(0) == LHS && - IP->getOperand(1) == RHS && !canGeneratePoison(&*IP)) + IP->getOperand(1) == RHS && !canGenerateIncompatiblePoison(&*IP)) return &*IP; if (IP == BlockBegin) break; } @@ -211,19 +212,25 @@ Value *SCEVExpander::InsertBinop(Instruction::BinaryOps Opcode, DebugLoc Loc = Builder.GetInsertPoint()->getDebugLoc(); SCEVInsertPointGuard Guard(Builder, this); - // Move the insertion point out of as many loops as we can. - while (const Loop *L = SE.LI.getLoopFor(Builder.GetInsertBlock())) { - if (!L->isLoopInvariant(LHS) || !L->isLoopInvariant(RHS)) break; - BasicBlock *Preheader = L->getLoopPreheader(); - if (!Preheader) break; + if (IsSafeToHoist) { + // Move the insertion point out of as many loops as we can. + while (const Loop *L = SE.LI.getLoopFor(Builder.GetInsertBlock())) { + if (!L->isLoopInvariant(LHS) || !L->isLoopInvariant(RHS)) break; + BasicBlock *Preheader = L->getLoopPreheader(); + if (!Preheader) break; - // Ok, move up a level. - Builder.SetInsertPoint(Preheader->getTerminator()); + // Ok, move up a level. + Builder.SetInsertPoint(Preheader->getTerminator()); + } } // If we haven't found this binop, insert it. Instruction *BO = cast(Builder.CreateBinOp(Opcode, LHS, RHS)); BO->setDebugLoc(Loc); + if (Flags & SCEV::FlagNUW) + BO->setHasNoUnsignedWrap(); + if (Flags & SCEV::FlagNSW) + BO->setHasNoSignedWrap(); rememberInstruction(BO); return BO; @@ -695,7 +702,7 @@ Value *SCEVExpander::visitAddExpr(const SCEVAddExpr *S) { // Sort by loop. Use a stable sort so that constants follow non-constants and // pointer operands precede non-pointer operands. - std::stable_sort(OpsAndLoops.begin(), OpsAndLoops.end(), LoopCompare(SE.DT)); + llvm::stable_sort(OpsAndLoops, LoopCompare(SE.DT)); // Emit instructions to add all the operands. Hoist as much as possible // out of loops, and form meaningful getelementptrs where possible. @@ -735,7 +742,8 @@ Value *SCEVExpander::visitAddExpr(const SCEVAddExpr *S) { // Instead of doing a negate and add, just do a subtract. Value *W = expandCodeFor(SE.getNegativeSCEV(Op), Ty); Sum = InsertNoopCastOfTo(Sum, Ty); - Sum = InsertBinop(Instruction::Sub, Sum, W); + Sum = InsertBinop(Instruction::Sub, Sum, W, SCEV::FlagAnyWrap, + /*IsSafeToHoist*/ true); ++I; } else { // A simple add. @@ -743,7 +751,8 @@ Value *SCEVExpander::visitAddExpr(const SCEVAddExpr *S) { Sum = InsertNoopCastOfTo(Sum, Ty); // Canonicalize a constant to the RHS. if (isa(Sum)) std::swap(Sum, W); - Sum = InsertBinop(Instruction::Add, Sum, W); + Sum = InsertBinop(Instruction::Add, Sum, W, S->getNoWrapFlags(), + /*IsSafeToHoist*/ true); ++I; } } @@ -762,7 +771,7 @@ Value *SCEVExpander::visitMulExpr(const SCEVMulExpr *S) { OpsAndLoops.push_back(std::make_pair(getRelevantLoop(*I), *I)); // Sort by loop. Use a stable sort so that constants follow non-constants. - std::stable_sort(OpsAndLoops.begin(), OpsAndLoops.end(), LoopCompare(SE.DT)); + llvm::stable_sort(OpsAndLoops, LoopCompare(SE.DT)); // Emit instructions to mul all the operands. Hoist as much as possible // out of loops. @@ -795,9 +804,13 @@ Value *SCEVExpander::visitMulExpr(const SCEVMulExpr *S) { if (Exponent & 1) Result = P; for (uint64_t BinExp = 2; BinExp <= Exponent; BinExp <<= 1) { - P = InsertBinop(Instruction::Mul, P, P); + P = InsertBinop(Instruction::Mul, P, P, SCEV::FlagAnyWrap, + /*IsSafeToHoist*/ true); if (Exponent & BinExp) - Result = Result ? InsertBinop(Instruction::Mul, Result, P) : P; + Result = Result ? InsertBinop(Instruction::Mul, Result, P, + SCEV::FlagAnyWrap, + /*IsSafeToHoist*/ true) + : P; } I = E; @@ -812,7 +825,8 @@ Value *SCEVExpander::visitMulExpr(const SCEVMulExpr *S) { } else if (I->second->isAllOnesValue()) { // Instead of doing a multiply by negative one, just do a negate. Prod = InsertNoopCastOfTo(Prod, Ty); - Prod = InsertBinop(Instruction::Sub, Constant::getNullValue(Ty), Prod); + Prod = InsertBinop(Instruction::Sub, Constant::getNullValue(Ty), Prod, + SCEV::FlagAnyWrap, /*IsSafeToHoist*/ true); ++I; } else { // A simple mul. @@ -824,10 +838,16 @@ Value *SCEVExpander::visitMulExpr(const SCEVMulExpr *S) { if (match(W, m_Power2(RHS))) { // Canonicalize Prod*(1<isVectorTy() && "vector types are not SCEVable"); + auto NWFlags = S->getNoWrapFlags(); + // clear nsw flag if shl will produce poison value. + if (RHS->logBase2() == RHS->getBitWidth() - 1) + NWFlags = ScalarEvolution::clearFlags(NWFlags, SCEV::FlagNSW); Prod = InsertBinop(Instruction::Shl, Prod, - ConstantInt::get(Ty, RHS->logBase2())); + ConstantInt::get(Ty, RHS->logBase2()), NWFlags, + /*IsSafeToHoist*/ true); } else { - Prod = InsertBinop(Instruction::Mul, Prod, W); + Prod = InsertBinop(Instruction::Mul, Prod, W, S->getNoWrapFlags(), + /*IsSafeToHoist*/ true); } } } @@ -843,11 +863,13 @@ Value *SCEVExpander::visitUDivExpr(const SCEVUDivExpr *S) { const APInt &RHS = SC->getAPInt(); if (RHS.isPowerOf2()) return InsertBinop(Instruction::LShr, LHS, - ConstantInt::get(Ty, RHS.logBase2())); + ConstantInt::get(Ty, RHS.logBase2()), + SCEV::FlagAnyWrap, /*IsSafeToHoist*/ true); } Value *RHS = expandCodeFor(S->getRHS(), Ty); - return InsertBinop(Instruction::UDiv, LHS, RHS); + return InsertBinop(Instruction::UDiv, LHS, RHS, SCEV::FlagAnyWrap, + /*IsSafeToHoist*/ SE.isKnownNonZero(S->getRHS())); } /// Move parts of Base into Rest to leave Base with the minimal @@ -1634,7 +1656,8 @@ Value *SCEVExpander::visitSMaxExpr(const SCEVSMaxExpr *S) { for (int i = S->getNumOperands()-2; i >= 0; --i) { // In the case of mixed integer and pointer types, do the // rest of the comparisons as integer. - if (S->getOperand(i)->getType() != Ty) { + Type *OpTy = S->getOperand(i)->getType(); + if (OpTy->isIntegerTy() != Ty->isIntegerTy()) { Ty = SE.getEffectiveSCEVType(Ty); LHS = InsertNoopCastOfTo(LHS, Ty); } @@ -1658,7 +1681,8 @@ Value *SCEVExpander::visitUMaxExpr(const SCEVUMaxExpr *S) { for (int i = S->getNumOperands()-2; i >= 0; --i) { // In the case of mixed integer and pointer types, do the // rest of the comparisons as integer. - if (S->getOperand(i)->getType() != Ty) { + Type *OpTy = S->getOperand(i)->getType(); + if (OpTy->isIntegerTy() != Ty->isIntegerTy()) { Ty = SE.getEffectiveSCEVType(Ty); LHS = InsertNoopCastOfTo(LHS, Ty); } @@ -1676,6 +1700,56 @@ Value *SCEVExpander::visitUMaxExpr(const SCEVUMaxExpr *S) { return LHS; } +Value *SCEVExpander::visitSMinExpr(const SCEVSMinExpr *S) { + Value *LHS = expand(S->getOperand(S->getNumOperands() - 1)); + Type *Ty = LHS->getType(); + for (int i = S->getNumOperands() - 2; i >= 0; --i) { + // In the case of mixed integer and pointer types, do the + // rest of the comparisons as integer. + Type *OpTy = S->getOperand(i)->getType(); + if (OpTy->isIntegerTy() != Ty->isIntegerTy()) { + Ty = SE.getEffectiveSCEVType(Ty); + LHS = InsertNoopCastOfTo(LHS, Ty); + } + Value *RHS = expandCodeFor(S->getOperand(i), Ty); + Value *ICmp = Builder.CreateICmpSLT(LHS, RHS); + rememberInstruction(ICmp); + Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "smin"); + rememberInstruction(Sel); + LHS = Sel; + } + // In the case of mixed integer and pointer types, cast the + // final result back to the pointer type. + if (LHS->getType() != S->getType()) + LHS = InsertNoopCastOfTo(LHS, S->getType()); + return LHS; +} + +Value *SCEVExpander::visitUMinExpr(const SCEVUMinExpr *S) { + Value *LHS = expand(S->getOperand(S->getNumOperands() - 1)); + Type *Ty = LHS->getType(); + for (int i = S->getNumOperands() - 2; i >= 0; --i) { + // In the case of mixed integer and pointer types, do the + // rest of the comparisons as integer. + Type *OpTy = S->getOperand(i)->getType(); + if (OpTy->isIntegerTy() != Ty->isIntegerTy()) { + Ty = SE.getEffectiveSCEVType(Ty); + LHS = InsertNoopCastOfTo(LHS, Ty); + } + Value *RHS = expandCodeFor(S->getOperand(i), Ty); + Value *ICmp = Builder.CreateICmpULT(LHS, RHS); + rememberInstruction(ICmp); + Value *Sel = Builder.CreateSelect(ICmp, LHS, RHS, "umin"); + rememberInstruction(Sel); + LHS = Sel; + } + // In the case of mixed integer and pointer types, cast the + // final result back to the pointer type. + if (LHS->getType() != S->getType()) + LHS = InsertNoopCastOfTo(LHS, S->getType()); + return LHS; +} + Value *SCEVExpander::expandCodeFor(const SCEV *SH, Type *Ty, Instruction *IP) { setInsertPoint(IP); @@ -1732,49 +1806,55 @@ Value *SCEVExpander::expand(const SCEV *S) { // Compute an insertion point for this SCEV object. Hoist the instructions // as far out in the loop nest as possible. Instruction *InsertPt = &*Builder.GetInsertPoint(); - for (Loop *L = SE.LI.getLoopFor(Builder.GetInsertBlock());; - L = L->getParentLoop()) - if (SE.isLoopInvariant(S, L)) { - if (!L) break; - if (BasicBlock *Preheader = L->getLoopPreheader()) - InsertPt = Preheader->getTerminator(); - else { - // LSR sets the insertion point for AddRec start/step values to the - // block start to simplify value reuse, even though it's an invalid - // position. SCEVExpander must correct for this in all cases. - InsertPt = &*L->getHeader()->getFirstInsertionPt(); - } - } else { - // We can move insertion point only if there is no div or rem operations - // otherwise we are risky to move it over the check for zero denominator. - auto SafeToHoist = [](const SCEV *S) { - return !SCEVExprContains(S, [](const SCEV *S) { - if (const auto *D = dyn_cast(S)) { - if (const auto *SC = dyn_cast(D->getRHS())) - // Division by non-zero constants can be hoisted. - return SC->getValue()->isZero(); - // All other divisions should not be moved as they may be - // divisions by zero and should be kept within the - // conditions of the surrounding loops that guard their - // execution (see PR35406). - return true; - } - return false; - }); - }; - // If the SCEV is computable at this level, insert it into the header - // after the PHIs (and after any other instructions that we've inserted - // there) so that it is guaranteed to dominate any user inside the loop. - if (L && SE.hasComputableLoopEvolution(S, L) && !PostIncLoops.count(L) && - SafeToHoist(S)) - InsertPt = &*L->getHeader()->getFirstInsertionPt(); - while (InsertPt->getIterator() != Builder.GetInsertPoint() && - (isInsertedInstruction(InsertPt) || - isa(InsertPt))) { - InsertPt = &*std::next(InsertPt->getIterator()); + + // We can move insertion point only if there is no div or rem operations + // otherwise we are risky to move it over the check for zero denominator. + auto SafeToHoist = [](const SCEV *S) { + return !SCEVExprContains(S, [](const SCEV *S) { + if (const auto *D = dyn_cast(S)) { + if (const auto *SC = dyn_cast(D->getRHS())) + // Division by non-zero constants can be hoisted. + return SC->getValue()->isZero(); + // All other divisions should not be moved as they may be + // divisions by zero and should be kept within the + // conditions of the surrounding loops that guard their + // execution (see PR35406). + return true; + } + return false; + }); + }; + if (SafeToHoist(S)) { + for (Loop *L = SE.LI.getLoopFor(Builder.GetInsertBlock());; + L = L->getParentLoop()) { + if (SE.isLoopInvariant(S, L)) { + if (!L) break; + if (BasicBlock *Preheader = L->getLoopPreheader()) + InsertPt = Preheader->getTerminator(); + else + // LSR sets the insertion point for AddRec start/step values to the + // block start to simplify value reuse, even though it's an invalid + // position. SCEVExpander must correct for this in all cases. + InsertPt = &*L->getHeader()->getFirstInsertionPt(); + } else { + // If the SCEV is computable at this level, insert it into the header + // after the PHIs (and after any other instructions that we've inserted + // there) so that it is guaranteed to dominate any user inside the loop. + if (L && SE.hasComputableLoopEvolution(S, L) && !PostIncLoops.count(L)) + InsertPt = &*L->getHeader()->getFirstInsertionPt(); + while (InsertPt->getIterator() != Builder.GetInsertPoint() && + (isInsertedInstruction(InsertPt) || + isa(InsertPt))) + InsertPt = &*std::next(InsertPt->getIterator()); + break; } - break; } + } + + // IndVarSimplify sometimes sets the insertion point at the block start, even + // when there are PHIs at that point. We must correct for this. + if (isa(*InsertPt)) + InsertPt = &*InsertPt->getParent()->getFirstInsertionPt(); // Check to see if we already expanded this here. auto I = InsertedExpressions.find(std::make_pair(S, InsertPt)); @@ -2071,10 +2151,13 @@ bool SCEVExpander::isHighCostExpansionHelper( if (auto *UDivExpr = dyn_cast(S)) { // If the divisor is a power of two and the SCEV type fits in a native - // integer, consider the division cheap irrespective of whether it occurs in - // the user code since it can be lowered into a right shift. + // integer (and the LHS not expensive), consider the division cheap + // irrespective of whether it occurs in the user code since it can be + // lowered into a right shift. if (auto *SC = dyn_cast(UDivExpr->getRHS())) if (SC->getAPInt().isPowerOf2()) { + if (isHighCostExpansionHelper(UDivExpr->getLHS(), L, At, Processed)) + return true; const DataLayout &DL = L->getHeader()->getParent()->getParent()->getDataLayout(); unsigned Width = cast(UDivExpr->getType())->getBitWidth(); @@ -2102,7 +2185,7 @@ bool SCEVExpander::isHighCostExpansionHelper( // HowManyLessThans uses a Max expression whenever the loop is not guarded by // the exit condition. - if (isa(S) || isa(S)) + if (isa(S)) return true; // Recurse past nary expressions, which commonly occur in the @@ -2339,6 +2422,24 @@ bool isSafeToExpand(const SCEV *S, ScalarEvolution &SE) { bool isSafeToExpandAt(const SCEV *S, const Instruction *InsertionPoint, ScalarEvolution &SE) { - return isSafeToExpand(S, SE) && SE.dominates(S, InsertionPoint->getParent()); + if (!isSafeToExpand(S, SE)) + return false; + // We have to prove that the expanded site of S dominates InsertionPoint. + // This is easy when not in the same block, but hard when S is an instruction + // to be expanded somewhere inside the same block as our insertion point. + // What we really need here is something analogous to an OrderedBasicBlock, + // but for the moment, we paper over the problem by handling two common and + // cheap to check cases. + if (SE.properlyDominates(S, InsertionPoint->getParent())) + return true; + if (SE.dominates(S, InsertionPoint->getParent())) { + if (InsertionPoint->getParent()->getTerminator() == InsertionPoint) + return true; + if (const SCEVUnknown *U = dyn_cast(S)) + for (const Value *V : InsertionPoint->operand_values()) + if (V == U->getValue()) + return true; + } + return false; } } diff --git a/lib/Analysis/ScalarEvolutionNormalization.cpp b/lib/Analysis/ScalarEvolutionNormalization.cpp index 3740039b8f86..209ae66ca53e 100644 --- a/lib/Analysis/ScalarEvolutionNormalization.cpp +++ b/lib/Analysis/ScalarEvolutionNormalization.cpp @@ -1,9 +1,8 @@ //===- ScalarEvolutionNormalization.cpp - See below -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/ScopedNoAliasAA.cpp b/lib/Analysis/ScopedNoAliasAA.cpp index 9a581fe46afc..094e4a3d5dc8 100644 --- a/lib/Analysis/ScopedNoAliasAA.cpp +++ b/lib/Analysis/ScopedNoAliasAA.cpp @@ -1,9 +1,8 @@ //===- ScopedNoAliasAA.cpp - Scoped No-Alias Alias Analysis ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -76,9 +75,10 @@ public: } // end anonymous namespace AliasResult ScopedNoAliasAAResult::alias(const MemoryLocation &LocA, - const MemoryLocation &LocB) { + const MemoryLocation &LocB, + AAQueryInfo &AAQI) { if (!EnableScopedNoAlias) - return AAResultBase::alias(LocA, LocB); + return AAResultBase::alias(LocA, LocB, AAQI); // Get the attached MDNodes. const MDNode *AScopes = LocA.AATags.Scope, *BScopes = LocB.AATags.Scope; @@ -92,13 +92,14 @@ AliasResult ScopedNoAliasAAResult::alias(const MemoryLocation &LocA, return NoAlias; // If they may alias, chain to the next AliasAnalysis. - return AAResultBase::alias(LocA, LocB); + return AAResultBase::alias(LocA, LocB, AAQI); } ModRefInfo ScopedNoAliasAAResult::getModRefInfo(const CallBase *Call, - const MemoryLocation &Loc) { + const MemoryLocation &Loc, + AAQueryInfo &AAQI) { if (!EnableScopedNoAlias) - return AAResultBase::getModRefInfo(Call, Loc); + return AAResultBase::getModRefInfo(Call, Loc, AAQI); if (!mayAliasInScopes(Loc.AATags.Scope, Call->getMetadata(LLVMContext::MD_noalias))) @@ -108,13 +109,14 @@ ModRefInfo ScopedNoAliasAAResult::getModRefInfo(const CallBase *Call, Loc.AATags.NoAlias)) return ModRefInfo::NoModRef; - return AAResultBase::getModRefInfo(Call, Loc); + return AAResultBase::getModRefInfo(Call, Loc, AAQI); } ModRefInfo ScopedNoAliasAAResult::getModRefInfo(const CallBase *Call1, - const CallBase *Call2) { + const CallBase *Call2, + AAQueryInfo &AAQI) { if (!EnableScopedNoAlias) - return AAResultBase::getModRefInfo(Call1, Call2); + return AAResultBase::getModRefInfo(Call1, Call2, AAQI); if (!mayAliasInScopes(Call1->getMetadata(LLVMContext::MD_alias_scope), Call2->getMetadata(LLVMContext::MD_noalias))) @@ -124,7 +126,7 @@ ModRefInfo ScopedNoAliasAAResult::getModRefInfo(const CallBase *Call1, Call1->getMetadata(LLVMContext::MD_noalias))) return ModRefInfo::NoModRef; - return AAResultBase::getModRefInfo(Call1, Call2); + return AAResultBase::getModRefInfo(Call1, Call2, AAQI); } static void collectMDInDomain(const MDNode *List, const MDNode *Domain, diff --git a/lib/Analysis/StackSafetyAnalysis.cpp b/lib/Analysis/StackSafetyAnalysis.cpp index 66b03845864f..4cf235db86eb 100644 --- a/lib/Analysis/StackSafetyAnalysis.cpp +++ b/lib/Analysis/StackSafetyAnalysis.cpp @@ -1,9 +1,8 @@ //===- StackSafetyAnalysis.cpp - Stack memory safety analysis -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -416,7 +415,9 @@ class StackSafetyDataFlowAnalysis { updateOneNode(F.first, F.second); } void runDataFlow(); +#ifndef NDEBUG void verifyFixedPoint(); +#endif public: StackSafetyDataFlowAnalysis( @@ -527,11 +528,13 @@ void StackSafetyDataFlowAnalysis::runDataFlow() { } } +#ifndef NDEBUG void StackSafetyDataFlowAnalysis::verifyFixedPoint() { WorkList.clear(); updateAllNodes(); assert(WorkList.empty()); } +#endif StackSafetyGlobalInfo StackSafetyDataFlowAnalysis::run() { runDataFlow(); diff --git a/lib/Analysis/StratifiedSets.h b/lib/Analysis/StratifiedSets.h index 2f20cd12506c..60ea2451b0ef 100644 --- a/lib/Analysis/StratifiedSets.h +++ b/lib/Analysis/StratifiedSets.h @@ -1,9 +1,8 @@ //===- StratifiedSets.h - Abstract stratified sets implementation. --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/Analysis/SyncDependenceAnalysis.cpp b/lib/Analysis/SyncDependenceAnalysis.cpp index e1a7e4476d12..3cf248a31142 100644 --- a/lib/Analysis/SyncDependenceAnalysis.cpp +++ b/lib/Analysis/SyncDependenceAnalysis.cpp @@ -1,10 +1,9 @@ //===- SyncDependenceAnalysis.cpp - Divergent Branch Dependence Calculation //--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -219,14 +218,9 @@ struct DivergencePropagator { template std::unique_ptr computeJoinPoints(const BasicBlock &RootBlock, - SuccessorIterable NodeSuccessors, const Loop *ParentLoop) { + SuccessorIterable NodeSuccessors, const Loop *ParentLoop, const BasicBlock * PdBoundBlock) { assert(JoinBlocks); - // immediate post dominator (no join block beyond that block) - const auto *PdNode = PDT.getNode(const_cast(&RootBlock)); - const auto *IpdNode = PdNode->getIDom(); - const auto *PdBoundBlock = IpdNode ? IpdNode->getBlock() : nullptr; - // bootstrap with branch targets for (const auto *SuccBlock : NodeSuccessors) { DefMap.emplace(SuccBlock, SuccBlock); @@ -341,13 +335,23 @@ const ConstBlockSet &SyncDependenceAnalysis::join_blocks(const Loop &Loop) { // already available in cache? auto ItCached = CachedLoopExitJoins.find(&Loop); - if (ItCached != CachedLoopExitJoins.end()) + if (ItCached != CachedLoopExitJoins.end()) { return *ItCached->second; + } + + // dont propagte beyond the immediate post dom of the loop + const auto *PdNode = PDT.getNode(const_cast(Loop.getHeader())); + const auto *IpdNode = PdNode->getIDom(); + const auto *PdBoundBlock = IpdNode ? IpdNode->getBlock() : nullptr; + while (PdBoundBlock && Loop.contains(PdBoundBlock)) { + IpdNode = IpdNode->getIDom(); + PdBoundBlock = IpdNode ? IpdNode->getBlock() : nullptr; + } // compute all join points DivergencePropagator Propagator{FuncRPOT, DT, PDT, LI}; auto JoinBlocks = Propagator.computeJoinPoints( - *Loop.getHeader(), LoopExits, Loop.getParentLoop()); + *Loop.getHeader(), LoopExits, Loop.getParentLoop(), PdBoundBlock); auto ItInserted = CachedLoopExitJoins.emplace(&Loop, std::move(JoinBlocks)); assert(ItInserted.second); @@ -366,11 +370,16 @@ SyncDependenceAnalysis::join_blocks(const Instruction &Term) { if (ItCached != CachedBranchJoins.end()) return *ItCached->second; + // dont propagate beyond the immediate post dominator of the branch + const auto *PdNode = PDT.getNode(const_cast(Term.getParent())); + const auto *IpdNode = PdNode->getIDom(); + const auto *PdBoundBlock = IpdNode ? IpdNode->getBlock() : nullptr; + // compute all join points DivergencePropagator Propagator{FuncRPOT, DT, PDT, LI}; const auto &TermBlock = *Term.getParent(); auto JoinBlocks = Propagator.computeJoinPoints( - TermBlock, successors(Term.getParent()), LI.getLoopFor(&TermBlock)); + TermBlock, successors(Term.getParent()), LI.getLoopFor(&TermBlock), PdBoundBlock); auto ItInserted = CachedBranchJoins.emplace(&Term, std::move(JoinBlocks)); assert(ItInserted.second); diff --git a/lib/Analysis/SyntheticCountsUtils.cpp b/lib/Analysis/SyntheticCountsUtils.cpp index c2d7bb11a4cf..22766e5f07f5 100644 --- a/lib/Analysis/SyntheticCountsUtils.cpp +++ b/lib/Analysis/SyntheticCountsUtils.cpp @@ -1,9 +1,8 @@ //===--- SyntheticCountsUtils.cpp - synthetic counts propagation utils ---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/TargetLibraryInfo.cpp b/lib/Analysis/TargetLibraryInfo.cpp index 4643f75da42d..ef139d3257d2 100644 --- a/lib/Analysis/TargetLibraryInfo.cpp +++ b/lib/Analysis/TargetLibraryInfo.cpp @@ -1,9 +1,8 @@ //===-- TargetLibraryInfo.cpp - Runtime library information ----------------==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -24,6 +23,8 @@ static cl::opt ClVectorLibrary( "No vector functions library"), clEnumValN(TargetLibraryInfoImpl::Accelerate, "Accelerate", "Accelerate framework"), + clEnumValN(TargetLibraryInfoImpl::MASSV, "MASSV", + "IBM MASS vector library"), clEnumValN(TargetLibraryInfoImpl::SVML, "SVML", "Intel SVML library"))); @@ -50,6 +51,16 @@ static bool hasSinCosPiStret(const Triple &T) { return true; } +static bool hasBcmp(const Triple &TT) { + // Posix removed support from bcmp() in 2001, but the glibc and several + // implementations of the libc still have it. + if (TT.isOSLinux()) + return TT.isGNUEnvironment() || TT.isMusl(); + // Both NetBSD and OpenBSD are planning to remove the function. Windows does + // not have it. + return TT.isOSFreeBSD() || TT.isOSSolaris() || TT.isOSDarwin(); +} + /// Initialize the set of available library functions based on the specified /// target triple. This should be carefully written so that a missing target /// triple gets a sane set of defaults. @@ -78,8 +89,8 @@ static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, ShouldSignExtI32Param = false; // PowerPC64, Sparc64, SystemZ need signext/zeroext on i32 parameters and // returns corresponding to C-level ints and unsigned ints. - if (T.getArch() == Triple::ppc64 || T.getArch() == Triple::ppc64le || - T.getArch() == Triple::sparcv9 || T.getArch() == Triple::systemz) { + if (T.isPPC64() || T.getArch() == Triple::sparcv9 || + T.getArch() == Triple::systemz) { ShouldExtI32Param = true; ShouldExtI32Return = true; } @@ -142,6 +153,9 @@ static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, TLI.setUnavailable(LibFunc_sincospif_stret); } + if (!hasBcmp(T)) + TLI.setUnavailable(LibFunc_bcmp); + if (T.isMacOSX() && T.getArch() == Triple::x86 && !T.isMacOSXVersionLT(10, 7)) { // x86-32 OSX has a scheme where fwrite and fputs (and some other functions @@ -153,33 +167,82 @@ static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, TLI.setAvailableWithName(LibFunc_fputs, "fputs$UNIX2003"); } - // iprintf and friends are only available on XCore and TCE. - if (T.getArch() != Triple::xcore && T.getArch() != Triple::tce) { + // iprintf and friends are only available on XCore, TCE, and Emscripten. + if (T.getArch() != Triple::xcore && T.getArch() != Triple::tce && + T.getOS() != Triple::Emscripten) { TLI.setUnavailable(LibFunc_iprintf); TLI.setUnavailable(LibFunc_siprintf); TLI.setUnavailable(LibFunc_fiprintf); } + // __small_printf and friends are only available on Emscripten. + if (T.getOS() != Triple::Emscripten) { + TLI.setUnavailable(LibFunc_small_printf); + TLI.setUnavailable(LibFunc_small_sprintf); + TLI.setUnavailable(LibFunc_small_fprintf); + } + if (T.isOSWindows() && !T.isOSCygMing()) { - // Win32 does not support long double + // XXX: The earliest documentation available at the moment is for VS2015/VC19: + // https://docs.microsoft.com/en-us/cpp/c-runtime-library/floating-point-support?view=vs-2015 + // XXX: In order to use an MSVCRT older than VC19, + // the specific library version must be explicit in the target triple, + // e.g., x86_64-pc-windows-msvc18. + bool hasPartialC99 = true; + if (T.isKnownWindowsMSVCEnvironment()) { + unsigned Major, Minor, Micro; + T.getEnvironmentVersion(Major, Minor, Micro); + hasPartialC99 = (Major == 0 || Major >= 19); + } + + // Latest targets support C89 math functions, in part. + bool isARM = (T.getArch() == Triple::aarch64 || + T.getArch() == Triple::arm); + bool hasPartialFloat = (isARM || + T.getArch() == Triple::x86_64); + + // Win32 does not support float C89 math functions, in general. + if (!hasPartialFloat) { + TLI.setUnavailable(LibFunc_acosf); + TLI.setUnavailable(LibFunc_asinf); + TLI.setUnavailable(LibFunc_atan2f); + TLI.setUnavailable(LibFunc_atanf); + TLI.setUnavailable(LibFunc_ceilf); + TLI.setUnavailable(LibFunc_cosf); + TLI.setUnavailable(LibFunc_coshf); + TLI.setUnavailable(LibFunc_expf); + TLI.setUnavailable(LibFunc_floorf); + TLI.setUnavailable(LibFunc_fmodf); + TLI.setUnavailable(LibFunc_log10f); + TLI.setUnavailable(LibFunc_logf); + TLI.setUnavailable(LibFunc_modff); + TLI.setUnavailable(LibFunc_powf); + TLI.setUnavailable(LibFunc_sinf); + TLI.setUnavailable(LibFunc_sinhf); + TLI.setUnavailable(LibFunc_sqrtf); + TLI.setUnavailable(LibFunc_tanf); + TLI.setUnavailable(LibFunc_tanhf); + } + if (!isARM) + TLI.setUnavailable(LibFunc_fabsf); + TLI.setUnavailable(LibFunc_frexpf); + TLI.setUnavailable(LibFunc_ldexpf); + + // Win32 does not support long double C89 math functions. TLI.setUnavailable(LibFunc_acosl); TLI.setUnavailable(LibFunc_asinl); - TLI.setUnavailable(LibFunc_atanl); TLI.setUnavailable(LibFunc_atan2l); + TLI.setUnavailable(LibFunc_atanl); TLI.setUnavailable(LibFunc_ceill); - TLI.setUnavailable(LibFunc_copysignl); TLI.setUnavailable(LibFunc_cosl); TLI.setUnavailable(LibFunc_coshl); TLI.setUnavailable(LibFunc_expl); - TLI.setUnavailable(LibFunc_fabsf); // Win32 and Win64 both lack fabsf TLI.setUnavailable(LibFunc_fabsl); TLI.setUnavailable(LibFunc_floorl); - TLI.setUnavailable(LibFunc_fmaxl); - TLI.setUnavailable(LibFunc_fminl); TLI.setUnavailable(LibFunc_fmodl); TLI.setUnavailable(LibFunc_frexpl); - TLI.setUnavailable(LibFunc_ldexpf); TLI.setUnavailable(LibFunc_ldexpl); + TLI.setUnavailable(LibFunc_log10l); TLI.setUnavailable(LibFunc_logl); TLI.setUnavailable(LibFunc_modfl); TLI.setUnavailable(LibFunc_powl); @@ -189,81 +252,66 @@ static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, TLI.setUnavailable(LibFunc_tanl); TLI.setUnavailable(LibFunc_tanhl); - // Win32 only has C89 math - TLI.setUnavailable(LibFunc_acosh); - TLI.setUnavailable(LibFunc_acoshf); + // Win32 does not fully support C99 math functions. + if (!hasPartialC99) { + TLI.setUnavailable(LibFunc_acosh); + TLI.setUnavailable(LibFunc_acoshf); + TLI.setUnavailable(LibFunc_asinh); + TLI.setUnavailable(LibFunc_asinhf); + TLI.setUnavailable(LibFunc_atanh); + TLI.setUnavailable(LibFunc_atanhf); + TLI.setAvailableWithName(LibFunc_cabs, "_cabs"); + TLI.setUnavailable(LibFunc_cabsf); + TLI.setUnavailable(LibFunc_cbrt); + TLI.setUnavailable(LibFunc_cbrtf); + TLI.setAvailableWithName(LibFunc_copysign, "_copysign"); + TLI.setAvailableWithName(LibFunc_copysignf, "_copysignf"); + TLI.setUnavailable(LibFunc_exp2); + TLI.setUnavailable(LibFunc_exp2f); + TLI.setUnavailable(LibFunc_expm1); + TLI.setUnavailable(LibFunc_expm1f); + TLI.setUnavailable(LibFunc_fmax); + TLI.setUnavailable(LibFunc_fmaxf); + TLI.setUnavailable(LibFunc_fmin); + TLI.setUnavailable(LibFunc_fminf); + TLI.setUnavailable(LibFunc_log1p); + TLI.setUnavailable(LibFunc_log1pf); + TLI.setUnavailable(LibFunc_log2); + TLI.setUnavailable(LibFunc_log2f); + TLI.setAvailableWithName(LibFunc_logb, "_logb"); + if (hasPartialFloat) + TLI.setAvailableWithName(LibFunc_logbf, "_logbf"); + else + TLI.setUnavailable(LibFunc_logbf); + TLI.setUnavailable(LibFunc_rint); + TLI.setUnavailable(LibFunc_rintf); + TLI.setUnavailable(LibFunc_round); + TLI.setUnavailable(LibFunc_roundf); + TLI.setUnavailable(LibFunc_trunc); + TLI.setUnavailable(LibFunc_truncf); + } + + // Win32 does not support long double C99 math functions. TLI.setUnavailable(LibFunc_acoshl); - TLI.setUnavailable(LibFunc_asinh); - TLI.setUnavailable(LibFunc_asinhf); TLI.setUnavailable(LibFunc_asinhl); - TLI.setUnavailable(LibFunc_atanh); - TLI.setUnavailable(LibFunc_atanhf); TLI.setUnavailable(LibFunc_atanhl); - TLI.setUnavailable(LibFunc_cabs); - TLI.setUnavailable(LibFunc_cabsf); TLI.setUnavailable(LibFunc_cabsl); - TLI.setUnavailable(LibFunc_cbrt); - TLI.setUnavailable(LibFunc_cbrtf); TLI.setUnavailable(LibFunc_cbrtl); - TLI.setUnavailable(LibFunc_exp2); - TLI.setUnavailable(LibFunc_exp2f); + TLI.setUnavailable(LibFunc_copysignl); TLI.setUnavailable(LibFunc_exp2l); - TLI.setUnavailable(LibFunc_expm1); - TLI.setUnavailable(LibFunc_expm1f); TLI.setUnavailable(LibFunc_expm1l); - TLI.setUnavailable(LibFunc_log2); - TLI.setUnavailable(LibFunc_log2f); - TLI.setUnavailable(LibFunc_log2l); - TLI.setUnavailable(LibFunc_log1p); - TLI.setUnavailable(LibFunc_log1pf); + TLI.setUnavailable(LibFunc_fmaxl); + TLI.setUnavailable(LibFunc_fminl); TLI.setUnavailable(LibFunc_log1pl); - TLI.setUnavailable(LibFunc_logb); - TLI.setUnavailable(LibFunc_logbf); + TLI.setUnavailable(LibFunc_log2l); TLI.setUnavailable(LibFunc_logbl); - TLI.setUnavailable(LibFunc_nearbyint); - TLI.setUnavailable(LibFunc_nearbyintf); TLI.setUnavailable(LibFunc_nearbyintl); - TLI.setUnavailable(LibFunc_rint); - TLI.setUnavailable(LibFunc_rintf); TLI.setUnavailable(LibFunc_rintl); - TLI.setUnavailable(LibFunc_round); - TLI.setUnavailable(LibFunc_roundf); TLI.setUnavailable(LibFunc_roundl); - TLI.setUnavailable(LibFunc_trunc); - TLI.setUnavailable(LibFunc_truncf); TLI.setUnavailable(LibFunc_truncl); - // Win32 provides some C99 math with mangled names - TLI.setAvailableWithName(LibFunc_copysign, "_copysign"); - - if (T.getArch() == Triple::x86) { - // Win32 on x86 implements single-precision math functions as macros - TLI.setUnavailable(LibFunc_acosf); - TLI.setUnavailable(LibFunc_asinf); - TLI.setUnavailable(LibFunc_atanf); - TLI.setUnavailable(LibFunc_atan2f); - TLI.setUnavailable(LibFunc_ceilf); - TLI.setUnavailable(LibFunc_copysignf); - TLI.setUnavailable(LibFunc_cosf); - TLI.setUnavailable(LibFunc_coshf); - TLI.setUnavailable(LibFunc_expf); - TLI.setUnavailable(LibFunc_floorf); - TLI.setUnavailable(LibFunc_fminf); - TLI.setUnavailable(LibFunc_fmaxf); - TLI.setUnavailable(LibFunc_fmodf); - TLI.setUnavailable(LibFunc_logf); - TLI.setUnavailable(LibFunc_log10f); - TLI.setUnavailable(LibFunc_modff); - TLI.setUnavailable(LibFunc_powf); - TLI.setUnavailable(LibFunc_sinf); - TLI.setUnavailable(LibFunc_sinhf); - TLI.setUnavailable(LibFunc_sqrtf); - TLI.setUnavailable(LibFunc_tanf); - TLI.setUnavailable(LibFunc_tanhf); - } - - // Win32 does *not* provide these functions, but they are - // generally available on POSIX-compliant systems: + // Win32 does not support these functions, but + // they are generally available on POSIX-compliant systems. TLI.setUnavailable(LibFunc_access); TLI.setUnavailable(LibFunc_bcmp); TLI.setUnavailable(LibFunc_bcopy); @@ -318,12 +366,6 @@ static void initialize(TargetLibraryInfoImpl &TLI, const Triple &T, TLI.setUnavailable(LibFunc_utime); TLI.setUnavailable(LibFunc_utimes); TLI.setUnavailable(LibFunc_write); - - // Win32 does *not* provide provide these functions, but they are - // specified by C99: - TLI.setUnavailable(LibFunc_atoll); - TLI.setUnavailable(LibFunc_frexpf); - TLI.setUnavailable(LibFunc_llabs); } switch (T.getOS()) { @@ -651,11 +693,21 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy, return ((NumParams == 2 || NumParams == 3) && FTy.getParamType(0)->isPointerTy() && FTy.getParamType(1)->isPointerTy()); + case LibFunc_strcat_chk: + --NumParams; + if (!IsSizeTTy(FTy.getParamType(NumParams))) + return false; + LLVM_FALLTHROUGH; case LibFunc_strcat: return (NumParams == 2 && FTy.getReturnType()->isPointerTy() && FTy.getParamType(0) == FTy.getReturnType() && FTy.getParamType(1) == FTy.getReturnType()); + case LibFunc_strncat_chk: + --NumParams; + if (!IsSizeTTy(FTy.getParamType(NumParams))) + return false; + LLVM_FALLTHROUGH; case LibFunc_strncat: return (NumParams == 3 && FTy.getReturnType()->isPointerTy() && FTy.getParamType(0) == FTy.getReturnType() && @@ -674,6 +726,19 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy, FTy.getParamType(0) == FTy.getParamType(1) && FTy.getParamType(0) == PCharTy); + case LibFunc_strlcat_chk: + case LibFunc_strlcpy_chk: + --NumParams; + if (!IsSizeTTy(FTy.getParamType(NumParams))) + return false; + LLVM_FALLTHROUGH; + case LibFunc_strlcat: + case LibFunc_strlcpy: + return NumParams == 3 && IsSizeTTy(FTy.getReturnType()) && + FTy.getParamType(0)->isPointerTy() && + FTy.getParamType(1)->isPointerTy() && + IsSizeTTy(FTy.getParamType(2)); + case LibFunc_strncpy_chk: case LibFunc_stpncpy_chk: --NumParams; @@ -739,14 +804,32 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy, case LibFunc_stat: case LibFunc_statvfs: case LibFunc_siprintf: + case LibFunc_small_sprintf: case LibFunc_sprintf: return (NumParams >= 2 && FTy.getParamType(0)->isPointerTy() && FTy.getParamType(1)->isPointerTy() && FTy.getReturnType()->isIntegerTy(32)); + + case LibFunc_sprintf_chk: + return NumParams == 4 && FTy.getParamType(0)->isPointerTy() && + FTy.getParamType(1)->isIntegerTy(32) && + IsSizeTTy(FTy.getParamType(2)) && + FTy.getParamType(3)->isPointerTy() && + FTy.getReturnType()->isIntegerTy(32); + case LibFunc_snprintf: return (NumParams == 3 && FTy.getParamType(0)->isPointerTy() && FTy.getParamType(2)->isPointerTy() && FTy.getReturnType()->isIntegerTy(32)); + + case LibFunc_snprintf_chk: + return NumParams == 5 && FTy.getParamType(0)->isPointerTy() && + IsSizeTTy(FTy.getParamType(1)) && + FTy.getParamType(2)->isIntegerTy(32) && + IsSizeTTy(FTy.getParamType(3)) && + FTy.getParamType(4)->isPointerTy() && + FTy.getReturnType()->isIntegerTy(32); + case LibFunc_setitimer: return (NumParams == 3 && FTy.getParamType(1)->isPointerTy() && FTy.getParamType(2)->isPointerTy()); @@ -795,6 +878,11 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy, FTy.getParamType(1)->isIntegerTy() && IsSizeTTy(FTy.getParamType(2))); + case LibFunc_memccpy_chk: + --NumParams; + if (!IsSizeTTy(FTy.getParamType(NumParams))) + return false; + LLVM_FALLTHROUGH; case LibFunc_memccpy: return (NumParams >= 2 && FTy.getParamType(1)->isPointerTy()); case LibFunc_memalign: @@ -836,6 +924,7 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy, case LibFunc_getenv: case LibFunc_getpwnam: case LibFunc_iprintf: + case LibFunc_small_printf: case LibFunc_pclose: case LibFunc_perror: case LibFunc_printf: @@ -915,6 +1004,7 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy, FTy.getParamType(1)->isPointerTy()); case LibFunc_fscanf: case LibFunc_fiprintf: + case LibFunc_small_fprintf: case LibFunc_fprintf: return (NumParams >= 2 && FTy.getReturnType()->isIntegerTy() && FTy.getParamType(0)->isPointerTy() && @@ -961,9 +1051,17 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy, case LibFunc_vsprintf: return (NumParams == 3 && FTy.getParamType(0)->isPointerTy() && FTy.getParamType(1)->isPointerTy()); + case LibFunc_vsprintf_chk: + return NumParams == 5 && FTy.getParamType(0)->isPointerTy() && + FTy.getParamType(1)->isIntegerTy(32) && + IsSizeTTy(FTy.getParamType(2)) && FTy.getParamType(3)->isPointerTy(); case LibFunc_vsnprintf: return (NumParams == 4 && FTy.getParamType(0)->isPointerTy() && FTy.getParamType(2)->isPointerTy()); + case LibFunc_vsnprintf_chk: + return NumParams == 6 && FTy.getParamType(0)->isPointerTy() && + FTy.getParamType(2)->isIntegerTy(32) && + IsSizeTTy(FTy.getParamType(3)) && FTy.getParamType(4)->isPointerTy(); case LibFunc_open: return (NumParams >= 2 && FTy.getParamType(0)->isPointerTy()); case LibFunc_opendir: @@ -1391,6 +1489,11 @@ bool TargetLibraryInfoImpl::isValidProtoForLibFunc(const FunctionType &FTy, bool TargetLibraryInfoImpl::getLibFunc(const Function &FDecl, LibFunc &F) const { + // Intrinsics don't overlap w/libcalls; if our module has a large number of + // intrinsics, this ends up being an interesting compile time win since we + // avoid string normalization and comparison. + if (FDecl.isIntrinsic()) return false; + const DataLayout *DL = FDecl.getParent() ? &FDecl.getParent()->getDataLayout() : nullptr; return getLibFunc(FDecl.getName(), F) && @@ -1430,151 +1533,24 @@ void TargetLibraryInfoImpl::addVectorizableFunctionsFromVecLib( switch (VecLib) { case Accelerate: { const VecDesc VecFuncs[] = { - // Floating-Point Arithmetic and Auxiliary Functions - {"ceilf", "vceilf", 4}, - {"fabsf", "vfabsf", 4}, - {"llvm.fabs.f32", "vfabsf", 4}, - {"floorf", "vfloorf", 4}, - {"sqrtf", "vsqrtf", 4}, - {"llvm.sqrt.f32", "vsqrtf", 4}, - - // Exponential and Logarithmic Functions - {"expf", "vexpf", 4}, - {"llvm.exp.f32", "vexpf", 4}, - {"expm1f", "vexpm1f", 4}, - {"logf", "vlogf", 4}, - {"llvm.log.f32", "vlogf", 4}, - {"log1pf", "vlog1pf", 4}, - {"log10f", "vlog10f", 4}, - {"llvm.log10.f32", "vlog10f", 4}, - {"logbf", "vlogbf", 4}, - - // Trigonometric Functions - {"sinf", "vsinf", 4}, - {"llvm.sin.f32", "vsinf", 4}, - {"cosf", "vcosf", 4}, - {"llvm.cos.f32", "vcosf", 4}, - {"tanf", "vtanf", 4}, - {"asinf", "vasinf", 4}, - {"acosf", "vacosf", 4}, - {"atanf", "vatanf", 4}, - - // Hyperbolic Functions - {"sinhf", "vsinhf", 4}, - {"coshf", "vcoshf", 4}, - {"tanhf", "vtanhf", 4}, - {"asinhf", "vasinhf", 4}, - {"acoshf", "vacoshf", 4}, - {"atanhf", "vatanhf", 4}, + #define TLI_DEFINE_ACCELERATE_VECFUNCS + #include "llvm/Analysis/VecFuncs.def" + }; + addVectorizableFunctions(VecFuncs); + break; + } + case MASSV: { + const VecDesc VecFuncs[] = { + #define TLI_DEFINE_MASSV_VECFUNCS + #include "llvm/Analysis/VecFuncs.def" }; addVectorizableFunctions(VecFuncs); break; } case SVML: { const VecDesc VecFuncs[] = { - {"sin", "__svml_sin2", 2}, - {"sin", "__svml_sin4", 4}, - {"sin", "__svml_sin8", 8}, - - {"sinf", "__svml_sinf4", 4}, - {"sinf", "__svml_sinf8", 8}, - {"sinf", "__svml_sinf16", 16}, - - {"llvm.sin.f64", "__svml_sin2", 2}, - {"llvm.sin.f64", "__svml_sin4", 4}, - {"llvm.sin.f64", "__svml_sin8", 8}, - - {"llvm.sin.f32", "__svml_sinf4", 4}, - {"llvm.sin.f32", "__svml_sinf8", 8}, - {"llvm.sin.f32", "__svml_sinf16", 16}, - - {"cos", "__svml_cos2", 2}, - {"cos", "__svml_cos4", 4}, - {"cos", "__svml_cos8", 8}, - - {"cosf", "__svml_cosf4", 4}, - {"cosf", "__svml_cosf8", 8}, - {"cosf", "__svml_cosf16", 16}, - - {"llvm.cos.f64", "__svml_cos2", 2}, - {"llvm.cos.f64", "__svml_cos4", 4}, - {"llvm.cos.f64", "__svml_cos8", 8}, - - {"llvm.cos.f32", "__svml_cosf4", 4}, - {"llvm.cos.f32", "__svml_cosf8", 8}, - {"llvm.cos.f32", "__svml_cosf16", 16}, - - {"pow", "__svml_pow2", 2}, - {"pow", "__svml_pow4", 4}, - {"pow", "__svml_pow8", 8}, - - {"powf", "__svml_powf4", 4}, - {"powf", "__svml_powf8", 8}, - {"powf", "__svml_powf16", 16}, - - { "__pow_finite", "__svml_pow2", 2 }, - { "__pow_finite", "__svml_pow4", 4 }, - { "__pow_finite", "__svml_pow8", 8 }, - - { "__powf_finite", "__svml_powf4", 4 }, - { "__powf_finite", "__svml_powf8", 8 }, - { "__powf_finite", "__svml_powf16", 16 }, - - {"llvm.pow.f64", "__svml_pow2", 2}, - {"llvm.pow.f64", "__svml_pow4", 4}, - {"llvm.pow.f64", "__svml_pow8", 8}, - - {"llvm.pow.f32", "__svml_powf4", 4}, - {"llvm.pow.f32", "__svml_powf8", 8}, - {"llvm.pow.f32", "__svml_powf16", 16}, - - {"exp", "__svml_exp2", 2}, - {"exp", "__svml_exp4", 4}, - {"exp", "__svml_exp8", 8}, - - {"expf", "__svml_expf4", 4}, - {"expf", "__svml_expf8", 8}, - {"expf", "__svml_expf16", 16}, - - { "__exp_finite", "__svml_exp2", 2 }, - { "__exp_finite", "__svml_exp4", 4 }, - { "__exp_finite", "__svml_exp8", 8 }, - - { "__expf_finite", "__svml_expf4", 4 }, - { "__expf_finite", "__svml_expf8", 8 }, - { "__expf_finite", "__svml_expf16", 16 }, - - {"llvm.exp.f64", "__svml_exp2", 2}, - {"llvm.exp.f64", "__svml_exp4", 4}, - {"llvm.exp.f64", "__svml_exp8", 8}, - - {"llvm.exp.f32", "__svml_expf4", 4}, - {"llvm.exp.f32", "__svml_expf8", 8}, - {"llvm.exp.f32", "__svml_expf16", 16}, - - {"log", "__svml_log2", 2}, - {"log", "__svml_log4", 4}, - {"log", "__svml_log8", 8}, - - {"logf", "__svml_logf4", 4}, - {"logf", "__svml_logf8", 8}, - {"logf", "__svml_logf16", 16}, - - { "__log_finite", "__svml_log2", 2 }, - { "__log_finite", "__svml_log4", 4 }, - { "__log_finite", "__svml_log8", 8 }, - - { "__logf_finite", "__svml_logf4", 4 }, - { "__logf_finite", "__svml_logf8", 8 }, - { "__logf_finite", "__svml_logf16", 16 }, - - {"llvm.log.f64", "__svml_log2", 2}, - {"llvm.log.f64", "__svml_log4", 4}, - {"llvm.log.f64", "__svml_log8", 8}, - - {"llvm.log.f32", "__svml_logf4", 4}, - {"llvm.log.f32", "__svml_logf8", 8}, - {"llvm.log.f32", "__svml_logf16", 16}, + #define TLI_DEFINE_SVML_VECFUNCS + #include "llvm/Analysis/VecFuncs.def" }; addVectorizableFunctions(VecFuncs); break; @@ -1589,9 +1565,8 @@ bool TargetLibraryInfoImpl::isFunctionVectorizable(StringRef funcName) const { if (funcName.empty()) return false; - std::vector::const_iterator I = std::lower_bound( - VectorDescs.begin(), VectorDescs.end(), funcName, - compareWithScalarFnName); + std::vector::const_iterator I = + llvm::lower_bound(VectorDescs, funcName, compareWithScalarFnName); return I != VectorDescs.end() && StringRef(I->ScalarFnName) == funcName; } @@ -1600,8 +1575,8 @@ StringRef TargetLibraryInfoImpl::getVectorizedFunction(StringRef F, F = sanitizeFunctionName(F); if (F.empty()) return F; - std::vector::const_iterator I = std::lower_bound( - VectorDescs.begin(), VectorDescs.end(), F, compareWithScalarFnName); + std::vector::const_iterator I = + llvm::lower_bound(VectorDescs, F, compareWithScalarFnName); while (I != VectorDescs.end() && StringRef(I->ScalarFnName) == F) { if (I->VectorizationFactor == VF) return I->VectorFnName; @@ -1616,8 +1591,8 @@ StringRef TargetLibraryInfoImpl::getScalarizedFunction(StringRef F, if (F.empty()) return F; - std::vector::const_iterator I = std::lower_bound( - ScalarDescs.begin(), ScalarDescs.end(), F, compareWithVectorFnName); + std::vector::const_iterator I = + llvm::lower_bound(ScalarDescs, F, compareWithVectorFnName); if (I == VectorDescs.end() || StringRef(I->VectorFnName) != F) return StringRef(); VF = I->VectorizationFactor; diff --git a/lib/Analysis/TargetTransformInfo.cpp b/lib/Analysis/TargetTransformInfo.cpp index 9151d46c6cce..eb04c34453fb 100644 --- a/lib/Analysis/TargetTransformInfo.cpp +++ b/lib/Analysis/TargetTransformInfo.cpp @@ -1,9 +1,8 @@ //===- llvm/Analysis/TargetTransformInfo.cpp ------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -19,6 +18,8 @@ #include "llvm/IR/PatternMatch.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/ErrorHandling.h" +#include "llvm/Analysis/CFG.h" +#include "llvm/Analysis/LoopIterator.h" #include using namespace llvm; @@ -41,6 +42,101 @@ struct NoTTIImpl : TargetTransformInfoImplCRTPBase { }; } +bool HardwareLoopInfo::canAnalyze(LoopInfo &LI) { + // If the loop has irreducible control flow, it can not be converted to + // Hardware loop. + LoopBlocksRPO RPOT(L); + RPOT.perform(&LI); + if (containsIrreducibleCFG(RPOT, LI)) + return false; + return true; +} + +bool HardwareLoopInfo::isHardwareLoopCandidate(ScalarEvolution &SE, + LoopInfo &LI, DominatorTree &DT, + bool ForceNestedLoop, + bool ForceHardwareLoopPHI) { + SmallVector ExitingBlocks; + L->getExitingBlocks(ExitingBlocks); + + for (SmallVectorImpl::iterator I = ExitingBlocks.begin(), + IE = ExitingBlocks.end(); + I != IE; ++I) { + BasicBlock *BB = *I; + + // If we pass the updated counter back through a phi, we need to know + // which latch the updated value will be coming from. + if (!L->isLoopLatch(BB)) { + if (ForceHardwareLoopPHI || CounterInReg) + continue; + } + + const SCEV *EC = SE.getExitCount(L, BB); + if (isa(EC)) + continue; + if (const SCEVConstant *ConstEC = dyn_cast(EC)) { + if (ConstEC->getValue()->isZero()) + continue; + } else if (!SE.isLoopInvariant(EC, L)) + continue; + + if (SE.getTypeSizeInBits(EC->getType()) > CountType->getBitWidth()) + continue; + + // If this exiting block is contained in a nested loop, it is not eligible + // for insertion of the branch-and-decrement since the inner loop would + // end up messing up the value in the CTR. + if (!IsNestingLegal && LI.getLoopFor(BB) != L && !ForceNestedLoop) + continue; + + // We now have a loop-invariant count of loop iterations (which is not the + // constant zero) for which we know that this loop will not exit via this + // existing block. + + // We need to make sure that this block will run on every loop iteration. + // For this to be true, we must dominate all blocks with backedges. Such + // blocks are in-loop predecessors to the header block. + bool NotAlways = false; + for (pred_iterator PI = pred_begin(L->getHeader()), + PIE = pred_end(L->getHeader()); + PI != PIE; ++PI) { + if (!L->contains(*PI)) + continue; + + if (!DT.dominates(*I, *PI)) { + NotAlways = true; + break; + } + } + + if (NotAlways) + continue; + + // Make sure this blocks ends with a conditional branch. + Instruction *TI = BB->getTerminator(); + if (!TI) + continue; + + if (BranchInst *BI = dyn_cast(TI)) { + if (!BI->isConditional()) + continue; + + ExitBranch = BI; + } else + continue; + + // Note that this block may not be the loop latch block, even if the loop + // has a latch block. + ExitBlock = *I; + ExitCount = EC; + break; + } + + if (!ExitBlock) + return false; + return true; +} + TargetTransformInfo::TargetTransformInfo(const DataLayout &DL) : TTIImpl(new Model(NoTTIImpl(DL))) {} @@ -61,15 +157,17 @@ int TargetTransformInfo::getOperationCost(unsigned Opcode, Type *Ty, return Cost; } -int TargetTransformInfo::getCallCost(FunctionType *FTy, int NumArgs) const { - int Cost = TTIImpl->getCallCost(FTy, NumArgs); +int TargetTransformInfo::getCallCost(FunctionType *FTy, int NumArgs, + const User *U) const { + int Cost = TTIImpl->getCallCost(FTy, NumArgs, U); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } int TargetTransformInfo::getCallCost(const Function *F, - ArrayRef Arguments) const { - int Cost = TTIImpl->getCallCost(F, Arguments); + ArrayRef Arguments, + const User *U) const { + int Cost = TTIImpl->getCallCost(F, Arguments, U); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } @@ -78,6 +176,10 @@ unsigned TargetTransformInfo::getInliningThresholdMultiplier() const { return TTIImpl->getInliningThresholdMultiplier(); } +int TargetTransformInfo::getInlinerVectorBonusPercent() const { + return TTIImpl->getInlinerVectorBonusPercent(); +} + int TargetTransformInfo::getGEPCost(Type *PointeeType, const Value *Ptr, ArrayRef Operands) const { return TTIImpl->getGEPCost(PointeeType, Ptr, Operands); @@ -89,8 +191,9 @@ int TargetTransformInfo::getExtCost(const Instruction *I, } int TargetTransformInfo::getIntrinsicCost( - Intrinsic::ID IID, Type *RetTy, ArrayRef Arguments) const { - int Cost = TTIImpl->getIntrinsicCost(IID, RetTy, Arguments); + Intrinsic::ID IID, Type *RetTy, ArrayRef Arguments, + const User *U) const { + int Cost = TTIImpl->getIntrinsicCost(IID, RetTy, Arguments, U); assert(Cost >= 0 && "TTI should not produce negative costs!"); return Cost; } @@ -128,6 +231,12 @@ bool TargetTransformInfo::isLoweredToCall(const Function *F) const { return TTIImpl->isLoweredToCall(F); } +bool TargetTransformInfo::isHardwareLoopProfitable( + Loop *L, ScalarEvolution &SE, AssumptionCache &AC, + TargetLibraryInfo *LibInfo, HardwareLoopInfo &HWLoopInfo) const { + return TTIImpl->isHardwareLoopProfitable(L, SE, AC, LibInfo, HWLoopInfo); +} + void TargetTransformInfo::getUnrollingPreferences( Loop *L, ScalarEvolution &SE, UnrollingPreferences &UP) const { return TTIImpl->getUnrollingPreferences(L, SE, UP); @@ -159,10 +268,21 @@ bool TargetTransformInfo::canMacroFuseCmp() const { return TTIImpl->canMacroFuseCmp(); } +bool TargetTransformInfo::canSaveCmp(Loop *L, BranchInst **BI, + ScalarEvolution *SE, LoopInfo *LI, + DominatorTree *DT, AssumptionCache *AC, + TargetLibraryInfo *LibInfo) const { + return TTIImpl->canSaveCmp(L, BI, SE, LI, DT, AC, LibInfo); +} + bool TargetTransformInfo::shouldFavorPostInc() const { return TTIImpl->shouldFavorPostInc(); } +bool TargetTransformInfo::shouldFavorBackedgeIndex(const Loop *L) const { + return TTIImpl->shouldFavorBackedgeIndex(L); +} + bool TargetTransformInfo::isLegalMaskedStore(Type *DataType) const { return TTIImpl->isLegalMaskedStore(DataType); } @@ -171,6 +291,16 @@ bool TargetTransformInfo::isLegalMaskedLoad(Type *DataType) const { return TTIImpl->isLegalMaskedLoad(DataType); } +bool TargetTransformInfo::isLegalNTStore(Type *DataType, + unsigned Alignment) const { + return TTIImpl->isLegalNTStore(DataType, Alignment); +} + +bool TargetTransformInfo::isLegalNTLoad(Type *DataType, + unsigned Alignment) const { + return TTIImpl->isLegalNTLoad(DataType, Alignment); +} + bool TargetTransformInfo::isLegalMaskedGather(Type *DataType) const { return TTIImpl->isLegalMaskedGather(DataType); } @@ -179,6 +309,14 @@ bool TargetTransformInfo::isLegalMaskedScatter(Type *DataType) const { return TTIImpl->isLegalMaskedScatter(DataType); } +bool TargetTransformInfo::isLegalMaskedCompressStore(Type *DataType) const { + return TTIImpl->isLegalMaskedCompressStore(DataType); +} + +bool TargetTransformInfo::isLegalMaskedExpandLoad(Type *DataType) const { + return TTIImpl->isLegalMaskedExpandLoad(DataType); +} + bool TargetTransformInfo::hasDivRemOp(Type *DataType, bool IsSigned) const { return TTIImpl->hasDivRemOp(DataType, IsSigned); } @@ -259,9 +397,9 @@ bool TargetTransformInfo::enableAggressiveInterleaving(bool LoopHasReductions) c return TTIImpl->enableAggressiveInterleaving(LoopHasReductions); } -const TargetTransformInfo::MemCmpExpansionOptions * -TargetTransformInfo::enableMemCmpExpansion(bool IsZeroCmp) const { - return TTIImpl->enableMemCmpExpansion(IsZeroCmp); +TargetTransformInfo::MemCmpExpansionOptions +TargetTransformInfo::enableMemCmpExpansion(bool OptSize, bool IsZeroCmp) const { + return TTIImpl->enableMemCmpExpansion(OptSize, IsZeroCmp); } bool TargetTransformInfo::enableInterleavedAccessVectorization() const { @@ -570,6 +708,12 @@ int TargetTransformInfo::getAddressComputationCost(Type *Tp, return Cost; } +int TargetTransformInfo::getMemcpyCost(const Instruction *I) const { + int Cost = TTIImpl->getMemcpyCost(I); + assert(Cost >= 0 && "TTI should not produce negative costs!"); + return Cost; +} + int TargetTransformInfo::getArithmeticReductionCost(unsigned Opcode, Type *Ty, bool IsPairwiseForm) const { int Cost = TTIImpl->getArithmeticReductionCost(Opcode, Ty, IsPairwiseForm); @@ -688,6 +832,10 @@ bool TargetTransformInfo::shouldExpandReduction(const IntrinsicInst *II) const { return TTIImpl->shouldExpandReduction(II); } +unsigned TargetTransformInfo::getGISelRematGlobalCost() const { + return TTIImpl->getGISelRematGlobalCost(); +} + int TargetTransformInfo::getInstructionLatency(const Instruction *I) const { return TTIImpl->getInstructionLatency(I); } @@ -1023,6 +1171,16 @@ int TargetTransformInfo::getInstructionThroughput(const Instruction *I) const { return getArithmeticInstrCost(I->getOpcode(), I->getType(), Op1VK, Op2VK, Op1VP, Op2VP, Operands); } + case Instruction::FNeg: { + TargetTransformInfo::OperandValueKind Op1VK, Op2VK; + TargetTransformInfo::OperandValueProperties Op1VP, Op2VP; + Op1VK = getOperandInfo(I->getOperand(0), Op1VP); + Op2VK = OK_AnyValue; + Op2VP = OP_None; + SmallVector Operands(I->operand_values()); + return getArithmeticInstrCost(I->getOpcode(), I->getType(), Op1VK, Op2VK, + Op1VP, Op2VP, Operands); + } case Instruction::Select: { const SelectInst *SI = cast(I); Type *CondTy = SI->getCondition()->getType(); diff --git a/lib/Analysis/Trace.cpp b/lib/Analysis/Trace.cpp index 4dec53151ed6..879c7172d038 100644 --- a/lib/Analysis/Trace.cpp +++ b/lib/Analysis/Trace.cpp @@ -1,9 +1,8 @@ //===- Trace.cpp - Implementation of Trace class --------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/TypeBasedAliasAnalysis.cpp b/lib/Analysis/TypeBasedAliasAnalysis.cpp index 83974da30a54..3b9040aa0f52 100644 --- a/lib/Analysis/TypeBasedAliasAnalysis.cpp +++ b/lib/Analysis/TypeBasedAliasAnalysis.cpp @@ -1,9 +1,8 @@ //===- TypeBasedAliasAnalysis.cpp - Type-Based Alias Analysis -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -368,26 +367,28 @@ static bool isStructPathTBAA(const MDNode *MD) { } AliasResult TypeBasedAAResult::alias(const MemoryLocation &LocA, - const MemoryLocation &LocB) { + const MemoryLocation &LocB, + AAQueryInfo &AAQI) { if (!EnableTBAA) - return AAResultBase::alias(LocA, LocB); + return AAResultBase::alias(LocA, LocB, AAQI); // If accesses may alias, chain to the next AliasAnalysis. if (Aliases(LocA.AATags.TBAA, LocB.AATags.TBAA)) - return AAResultBase::alias(LocA, LocB); + return AAResultBase::alias(LocA, LocB, AAQI); // Otherwise return a definitive result. return NoAlias; } bool TypeBasedAAResult::pointsToConstantMemory(const MemoryLocation &Loc, + AAQueryInfo &AAQI, bool OrLocal) { if (!EnableTBAA) - return AAResultBase::pointsToConstantMemory(Loc, OrLocal); + return AAResultBase::pointsToConstantMemory(Loc, AAQI, OrLocal); const MDNode *M = Loc.AATags.TBAA; if (!M) - return AAResultBase::pointsToConstantMemory(Loc, OrLocal); + return AAResultBase::pointsToConstantMemory(Loc, AAQI, OrLocal); // If this is an "immutable" type, we can assume the pointer is pointing // to constant memory. @@ -395,7 +396,7 @@ bool TypeBasedAAResult::pointsToConstantMemory(const MemoryLocation &Loc, (isStructPathTBAA(M) && TBAAStructTagNode(M).isTypeImmutable())) return true; - return AAResultBase::pointsToConstantMemory(Loc, OrLocal); + return AAResultBase::pointsToConstantMemory(Loc, AAQI, OrLocal); } FunctionModRefBehavior @@ -421,29 +422,31 @@ FunctionModRefBehavior TypeBasedAAResult::getModRefBehavior(const Function *F) { } ModRefInfo TypeBasedAAResult::getModRefInfo(const CallBase *Call, - const MemoryLocation &Loc) { + const MemoryLocation &Loc, + AAQueryInfo &AAQI) { if (!EnableTBAA) - return AAResultBase::getModRefInfo(Call, Loc); + return AAResultBase::getModRefInfo(Call, Loc, AAQI); if (const MDNode *L = Loc.AATags.TBAA) if (const MDNode *M = Call->getMetadata(LLVMContext::MD_tbaa)) if (!Aliases(L, M)) return ModRefInfo::NoModRef; - return AAResultBase::getModRefInfo(Call, Loc); + return AAResultBase::getModRefInfo(Call, Loc, AAQI); } ModRefInfo TypeBasedAAResult::getModRefInfo(const CallBase *Call1, - const CallBase *Call2) { + const CallBase *Call2, + AAQueryInfo &AAQI) { if (!EnableTBAA) - return AAResultBase::getModRefInfo(Call1, Call2); + return AAResultBase::getModRefInfo(Call1, Call2, AAQI); if (const MDNode *M1 = Call1->getMetadata(LLVMContext::MD_tbaa)) if (const MDNode *M2 = Call2->getMetadata(LLVMContext::MD_tbaa)) if (!Aliases(M1, M2)) return ModRefInfo::NoModRef; - return AAResultBase::getModRefInfo(Call1, Call2); + return AAResultBase::getModRefInfo(Call1, Call2, AAQI); } bool MDNode::isTBAAVtableAccess() const { diff --git a/lib/Analysis/TypeMetadataUtils.cpp b/lib/Analysis/TypeMetadataUtils.cpp index bd13a43b8d46..9311dfbc6eba 100644 --- a/lib/Analysis/TypeMetadataUtils.cpp +++ b/lib/Analysis/TypeMetadataUtils.cpp @@ -1,9 +1,8 @@ //===- TypeMetadataUtils.cpp - Utilities related to type metadata ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/ValueLattice.cpp b/lib/Analysis/ValueLattice.cpp index 7de437ca480e..a0115a0eec36 100644 --- a/lib/Analysis/ValueLattice.cpp +++ b/lib/Analysis/ValueLattice.cpp @@ -1,9 +1,8 @@ //===- ValueLattice.cpp - Value constraint analysis -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/Analysis/ValueLatticeUtils.cpp b/lib/Analysis/ValueLatticeUtils.cpp index 22c9de4fe94d..3f9287e26ce7 100644 --- a/lib/Analysis/ValueLatticeUtils.cpp +++ b/lib/Analysis/ValueLatticeUtils.cpp @@ -1,9 +1,8 @@ //===-- ValueLatticeUtils.cpp - Utils for solving lattices ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Analysis/ValueTracking.cpp b/lib/Analysis/ValueTracking.cpp index 0446426c0e66..c70906dcc629 100644 --- a/lib/Analysis/ValueTracking.cpp +++ b/lib/Analysis/ValueTracking.cpp @@ -1,9 +1,8 @@ //===- ValueTracking.cpp - Walk computations to compute properties --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -39,7 +38,6 @@ #include "llvm/IR/Constant.h" #include "llvm/IR/ConstantRange.h" #include "llvm/IR/Constants.h" -#include "llvm/IR/DataLayout.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/DiagnosticInfo.h" #include "llvm/IR/Dominators.h" @@ -617,237 +615,242 @@ static void computeKnownBitsFromAssume(const Value *V, KnownBits &Known, if (Depth == MaxDepth) continue; + ICmpInst *Cmp = dyn_cast(Arg); + if (!Cmp) + continue; + Value *A, *B; - auto m_V = m_CombineOr(m_Specific(V), - m_CombineOr(m_PtrToInt(m_Specific(V)), - m_BitCast(m_Specific(V)))); + auto m_V = m_CombineOr(m_Specific(V), m_PtrToInt(m_Specific(V))); CmpInst::Predicate Pred; uint64_t C; - // assume(v = a) - if (match(Arg, m_c_ICmp(Pred, m_V, m_Value(A))) && - Pred == ICmpInst::ICMP_EQ && isValidAssumeForContext(I, Q.CxtI, Q.DT)) { - KnownBits RHSKnown(BitWidth); - computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); - Known.Zero |= RHSKnown.Zero; - Known.One |= RHSKnown.One; - // assume(v & b = a) - } else if (match(Arg, - m_c_ICmp(Pred, m_c_And(m_V, m_Value(B)), m_Value(A))) && - Pred == ICmpInst::ICMP_EQ && - isValidAssumeForContext(I, Q.CxtI, Q.DT)) { - KnownBits RHSKnown(BitWidth); - computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); - KnownBits MaskKnown(BitWidth); - computeKnownBits(B, MaskKnown, Depth+1, Query(Q, I)); - - // For those bits in the mask that are known to be one, we can propagate - // known bits from the RHS to V. - Known.Zero |= RHSKnown.Zero & MaskKnown.One; - Known.One |= RHSKnown.One & MaskKnown.One; - // assume(~(v & b) = a) - } else if (match(Arg, m_c_ICmp(Pred, m_Not(m_c_And(m_V, m_Value(B))), - m_Value(A))) && - Pred == ICmpInst::ICMP_EQ && - isValidAssumeForContext(I, Q.CxtI, Q.DT)) { - KnownBits RHSKnown(BitWidth); - computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); - KnownBits MaskKnown(BitWidth); - computeKnownBits(B, MaskKnown, Depth+1, Query(Q, I)); - - // For those bits in the mask that are known to be one, we can propagate - // inverted known bits from the RHS to V. - Known.Zero |= RHSKnown.One & MaskKnown.One; - Known.One |= RHSKnown.Zero & MaskKnown.One; - // assume(v | b = a) - } else if (match(Arg, - m_c_ICmp(Pred, m_c_Or(m_V, m_Value(B)), m_Value(A))) && - Pred == ICmpInst::ICMP_EQ && - isValidAssumeForContext(I, Q.CxtI, Q.DT)) { - KnownBits RHSKnown(BitWidth); - computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); - KnownBits BKnown(BitWidth); - computeKnownBits(B, BKnown, Depth+1, Query(Q, I)); - - // For those bits in B that are known to be zero, we can propagate known - // bits from the RHS to V. - Known.Zero |= RHSKnown.Zero & BKnown.Zero; - Known.One |= RHSKnown.One & BKnown.Zero; - // assume(~(v | b) = a) - } else if (match(Arg, m_c_ICmp(Pred, m_Not(m_c_Or(m_V, m_Value(B))), - m_Value(A))) && - Pred == ICmpInst::ICMP_EQ && - isValidAssumeForContext(I, Q.CxtI, Q.DT)) { - KnownBits RHSKnown(BitWidth); - computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); - KnownBits BKnown(BitWidth); - computeKnownBits(B, BKnown, Depth+1, Query(Q, I)); - - // For those bits in B that are known to be zero, we can propagate - // inverted known bits from the RHS to V. - Known.Zero |= RHSKnown.One & BKnown.Zero; - Known.One |= RHSKnown.Zero & BKnown.Zero; - // assume(v ^ b = a) - } else if (match(Arg, - m_c_ICmp(Pred, m_c_Xor(m_V, m_Value(B)), m_Value(A))) && - Pred == ICmpInst::ICMP_EQ && - isValidAssumeForContext(I, Q.CxtI, Q.DT)) { - KnownBits RHSKnown(BitWidth); - computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); - KnownBits BKnown(BitWidth); - computeKnownBits(B, BKnown, Depth+1, Query(Q, I)); - - // For those bits in B that are known to be zero, we can propagate known - // bits from the RHS to V. For those bits in B that are known to be one, - // we can propagate inverted known bits from the RHS to V. - Known.Zero |= RHSKnown.Zero & BKnown.Zero; - Known.One |= RHSKnown.One & BKnown.Zero; - Known.Zero |= RHSKnown.One & BKnown.One; - Known.One |= RHSKnown.Zero & BKnown.One; - // assume(~(v ^ b) = a) - } else if (match(Arg, m_c_ICmp(Pred, m_Not(m_c_Xor(m_V, m_Value(B))), - m_Value(A))) && - Pred == ICmpInst::ICMP_EQ && - isValidAssumeForContext(I, Q.CxtI, Q.DT)) { - KnownBits RHSKnown(BitWidth); - computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); - KnownBits BKnown(BitWidth); - computeKnownBits(B, BKnown, Depth+1, Query(Q, I)); - - // For those bits in B that are known to be zero, we can propagate - // inverted known bits from the RHS to V. For those bits in B that are - // known to be one, we can propagate known bits from the RHS to V. - Known.Zero |= RHSKnown.One & BKnown.Zero; - Known.One |= RHSKnown.Zero & BKnown.Zero; - Known.Zero |= RHSKnown.Zero & BKnown.One; - Known.One |= RHSKnown.One & BKnown.One; - // assume(v << c = a) - } else if (match(Arg, m_c_ICmp(Pred, m_Shl(m_V, m_ConstantInt(C)), - m_Value(A))) && - Pred == ICmpInst::ICMP_EQ && - isValidAssumeForContext(I, Q.CxtI, Q.DT) && - C < BitWidth) { - KnownBits RHSKnown(BitWidth); - computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); - // For those bits in RHS that are known, we can propagate them to known - // bits in V shifted to the right by C. - RHSKnown.Zero.lshrInPlace(C); - Known.Zero |= RHSKnown.Zero; - RHSKnown.One.lshrInPlace(C); - Known.One |= RHSKnown.One; - // assume(~(v << c) = a) - } else if (match(Arg, m_c_ICmp(Pred, m_Not(m_Shl(m_V, m_ConstantInt(C))), - m_Value(A))) && - Pred == ICmpInst::ICMP_EQ && - isValidAssumeForContext(I, Q.CxtI, Q.DT) && - C < BitWidth) { - KnownBits RHSKnown(BitWidth); - computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); - // For those bits in RHS that are known, we can propagate them inverted - // to known bits in V shifted to the right by C. - RHSKnown.One.lshrInPlace(C); - Known.Zero |= RHSKnown.One; - RHSKnown.Zero.lshrInPlace(C); - Known.One |= RHSKnown.Zero; - // assume(v >> c = a) - } else if (match(Arg, - m_c_ICmp(Pred, m_Shr(m_V, m_ConstantInt(C)), - m_Value(A))) && - Pred == ICmpInst::ICMP_EQ && - isValidAssumeForContext(I, Q.CxtI, Q.DT) && - C < BitWidth) { - KnownBits RHSKnown(BitWidth); - computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); - // For those bits in RHS that are known, we can propagate them to known - // bits in V shifted to the right by C. - Known.Zero |= RHSKnown.Zero << C; - Known.One |= RHSKnown.One << C; - // assume(~(v >> c) = a) - } else if (match(Arg, m_c_ICmp(Pred, m_Not(m_Shr(m_V, m_ConstantInt(C))), - m_Value(A))) && - Pred == ICmpInst::ICMP_EQ && - isValidAssumeForContext(I, Q.CxtI, Q.DT) && - C < BitWidth) { - KnownBits RHSKnown(BitWidth); - computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); - // For those bits in RHS that are known, we can propagate them inverted - // to known bits in V shifted to the right by C. - Known.Zero |= RHSKnown.One << C; - Known.One |= RHSKnown.Zero << C; - // assume(v >=_s c) where c is non-negative - } else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) && - Pred == ICmpInst::ICMP_SGE && - isValidAssumeForContext(I, Q.CxtI, Q.DT)) { - KnownBits RHSKnown(BitWidth); - computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); - - if (RHSKnown.isNonNegative()) { - // We know that the sign bit is zero. - Known.makeNonNegative(); + switch (Cmp->getPredicate()) { + default: + break; + case ICmpInst::ICMP_EQ: + // assume(v = a) + if (match(Cmp, m_c_ICmp(Pred, m_V, m_Value(A))) && + isValidAssumeForContext(I, Q.CxtI, Q.DT)) { + KnownBits RHSKnown(BitWidth); + computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); + Known.Zero |= RHSKnown.Zero; + Known.One |= RHSKnown.One; + // assume(v & b = a) + } else if (match(Cmp, + m_c_ICmp(Pred, m_c_And(m_V, m_Value(B)), m_Value(A))) && + isValidAssumeForContext(I, Q.CxtI, Q.DT)) { + KnownBits RHSKnown(BitWidth); + computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); + KnownBits MaskKnown(BitWidth); + computeKnownBits(B, MaskKnown, Depth+1, Query(Q, I)); + + // For those bits in the mask that are known to be one, we can propagate + // known bits from the RHS to V. + Known.Zero |= RHSKnown.Zero & MaskKnown.One; + Known.One |= RHSKnown.One & MaskKnown.One; + // assume(~(v & b) = a) + } else if (match(Cmp, m_c_ICmp(Pred, m_Not(m_c_And(m_V, m_Value(B))), + m_Value(A))) && + isValidAssumeForContext(I, Q.CxtI, Q.DT)) { + KnownBits RHSKnown(BitWidth); + computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); + KnownBits MaskKnown(BitWidth); + computeKnownBits(B, MaskKnown, Depth+1, Query(Q, I)); + + // For those bits in the mask that are known to be one, we can propagate + // inverted known bits from the RHS to V. + Known.Zero |= RHSKnown.One & MaskKnown.One; + Known.One |= RHSKnown.Zero & MaskKnown.One; + // assume(v | b = a) + } else if (match(Cmp, + m_c_ICmp(Pred, m_c_Or(m_V, m_Value(B)), m_Value(A))) && + isValidAssumeForContext(I, Q.CxtI, Q.DT)) { + KnownBits RHSKnown(BitWidth); + computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); + KnownBits BKnown(BitWidth); + computeKnownBits(B, BKnown, Depth+1, Query(Q, I)); + + // For those bits in B that are known to be zero, we can propagate known + // bits from the RHS to V. + Known.Zero |= RHSKnown.Zero & BKnown.Zero; + Known.One |= RHSKnown.One & BKnown.Zero; + // assume(~(v | b) = a) + } else if (match(Cmp, m_c_ICmp(Pred, m_Not(m_c_Or(m_V, m_Value(B))), + m_Value(A))) && + isValidAssumeForContext(I, Q.CxtI, Q.DT)) { + KnownBits RHSKnown(BitWidth); + computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); + KnownBits BKnown(BitWidth); + computeKnownBits(B, BKnown, Depth+1, Query(Q, I)); + + // For those bits in B that are known to be zero, we can propagate + // inverted known bits from the RHS to V. + Known.Zero |= RHSKnown.One & BKnown.Zero; + Known.One |= RHSKnown.Zero & BKnown.Zero; + // assume(v ^ b = a) + } else if (match(Cmp, + m_c_ICmp(Pred, m_c_Xor(m_V, m_Value(B)), m_Value(A))) && + isValidAssumeForContext(I, Q.CxtI, Q.DT)) { + KnownBits RHSKnown(BitWidth); + computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); + KnownBits BKnown(BitWidth); + computeKnownBits(B, BKnown, Depth+1, Query(Q, I)); + + // For those bits in B that are known to be zero, we can propagate known + // bits from the RHS to V. For those bits in B that are known to be one, + // we can propagate inverted known bits from the RHS to V. + Known.Zero |= RHSKnown.Zero & BKnown.Zero; + Known.One |= RHSKnown.One & BKnown.Zero; + Known.Zero |= RHSKnown.One & BKnown.One; + Known.One |= RHSKnown.Zero & BKnown.One; + // assume(~(v ^ b) = a) + } else if (match(Cmp, m_c_ICmp(Pred, m_Not(m_c_Xor(m_V, m_Value(B))), + m_Value(A))) && + isValidAssumeForContext(I, Q.CxtI, Q.DT)) { + KnownBits RHSKnown(BitWidth); + computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); + KnownBits BKnown(BitWidth); + computeKnownBits(B, BKnown, Depth+1, Query(Q, I)); + + // For those bits in B that are known to be zero, we can propagate + // inverted known bits from the RHS to V. For those bits in B that are + // known to be one, we can propagate known bits from the RHS to V. + Known.Zero |= RHSKnown.One & BKnown.Zero; + Known.One |= RHSKnown.Zero & BKnown.Zero; + Known.Zero |= RHSKnown.Zero & BKnown.One; + Known.One |= RHSKnown.One & BKnown.One; + // assume(v << c = a) + } else if (match(Cmp, m_c_ICmp(Pred, m_Shl(m_V, m_ConstantInt(C)), + m_Value(A))) && + isValidAssumeForContext(I, Q.CxtI, Q.DT) && C < BitWidth) { + KnownBits RHSKnown(BitWidth); + computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); + // For those bits in RHS that are known, we can propagate them to known + // bits in V shifted to the right by C. + RHSKnown.Zero.lshrInPlace(C); + Known.Zero |= RHSKnown.Zero; + RHSKnown.One.lshrInPlace(C); + Known.One |= RHSKnown.One; + // assume(~(v << c) = a) + } else if (match(Cmp, m_c_ICmp(Pred, m_Not(m_Shl(m_V, m_ConstantInt(C))), + m_Value(A))) && + isValidAssumeForContext(I, Q.CxtI, Q.DT) && C < BitWidth) { + KnownBits RHSKnown(BitWidth); + computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); + // For those bits in RHS that are known, we can propagate them inverted + // to known bits in V shifted to the right by C. + RHSKnown.One.lshrInPlace(C); + Known.Zero |= RHSKnown.One; + RHSKnown.Zero.lshrInPlace(C); + Known.One |= RHSKnown.Zero; + // assume(v >> c = a) + } else if (match(Cmp, m_c_ICmp(Pred, m_Shr(m_V, m_ConstantInt(C)), + m_Value(A))) && + isValidAssumeForContext(I, Q.CxtI, Q.DT) && C < BitWidth) { + KnownBits RHSKnown(BitWidth); + computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); + // For those bits in RHS that are known, we can propagate them to known + // bits in V shifted to the right by C. + Known.Zero |= RHSKnown.Zero << C; + Known.One |= RHSKnown.One << C; + // assume(~(v >> c) = a) + } else if (match(Cmp, m_c_ICmp(Pred, m_Not(m_Shr(m_V, m_ConstantInt(C))), + m_Value(A))) && + isValidAssumeForContext(I, Q.CxtI, Q.DT) && C < BitWidth) { + KnownBits RHSKnown(BitWidth); + computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); + // For those bits in RHS that are known, we can propagate them inverted + // to known bits in V shifted to the right by C. + Known.Zero |= RHSKnown.One << C; + Known.One |= RHSKnown.Zero << C; } - // assume(v >_s c) where c is at least -1. - } else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) && - Pred == ICmpInst::ICMP_SGT && - isValidAssumeForContext(I, Q.CxtI, Q.DT)) { - KnownBits RHSKnown(BitWidth); - computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); - - if (RHSKnown.isAllOnes() || RHSKnown.isNonNegative()) { - // We know that the sign bit is zero. - Known.makeNonNegative(); + break; + case ICmpInst::ICMP_SGE: + // assume(v >=_s c) where c is non-negative + if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A))) && + isValidAssumeForContext(I, Q.CxtI, Q.DT)) { + KnownBits RHSKnown(BitWidth); + computeKnownBits(A, RHSKnown, Depth + 1, Query(Q, I)); + + if (RHSKnown.isNonNegative()) { + // We know that the sign bit is zero. + Known.makeNonNegative(); + } } - // assume(v <=_s c) where c is negative - } else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) && - Pred == ICmpInst::ICMP_SLE && - isValidAssumeForContext(I, Q.CxtI, Q.DT)) { - KnownBits RHSKnown(BitWidth); - computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); - - if (RHSKnown.isNegative()) { - // We know that the sign bit is one. - Known.makeNegative(); + break; + case ICmpInst::ICMP_SGT: + // assume(v >_s c) where c is at least -1. + if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A))) && + isValidAssumeForContext(I, Q.CxtI, Q.DT)) { + KnownBits RHSKnown(BitWidth); + computeKnownBits(A, RHSKnown, Depth + 1, Query(Q, I)); + + if (RHSKnown.isAllOnes() || RHSKnown.isNonNegative()) { + // We know that the sign bit is zero. + Known.makeNonNegative(); + } } - // assume(v <_s c) where c is non-positive - } else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) && - Pred == ICmpInst::ICMP_SLT && - isValidAssumeForContext(I, Q.CxtI, Q.DT)) { - KnownBits RHSKnown(BitWidth); - computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); - - if (RHSKnown.isZero() || RHSKnown.isNegative()) { - // We know that the sign bit is one. - Known.makeNegative(); + break; + case ICmpInst::ICMP_SLE: + // assume(v <=_s c) where c is negative + if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A))) && + isValidAssumeForContext(I, Q.CxtI, Q.DT)) { + KnownBits RHSKnown(BitWidth); + computeKnownBits(A, RHSKnown, Depth + 1, Query(Q, I)); + + if (RHSKnown.isNegative()) { + // We know that the sign bit is one. + Known.makeNegative(); + } } - // assume(v <=_u c) - } else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) && - Pred == ICmpInst::ICMP_ULE && - isValidAssumeForContext(I, Q.CxtI, Q.DT)) { - KnownBits RHSKnown(BitWidth); - computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); - - // Whatever high bits in c are zero are known to be zero. - Known.Zero.setHighBits(RHSKnown.countMinLeadingZeros()); - // assume(v <_u c) - } else if (match(Arg, m_ICmp(Pred, m_V, m_Value(A))) && - Pred == ICmpInst::ICMP_ULT && - isValidAssumeForContext(I, Q.CxtI, Q.DT)) { - KnownBits RHSKnown(BitWidth); - computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); - - // If the RHS is known zero, then this assumption must be wrong (nothing - // is unsigned less than zero). Signal a conflict and get out of here. - if (RHSKnown.isZero()) { - Known.Zero.setAllBits(); - Known.One.setAllBits(); - break; + break; + case ICmpInst::ICMP_SLT: + // assume(v <_s c) where c is non-positive + if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A))) && + isValidAssumeForContext(I, Q.CxtI, Q.DT)) { + KnownBits RHSKnown(BitWidth); + computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); + + if (RHSKnown.isZero() || RHSKnown.isNegative()) { + // We know that the sign bit is one. + Known.makeNegative(); + } } - - // Whatever high bits in c are zero are known to be zero (if c is a power - // of 2, then one more). - if (isKnownToBeAPowerOfTwo(A, false, Depth + 1, Query(Q, I))) - Known.Zero.setHighBits(RHSKnown.countMinLeadingZeros() + 1); - else + break; + case ICmpInst::ICMP_ULE: + // assume(v <=_u c) + if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A))) && + isValidAssumeForContext(I, Q.CxtI, Q.DT)) { + KnownBits RHSKnown(BitWidth); + computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); + + // Whatever high bits in c are zero are known to be zero. Known.Zero.setHighBits(RHSKnown.countMinLeadingZeros()); + } + break; + case ICmpInst::ICMP_ULT: + // assume(v <_u c) + if (match(Cmp, m_ICmp(Pred, m_V, m_Value(A))) && + isValidAssumeForContext(I, Q.CxtI, Q.DT)) { + KnownBits RHSKnown(BitWidth); + computeKnownBits(A, RHSKnown, Depth+1, Query(Q, I)); + + // If the RHS is known zero, then this assumption must be wrong (nothing + // is unsigned less than zero). Signal a conflict and get out of here. + if (RHSKnown.isZero()) { + Known.Zero.setAllBits(); + Known.One.setAllBits(); + break; + } + + // Whatever high bits in c are zero are known to be zero (if c is a power + // of 2, then one more). + if (isKnownToBeAPowerOfTwo(A, false, Depth + 1, Query(Q, I))) + Known.Zero.setHighBits(RHSKnown.countMinLeadingZeros() + 1); + else + Known.Zero.setHighBits(RHSKnown.countMinLeadingZeros()); + } + break; } } @@ -1129,12 +1132,9 @@ static void computeKnownBitsFromOperator(const Operator *I, KnownBits &Known, Q.DL.getTypeSizeInBits(ScalarTy); assert(SrcBitWidth && "SrcBitWidth can't be zero"); - Known = Known.zextOrTrunc(SrcBitWidth); + Known = Known.zextOrTrunc(SrcBitWidth, false); computeKnownBits(I->getOperand(0), Known, Depth + 1, Q); - Known = Known.zextOrTrunc(BitWidth); - // Any top bits are known to be zero. - if (BitWidth > SrcBitWidth) - Known.Zero.setBitsFrom(SrcBitWidth); + Known = Known.zextOrTrunc(BitWidth, true /* ExtendedBitsAreKnownZero */); break; } case Instruction::BitCast: { @@ -1527,6 +1527,37 @@ static void computeKnownBitsFromOperator(const Operator *I, KnownBits &Known, Known2.One.shl(ShiftAmt) | Known3.One.lshr(BitWidth - ShiftAmt); break; } + case Intrinsic::uadd_sat: + case Intrinsic::usub_sat: { + bool IsAdd = II->getIntrinsicID() == Intrinsic::uadd_sat; + computeKnownBits(I->getOperand(0), Known, Depth + 1, Q); + computeKnownBits(I->getOperand(1), Known2, Depth + 1, Q); + + // Add: Leading ones of either operand are preserved. + // Sub: Leading zeros of LHS and leading ones of RHS are preserved + // as leading zeros in the result. + unsigned LeadingKnown; + if (IsAdd) + LeadingKnown = std::max(Known.countMinLeadingOnes(), + Known2.countMinLeadingOnes()); + else + LeadingKnown = std::max(Known.countMinLeadingZeros(), + Known2.countMinLeadingOnes()); + + Known = KnownBits::computeForAddSub( + IsAdd, /* NSW */ false, Known, Known2); + + // We select between the operation result and all-ones/zero + // respectively, so we can preserve known ones/zeros. + if (IsAdd) { + Known.One.setHighBits(LeadingKnown); + Known.Zero.clearAllBits(); + } else { + Known.Zero.setHighBits(LeadingKnown); + Known.One.clearAllBits(); + } + break; + } case Intrinsic::x86_sse42_crc32_64_64: Known.Zero.setBitsFrom(32); break; @@ -1967,6 +1998,15 @@ bool isKnownNonZero(const Value *V, unsigned Depth, const Query &Q) { // Must be non-zero due to null test above. return true; + if (auto *CE = dyn_cast(C)) { + // See the comment for IntToPtr/PtrToInt instructions below. + if (CE->getOpcode() == Instruction::IntToPtr || + CE->getOpcode() == Instruction::PtrToInt) + if (Q.DL.getTypeSizeInBits(CE->getOperand(0)->getType()) <= + Q.DL.getTypeSizeInBits(CE->getType())) + return isKnownNonZero(CE->getOperand(0), Depth, Q); + } + // For constant vectors, check that all elements are undefined or known // non-zero to determine that the whole vector is known non-zero. if (auto *VecTy = dyn_cast(C->getType())) { @@ -2037,11 +2077,33 @@ bool isKnownNonZero(const Value *V, unsigned Depth, const Query &Q) { if (isKnownNonNullFromDominatingCondition(V, Q.CxtI, Q.DT)) return true; + // Look through bitcast operations, GEPs, and int2ptr instructions as they + // do not alter the value, or at least not the nullness property of the + // value, e.g., int2ptr is allowed to zero/sign extend the value. + // + // Note that we have to take special care to avoid looking through + // truncating casts, e.g., int2ptr/ptr2int with appropriate sizes, as well + // as casts that can alter the value, e.g., AddrSpaceCasts. if (const GEPOperator *GEP = dyn_cast(V)) if (isGEPKnownNonNull(GEP, Depth, Q)) return true; + + if (auto *BCO = dyn_cast(V)) + return isKnownNonZero(BCO->getOperand(0), Depth, Q); + + if (auto *I2P = dyn_cast(V)) + if (Q.DL.getTypeSizeInBits(I2P->getSrcTy()) <= + Q.DL.getTypeSizeInBits(I2P->getDestTy())) + return isKnownNonZero(I2P->getOperand(0), Depth, Q); } + // Similar to int2ptr above, we can look through ptr2int here if the cast + // is a no-op or an extend and not a truncate. + if (auto *P2I = dyn_cast(V)) + if (Q.DL.getTypeSizeInBits(P2I->getSrcTy()) <= + Q.DL.getTypeSizeInBits(P2I->getDestTy())) + return isKnownNonZero(P2I->getOperand(0), Depth, Q); + unsigned BitWidth = getBitWidth(V->getType()->getScalarType(), Q.DL); // X | Y != 0 if X != 0 or Y != 0. @@ -3082,6 +3144,11 @@ bool llvm::isKnownNeverNaN(const Value *V, const TargetLibraryInfo *TLI, case Intrinsic::sqrt: return isKnownNeverNaN(II->getArgOperand(0), TLI, Depth + 1) && CannotBeOrderedLessThanZero(II->getArgOperand(0), TLI); + case Intrinsic::minnum: + case Intrinsic::maxnum: + // If either operand is not NaN, the result is not NaN. + return isKnownNeverNaN(II->getArgOperand(0), TLI, Depth + 1) || + isKnownNeverNaN(II->getArgOperand(1), TLI, Depth + 1); default: return false; } @@ -3107,7 +3174,7 @@ bool llvm::isKnownNeverNaN(const Value *V, const TargetLibraryInfo *TLI, return true; } -Value *llvm::isBytewiseValue(Value *V) { +Value *llvm::isBytewiseValue(Value *V, const DataLayout &DL) { // All byte-wide stores are splatable, even of arbitrary variables. if (V->getType()->isIntegerTy(8)) @@ -3120,6 +3187,10 @@ Value *llvm::isBytewiseValue(Value *V) { if (isa(V)) return UndefInt8; + const uint64_t Size = DL.getTypeStoreSize(V->getType()); + if (!Size) + return UndefInt8; + Constant *C = dyn_cast(V); if (!C) { // Conceptually, we could handle things like: @@ -3146,7 +3217,8 @@ Value *llvm::isBytewiseValue(Value *V) { else if (CFP->getType()->isDoubleTy()) Ty = Type::getInt64Ty(Ctx); // Don't handle long double formats, which have strange constraints. - return Ty ? isBytewiseValue(ConstantExpr::getBitCast(CFP, Ty)) : nullptr; + return Ty ? isBytewiseValue(ConstantExpr::getBitCast(CFP, Ty), DL) + : nullptr; } // We can handle constant integers that are multiple of 8 bits. @@ -3159,6 +3231,17 @@ Value *llvm::isBytewiseValue(Value *V) { } } + if (auto *CE = dyn_cast(C)) { + if (CE->getOpcode() == Instruction::IntToPtr) { + auto PS = DL.getPointerSizeInBits( + cast(CE->getType())->getAddressSpace()); + return isBytewiseValue( + ConstantExpr::getIntegerCast(CE->getOperand(0), + Type::getIntNTy(Ctx, PS), false), + DL); + } + } + auto Merge = [&](Value *LHS, Value *RHS) -> Value * { if (LHS == RHS) return LHS; @@ -3174,20 +3257,15 @@ Value *llvm::isBytewiseValue(Value *V) { if (ConstantDataSequential *CA = dyn_cast(C)) { Value *Val = UndefInt8; for (unsigned I = 0, E = CA->getNumElements(); I != E; ++I) - if (!(Val = Merge(Val, isBytewiseValue(CA->getElementAsConstant(I))))) + if (!(Val = Merge(Val, isBytewiseValue(CA->getElementAsConstant(I), DL)))) return nullptr; return Val; } - if (isa(C)) { - Constant *Splat = cast(C)->getSplatValue(); - return Splat ? isBytewiseValue(Splat) : nullptr; - } - - if (isa(C) || isa(C)) { + if (isa(C)) { Value *Val = UndefInt8; for (unsigned I = 0, E = C->getNumOperands(); I != E; ++I) - if (!(Val = Merge(Val, isBytewiseValue(C->getOperand(I))))) + if (!(Val = Merge(Val, isBytewiseValue(C->getOperand(I), DL)))) return nullptr; return Val; } @@ -3363,57 +3441,6 @@ Value *llvm::FindInsertedValue(Value *V, ArrayRef idx_range, return nullptr; } -/// Analyze the specified pointer to see if it can be expressed as a base -/// pointer plus a constant offset. Return the base and offset to the caller. -Value *llvm::GetPointerBaseWithConstantOffset(Value *Ptr, int64_t &Offset, - const DataLayout &DL) { - unsigned BitWidth = DL.getIndexTypeSizeInBits(Ptr->getType()); - APInt ByteOffset(BitWidth, 0); - - // We walk up the defs but use a visited set to handle unreachable code. In - // that case, we stop after accumulating the cycle once (not that it - // matters). - SmallPtrSet Visited; - while (Visited.insert(Ptr).second) { - if (Ptr->getType()->isVectorTy()) - break; - - if (GEPOperator *GEP = dyn_cast(Ptr)) { - // If one of the values we have visited is an addrspacecast, then - // the pointer type of this GEP may be different from the type - // of the Ptr parameter which was passed to this function. This - // means when we construct GEPOffset, we need to use the size - // of GEP's pointer type rather than the size of the original - // pointer type. - APInt GEPOffset(DL.getIndexTypeSizeInBits(Ptr->getType()), 0); - if (!GEP->accumulateConstantOffset(DL, GEPOffset)) - break; - - APInt OrigByteOffset(ByteOffset); - ByteOffset += GEPOffset.sextOrTrunc(ByteOffset.getBitWidth()); - if (ByteOffset.getMinSignedBits() > 64) { - // Stop traversal if the pointer offset wouldn't fit into int64_t - // (this should be removed if Offset is updated to an APInt) - ByteOffset = OrigByteOffset; - break; - } - - Ptr = GEP->getPointerOperand(); - } else if (Operator::getOpcode(Ptr) == Instruction::BitCast || - Operator::getOpcode(Ptr) == Instruction::AddrSpaceCast) { - Ptr = cast(Ptr)->getOperand(0); - } else if (GlobalAlias *GA = dyn_cast(Ptr)) { - if (GA->isInterposable()) - break; - Ptr = GA->getAliasee(); - } else { - break; - } - } - Offset = ByteOffset.getSExtValue(); - return Ptr; -} - bool llvm::isGEPBasedOnPointerToString(const GEPOperator *GEP, unsigned CharSize) { // Make sure the GEP has exactly three arguments. @@ -3638,7 +3665,9 @@ const Value *llvm::getArgumentAliasingToReturnedPointer(const CallBase *Call) { bool llvm::isIntrinsicReturningPointerAliasingArgumentWithoutCapturing( const CallBase *Call) { return Call->getIntrinsicID() == Intrinsic::launder_invariant_group || - Call->getIntrinsicID() == Intrinsic::strip_invariant_group; + Call->getIntrinsicID() == Intrinsic::strip_invariant_group || + Call->getIntrinsicID() == Intrinsic::aarch64_irg || + Call->getIntrinsicID() == Intrinsic::aarch64_tagp; } /// \p PN defines a loop-variant pointer to an object. Check if the @@ -3717,26 +3746,27 @@ Value *llvm::GetUnderlyingObject(Value *V, const DataLayout &DL, return V; } -void llvm::GetUnderlyingObjects(Value *V, SmallVectorImpl &Objects, +void llvm::GetUnderlyingObjects(const Value *V, + SmallVectorImpl &Objects, const DataLayout &DL, LoopInfo *LI, unsigned MaxLookup) { - SmallPtrSet Visited; - SmallVector Worklist; + SmallPtrSet Visited; + SmallVector Worklist; Worklist.push_back(V); do { - Value *P = Worklist.pop_back_val(); + const Value *P = Worklist.pop_back_val(); P = GetUnderlyingObject(P, DL, MaxLookup); if (!Visited.insert(P).second) continue; - if (SelectInst *SI = dyn_cast(P)) { + if (auto *SI = dyn_cast(P)) { Worklist.push_back(SI->getTrueValue()); Worklist.push_back(SI->getFalseValue()); continue; } - if (PHINode *PN = dyn_cast(P)) { + if (auto *PN = dyn_cast(P)) { // If this PHI changes the underlying object in every iteration of the // loop, don't look through it. Consider: // int **A; @@ -3797,10 +3827,10 @@ bool llvm::getUnderlyingObjectsForCodeGen(const Value *V, do { V = Working.pop_back_val(); - SmallVector Objs; - GetUnderlyingObjects(const_cast(V), Objs, DL); + SmallVector Objs; + GetUnderlyingObjects(V, Objs, DL); - for (Value *V : Objs) { + for (const Value *V : Objs) { if (!Visited.insert(V).second) continue; if (Operator::getOpcode(V) == Instruction::IntToPtr) { @@ -3888,7 +3918,8 @@ bool llvm::isSafeToSpeculativelyExecute(const Value *V, return false; const DataLayout &DL = LI->getModule()->getDataLayout(); return isDereferenceableAndAlignedPointer(LI->getPointerOperand(), - LI->getAlignment(), DL, CtxI, DT); + LI->getType(), LI->getAlignment(), + DL, CtxI, DT); } case Instruction::Call: { auto *CI = cast(Inst); @@ -3901,6 +3932,7 @@ bool llvm::isSafeToSpeculativelyExecute(const Value *V, case Instruction::VAArg: case Instruction::Alloca: case Instruction::Invoke: + case Instruction::CallBr: case Instruction::PHI: case Instruction::Store: case Instruction::Ret: @@ -3926,51 +3958,46 @@ bool llvm::mayBeMemoryDependent(const Instruction &I) { return I.mayReadOrWriteMemory() || !isSafeToSpeculativelyExecute(&I); } +/// Convert ConstantRange OverflowResult into ValueTracking OverflowResult. +static OverflowResult mapOverflowResult(ConstantRange::OverflowResult OR) { + switch (OR) { + case ConstantRange::OverflowResult::MayOverflow: + return OverflowResult::MayOverflow; + case ConstantRange::OverflowResult::AlwaysOverflowsLow: + return OverflowResult::AlwaysOverflowsLow; + case ConstantRange::OverflowResult::AlwaysOverflowsHigh: + return OverflowResult::AlwaysOverflowsHigh; + case ConstantRange::OverflowResult::NeverOverflows: + return OverflowResult::NeverOverflows; + } + llvm_unreachable("Unknown OverflowResult"); +} + +/// Combine constant ranges from computeConstantRange() and computeKnownBits(). +static ConstantRange computeConstantRangeIncludingKnownBits( + const Value *V, bool ForSigned, const DataLayout &DL, unsigned Depth, + AssumptionCache *AC, const Instruction *CxtI, const DominatorTree *DT, + OptimizationRemarkEmitter *ORE = nullptr, bool UseInstrInfo = true) { + KnownBits Known = computeKnownBits( + V, DL, Depth, AC, CxtI, DT, ORE, UseInstrInfo); + ConstantRange CR1 = ConstantRange::fromKnownBits(Known, ForSigned); + ConstantRange CR2 = computeConstantRange(V, UseInstrInfo); + ConstantRange::PreferredRangeType RangeType = + ForSigned ? ConstantRange::Signed : ConstantRange::Unsigned; + return CR1.intersectWith(CR2, RangeType); +} + OverflowResult llvm::computeOverflowForUnsignedMul( const Value *LHS, const Value *RHS, const DataLayout &DL, AssumptionCache *AC, const Instruction *CxtI, const DominatorTree *DT, bool UseInstrInfo) { - // Multiplying n * m significant bits yields a result of n + m significant - // bits. If the total number of significant bits does not exceed the - // result bit width (minus 1), there is no overflow. - // This means if we have enough leading zero bits in the operands - // we can guarantee that the result does not overflow. - // Ref: "Hacker's Delight" by Henry Warren - unsigned BitWidth = LHS->getType()->getScalarSizeInBits(); - KnownBits LHSKnown(BitWidth); - KnownBits RHSKnown(BitWidth); - computeKnownBits(LHS, LHSKnown, DL, /*Depth=*/0, AC, CxtI, DT, nullptr, - UseInstrInfo); - computeKnownBits(RHS, RHSKnown, DL, /*Depth=*/0, AC, CxtI, DT, nullptr, - UseInstrInfo); - // Note that underestimating the number of zero bits gives a more - // conservative answer. - unsigned ZeroBits = LHSKnown.countMinLeadingZeros() + - RHSKnown.countMinLeadingZeros(); - // First handle the easy case: if we have enough zero bits there's - // definitely no overflow. - if (ZeroBits >= BitWidth) - return OverflowResult::NeverOverflows; - - // Get the largest possible values for each operand. - APInt LHSMax = ~LHSKnown.Zero; - APInt RHSMax = ~RHSKnown.Zero; - - // We know the multiply operation doesn't overflow if the maximum values for - // each operand will not overflow after we multiply them together. - bool MaxOverflow; - (void)LHSMax.umul_ov(RHSMax, MaxOverflow); - if (!MaxOverflow) - return OverflowResult::NeverOverflows; - - // We know it always overflows if multiplying the smallest possible values for - // the operands also results in overflow. - bool MinOverflow; - (void)LHSKnown.One.umul_ov(RHSKnown.One, MinOverflow); - if (MinOverflow) - return OverflowResult::AlwaysOverflows; - - return OverflowResult::MayOverflow; + KnownBits LHSKnown = computeKnownBits(LHS, DL, /*Depth=*/0, AC, CxtI, DT, + nullptr, UseInstrInfo); + KnownBits RHSKnown = computeKnownBits(RHS, DL, /*Depth=*/0, AC, CxtI, DT, + nullptr, UseInstrInfo); + ConstantRange LHSRange = ConstantRange::fromKnownBits(LHSKnown, false); + ConstantRange RHSRange = ConstantRange::fromKnownBits(RHSKnown, false); + return mapOverflowResult(LHSRange.unsignedMulMayOverflow(RHSRange)); } OverflowResult @@ -4020,69 +4047,13 @@ OverflowResult llvm::computeOverflowForUnsignedAdd( const Value *LHS, const Value *RHS, const DataLayout &DL, AssumptionCache *AC, const Instruction *CxtI, const DominatorTree *DT, bool UseInstrInfo) { - KnownBits LHSKnown = computeKnownBits(LHS, DL, /*Depth=*/0, AC, CxtI, DT, - nullptr, UseInstrInfo); - if (LHSKnown.isNonNegative() || LHSKnown.isNegative()) { - KnownBits RHSKnown = computeKnownBits(RHS, DL, /*Depth=*/0, AC, CxtI, DT, - nullptr, UseInstrInfo); - - if (LHSKnown.isNegative() && RHSKnown.isNegative()) { - // The sign bit is set in both cases: this MUST overflow. - return OverflowResult::AlwaysOverflows; - } - - if (LHSKnown.isNonNegative() && RHSKnown.isNonNegative()) { - // The sign bit is clear in both cases: this CANNOT overflow. - return OverflowResult::NeverOverflows; - } - } - - return OverflowResult::MayOverflow; -} - -/// Return true if we can prove that adding the two values of the -/// knownbits will not overflow. -/// Otherwise return false. -static bool checkRippleForSignedAdd(const KnownBits &LHSKnown, - const KnownBits &RHSKnown) { - // Addition of two 2's complement numbers having opposite signs will never - // overflow. - if ((LHSKnown.isNegative() && RHSKnown.isNonNegative()) || - (LHSKnown.isNonNegative() && RHSKnown.isNegative())) - return true; - - // If either of the values is known to be non-negative, adding them can only - // overflow if the second is also non-negative, so we can assume that. - // Two non-negative numbers will only overflow if there is a carry to the - // sign bit, so we can check if even when the values are as big as possible - // there is no overflow to the sign bit. - if (LHSKnown.isNonNegative() || RHSKnown.isNonNegative()) { - APInt MaxLHS = ~LHSKnown.Zero; - MaxLHS.clearSignBit(); - APInt MaxRHS = ~RHSKnown.Zero; - MaxRHS.clearSignBit(); - APInt Result = std::move(MaxLHS) + std::move(MaxRHS); - return Result.isSignBitClear(); - } - - // If either of the values is known to be negative, adding them can only - // overflow if the second is also negative, so we can assume that. - // Two negative number will only overflow if there is no carry to the sign - // bit, so we can check if even when the values are as small as possible - // there is overflow to the sign bit. - if (LHSKnown.isNegative() || RHSKnown.isNegative()) { - APInt MinLHS = LHSKnown.One; - MinLHS.clearSignBit(); - APInt MinRHS = RHSKnown.One; - MinRHS.clearSignBit(); - APInt Result = std::move(MinLHS) + std::move(MinRHS); - return Result.isSignBitSet(); - } - - // If we reached here it means that we know nothing about the sign bits. - // In this case we can't know if there will be an overflow, since by - // changing the sign bits any two values can be made to overflow. - return false; + ConstantRange LHSRange = computeConstantRangeIncludingKnownBits( + LHS, /*ForSigned=*/false, DL, /*Depth=*/0, AC, CxtI, DT, + nullptr, UseInstrInfo); + ConstantRange RHSRange = computeConstantRangeIncludingKnownBits( + RHS, /*ForSigned=*/false, DL, /*Depth=*/0, AC, CxtI, DT, + nullptr, UseInstrInfo); + return mapOverflowResult(LHSRange.unsignedAddMayOverflow(RHSRange)); } static OverflowResult computeOverflowForSignedAdd(const Value *LHS, @@ -4114,30 +4085,35 @@ static OverflowResult computeOverflowForSignedAdd(const Value *LHS, ComputeNumSignBits(RHS, DL, 0, AC, CxtI, DT) > 1) return OverflowResult::NeverOverflows; - KnownBits LHSKnown = computeKnownBits(LHS, DL, /*Depth=*/0, AC, CxtI, DT); - KnownBits RHSKnown = computeKnownBits(RHS, DL, /*Depth=*/0, AC, CxtI, DT); - - if (checkRippleForSignedAdd(LHSKnown, RHSKnown)) - return OverflowResult::NeverOverflows; + ConstantRange LHSRange = computeConstantRangeIncludingKnownBits( + LHS, /*ForSigned=*/true, DL, /*Depth=*/0, AC, CxtI, DT); + ConstantRange RHSRange = computeConstantRangeIncludingKnownBits( + RHS, /*ForSigned=*/true, DL, /*Depth=*/0, AC, CxtI, DT); + OverflowResult OR = + mapOverflowResult(LHSRange.signedAddMayOverflow(RHSRange)); + if (OR != OverflowResult::MayOverflow) + return OR; // The remaining code needs Add to be available. Early returns if not so. if (!Add) return OverflowResult::MayOverflow; // If the sign of Add is the same as at least one of the operands, this add - // CANNOT overflow. This is particularly useful when the sum is - // @llvm.assume'ed non-negative rather than proved so from analyzing its - // operands. + // CANNOT overflow. If this can be determined from the known bits of the + // operands the above signedAddMayOverflow() check will have already done so. + // The only other way to improve on the known bits is from an assumption, so + // call computeKnownBitsFromAssume() directly. bool LHSOrRHSKnownNonNegative = - (LHSKnown.isNonNegative() || RHSKnown.isNonNegative()); + (LHSRange.isAllNonNegative() || RHSRange.isAllNonNegative()); bool LHSOrRHSKnownNegative = - (LHSKnown.isNegative() || RHSKnown.isNegative()); + (LHSRange.isAllNegative() || RHSRange.isAllNegative()); if (LHSOrRHSKnownNonNegative || LHSOrRHSKnownNegative) { - KnownBits AddKnown = computeKnownBits(Add, DL, /*Depth=*/0, AC, CxtI, DT); + KnownBits AddKnown(LHSRange.getBitWidth()); + computeKnownBitsFromAssume( + Add, AddKnown, /*Depth=*/0, Query(DL, AC, CxtI, DT, true)); if ((AddKnown.isNonNegative() && LHSOrRHSKnownNonNegative) || - (AddKnown.isNegative() && LHSOrRHSKnownNegative)) { + (AddKnown.isNegative() && LHSOrRHSKnownNegative)) return OverflowResult::NeverOverflows; - } } return OverflowResult::MayOverflow; @@ -4149,20 +4125,11 @@ OverflowResult llvm::computeOverflowForUnsignedSub(const Value *LHS, AssumptionCache *AC, const Instruction *CxtI, const DominatorTree *DT) { - KnownBits LHSKnown = computeKnownBits(LHS, DL, /*Depth=*/0, AC, CxtI, DT); - if (LHSKnown.isNonNegative() || LHSKnown.isNegative()) { - KnownBits RHSKnown = computeKnownBits(RHS, DL, /*Depth=*/0, AC, CxtI, DT); - - // If the LHS is negative and the RHS is non-negative, no unsigned wrap. - if (LHSKnown.isNegative() && RHSKnown.isNonNegative()) - return OverflowResult::NeverOverflows; - - // If the LHS is non-negative and the RHS negative, we always wrap. - if (LHSKnown.isNonNegative() && RHSKnown.isNegative()) - return OverflowResult::AlwaysOverflows; - } - - return OverflowResult::MayOverflow; + ConstantRange LHSRange = computeConstantRangeIncludingKnownBits( + LHS, /*ForSigned=*/false, DL, /*Depth=*/0, AC, CxtI, DT); + ConstantRange RHSRange = computeConstantRangeIncludingKnownBits( + RHS, /*ForSigned=*/false, DL, /*Depth=*/0, AC, CxtI, DT); + return mapOverflowResult(LHSRange.unsignedSubMayOverflow(RHSRange)); } OverflowResult llvm::computeOverflowForSignedSub(const Value *LHS, @@ -4177,37 +4144,19 @@ OverflowResult llvm::computeOverflowForSignedSub(const Value *LHS, ComputeNumSignBits(RHS, DL, 0, AC, CxtI, DT) > 1) return OverflowResult::NeverOverflows; - KnownBits LHSKnown = computeKnownBits(LHS, DL, 0, AC, CxtI, DT); - - KnownBits RHSKnown = computeKnownBits(RHS, DL, 0, AC, CxtI, DT); - - // Subtraction of two 2's complement numbers having identical signs will - // never overflow. - if ((LHSKnown.isNegative() && RHSKnown.isNegative()) || - (LHSKnown.isNonNegative() && RHSKnown.isNonNegative())) - return OverflowResult::NeverOverflows; - - // TODO: implement logic similar to checkRippleForAdd - return OverflowResult::MayOverflow; + ConstantRange LHSRange = computeConstantRangeIncludingKnownBits( + LHS, /*ForSigned=*/true, DL, /*Depth=*/0, AC, CxtI, DT); + ConstantRange RHSRange = computeConstantRangeIncludingKnownBits( + RHS, /*ForSigned=*/true, DL, /*Depth=*/0, AC, CxtI, DT); + return mapOverflowResult(LHSRange.signedSubMayOverflow(RHSRange)); } -bool llvm::isOverflowIntrinsicNoWrap(const IntrinsicInst *II, +bool llvm::isOverflowIntrinsicNoWrap(const WithOverflowInst *WO, const DominatorTree &DT) { -#ifndef NDEBUG - auto IID = II->getIntrinsicID(); - assert((IID == Intrinsic::sadd_with_overflow || - IID == Intrinsic::uadd_with_overflow || - IID == Intrinsic::ssub_with_overflow || - IID == Intrinsic::usub_with_overflow || - IID == Intrinsic::smul_with_overflow || - IID == Intrinsic::umul_with_overflow) && - "Not an overflow intrinsic!"); -#endif - SmallVector GuardingBranches; SmallVector Results; - for (const User *U : II->users()) { + for (const User *U : WO->users()) { if (const auto *EVI = dyn_cast(U)) { assert(EVI->getNumIndices() == 1 && "Obvious from CI's type"); @@ -4307,6 +4256,11 @@ bool llvm::isGuaranteedToTransferExecutionToSuccessor(const Instruction *I) { if (!CS.doesNotThrow()) return false; + // A function which doens't throw and has "willreturn" attribute will + // always return. + if (CS.hasFnAttr(Attribute::WillReturn)) + return true; + // Non-throwing call sites can loop infinitely, call exit/pthread_exit // etc. and thus not return. However, LLVM already assumes that // @@ -4325,7 +4279,8 @@ bool llvm::isGuaranteedToTransferExecutionToSuccessor(const Instruction *I) { // is guaranteed to return. return CS.onlyReadsMemory() || CS.onlyAccessesArgMemory() || match(I, m_Intrinsic()) || - match(I, m_Intrinsic()); + match(I, m_Intrinsic()) || + match(I, m_Intrinsic()); } // Other instructions return normally. @@ -4333,7 +4288,7 @@ bool llvm::isGuaranteedToTransferExecutionToSuccessor(const Instruction *I) { } bool llvm::isGuaranteedToTransferExecutionToSuccessor(const BasicBlock *BB) { - // TODO: This is slightly consdervative for invoke instruction since exiting + // TODO: This is slightly conservative for invoke instruction since exiting // via an exception *is* normal control for them. for (auto I = BB->begin(), E = BB->end(); I != E; ++I) if (!isGuaranteedToTransferExecutionToSuccessor(&*I)) @@ -4357,6 +4312,8 @@ bool llvm::isGuaranteedToExecuteForEveryIteration(const Instruction *I, } bool llvm::propagatesFullPoison(const Instruction *I) { + // TODO: This should include all instructions apart from phis, selects and + // call-like instructions. switch (I->getOpcode()) { case Instruction::Add: case Instruction::Sub: @@ -4409,10 +4366,21 @@ const Value *llvm::getGuaranteedNonFullPoisonOp(const Instruction *I) { return I->getOperand(1); default: + // Note: It's really tempting to think that a conditional branch or + // switch should be listed here, but that's incorrect. It's not + // branching off of poison which is UB, it is executing a side effecting + // instruction which follows the branch. return nullptr; } } +bool llvm::mustTriggerUB(const Instruction *I, + const SmallSet& KnownPoison) { + auto *NotPoison = getGuaranteedNonFullPoisonOp(I); + return (NotPoison && KnownPoison.count(NotPoison)); +} + + bool llvm::programUndefinedIfFullPoison(const Instruction *PoisonI) { // We currently only look for uses of poison values within the same basic // block, as that makes it easier to guarantee that the uses will be @@ -4436,8 +4404,7 @@ bool llvm::programUndefinedIfFullPoison(const Instruction *PoisonI) { while (Iter++ < MaxDepth) { for (auto &I : make_range(Begin, End)) { if (&I != PoisonI) { - const Value *NotPoison = getGuaranteedNonFullPoisonOp(&I); - if (NotPoison != nullptr && YieldsPoison.count(NotPoison)) + if (mustTriggerUB(&I, YieldsPoison)) return true; if (!isGuaranteedToTransferExecutionToSuccessor(&I)) return false; @@ -4926,6 +4893,10 @@ static SelectPatternResult matchSelectPattern(CmpInst::Predicate Pred, if (Pred == ICmpInst::ICMP_SGT && match(CmpRHS, ZeroOrAllOnes)) return {SPF_ABS, SPNB_NA, false}; + // (X >=s 0) ? X : -X or (X >=s 1) ? X : -X --> ABS(X) + if (Pred == ICmpInst::ICMP_SGE && match(CmpRHS, ZeroOrOne)) + return {SPF_ABS, SPNB_NA, false}; + // (X NABS(X) // (-X NABS(X) if (Pred == ICmpInst::ICMP_SLT && match(CmpRHS, ZeroOrOne)) @@ -5084,11 +5055,19 @@ SelectPatternResult llvm::matchSelectPattern(Value *V, Value *&LHS, Value *&RHS, CmpInst *CmpI = dyn_cast(SI->getCondition()); if (!CmpI) return {SPF_UNKNOWN, SPNB_NA, false}; + Value *TrueVal = SI->getTrueValue(); + Value *FalseVal = SI->getFalseValue(); + + return llvm::matchDecomposedSelectPattern(CmpI, TrueVal, FalseVal, LHS, RHS, + CastOp, Depth); +} + +SelectPatternResult llvm::matchDecomposedSelectPattern( + CmpInst *CmpI, Value *TrueVal, Value *FalseVal, Value *&LHS, Value *&RHS, + Instruction::CastOps *CastOp, unsigned Depth) { CmpInst::Predicate Pred = CmpI->getPredicate(); Value *CmpLHS = CmpI->getOperand(0); Value *CmpRHS = CmpI->getOperand(1); - Value *TrueVal = SI->getTrueValue(); - Value *FalseVal = SI->getFalseValue(); FastMathFlags FMF; if (isa(CmpI)) FMF = CmpI->getFastMathFlags(); @@ -5430,3 +5409,298 @@ Optional llvm::isImpliedByDomCondition(const Value *Cond, bool CondIsTrue = TrueBB == ContextBB; return isImpliedCondition(PredCond, Cond, DL, CondIsTrue); } + +static void setLimitsForBinOp(const BinaryOperator &BO, APInt &Lower, + APInt &Upper, const InstrInfoQuery &IIQ) { + unsigned Width = Lower.getBitWidth(); + const APInt *C; + switch (BO.getOpcode()) { + case Instruction::Add: + if (match(BO.getOperand(1), m_APInt(C)) && !C->isNullValue()) { + // FIXME: If we have both nuw and nsw, we should reduce the range further. + if (IIQ.hasNoUnsignedWrap(cast(&BO))) { + // 'add nuw x, C' produces [C, UINT_MAX]. + Lower = *C; + } else if (IIQ.hasNoSignedWrap(cast(&BO))) { + if (C->isNegative()) { + // 'add nsw x, -C' produces [SINT_MIN, SINT_MAX - C]. + Lower = APInt::getSignedMinValue(Width); + Upper = APInt::getSignedMaxValue(Width) + *C + 1; + } else { + // 'add nsw x, +C' produces [SINT_MIN + C, SINT_MAX]. + Lower = APInt::getSignedMinValue(Width) + *C; + Upper = APInt::getSignedMaxValue(Width) + 1; + } + } + } + break; + + case Instruction::And: + if (match(BO.getOperand(1), m_APInt(C))) + // 'and x, C' produces [0, C]. + Upper = *C + 1; + break; + + case Instruction::Or: + if (match(BO.getOperand(1), m_APInt(C))) + // 'or x, C' produces [C, UINT_MAX]. + Lower = *C; + break; + + case Instruction::AShr: + if (match(BO.getOperand(1), m_APInt(C)) && C->ult(Width)) { + // 'ashr x, C' produces [INT_MIN >> C, INT_MAX >> C]. + Lower = APInt::getSignedMinValue(Width).ashr(*C); + Upper = APInt::getSignedMaxValue(Width).ashr(*C) + 1; + } else if (match(BO.getOperand(0), m_APInt(C))) { + unsigned ShiftAmount = Width - 1; + if (!C->isNullValue() && IIQ.isExact(&BO)) + ShiftAmount = C->countTrailingZeros(); + if (C->isNegative()) { + // 'ashr C, x' produces [C, C >> (Width-1)] + Lower = *C; + Upper = C->ashr(ShiftAmount) + 1; + } else { + // 'ashr C, x' produces [C >> (Width-1), C] + Lower = C->ashr(ShiftAmount); + Upper = *C + 1; + } + } + break; + + case Instruction::LShr: + if (match(BO.getOperand(1), m_APInt(C)) && C->ult(Width)) { + // 'lshr x, C' produces [0, UINT_MAX >> C]. + Upper = APInt::getAllOnesValue(Width).lshr(*C) + 1; + } else if (match(BO.getOperand(0), m_APInt(C))) { + // 'lshr C, x' produces [C >> (Width-1), C]. + unsigned ShiftAmount = Width - 1; + if (!C->isNullValue() && IIQ.isExact(&BO)) + ShiftAmount = C->countTrailingZeros(); + Lower = C->lshr(ShiftAmount); + Upper = *C + 1; + } + break; + + case Instruction::Shl: + if (match(BO.getOperand(0), m_APInt(C))) { + if (IIQ.hasNoUnsignedWrap(&BO)) { + // 'shl nuw C, x' produces [C, C << CLZ(C)] + Lower = *C; + Upper = Lower.shl(Lower.countLeadingZeros()) + 1; + } else if (BO.hasNoSignedWrap()) { // TODO: What if both nuw+nsw? + if (C->isNegative()) { + // 'shl nsw C, x' produces [C << CLO(C)-1, C] + unsigned ShiftAmount = C->countLeadingOnes() - 1; + Lower = C->shl(ShiftAmount); + Upper = *C + 1; + } else { + // 'shl nsw C, x' produces [C, C << CLZ(C)-1] + unsigned ShiftAmount = C->countLeadingZeros() - 1; + Lower = *C; + Upper = C->shl(ShiftAmount) + 1; + } + } + } + break; + + case Instruction::SDiv: + if (match(BO.getOperand(1), m_APInt(C))) { + APInt IntMin = APInt::getSignedMinValue(Width); + APInt IntMax = APInt::getSignedMaxValue(Width); + if (C->isAllOnesValue()) { + // 'sdiv x, -1' produces [INT_MIN + 1, INT_MAX] + // where C != -1 and C != 0 and C != 1 + Lower = IntMin + 1; + Upper = IntMax + 1; + } else if (C->countLeadingZeros() < Width - 1) { + // 'sdiv x, C' produces [INT_MIN / C, INT_MAX / C] + // where C != -1 and C != 0 and C != 1 + Lower = IntMin.sdiv(*C); + Upper = IntMax.sdiv(*C); + if (Lower.sgt(Upper)) + std::swap(Lower, Upper); + Upper = Upper + 1; + assert(Upper != Lower && "Upper part of range has wrapped!"); + } + } else if (match(BO.getOperand(0), m_APInt(C))) { + if (C->isMinSignedValue()) { + // 'sdiv INT_MIN, x' produces [INT_MIN, INT_MIN / -2]. + Lower = *C; + Upper = Lower.lshr(1) + 1; + } else { + // 'sdiv C, x' produces [-|C|, |C|]. + Upper = C->abs() + 1; + Lower = (-Upper) + 1; + } + } + break; + + case Instruction::UDiv: + if (match(BO.getOperand(1), m_APInt(C)) && !C->isNullValue()) { + // 'udiv x, C' produces [0, UINT_MAX / C]. + Upper = APInt::getMaxValue(Width).udiv(*C) + 1; + } else if (match(BO.getOperand(0), m_APInt(C))) { + // 'udiv C, x' produces [0, C]. + Upper = *C + 1; + } + break; + + case Instruction::SRem: + if (match(BO.getOperand(1), m_APInt(C))) { + // 'srem x, C' produces (-|C|, |C|). + Upper = C->abs(); + Lower = (-Upper) + 1; + } + break; + + case Instruction::URem: + if (match(BO.getOperand(1), m_APInt(C))) + // 'urem x, C' produces [0, C). + Upper = *C; + break; + + default: + break; + } +} + +static void setLimitsForIntrinsic(const IntrinsicInst &II, APInt &Lower, + APInt &Upper) { + unsigned Width = Lower.getBitWidth(); + const APInt *C; + switch (II.getIntrinsicID()) { + case Intrinsic::uadd_sat: + // uadd.sat(x, C) produces [C, UINT_MAX]. + if (match(II.getOperand(0), m_APInt(C)) || + match(II.getOperand(1), m_APInt(C))) + Lower = *C; + break; + case Intrinsic::sadd_sat: + if (match(II.getOperand(0), m_APInt(C)) || + match(II.getOperand(1), m_APInt(C))) { + if (C->isNegative()) { + // sadd.sat(x, -C) produces [SINT_MIN, SINT_MAX + (-C)]. + Lower = APInt::getSignedMinValue(Width); + Upper = APInt::getSignedMaxValue(Width) + *C + 1; + } else { + // sadd.sat(x, +C) produces [SINT_MIN + C, SINT_MAX]. + Lower = APInt::getSignedMinValue(Width) + *C; + Upper = APInt::getSignedMaxValue(Width) + 1; + } + } + break; + case Intrinsic::usub_sat: + // usub.sat(C, x) produces [0, C]. + if (match(II.getOperand(0), m_APInt(C))) + Upper = *C + 1; + // usub.sat(x, C) produces [0, UINT_MAX - C]. + else if (match(II.getOperand(1), m_APInt(C))) + Upper = APInt::getMaxValue(Width) - *C + 1; + break; + case Intrinsic::ssub_sat: + if (match(II.getOperand(0), m_APInt(C))) { + if (C->isNegative()) { + // ssub.sat(-C, x) produces [SINT_MIN, -SINT_MIN + (-C)]. + Lower = APInt::getSignedMinValue(Width); + Upper = *C - APInt::getSignedMinValue(Width) + 1; + } else { + // ssub.sat(+C, x) produces [-SINT_MAX + C, SINT_MAX]. + Lower = *C - APInt::getSignedMaxValue(Width); + Upper = APInt::getSignedMaxValue(Width) + 1; + } + } else if (match(II.getOperand(1), m_APInt(C))) { + if (C->isNegative()) { + // ssub.sat(x, -C) produces [SINT_MIN - (-C), SINT_MAX]: + Lower = APInt::getSignedMinValue(Width) - *C; + Upper = APInt::getSignedMaxValue(Width) + 1; + } else { + // ssub.sat(x, +C) produces [SINT_MIN, SINT_MAX - C]. + Lower = APInt::getSignedMinValue(Width); + Upper = APInt::getSignedMaxValue(Width) - *C + 1; + } + } + break; + default: + break; + } +} + +static void setLimitsForSelectPattern(const SelectInst &SI, APInt &Lower, + APInt &Upper) { + const Value *LHS, *RHS; + SelectPatternResult R = matchSelectPattern(&SI, LHS, RHS); + if (R.Flavor == SPF_UNKNOWN) + return; + + unsigned BitWidth = SI.getType()->getScalarSizeInBits(); + + if (R.Flavor == SelectPatternFlavor::SPF_ABS) { + // If the negation part of the abs (in RHS) has the NSW flag, + // then the result of abs(X) is [0..SIGNED_MAX], + // otherwise it is [0..SIGNED_MIN], as -SIGNED_MIN == SIGNED_MIN. + Lower = APInt::getNullValue(BitWidth); + if (cast(RHS)->hasNoSignedWrap()) + Upper = APInt::getSignedMaxValue(BitWidth) + 1; + else + Upper = APInt::getSignedMinValue(BitWidth) + 1; + return; + } + + if (R.Flavor == SelectPatternFlavor::SPF_NABS) { + // The result of -abs(X) is <= 0. + Lower = APInt::getSignedMinValue(BitWidth); + Upper = APInt(BitWidth, 1); + return; + } + + const APInt *C; + if (!match(LHS, m_APInt(C)) && !match(RHS, m_APInt(C))) + return; + + switch (R.Flavor) { + case SPF_UMIN: + Upper = *C + 1; + break; + case SPF_UMAX: + Lower = *C; + break; + case SPF_SMIN: + Lower = APInt::getSignedMinValue(BitWidth); + Upper = *C + 1; + break; + case SPF_SMAX: + Lower = *C; + Upper = APInt::getSignedMaxValue(BitWidth) + 1; + break; + default: + break; + } +} + +ConstantRange llvm::computeConstantRange(const Value *V, bool UseInstrInfo) { + assert(V->getType()->isIntOrIntVectorTy() && "Expected integer instruction"); + + const APInt *C; + if (match(V, m_APInt(C))) + return ConstantRange(*C); + + InstrInfoQuery IIQ(UseInstrInfo); + unsigned BitWidth = V->getType()->getScalarSizeInBits(); + APInt Lower = APInt(BitWidth, 0); + APInt Upper = APInt(BitWidth, 0); + if (auto *BO = dyn_cast(V)) + setLimitsForBinOp(*BO, Lower, Upper, IIQ); + else if (auto *II = dyn_cast(V)) + setLimitsForIntrinsic(*II, Lower, Upper); + else if (auto *SI = dyn_cast(V)) + setLimitsForSelectPattern(*SI, Lower, Upper); + + ConstantRange CR = ConstantRange::getNonEmpty(Lower, Upper); + + if (auto *I = dyn_cast(V)) + if (auto *Range = IIQ.getMetadata(I, LLVMContext::MD_range)) + CR = CR.intersectWith(getConstantRangeFromMetadata(*Range)); + + return CR; +} diff --git a/lib/Analysis/VectorUtils.cpp b/lib/Analysis/VectorUtils.cpp index 5656a19d7e0d..986756eb2627 100644 --- a/lib/Analysis/VectorUtils.cpp +++ b/lib/Analysis/VectorUtils.cpp @@ -1,9 +1,8 @@ //===----------- VectorUtils.cpp - Vectorizer utility functions -----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -38,8 +37,9 @@ static cl::opt MaxInterleaveGroupFactor( cl::init(8)); /// Return true if all of the intrinsic's arguments and return type are scalars -/// for the scalar form of the intrinsic and vectors for the vector form of the -/// intrinsic. +/// for the scalar form of the intrinsic, and vectors for the vector form of the +/// intrinsic (except operands that are marked as always being scalar by +/// hasVectorInstrinsicScalarOpd). bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) { switch (ID) { case Intrinsic::bswap: // Begin integer bit-manipulation. @@ -49,6 +49,13 @@ bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) { case Intrinsic::cttz: case Intrinsic::fshl: case Intrinsic::fshr: + case Intrinsic::sadd_sat: + case Intrinsic::ssub_sat: + case Intrinsic::uadd_sat: + case Intrinsic::usub_sat: + case Intrinsic::smul_fix: + case Intrinsic::smul_fix_sat: + case Intrinsic::umul_fix: case Intrinsic::sqrt: // Begin floating-point. case Intrinsic::sin: case Intrinsic::cos: @@ -74,18 +81,13 @@ bool llvm::isTriviallyVectorizable(Intrinsic::ID ID) { case Intrinsic::fmuladd: case Intrinsic::powi: case Intrinsic::canonicalize: - case Intrinsic::sadd_sat: - case Intrinsic::ssub_sat: - case Intrinsic::uadd_sat: - case Intrinsic::usub_sat: return true; default: return false; } } -/// Identifies if the intrinsic has a scalar operand. It check for -/// ctlz,cttz and powi special intrinsics whose argument is scalar. +/// Identifies if the vector form of the intrinsic has a scalar operand. bool llvm::hasVectorInstrinsicScalarOpd(Intrinsic::ID ID, unsigned ScalarOpdIdx) { switch (ID) { @@ -93,6 +95,10 @@ bool llvm::hasVectorInstrinsicScalarOpd(Intrinsic::ID ID, case Intrinsic::cttz: case Intrinsic::powi: return (ScalarOpdIdx == 1); + case Intrinsic::smul_fix: + case Intrinsic::smul_fix_sat: + case Intrinsic::umul_fix: + return (ScalarOpdIdx == 2); default: return false; } @@ -300,30 +306,60 @@ Value *llvm::findScalarElement(Value *V, unsigned EltNo) { /// Get splat value if the input is a splat vector or return nullptr. /// This function is not fully general. It checks only 2 cases: -/// the input value is (1) a splat constants vector or (2) a sequence -/// of instructions that broadcast a single value into a vector. -/// +/// the input value is (1) a splat constant vector or (2) a sequence +/// of instructions that broadcasts a scalar at element 0. const llvm::Value *llvm::getSplatValue(const Value *V) { - - if (auto *C = dyn_cast(V)) - if (isa(V->getType())) + if (isa(V->getType())) + if (auto *C = dyn_cast(V)) return C->getSplatValue(); - auto *ShuffleInst = dyn_cast(V); - if (!ShuffleInst) - return nullptr; - // All-zero (or undef) shuffle mask elements. - for (int MaskElt : ShuffleInst->getShuffleMask()) - if (MaskElt != 0 && MaskElt != -1) - return nullptr; - // The first shuffle source is 'insertelement' with index 0. - auto *InsertEltInst = - dyn_cast(ShuffleInst->getOperand(0)); - if (!InsertEltInst || !isa(InsertEltInst->getOperand(2)) || - !cast(InsertEltInst->getOperand(2))->isZero()) - return nullptr; + // shuf (inselt ?, Splat, 0), ?, <0, undef, 0, ...> + Value *Splat; + if (match(V, m_ShuffleVector(m_InsertElement(m_Value(), m_Value(Splat), + m_ZeroInt()), + m_Value(), m_ZeroInt()))) + return Splat; - return InsertEltInst->getOperand(1); + return nullptr; +} + +// This setting is based on its counterpart in value tracking, but it could be +// adjusted if needed. +const unsigned MaxDepth = 6; + +bool llvm::isSplatValue(const Value *V, unsigned Depth) { + assert(Depth <= MaxDepth && "Limit Search Depth"); + + if (isa(V->getType())) { + if (isa(V)) + return true; + // FIXME: Constant splat analysis does not allow undef elements. + if (auto *C = dyn_cast(V)) + return C->getSplatValue() != nullptr; + } + + // FIXME: Constant splat analysis does not allow undef elements. + Constant *Mask; + if (match(V, m_ShuffleVector(m_Value(), m_Value(), m_Constant(Mask)))) + return Mask->getSplatValue() != nullptr; + + // The remaining tests are all recursive, so bail out if we hit the limit. + if (Depth++ == MaxDepth) + return false; + + // If both operands of a binop are splats, the result is a splat. + Value *X, *Y, *Z; + if (match(V, m_BinOp(m_Value(X), m_Value(Y)))) + return isSplatValue(X, Depth) && isSplatValue(Y, Depth); + + // If all operands of a select are splats, the result is a splat. + if (match(V, m_Select(m_Value(X), m_Value(Y), m_Value(Z)))) + return isSplatValue(X, Depth) && isSplatValue(Y, Depth) && + isSplatValue(Z, Depth); + + // TODO: Add support for unary ops (fneg), casts, intrinsics (overflow ops). + + return false; } MapVector @@ -711,6 +747,52 @@ Value *llvm::concatenateVectors(IRBuilder<> &Builder, ArrayRef Vecs) { return ResList[0]; } +bool llvm::maskIsAllZeroOrUndef(Value *Mask) { + auto *ConstMask = dyn_cast(Mask); + if (!ConstMask) + return false; + if (ConstMask->isNullValue() || isa(ConstMask)) + return true; + for (unsigned I = 0, E = ConstMask->getType()->getVectorNumElements(); I != E; + ++I) { + if (auto *MaskElt = ConstMask->getAggregateElement(I)) + if (MaskElt->isNullValue() || isa(MaskElt)) + continue; + return false; + } + return true; +} + + +bool llvm::maskIsAllOneOrUndef(Value *Mask) { + auto *ConstMask = dyn_cast(Mask); + if (!ConstMask) + return false; + if (ConstMask->isAllOnesValue() || isa(ConstMask)) + return true; + for (unsigned I = 0, E = ConstMask->getType()->getVectorNumElements(); I != E; + ++I) { + if (auto *MaskElt = ConstMask->getAggregateElement(I)) + if (MaskElt->isAllOnesValue() || isa(MaskElt)) + continue; + return false; + } + return true; +} + +/// TODO: This is a lot like known bits, but for +/// vectors. Is there something we can common this with? +APInt llvm::possiblyDemandedEltsInMask(Value *Mask) { + + const unsigned VWidth = cast(Mask->getType())->getNumElements(); + APInt DemandedElts = APInt::getAllOnesValue(VWidth); + if (auto *CV = dyn_cast(Mask)) + for (unsigned i = 0; i < VWidth; i++) + if (CV->getAggregateElement(i)->isNullValue()) + DemandedElts.clearBit(i); + return DemandedElts; +} + bool InterleavedAccessInfo::isStrided(int Stride) { unsigned Factor = std::abs(Stride); return Factor >= 2 && Factor <= MaxInterleaveGroupFactor; @@ -992,7 +1074,7 @@ void InterleavedAccessInfo::analyzeInterleaving( // that all the pointers in the group don't wrap. // So we check only group member 0 (which is always guaranteed to exist), // and group member Factor - 1; If the latter doesn't exist we rely on - // peeling (if it is a non-reveresed accsess -- see Case 3). + // peeling (if it is a non-reversed accsess -- see Case 3). Value *FirstMemberPtr = getLoadStorePointerOperand(Group->getMember(0)); if (!getPtrStride(PSE, FirstMemberPtr, TheLoop, Strides, /*Assume=*/false, /*ShouldCheckWrap=*/true)) { diff --git a/lib/AsmParser/LLLexer.cpp b/lib/AsmParser/LLLexer.cpp index eab7ec819536..72d2357c2933 100644 --- a/lib/AsmParser/LLLexer.cpp +++ b/lib/AsmParser/LLLexer.cpp @@ -1,9 +1,8 @@ //===- LLLexer.cpp - Lexer for .ll Files ----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -571,6 +570,7 @@ lltok::Kind LLLexer::LexIdentifier() { KEYWORD(align); KEYWORD(addrspace); KEYWORD(section); + KEYWORD(partition); KEYWORD(alias); KEYWORD(ifunc); KEYWORD(module); @@ -650,6 +650,7 @@ lltok::Kind LLLexer::LexIdentifier() { KEYWORD(nobuiltin); KEYWORD(nocapture); KEYWORD(noduplicate); + KEYWORD(nofree); KEYWORD(noimplicitfloat); KEYWORD(noinline); KEYWORD(norecurse); @@ -657,6 +658,7 @@ lltok::Kind LLLexer::LexIdentifier() { KEYWORD(nonnull); KEYWORD(noredzone); KEYWORD(noreturn); + KEYWORD(nosync); KEYWORD(nocf_check); KEYWORD(nounwind); KEYWORD(optforfuzzing); @@ -677,14 +679,17 @@ lltok::Kind LLLexer::LexIdentifier() { KEYWORD(shadowcallstack); KEYWORD(sanitize_address); KEYWORD(sanitize_hwaddress); + KEYWORD(sanitize_memtag); KEYWORD(sanitize_thread); KEYWORD(sanitize_memory); KEYWORD(speculative_load_hardening); KEYWORD(swifterror); KEYWORD(swiftself); KEYWORD(uwtable); + KEYWORD(willreturn); KEYWORD(writeonly); KEYWORD(zeroext); + KEYWORD(immarg); KEYWORD(type); KEYWORD(opaque); @@ -706,6 +711,7 @@ lltok::Kind LLLexer::LexIdentifier() { KEYWORD(xchg); KEYWORD(nand); KEYWORD(max); KEYWORD(min); KEYWORD(umax); KEYWORD(umin); + KEYWORD(vscale); KEYWORD(x); KEYWORD(blockaddress); @@ -733,6 +739,7 @@ lltok::Kind LLLexer::LexIdentifier() { KEYWORD(notEligibleToImport); KEYWORD(live); KEYWORD(dsoLocal); + KEYWORD(canAutoHide); KEYWORD(function); KEYWORD(insts); KEYWORD(funcFlags); @@ -749,6 +756,8 @@ lltok::Kind LLLexer::LexIdentifier() { KEYWORD(critical); KEYWORD(relbf); KEYWORD(variable); + KEYWORD(vTableFuncs); + KEYWORD(virtFunc); KEYWORD(aliasee); KEYWORD(refs); KEYWORD(typeIdInfo); @@ -761,6 +770,7 @@ lltok::Kind LLLexer::LexIdentifier() { KEYWORD(offset); KEYWORD(args); KEYWORD(typeid); + KEYWORD(typeidCompatibleVTable); KEYWORD(summary); KEYWORD(typeTestRes); KEYWORD(kind); @@ -859,6 +869,7 @@ lltok::Kind LLLexer::LexIdentifier() { INSTKEYWORD(invoke, Invoke); INSTKEYWORD(resume, Resume); INSTKEYWORD(unreachable, Unreachable); + INSTKEYWORD(callbr, CallBr); INSTKEYWORD(alloca, Alloca); INSTKEYWORD(load, Load); @@ -1047,7 +1058,17 @@ lltok::Kind LLLexer::LexDigitOrNegative() { for (; isdigit(static_cast(CurPtr[0])); ++CurPtr) /*empty*/; - // Check to see if this really is a label afterall, e.g. "-1:". + // Check if this is a fully-numeric label: + if (isdigit(TokStart[0]) && CurPtr[0] == ':') { + uint64_t Val = atoull(TokStart, CurPtr); + ++CurPtr; // Skip the colon. + if ((unsigned)Val != Val) + Error("invalid value number (too large)!"); + UIntVal = unsigned(Val); + return lltok::LabelID; + } + + // Check to see if this really is a string label, e.g. "-1:". if (isLabelChar(CurPtr[0]) || CurPtr[0] == ':') { if (const char *End = isLabelTail(CurPtr)) { StrVal.assign(TokStart, End-1); diff --git a/lib/AsmParser/LLLexer.h b/lib/AsmParser/LLLexer.h index 21deb6e08910..4d3a2920e937 100644 --- a/lib/AsmParser/LLLexer.h +++ b/lib/AsmParser/LLLexer.h @@ -1,9 +1,8 @@ //===- LLLexer.h - Lexer for LLVM Assembly Files ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/AsmParser/LLParser.cpp b/lib/AsmParser/LLParser.cpp index ee634505581e..87dff6468f2d 100644 --- a/lib/AsmParser/LLParser.cpp +++ b/lib/AsmParser/LLParser.cpp @@ -1,9 +1,8 @@ //===-- LLParser.cpp - Parser Class ---------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -164,6 +163,14 @@ bool LLParser::ValidateEndOfModule() { AS = AS.addAttributes(Context, AttributeList::FunctionIndex, AttributeSet::get(Context, FnAttrs)); II->setAttributes(AS); + } else if (CallBrInst *CBI = dyn_cast(V)) { + AttributeList AS = CBI->getAttributes(); + AttrBuilder FnAttrs(AS.getFnAttributes()); + AS = AS.removeAttributes(Context, AttributeList::FunctionIndex); + FnAttrs.merge(B); + AS = AS.addAttributes(Context, AttributeList::FunctionIndex, + AttributeSet::get(Context, FnAttrs)); + CBI->setAttributes(AS); } else if (auto *GV = dyn_cast(V)) { AttrBuilder Attrs(GV->getAttributes()); Attrs.merge(B); @@ -814,19 +821,26 @@ bool LLParser::ParseSummaryEntry() { if (!Index) return SkipModuleSummaryEntry(); + bool result = false; switch (Lex.getKind()) { case lltok::kw_gv: - return ParseGVEntry(SummaryID); + result = ParseGVEntry(SummaryID); + break; case lltok::kw_module: - return ParseModuleEntry(SummaryID); + result = ParseModuleEntry(SummaryID); + break; case lltok::kw_typeid: - return ParseTypeIdEntry(SummaryID); + result = ParseTypeIdEntry(SummaryID); + break; + case lltok::kw_typeidCompatibleVTable: + result = ParseTypeIdCompatibleVtableEntry(SummaryID); break; default: - return Error(Lex.getLoc(), "unexpected summary kind"); + result = Error(Lex.getLoc(), "unexpected summary kind"); + break; } Lex.setIgnoreColonInIdentifiers(false); - return false; + return result; } static bool isValidVisibilityForLinkage(unsigned V, unsigned L) { @@ -845,11 +859,14 @@ static void maybeSetDSOLocal(bool DSOLocal, GlobalValue &GV) { /// ::= GlobalVar '=' OptionalLinkage OptionalPreemptionSpecifier /// OptionalVisibility OptionalDLLStorageClass /// OptionalThreadLocal OptionalUnnamedAddr -// 'alias|ifunc' IndirectSymbol +/// 'alias|ifunc' IndirectSymbol IndirectSymbolAttr* /// /// IndirectSymbol /// ::= TypeAndValue /// +/// IndirectSymbolAttr +/// ::= ',' 'partition' StringConstant +/// /// Everything through OptionalUnnamedAddr has already been parsed. /// bool LLParser::parseIndirectSymbol(const std::string &Name, LocTy NameLoc, @@ -949,6 +966,21 @@ bool LLParser::parseIndirectSymbol(const std::string &Name, LocTy NameLoc, GA->setUnnamedAddr(UnnamedAddr); maybeSetDSOLocal(DSOLocal, *GA); + // At this point we've parsed everything except for the IndirectSymbolAttrs. + // Now parse them if there are any. + while (Lex.getKind() == lltok::comma) { + Lex.Lex(); + + if (Lex.getKind() == lltok::kw_partition) { + Lex.Lex(); + GA->setPartition(Lex.getStrVal()); + if (ParseToken(lltok::StringConstant, "expected partition string")) + return true; + } else { + return TokError("unknown alias or ifunc property!"); + } + } + if (Name.empty()) NumberedVals.push_back(GA.get()); @@ -1084,6 +1116,11 @@ bool LLParser::ParseGlobal(const std::string &Name, LocTy NameLoc, GV->setSection(Lex.getStrVal()); if (ParseToken(lltok::StringConstant, "expected global section string")) return true; + } else if (Lex.getKind() == lltok::kw_partition) { + Lex.Lex(); + GV->setPartition(Lex.getStrVal()); + if (ParseToken(lltok::StringConstant, "expected partition string")) + return true; } else if (Lex.getKind() == lltok::kw_align) { unsigned Alignment; if (ParseOptionalAlignment(Alignment)) return true; @@ -1243,12 +1280,14 @@ bool LLParser::ParseFnAttributeValuePairs(AttrBuilder &B, case lltok::kw_naked: B.addAttribute(Attribute::Naked); break; case lltok::kw_nobuiltin: B.addAttribute(Attribute::NoBuiltin); break; case lltok::kw_noduplicate: B.addAttribute(Attribute::NoDuplicate); break; + case lltok::kw_nofree: B.addAttribute(Attribute::NoFree); break; case lltok::kw_noimplicitfloat: B.addAttribute(Attribute::NoImplicitFloat); break; case lltok::kw_noinline: B.addAttribute(Attribute::NoInline); break; case lltok::kw_nonlazybind: B.addAttribute(Attribute::NonLazyBind); break; case lltok::kw_noredzone: B.addAttribute(Attribute::NoRedZone); break; case lltok::kw_noreturn: B.addAttribute(Attribute::NoReturn); break; + case lltok::kw_nosync: B.addAttribute(Attribute::NoSync); break; case lltok::kw_nocf_check: B.addAttribute(Attribute::NoCfCheck); break; case lltok::kw_norecurse: B.addAttribute(Attribute::NoRecurse); break; case lltok::kw_nounwind: B.addAttribute(Attribute::NoUnwind); break; @@ -1272,6 +1311,8 @@ bool LLParser::ParseFnAttributeValuePairs(AttrBuilder &B, B.addAttribute(Attribute::SanitizeAddress); break; case lltok::kw_sanitize_hwaddress: B.addAttribute(Attribute::SanitizeHWAddress); break; + case lltok::kw_sanitize_memtag: + B.addAttribute(Attribute::SanitizeMemTag); break; case lltok::kw_sanitize_thread: B.addAttribute(Attribute::SanitizeThread); break; case lltok::kw_sanitize_memory: @@ -1281,6 +1322,7 @@ bool LLParser::ParseFnAttributeValuePairs(AttrBuilder &B, break; case lltok::kw_strictfp: B.addAttribute(Attribute::StrictFP); break; case lltok::kw_uwtable: B.addAttribute(Attribute::UWTable); break; + case lltok::kw_willreturn: B.addAttribute(Attribute::WillReturn); break; case lltok::kw_writeonly: B.addAttribute(Attribute::WriteOnly); break; // Error handling. @@ -1303,6 +1345,7 @@ bool LLParser::ParseFnAttributeValuePairs(AttrBuilder &B, case lltok::kw_sret: case lltok::kw_swifterror: case lltok::kw_swiftself: + case lltok::kw_immarg: HaveError |= Error(Lex.getLoc(), "invalid use of parameter-only attribute on a function"); @@ -1566,7 +1609,13 @@ bool LLParser::ParseOptionalParamAttrs(AttrBuilder &B) { B.addAlignmentAttr(Alignment); continue; } - case lltok::kw_byval: B.addAttribute(Attribute::ByVal); break; + case lltok::kw_byval: { + Type *Ty; + if (ParseByValWithOptionalType(Ty)) + return true; + B.addByValAttr(Ty); + continue; + } case lltok::kw_dereferenceable: { uint64_t Bytes; if (ParseOptionalDerefAttrBytes(lltok::kw_dereferenceable, Bytes)) @@ -1596,6 +1645,7 @@ bool LLParser::ParseOptionalParamAttrs(AttrBuilder &B) { case lltok::kw_swiftself: B.addAttribute(Attribute::SwiftSelf); break; case lltok::kw_writeonly: B.addAttribute(Attribute::WriteOnly); break; case lltok::kw_zeroext: B.addAttribute(Attribute::ZExt); break; + case lltok::kw_immarg: B.addAttribute(Attribute::ImmArg); break; case lltok::kw_alignstack: case lltok::kw_alwaysinline: @@ -1620,6 +1670,7 @@ bool LLParser::ParseOptionalParamAttrs(AttrBuilder &B) { case lltok::kw_returns_twice: case lltok::kw_sanitize_address: case lltok::kw_sanitize_hwaddress: + case lltok::kw_sanitize_memtag: case lltok::kw_sanitize_memory: case lltok::kw_sanitize_thread: case lltok::kw_speculative_load_hardening: @@ -1690,6 +1741,7 @@ bool LLParser::ParseOptionalReturnAttrs(AttrBuilder &B) { case lltok::kw_sret: case lltok::kw_swifterror: case lltok::kw_swiftself: + case lltok::kw_immarg: HaveError |= Error(Lex.getLoc(), "invalid use of parameter-only attribute"); break; @@ -1717,6 +1769,7 @@ bool LLParser::ParseOptionalReturnAttrs(AttrBuilder &B) { case lltok::kw_returns_twice: case lltok::kw_sanitize_address: case lltok::kw_sanitize_hwaddress: + case lltok::kw_sanitize_memtag: case lltok::kw_sanitize_memory: case lltok::kw_sanitize_thread: case lltok::kw_speculative_load_hardening: @@ -2417,6 +2470,22 @@ bool LLParser::ParseParameterList(SmallVectorImpl &ArgList, return false; } +/// ParseByValWithOptionalType +/// ::= byval +/// ::= byval() +bool LLParser::ParseByValWithOptionalType(Type *&Result) { + Result = nullptr; + if (!EatIfPresent(lltok::kw_byval)) + return true; + if (!EatIfPresent(lltok::lparen)) + return false; + if (ParseType(Result)) + return true; + if (!EatIfPresent(lltok::rparen)) + return Error(Lex.getLoc(), "expected ')'"); + return false; +} + /// ParseOptionalOperandBundles /// ::= /*empty*/ /// ::= '[' OperandBundle [, OperandBundle ]* ']' @@ -2684,7 +2753,18 @@ bool LLParser::ParseStructBody(SmallVectorImpl &Body) { /// Type /// ::= '[' APSINTVAL 'x' Types ']' /// ::= '<' APSINTVAL 'x' Types '>' +/// ::= '<' 'vscale' 'x' APSINTVAL 'x' Types '>' bool LLParser::ParseArrayVectorType(Type *&Result, bool isVector) { + bool Scalable = false; + + if (isVector && Lex.getKind() == lltok::kw_vscale) { + Lex.Lex(); // consume the 'vscale' + if (ParseToken(lltok::kw_x, "expected 'x' after vscale")) + return true; + + Scalable = true; + } + if (Lex.getKind() != lltok::APSInt || Lex.getAPSIntVal().isSigned() || Lex.getAPSIntVal().getBitWidth() > 64) return TokError("expected number in address space"); @@ -2711,7 +2791,7 @@ bool LLParser::ParseArrayVectorType(Type *&Result, bool isVector) { return Error(SizeLoc, "size too large for vector"); if (!VectorType::isValidElementType(EltTy)) return Error(TypeLoc, "invalid vector element type"); - Result = VectorType::get(EltTy, unsigned(Size)); + Result = VectorType::get(EltTy, unsigned(Size), Scalable); } else { if (!ArrayType::isValidElementType(EltTy)) return Error(TypeLoc, "invalid array element type"); @@ -2916,13 +2996,27 @@ BasicBlock *LLParser::PerFunctionState::GetBB(unsigned ID, LocTy Loc) { /// unnamed. If there is an error, this returns null otherwise it returns /// the block being defined. BasicBlock *LLParser::PerFunctionState::DefineBB(const std::string &Name, - LocTy Loc) { + int NameID, LocTy Loc) { BasicBlock *BB; - if (Name.empty()) + if (Name.empty()) { + if (NameID != -1 && unsigned(NameID) != NumberedVals.size()) { + P.Error(Loc, "label expected to be numbered '" + + Twine(NumberedVals.size()) + "'"); + return nullptr; + } BB = GetBB(NumberedVals.size(), Loc); - else + if (!BB) { + P.Error(Loc, "unable to create block numbered '" + + Twine(NumberedVals.size()) + "'"); + return nullptr; + } + } else { BB = GetBB(Name, Loc); - if (!BB) return nullptr; // Already diagnosed error. + if (!BB) { + P.Error(Loc, "unable to create block named '" + Name + "'"); + return nullptr; + } + } // Move the block to the end of the function. Forward ref'd blocks are // inserted wherever they happen to be referenced. @@ -3342,7 +3436,6 @@ bool LLParser::ParseValID(ValID &ID, PerFunctionState *PFS) { unsigned Opc = Lex.getUIntVal(); Constant *Val0, *Val1; Lex.Lex(); - LocTy ModifierLoc = Lex.getLoc(); if (Opc == Instruction::Add || Opc == Instruction::Sub || Opc == Instruction::Mul || Opc == Instruction::Shl) { if (EatIfPresent(lltok::kw_nuw)) @@ -3365,12 +3458,6 @@ bool LLParser::ParseValID(ValID &ID, PerFunctionState *PFS) { return true; if (Val0->getType() != Val1->getType()) return Error(ID.Loc, "operands of constexpr must have same type"); - if (!Val0->getType()->isIntOrIntVectorTy()) { - if (NUW) - return Error(ModifierLoc, "nuw only applies to integer operations"); - if (NSW) - return Error(ModifierLoc, "nsw only applies to integer operations"); - } // Check that the type is valid for the operator. switch (Opc) { case Instruction::Add: @@ -4640,6 +4727,24 @@ bool LLParser::ParseDILexicalBlockFile(MDNode *&Result, bool IsDistinct) { return false; } +/// ParseDICommonBlock: +/// ::= !DICommonBlock(scope: !0, file: !2, name: "COMMON name", line: 9) +bool LLParser::ParseDICommonBlock(MDNode *&Result, bool IsDistinct) { +#define VISIT_MD_FIELDS(OPTIONAL, REQUIRED) \ + REQUIRED(scope, MDField, ); \ + OPTIONAL(declaration, MDField, ); \ + OPTIONAL(name, MDStringField, ); \ + OPTIONAL(file, MDField, ); \ + OPTIONAL(line, LineField, ); + PARSE_MD_FIELDS(); +#undef VISIT_MD_FIELDS + + Result = GET_OR_DISTINCT(DICommonBlock, + (Context, scope.Val, declaration.Val, name.Val, + file.Val, line.Val)); + return false; +} + /// ParseDINamespace: /// ::= !DINamespace(scope: !0, file: !2, name: "SomeNamespace", line: 9) bool LLParser::ParseDINamespace(MDNode *&Result, bool IsDistinct) { @@ -4828,6 +4933,15 @@ bool LLParser::ParseDIExpression(MDNode *&Result, bool IsDistinct) { return TokError(Twine("invalid DWARF op '") + Lex.getStrVal() + "'"); } + if (Lex.getKind() == lltok::DwarfAttEncoding) { + if (unsigned Op = dwarf::getAttributeEncoding(Lex.getStrVal())) { + Lex.Lex(); + Elements.push_back(Op); + continue; + } + return TokError(Twine("invalid DWARF attribute encoding '") + Lex.getStrVal() + "'"); + } + if (Lex.getKind() != lltok::APSInt || Lex.getAPSIntVal().isSigned()) return TokError("expected unsigned integer"); @@ -5239,6 +5353,7 @@ bool LLParser::ParseFunctionHeader(Function *&Fn, bool isDefine) { std::vector FwdRefAttrGrps; LocTy BuiltinLoc; std::string Section; + std::string Partition; unsigned Alignment; std::string GC; GlobalValue::UnnamedAddr UnnamedAddr = GlobalValue::UnnamedAddr::None; @@ -5255,6 +5370,8 @@ bool LLParser::ParseFunctionHeader(Function *&Fn, bool isDefine) { BuiltinLoc) || (EatIfPresent(lltok::kw_section) && ParseStringConstant(Section)) || + (EatIfPresent(lltok::kw_partition) && + ParseStringConstant(Partition)) || parseOptionalComdat(FunctionName, C) || ParseOptionalAlignment(Alignment) || (EatIfPresent(lltok::kw_gc) && @@ -5356,6 +5473,7 @@ bool LLParser::ParseFunctionHeader(Function *&Fn, bool isDefine) { Fn->setUnnamedAddr(UnnamedAddr); Fn->setAlignment(Alignment); Fn->setSection(Section); + Fn->setPartition(Partition); Fn->setComdat(C); Fn->setPersonalityFn(PersonalityFn); if (!GC.empty()) Fn->setGC(GC); @@ -5470,20 +5588,23 @@ bool LLParser::ParseFunctionBody(Function &Fn) { } /// ParseBasicBlock -/// ::= LabelStr? Instruction* +/// ::= (LabelStr|LabelID)? Instruction* bool LLParser::ParseBasicBlock(PerFunctionState &PFS) { // If this basic block starts out with a name, remember it. std::string Name; + int NameID = -1; LocTy NameLoc = Lex.getLoc(); if (Lex.getKind() == lltok::LabelStr) { Name = Lex.getStrVal(); Lex.Lex(); + } else if (Lex.getKind() == lltok::LabelID) { + NameID = Lex.getUIntVal(); + Lex.Lex(); } - BasicBlock *BB = PFS.DefineBB(Name, NameLoc); + BasicBlock *BB = PFS.DefineBB(Name, NameID, NameLoc); if (!BB) - return Error(NameLoc, - "unable to create block named '" + Name + "'"); + return true; std::string NameStr; @@ -5567,10 +5688,11 @@ int LLParser::ParseInstruction(Instruction *&Inst, BasicBlock *BB, case lltok::kw_catchswitch: return ParseCatchSwitch(Inst, PFS); case lltok::kw_catchpad: return ParseCatchPad(Inst, PFS); case lltok::kw_cleanuppad: return ParseCleanupPad(Inst, PFS); + case lltok::kw_callbr: return ParseCallBr(Inst, PFS); // Unary Operators. case lltok::kw_fneg: { FastMathFlags FMF = EatFastMathFlagsIfPresent(); - int Res = ParseUnaryOp(Inst, PFS, KeywordVal, 2); + int Res = ParseUnaryOp(Inst, PFS, KeywordVal, /*IsFP*/true); if (Res != 0) return Res; if (FMF.any()) @@ -5586,7 +5708,7 @@ int LLParser::ParseInstruction(Instruction *&Inst, BasicBlock *BB, bool NSW = EatIfPresent(lltok::kw_nsw); if (!NUW) NUW = EatIfPresent(lltok::kw_nuw); - if (ParseArithmetic(Inst, PFS, KeywordVal, 1)) return true; + if (ParseArithmetic(Inst, PFS, KeywordVal, /*IsFP*/false)) return true; if (NUW) cast(Inst)->setHasNoUnsignedWrap(true); if (NSW) cast(Inst)->setHasNoSignedWrap(true); @@ -5598,7 +5720,7 @@ int LLParser::ParseInstruction(Instruction *&Inst, BasicBlock *BB, case lltok::kw_fdiv: case lltok::kw_frem: { FastMathFlags FMF = EatFastMathFlagsIfPresent(); - int Res = ParseArithmetic(Inst, PFS, KeywordVal, 2); + int Res = ParseArithmetic(Inst, PFS, KeywordVal, /*IsFP*/true); if (Res != 0) return Res; if (FMF.any()) @@ -5612,13 +5734,14 @@ int LLParser::ParseInstruction(Instruction *&Inst, BasicBlock *BB, case lltok::kw_ashr: { bool Exact = EatIfPresent(lltok::kw_exact); - if (ParseArithmetic(Inst, PFS, KeywordVal, 1)) return true; + if (ParseArithmetic(Inst, PFS, KeywordVal, /*IsFP*/false)) return true; if (Exact) cast(Inst)->setIsExact(true); return false; } case lltok::kw_urem: - case lltok::kw_srem: return ParseArithmetic(Inst, PFS, KeywordVal, 1); + case lltok::kw_srem: return ParseArithmetic(Inst, PFS, KeywordVal, + /*IsFP*/false); case lltok::kw_and: case lltok::kw_or: case lltok::kw_xor: return ParseLogical(Inst, PFS, KeywordVal); @@ -5648,7 +5771,19 @@ int LLParser::ParseInstruction(Instruction *&Inst, BasicBlock *BB, case lltok::kw_inttoptr: case lltok::kw_ptrtoint: return ParseCast(Inst, PFS, KeywordVal); // Other. - case lltok::kw_select: return ParseSelect(Inst, PFS); + case lltok::kw_select: { + FastMathFlags FMF = EatFastMathFlagsIfPresent(); + int Res = ParseSelect(Inst, PFS); + if (Res != 0) + return Res; + if (FMF.any()) { + if (!Inst->getType()->isFPOrFPVectorTy()) + return Error(Loc, "fast-math-flags specified for select without " + "floating-point scalar or vector return type"); + Inst->setFastMathFlags(FMF); + } + return 0; + } case lltok::kw_va_arg: return ParseVA_Arg(Inst, PFS); case lltok::kw_extractelement: return ParseExtractElement(Inst, PFS); case lltok::kw_insertelement: return ParseInsertElement(Inst, PFS); @@ -6155,28 +6290,16 @@ bool LLParser::ParseCleanupPad(Instruction *&Inst, PerFunctionState &PFS) { /// ParseUnaryOp /// ::= UnaryOp TypeAndValue ',' Value /// -/// If OperandType is 0, then any FP or integer operand is allowed. If it is 1, -/// then any integer operand is allowed, if it is 2, any fp operand is allowed. +/// If IsFP is false, then any integer operand is allowed, if it is true, any fp +/// operand is allowed. bool LLParser::ParseUnaryOp(Instruction *&Inst, PerFunctionState &PFS, - unsigned Opc, unsigned OperandType) { + unsigned Opc, bool IsFP) { LocTy Loc; Value *LHS; if (ParseTypeAndValue(LHS, Loc, PFS)) return true; - bool Valid; - switch (OperandType) { - default: llvm_unreachable("Unknown operand type!"); - case 0: // int or FP. - Valid = LHS->getType()->isIntOrIntVectorTy() || - LHS->getType()->isFPOrFPVectorTy(); - break; - case 1: - Valid = LHS->getType()->isIntOrIntVectorTy(); - break; - case 2: - Valid = LHS->getType()->isFPOrFPVectorTy(); - break; - } + bool Valid = IsFP ? LHS->getType()->isFPOrFPVectorTy() + : LHS->getType()->isIntOrIntVectorTy(); if (!Valid) return Error(Loc, "invalid operand type for instruction"); @@ -6185,6 +6308,124 @@ bool LLParser::ParseUnaryOp(Instruction *&Inst, PerFunctionState &PFS, return false; } +/// ParseCallBr +/// ::= 'callbr' OptionalCallingConv OptionalAttrs Type Value ParamList +/// OptionalAttrs OptionalOperandBundles 'to' TypeAndValue +/// '[' LabelList ']' +bool LLParser::ParseCallBr(Instruction *&Inst, PerFunctionState &PFS) { + LocTy CallLoc = Lex.getLoc(); + AttrBuilder RetAttrs, FnAttrs; + std::vector FwdRefAttrGrps; + LocTy NoBuiltinLoc; + unsigned CC; + Type *RetType = nullptr; + LocTy RetTypeLoc; + ValID CalleeID; + SmallVector ArgList; + SmallVector BundleList; + + BasicBlock *DefaultDest; + if (ParseOptionalCallingConv(CC) || ParseOptionalReturnAttrs(RetAttrs) || + ParseType(RetType, RetTypeLoc, true /*void allowed*/) || + ParseValID(CalleeID) || ParseParameterList(ArgList, PFS) || + ParseFnAttributeValuePairs(FnAttrs, FwdRefAttrGrps, false, + NoBuiltinLoc) || + ParseOptionalOperandBundles(BundleList, PFS) || + ParseToken(lltok::kw_to, "expected 'to' in callbr") || + ParseTypeAndBasicBlock(DefaultDest, PFS) || + ParseToken(lltok::lsquare, "expected '[' in callbr")) + return true; + + // Parse the destination list. + SmallVector IndirectDests; + + if (Lex.getKind() != lltok::rsquare) { + BasicBlock *DestBB; + if (ParseTypeAndBasicBlock(DestBB, PFS)) + return true; + IndirectDests.push_back(DestBB); + + while (EatIfPresent(lltok::comma)) { + if (ParseTypeAndBasicBlock(DestBB, PFS)) + return true; + IndirectDests.push_back(DestBB); + } + } + + if (ParseToken(lltok::rsquare, "expected ']' at end of block list")) + return true; + + // If RetType is a non-function pointer type, then this is the short syntax + // for the call, which means that RetType is just the return type. Infer the + // rest of the function argument types from the arguments that are present. + FunctionType *Ty = dyn_cast(RetType); + if (!Ty) { + // Pull out the types of all of the arguments... + std::vector ParamTypes; + for (unsigned i = 0, e = ArgList.size(); i != e; ++i) + ParamTypes.push_back(ArgList[i].V->getType()); + + if (!FunctionType::isValidReturnType(RetType)) + return Error(RetTypeLoc, "Invalid result type for LLVM function"); + + Ty = FunctionType::get(RetType, ParamTypes, false); + } + + CalleeID.FTy = Ty; + + // Look up the callee. + Value *Callee; + if (ConvertValIDToValue(PointerType::getUnqual(Ty), CalleeID, Callee, &PFS, + /*IsCall=*/true)) + return true; + + if (isa(Callee) && !Ty->getReturnType()->isVoidTy()) + return Error(RetTypeLoc, "asm-goto outputs not supported"); + + // Set up the Attribute for the function. + SmallVector Args; + SmallVector ArgAttrs; + + // Loop through FunctionType's arguments and ensure they are specified + // correctly. Also, gather any parameter attributes. + FunctionType::param_iterator I = Ty->param_begin(); + FunctionType::param_iterator E = Ty->param_end(); + for (unsigned i = 0, e = ArgList.size(); i != e; ++i) { + Type *ExpectedTy = nullptr; + if (I != E) { + ExpectedTy = *I++; + } else if (!Ty->isVarArg()) { + return Error(ArgList[i].Loc, "too many arguments specified"); + } + + if (ExpectedTy && ExpectedTy != ArgList[i].V->getType()) + return Error(ArgList[i].Loc, "argument is not of expected type '" + + getTypeString(ExpectedTy) + "'"); + Args.push_back(ArgList[i].V); + ArgAttrs.push_back(ArgList[i].Attrs); + } + + if (I != E) + return Error(CallLoc, "not enough parameters specified for call"); + + if (FnAttrs.hasAlignmentAttr()) + return Error(CallLoc, "callbr instructions may not have an alignment"); + + // Finish off the Attribute and check them + AttributeList PAL = + AttributeList::get(Context, AttributeSet::get(Context, FnAttrs), + AttributeSet::get(Context, RetAttrs), ArgAttrs); + + CallBrInst *CBI = + CallBrInst::Create(Ty, Callee, DefaultDest, IndirectDests, Args, + BundleList); + CBI->setCallingConv(CC); + CBI->setAttributes(PAL); + ForwardRefAttrGroups[CBI] = FwdRefAttrGrps; + Inst = CBI; + return false; +} + //===----------------------------------------------------------------------===// // Binary Operators. //===----------------------------------------------------------------------===// @@ -6192,26 +6433,18 @@ bool LLParser::ParseUnaryOp(Instruction *&Inst, PerFunctionState &PFS, /// ParseArithmetic /// ::= ArithmeticOps TypeAndValue ',' Value /// -/// If OperandType is 0, then any FP or integer operand is allowed. If it is 1, -/// then any integer operand is allowed, if it is 2, any fp operand is allowed. +/// If IsFP is false, then any integer operand is allowed, if it is true, any fp +/// operand is allowed. bool LLParser::ParseArithmetic(Instruction *&Inst, PerFunctionState &PFS, - unsigned Opc, unsigned OperandType) { + unsigned Opc, bool IsFP) { LocTy Loc; Value *LHS, *RHS; if (ParseTypeAndValue(LHS, Loc, PFS) || ParseToken(lltok::comma, "expected ',' in arithmetic operation") || ParseValue(LHS->getType(), RHS, PFS)) return true; - bool Valid; - switch (OperandType) { - default: llvm_unreachable("Unknown operand type!"); - case 0: // int or FP. - Valid = LHS->getType()->isIntOrIntVectorTy() || - LHS->getType()->isFPOrFPVectorTy(); - break; - case 1: Valid = LHS->getType()->isIntOrIntVectorTy(); break; - case 2: Valid = LHS->getType()->isFPOrFPVectorTy(); break; - } + bool Valid = IsFP ? LHS->getType()->isFPOrFPVectorTy() + : LHS->getType()->isIntOrIntVectorTy(); if (!Valid) return Error(Loc, "invalid operand type for instruction"); @@ -6816,6 +7049,7 @@ int LLParser::ParseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) { AtomicOrdering Ordering = AtomicOrdering::NotAtomic; SyncScope::ID SSID = SyncScope::System; bool isVolatile = false; + bool IsFP = false; AtomicRMWInst::BinOp Operation; if (EatIfPresent(lltok::kw_volatile)) @@ -6834,6 +7068,14 @@ int LLParser::ParseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) { case lltok::kw_min: Operation = AtomicRMWInst::Min; break; case lltok::kw_umax: Operation = AtomicRMWInst::UMax; break; case lltok::kw_umin: Operation = AtomicRMWInst::UMin; break; + case lltok::kw_fadd: + Operation = AtomicRMWInst::FAdd; + IsFP = true; + break; + case lltok::kw_fsub: + Operation = AtomicRMWInst::FSub; + IsFP = true; + break; } Lex.Lex(); // Eat the operation. @@ -6850,10 +7092,25 @@ int LLParser::ParseAtomicRMW(Instruction *&Inst, PerFunctionState &PFS) { if (cast(Ptr->getType())->getElementType() != Val->getType()) return Error(ValLoc, "atomicrmw value and pointer type do not match"); - if (!Val->getType()->isIntegerTy()) { - return Error(ValLoc, "atomicrmw " + - AtomicRMWInst::getOperationName(Operation) + - " operand must be an integer"); + if (Operation == AtomicRMWInst::Xchg) { + if (!Val->getType()->isIntegerTy() && + !Val->getType()->isFloatingPointTy()) { + return Error(ValLoc, "atomicrmw " + + AtomicRMWInst::getOperationName(Operation) + + " operand must be an integer or floating point type"); + } + } else if (IsFP) { + if (!Val->getType()->isFloatingPointTy()) { + return Error(ValLoc, "atomicrmw " + + AtomicRMWInst::getOperationName(Operation) + + " operand must be a floating point type"); + } + } else { + if (!Val->getType()->isIntegerTy()) { + return Error(ValLoc, "atomicrmw " + + AtomicRMWInst::getOperationName(Operation) + + " operand must be an integer"); + } } unsigned Size = Val->getType()->getPrimitiveSizeInBits(); @@ -7249,6 +7506,92 @@ bool LLParser::ParseTypeIdSummary(TypeIdSummary &TIS) { return false; } +static ValueInfo EmptyVI = + ValueInfo(false, (GlobalValueSummaryMapTy::value_type *)-8); + +/// TypeIdCompatibleVtableEntry +/// ::= 'typeidCompatibleVTable' ':' '(' 'name' ':' STRINGCONSTANT ',' +/// TypeIdCompatibleVtableInfo +/// ')' +bool LLParser::ParseTypeIdCompatibleVtableEntry(unsigned ID) { + assert(Lex.getKind() == lltok::kw_typeidCompatibleVTable); + Lex.Lex(); + + std::string Name; + if (ParseToken(lltok::colon, "expected ':' here") || + ParseToken(lltok::lparen, "expected '(' here") || + ParseToken(lltok::kw_name, "expected 'name' here") || + ParseToken(lltok::colon, "expected ':' here") || + ParseStringConstant(Name)) + return true; + + TypeIdCompatibleVtableInfo &TI = + Index->getOrInsertTypeIdCompatibleVtableSummary(Name); + if (ParseToken(lltok::comma, "expected ',' here") || + ParseToken(lltok::kw_summary, "expected 'summary' here") || + ParseToken(lltok::colon, "expected ':' here") || + ParseToken(lltok::lparen, "expected '(' here")) + return true; + + IdToIndexMapType IdToIndexMap; + // Parse each call edge + do { + uint64_t Offset; + if (ParseToken(lltok::lparen, "expected '(' here") || + ParseToken(lltok::kw_offset, "expected 'offset' here") || + ParseToken(lltok::colon, "expected ':' here") || ParseUInt64(Offset) || + ParseToken(lltok::comma, "expected ',' here")) + return true; + + LocTy Loc = Lex.getLoc(); + unsigned GVId; + ValueInfo VI; + if (ParseGVReference(VI, GVId)) + return true; + + // Keep track of the TypeIdCompatibleVtableInfo array index needing a + // forward reference. We will save the location of the ValueInfo needing an + // update, but can only do so once the std::vector is finalized. + if (VI == EmptyVI) + IdToIndexMap[GVId].push_back(std::make_pair(TI.size(), Loc)); + TI.push_back({Offset, VI}); + + if (ParseToken(lltok::rparen, "expected ')' in call")) + return true; + } while (EatIfPresent(lltok::comma)); + + // Now that the TI vector is finalized, it is safe to save the locations + // of any forward GV references that need updating later. + for (auto I : IdToIndexMap) { + for (auto P : I.second) { + assert(TI[P.first].VTableVI == EmptyVI && + "Forward referenced ValueInfo expected to be empty"); + auto FwdRef = ForwardRefValueInfos.insert(std::make_pair( + I.first, std::vector>())); + FwdRef.first->second.push_back( + std::make_pair(&TI[P.first].VTableVI, P.second)); + } + } + + if (ParseToken(lltok::rparen, "expected ')' here") || + ParseToken(lltok::rparen, "expected ')' here")) + return true; + + // Check if this ID was forward referenced, and if so, update the + // corresponding GUIDs. + auto FwdRefTIDs = ForwardRefTypeIds.find(ID); + if (FwdRefTIDs != ForwardRefTypeIds.end()) { + for (auto TIDRef : FwdRefTIDs->second) { + assert(!*TIDRef.first && + "Forward referenced type id GUID expected to be 0"); + *TIDRef.first = GlobalValue::getGUID(Name); + } + ForwardRefTypeIds.erase(FwdRefTIDs); + } + + return false; +} + /// TypeTestResolution /// ::= 'typeTestRes' ':' '(' 'kind' ':' /// ( 'unsat' | 'byteArray' | 'inline' | 'single' | 'allOnes' ) ',' @@ -7523,9 +7866,13 @@ static const auto FwdVIRef = (GlobalValueSummaryMapTy::value_type *)-8; static void resolveFwdRef(ValueInfo *Fwd, ValueInfo &Resolved) { bool ReadOnly = Fwd->isReadOnly(); + bool WriteOnly = Fwd->isWriteOnly(); + assert(!(ReadOnly && WriteOnly)); *Fwd = Resolved; if (ReadOnly) Fwd->setReadOnly(); + if (WriteOnly) + Fwd->setWriteOnly(); } /// Stores the given Name/GUID and associated summary into the Index. @@ -7554,10 +7901,6 @@ void LLParser::AddGlobalValueToIndex( } } - // Add the summary if one was provided. - if (Summary) - Index->addGlobalValueSummary(VI, std::move(Summary)); - // Resolve forward references from calls/refs auto FwdRefVIs = ForwardRefValueInfos.find(ID); if (FwdRefVIs != ForwardRefValueInfos.end()) { @@ -7575,11 +7918,16 @@ void LLParser::AddGlobalValueToIndex( for (auto AliaseeRef : FwdRefAliasees->second) { assert(!AliaseeRef.first->hasAliasee() && "Forward referencing alias already has aliasee"); - AliaseeRef.first->setAliasee(VI.getSummaryList().front().get()); + assert(Summary && "Aliasee must be a definition"); + AliaseeRef.first->setAliasee(VI, Summary.get()); } ForwardRefAliasees.erase(FwdRefAliasees); } + // Add the summary if one was provided. + if (Summary) + Index->addGlobalValueSummary(VI, std::move(Summary)); + // Save the associated ValueInfo for use in later references by ID. if (ID == NumberedValueInfos.size()) NumberedValueInfos.push_back(VI); @@ -7683,7 +8031,7 @@ bool LLParser::ParseFunctionSummary(std::string Name, GlobalValue::GUID GUID, StringRef ModulePath; GlobalValueSummary::GVFlags GVFlags = GlobalValueSummary::GVFlags( /*Linkage=*/GlobalValue::ExternalLinkage, /*NotEligibleToImport=*/false, - /*Live=*/false, /*IsLocal=*/false); + /*Live=*/false, /*IsLocal=*/false, /*CanAutoHide=*/false); unsigned InstCount; std::vector Calls; FunctionSummary::TypeIdInfo TypeIdInfo; @@ -7753,9 +8101,11 @@ bool LLParser::ParseVariableSummary(std::string Name, GlobalValue::GUID GUID, StringRef ModulePath; GlobalValueSummary::GVFlags GVFlags = GlobalValueSummary::GVFlags( /*Linkage=*/GlobalValue::ExternalLinkage, /*NotEligibleToImport=*/false, - /*Live=*/false, /*IsLocal=*/false); - GlobalVarSummary::GVarFlags GVarFlags(/*ReadOnly*/ false); + /*Live=*/false, /*IsLocal=*/false, /*CanAutoHide=*/false); + GlobalVarSummary::GVarFlags GVarFlags(/*ReadOnly*/ false, + /* WriteOnly */ false); std::vector Refs; + VTableFuncList VTableFuncs; if (ParseToken(lltok::colon, "expected ':' here") || ParseToken(lltok::lparen, "expected '(' here") || ParseModuleReference(ModulePath) || @@ -7764,10 +8114,20 @@ bool LLParser::ParseVariableSummary(std::string Name, GlobalValue::GUID GUID, ParseGVarFlags(GVarFlags)) return true; - // Parse optional refs field - if (EatIfPresent(lltok::comma)) { - if (ParseOptionalRefs(Refs)) - return true; + // Parse optional fields + while (EatIfPresent(lltok::comma)) { + switch (Lex.getKind()) { + case lltok::kw_vTableFuncs: + if (ParseOptionalVTableFuncs(VTableFuncs)) + return true; + break; + case lltok::kw_refs: + if (ParseOptionalRefs(Refs)) + return true; + break; + default: + return Error(Lex.getLoc(), "expected optional variable summary field"); + } } if (ParseToken(lltok::rparen, "expected ')' here")) @@ -7777,6 +8137,7 @@ bool LLParser::ParseVariableSummary(std::string Name, GlobalValue::GUID GUID, llvm::make_unique(GVFlags, GVarFlags, std::move(Refs)); GS->setModulePath(ModulePath); + GS->setVTableFuncs(std::move(VTableFuncs)); AddGlobalValueToIndex(Name, GUID, (GlobalValue::LinkageTypes)GVFlags.Linkage, ID, std::move(GS)); @@ -7796,7 +8157,7 @@ bool LLParser::ParseAliasSummary(std::string Name, GlobalValue::GUID GUID, StringRef ModulePath; GlobalValueSummary::GVFlags GVFlags = GlobalValueSummary::GVFlags( /*Linkage=*/GlobalValue::ExternalLinkage, /*NotEligibleToImport=*/false, - /*Live=*/false, /*IsLocal=*/false); + /*Live=*/false, /*IsLocal=*/false, /*CanAutoHide=*/false); if (ParseToken(lltok::colon, "expected ':' here") || ParseToken(lltok::lparen, "expected '(' here") || ParseModuleReference(ModulePath) || @@ -7823,8 +8184,11 @@ bool LLParser::ParseAliasSummary(std::string Name, GlobalValue::GUID GUID, auto FwdRef = ForwardRefAliasees.insert( std::make_pair(GVId, std::vector>())); FwdRef.first->second.push_back(std::make_pair(AS.get(), Loc)); - } else - AS->setAliasee(AliaseeVI.getSummaryList().front().get()); + } else { + auto Summary = Index->findSummaryInModule(AliaseeVI, ModulePath); + assert(Summary && "Aliasee must be a definition"); + AS->setAliasee(AliaseeVI, Summary); + } AddGlobalValueToIndex(Name, GUID, (GlobalValue::LinkageTypes)GVFlags.Linkage, ID, std::move(AS)); @@ -7856,7 +8220,7 @@ bool LLParser::ParseOptionalFFlags(FunctionSummary::FFlags &FFlags) { return true; do { - unsigned Val; + unsigned Val = 0; switch (Lex.getKind()) { case lltok::kw_readNone: Lex.Lex(); @@ -7994,6 +8358,67 @@ bool LLParser::ParseHotness(CalleeInfo::HotnessType &Hotness) { return false; } +/// OptionalVTableFuncs +/// := 'vTableFuncs' ':' '(' VTableFunc [',' VTableFunc]* ')' +/// VTableFunc ::= '(' 'virtFunc' ':' GVReference ',' 'offset' ':' UInt64 ')' +bool LLParser::ParseOptionalVTableFuncs(VTableFuncList &VTableFuncs) { + assert(Lex.getKind() == lltok::kw_vTableFuncs); + Lex.Lex(); + + if (ParseToken(lltok::colon, "expected ':' in vTableFuncs") | + ParseToken(lltok::lparen, "expected '(' in vTableFuncs")) + return true; + + IdToIndexMapType IdToIndexMap; + // Parse each virtual function pair + do { + ValueInfo VI; + if (ParseToken(lltok::lparen, "expected '(' in vTableFunc") || + ParseToken(lltok::kw_virtFunc, "expected 'callee' in vTableFunc") || + ParseToken(lltok::colon, "expected ':'")) + return true; + + LocTy Loc = Lex.getLoc(); + unsigned GVId; + if (ParseGVReference(VI, GVId)) + return true; + + uint64_t Offset; + if (ParseToken(lltok::comma, "expected comma") || + ParseToken(lltok::kw_offset, "expected offset") || + ParseToken(lltok::colon, "expected ':'") || ParseUInt64(Offset)) + return true; + + // Keep track of the VTableFuncs array index needing a forward reference. + // We will save the location of the ValueInfo needing an update, but + // can only do so once the std::vector is finalized. + if (VI == EmptyVI) + IdToIndexMap[GVId].push_back(std::make_pair(VTableFuncs.size(), Loc)); + VTableFuncs.push_back({VI, Offset}); + + if (ParseToken(lltok::rparen, "expected ')' in vTableFunc")) + return true; + } while (EatIfPresent(lltok::comma)); + + // Now that the VTableFuncs vector is finalized, it is safe to save the + // locations of any forward GV references that need updating later. + for (auto I : IdToIndexMap) { + for (auto P : I.second) { + assert(VTableFuncs[P.first].FuncVI == EmptyVI && + "Forward referenced ValueInfo expected to be empty"); + auto FwdRef = ForwardRefValueInfos.insert(std::make_pair( + I.first, std::vector>())); + FwdRef.first->second.push_back( + std::make_pair(&VTableFuncs[P.first].FuncVI, P.second)); + } + } + + if (ParseToken(lltok::rparen, "expected ')' in vTableFuncs")) + return true; + + return false; +} + /// OptionalRefs /// := 'refs' ':' '(' GVReference [',' GVReference]* ')' bool LLParser::ParseOptionalRefs(std::vector &Refs) { @@ -8019,10 +8444,11 @@ bool LLParser::ParseOptionalRefs(std::vector &Refs) { VContexts.push_back(VC); } while (EatIfPresent(lltok::comma)); - // Sort value contexts so that ones with readonly ValueInfo are at the end - // of VContexts vector. This is needed to match immutableRefCount() behavior. + // Sort value contexts so that ones with writeonly + // and readonly ValueInfo are at the end of VContexts vector. + // See FunctionSummary::specialRefCounts() llvm::sort(VContexts, [](const ValueContext &VC1, const ValueContext &VC2) { - return VC1.VI.isReadOnly() < VC2.VI.isReadOnly(); + return VC1.VI.getAccessSpecifier() < VC2.VI.getAccessSpecifier(); }); IdToIndexMapType IdToIndexMap; @@ -8283,41 +8709,55 @@ bool LLParser::ParseVFuncId(FunctionSummary::VFuncId &VFuncId, /// GVFlags /// ::= 'flags' ':' '(' 'linkage' ':' OptionalLinkageAux ',' /// 'notEligibleToImport' ':' Flag ',' 'live' ':' Flag ',' -/// 'dsoLocal' ':' Flag ')' +/// 'dsoLocal' ':' Flag ',' 'canAutoHide' ':' Flag ')' bool LLParser::ParseGVFlags(GlobalValueSummary::GVFlags &GVFlags) { assert(Lex.getKind() == lltok::kw_flags); Lex.Lex(); - bool HasLinkage; if (ParseToken(lltok::colon, "expected ':' here") || - ParseToken(lltok::lparen, "expected '(' here") || - ParseToken(lltok::kw_linkage, "expected 'linkage' here") || - ParseToken(lltok::colon, "expected ':' here")) - return true; - - GVFlags.Linkage = parseOptionalLinkageAux(Lex.getKind(), HasLinkage); - assert(HasLinkage && "Linkage not optional in summary entry"); - Lex.Lex(); - - unsigned Flag; - if (ParseToken(lltok::comma, "expected ',' here") || - ParseToken(lltok::kw_notEligibleToImport, - "expected 'notEligibleToImport' here") || - ParseToken(lltok::colon, "expected ':' here") || ParseFlag(Flag)) - return true; - GVFlags.NotEligibleToImport = Flag; - - if (ParseToken(lltok::comma, "expected ',' here") || - ParseToken(lltok::kw_live, "expected 'live' here") || - ParseToken(lltok::colon, "expected ':' here") || ParseFlag(Flag)) + ParseToken(lltok::lparen, "expected '(' here")) return true; - GVFlags.Live = Flag; - if (ParseToken(lltok::comma, "expected ',' here") || - ParseToken(lltok::kw_dsoLocal, "expected 'dsoLocal' here") || - ParseToken(lltok::colon, "expected ':' here") || ParseFlag(Flag)) - return true; - GVFlags.DSOLocal = Flag; + do { + unsigned Flag = 0; + switch (Lex.getKind()) { + case lltok::kw_linkage: + Lex.Lex(); + if (ParseToken(lltok::colon, "expected ':'")) + return true; + bool HasLinkage; + GVFlags.Linkage = parseOptionalLinkageAux(Lex.getKind(), HasLinkage); + assert(HasLinkage && "Linkage not optional in summary entry"); + Lex.Lex(); + break; + case lltok::kw_notEligibleToImport: + Lex.Lex(); + if (ParseToken(lltok::colon, "expected ':'") || ParseFlag(Flag)) + return true; + GVFlags.NotEligibleToImport = Flag; + break; + case lltok::kw_live: + Lex.Lex(); + if (ParseToken(lltok::colon, "expected ':'") || ParseFlag(Flag)) + return true; + GVFlags.Live = Flag; + break; + case lltok::kw_dsoLocal: + Lex.Lex(); + if (ParseToken(lltok::colon, "expected ':'") || ParseFlag(Flag)) + return true; + GVFlags.DSOLocal = Flag; + break; + case lltok::kw_canAutoHide: + Lex.Lex(); + if (ParseToken(lltok::colon, "expected ':'") || ParseFlag(Flag)) + return true; + GVFlags.CanAutoHide = Flag; + break; + default: + return Error(Lex.getLoc(), "expected gv flag type"); + } + } while (EatIfPresent(lltok::comma)); if (ParseToken(lltok::rparen, "expected ')' here")) return true; @@ -8326,24 +8766,41 @@ bool LLParser::ParseGVFlags(GlobalValueSummary::GVFlags &GVFlags) { } /// GVarFlags -/// ::= 'varFlags' ':' '(' 'readonly' ':' Flag ')' +/// ::= 'varFlags' ':' '(' 'readonly' ':' Flag +/// ',' 'writeonly' ':' Flag ')' bool LLParser::ParseGVarFlags(GlobalVarSummary::GVarFlags &GVarFlags) { assert(Lex.getKind() == lltok::kw_varFlags); Lex.Lex(); - unsigned Flag; if (ParseToken(lltok::colon, "expected ':' here") || - ParseToken(lltok::lparen, "expected '(' here") || - ParseToken(lltok::kw_readonly, "expected 'readonly' here") || - ParseToken(lltok::colon, "expected ':' here")) + ParseToken(lltok::lparen, "expected '(' here")) return true; - ParseFlag(Flag); - GVarFlags.ReadOnly = Flag; + auto ParseRest = [this](unsigned int &Val) { + Lex.Lex(); + if (ParseToken(lltok::colon, "expected ':'")) + return true; + return ParseFlag(Val); + }; - if (ParseToken(lltok::rparen, "expected ')' here")) - return true; - return false; + do { + unsigned Flag = 0; + switch (Lex.getKind()) { + case lltok::kw_readonly: + if (ParseRest(Flag)) + return true; + GVarFlags.MaybeReadOnly = Flag; + break; + case lltok::kw_writeonly: + if (ParseRest(Flag)) + return true; + GVarFlags.MaybeWriteOnly = Flag; + break; + default: + return Error(Lex.getLoc(), "expected gvar flag type"); + } + } while (EatIfPresent(lltok::comma)); + return ParseToken(lltok::rparen, "expected ')' here"); } /// ModuleReference @@ -8366,7 +8823,9 @@ bool LLParser::ParseModuleReference(StringRef &ModulePath) { /// GVReference /// ::= SummaryID bool LLParser::ParseGVReference(ValueInfo &VI, unsigned &GVId) { - bool ReadOnly = EatIfPresent(lltok::kw_readonly); + bool WriteOnly = false, ReadOnly = EatIfPresent(lltok::kw_readonly); + if (!ReadOnly) + WriteOnly = EatIfPresent(lltok::kw_writeonly); if (ParseToken(lltok::SummaryID, "expected GV ID")) return true; @@ -8381,5 +8840,7 @@ bool LLParser::ParseGVReference(ValueInfo &VI, unsigned &GVId) { if (ReadOnly) VI.setReadOnly(); + if (WriteOnly) + VI.setWriteOnly(); return false; } diff --git a/lib/AsmParser/LLParser.h b/lib/AsmParser/LLParser.h index 5a0fc297265d..610e2e262008 100644 --- a/lib/AsmParser/LLParser.h +++ b/lib/AsmParser/LLParser.h @@ -1,9 +1,8 @@ //===-- LLParser.h - Parser Class -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -340,6 +339,7 @@ namespace llvm { bool ParseFnAttributeValuePairs(AttrBuilder &B, std::vector &FwdRefAttrGrps, bool inAttrGrp, LocTy &BuiltinLoc); + bool ParseByValWithOptionalType(Type *&Result); // Module Summary Index Parsing. bool SkipModuleSummaryEntry(); @@ -369,9 +369,11 @@ namespace llvm { IdToIndexMapType &IdToIndexMap, unsigned Index); bool ParseVFuncId(FunctionSummary::VFuncId &VFuncId, IdToIndexMapType &IdToIndexMap, unsigned Index); + bool ParseOptionalVTableFuncs(VTableFuncList &VTableFuncs); bool ParseOptionalRefs(std::vector &Refs); bool ParseTypeIdEntry(unsigned ID); bool ParseTypeIdSummary(TypeIdSummary &TIS); + bool ParseTypeIdCompatibleVtableEntry(unsigned ID); bool ParseTypeTestResolution(TypeTestResolution &TTRes); bool ParseOptionalWpdResolutions( std::map &WPDResMap); @@ -446,7 +448,7 @@ namespace llvm { /// DefineBB - Define the specified basic block, which is either named or /// unnamed. If there is an error, this returns null otherwise it returns /// the block being defined. - BasicBlock *DefineBB(const std::string &Name, LocTy Loc); + BasicBlock *DefineBB(const std::string &Name, int NameID, LocTy Loc); bool resolveForwardRefBlockAddresses(); }; @@ -571,11 +573,12 @@ namespace llvm { bool ParseCatchSwitch(Instruction *&Inst, PerFunctionState &PFS); bool ParseCatchPad(Instruction *&Inst, PerFunctionState &PFS); bool ParseCleanupPad(Instruction *&Inst, PerFunctionState &PFS); + bool ParseCallBr(Instruction *&Inst, PerFunctionState &PFS); bool ParseUnaryOp(Instruction *&Inst, PerFunctionState &PFS, unsigned Opc, - unsigned OperandType); + bool IsFP); bool ParseArithmetic(Instruction *&Inst, PerFunctionState &PFS, unsigned Opc, - unsigned OperandType); + bool IsFP); bool ParseLogical(Instruction *&Inst, PerFunctionState &PFS, unsigned Opc); bool ParseCompare(Instruction *&Inst, PerFunctionState &PFS, unsigned Opc); bool ParseCast(Instruction *&Inst, PerFunctionState &PFS, unsigned Opc); diff --git a/lib/AsmParser/LLToken.h b/lib/AsmParser/LLToken.h index c2e2795a9467..0e9ba4db4742 100644 --- a/lib/AsmParser/LLToken.h +++ b/lib/AsmParser/LLToken.h @@ -1,9 +1,8 @@ //===- LLToken.h - Token Codes for LLVM Assembly Files ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -38,6 +37,7 @@ enum Kind { bar, // | colon, // : + kw_vscale, kw_x, kw_true, kw_false, @@ -114,6 +114,7 @@ enum Kind { kw_align, kw_addrspace, kw_section, + kw_partition, kw_alias, kw_ifunc, kw_module, @@ -175,6 +176,7 @@ enum Kind { kw_argmemonly, kw_sanitize_address, kw_sanitize_hwaddress, + kw_sanitize_memtag, kw_builtin, kw_byval, kw_inalloca, @@ -194,6 +196,7 @@ enum Kind { kw_nobuiltin, kw_nocapture, kw_noduplicate, + kw_nofree, kw_noimplicitfloat, kw_noinline, kw_norecurse, @@ -201,6 +204,7 @@ enum Kind { kw_nonnull, kw_noredzone, kw_noreturn, + kw_nosync, kw_nocf_check, kw_nounwind, kw_optforfuzzing, @@ -225,8 +229,10 @@ enum Kind { kw_swifterror, kw_swiftself, kw_uwtable, + kw_willreturn, kw_writeonly, kw_zeroext, + kw_immarg, kw_type, kw_opaque, @@ -328,6 +334,7 @@ enum Kind { kw_catchret, kw_catchpad, kw_cleanuppad, + kw_callbr, kw_alloca, kw_load, @@ -363,6 +370,7 @@ enum Kind { kw_notEligibleToImport, kw_live, kw_dsoLocal, + kw_canAutoHide, kw_function, kw_insts, kw_funcFlags, @@ -379,6 +387,8 @@ enum Kind { kw_critical, kw_relbf, kw_variable, + kw_vTableFuncs, + kw_virtFunc, kw_aliasee, kw_refs, kw_typeIdInfo, @@ -391,6 +401,7 @@ enum Kind { kw_offset, kw_args, kw_typeid, + kw_typeidCompatibleVTable, kw_summary, kw_typeTestRes, kw_kind, @@ -421,6 +432,7 @@ enum Kind { kw_varFlags, // Unsigned Valued tokens (UIntVal). + LabelID, // 42: GlobalID, // @42 LocalVarID, // %42 AttrGrpID, // #42 diff --git a/lib/AsmParser/Parser.cpp b/lib/AsmParser/Parser.cpp index 1205dff24e8a..b13c6237f411 100644 --- a/lib/AsmParser/Parser.cpp +++ b/lib/AsmParser/Parser.cpp @@ -1,9 +1,8 @@ //===- Parser.cpp - Main dispatch module for the Parser library -----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/BinaryFormat/AMDGPUMetadataVerifier.cpp b/lib/BinaryFormat/AMDGPUMetadataVerifier.cpp index b789f646b5f6..3f36dff9f55c 100644 --- a/lib/BinaryFormat/AMDGPUMetadataVerifier.cpp +++ b/lib/BinaryFormat/AMDGPUMetadataVerifier.cpp @@ -1,9 +1,8 @@ //===- AMDGPUMetadataVerifier.cpp - MsgPack Types ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -21,98 +20,92 @@ namespace HSAMD { namespace V3 { bool MetadataVerifier::verifyScalar( - msgpack::Node &Node, msgpack::ScalarNode::ScalarKind SKind, - function_ref verifyValue) { - auto ScalarPtr = dyn_cast(&Node); - if (!ScalarPtr) - return false; - auto &Scalar = *ScalarPtr; - // Do not output extraneous tags for types we know from the spec. - Scalar.IgnoreTag = true; - if (Scalar.getScalarKind() != SKind) { + msgpack::DocNode &Node, msgpack::Type SKind, + function_ref verifyValue) { + if (!Node.isScalar()) + return false; + if (Node.getKind() != SKind) { if (Strict) return false; // If we are not strict, we interpret string values as "implicitly typed" // and attempt to coerce them to the expected type here. - if (Scalar.getScalarKind() != msgpack::ScalarNode::SK_String) + if (Node.getKind() != msgpack::Type::String) return false; - std::string StringValue = Scalar.getString(); - Scalar.setScalarKind(SKind); - if (Scalar.inputYAML(StringValue) != StringRef()) + StringRef StringValue = Node.getString(); + Node.fromString(StringValue); + if (Node.getKind() != SKind) return false; } if (verifyValue) - return verifyValue(Scalar); + return verifyValue(Node); return true; } -bool MetadataVerifier::verifyInteger(msgpack::Node &Node) { - if (!verifyScalar(Node, msgpack::ScalarNode::SK_UInt)) - if (!verifyScalar(Node, msgpack::ScalarNode::SK_Int)) +bool MetadataVerifier::verifyInteger(msgpack::DocNode &Node) { + if (!verifyScalar(Node, msgpack::Type::UInt)) + if (!verifyScalar(Node, msgpack::Type::Int)) return false; return true; } bool MetadataVerifier::verifyArray( - msgpack::Node &Node, function_ref verifyNode, + msgpack::DocNode &Node, function_ref verifyNode, Optional Size) { - auto ArrayPtr = dyn_cast(&Node); - if (!ArrayPtr) + if (!Node.isArray()) return false; - auto &Array = *ArrayPtr; + auto &Array = Node.getArray(); if (Size && Array.size() != *Size) return false; for (auto &Item : Array) - if (!verifyNode(*Item.get())) + if (!verifyNode(Item)) return false; return true; } bool MetadataVerifier::verifyEntry( - msgpack::MapNode &MapNode, StringRef Key, bool Required, - function_ref verifyNode) { + msgpack::MapDocNode &MapNode, StringRef Key, bool Required, + function_ref verifyNode) { auto Entry = MapNode.find(Key); if (Entry == MapNode.end()) return !Required; - return verifyNode(*Entry->second.get()); + return verifyNode(Entry->second); } bool MetadataVerifier::verifyScalarEntry( - msgpack::MapNode &MapNode, StringRef Key, bool Required, - msgpack::ScalarNode::ScalarKind SKind, - function_ref verifyValue) { - return verifyEntry(MapNode, Key, Required, [=](msgpack::Node &Node) { + msgpack::MapDocNode &MapNode, StringRef Key, bool Required, + msgpack::Type SKind, + function_ref verifyValue) { + return verifyEntry(MapNode, Key, Required, [=](msgpack::DocNode &Node) { return verifyScalar(Node, SKind, verifyValue); }); } -bool MetadataVerifier::verifyIntegerEntry(msgpack::MapNode &MapNode, +bool MetadataVerifier::verifyIntegerEntry(msgpack::MapDocNode &MapNode, StringRef Key, bool Required) { - return verifyEntry(MapNode, Key, Required, [this](msgpack::Node &Node) { + return verifyEntry(MapNode, Key, Required, [this](msgpack::DocNode &Node) { return verifyInteger(Node); }); } -bool MetadataVerifier::verifyKernelArgs(msgpack::Node &Node) { - auto ArgsMapPtr = dyn_cast(&Node); - if (!ArgsMapPtr) +bool MetadataVerifier::verifyKernelArgs(msgpack::DocNode &Node) { + if (!Node.isMap()) return false; - auto &ArgsMap = *ArgsMapPtr; + auto &ArgsMap = Node.getMap(); if (!verifyScalarEntry(ArgsMap, ".name", false, - msgpack::ScalarNode::SK_String)) + msgpack::Type::String)) return false; if (!verifyScalarEntry(ArgsMap, ".type_name", false, - msgpack::ScalarNode::SK_String)) + msgpack::Type::String)) return false; if (!verifyIntegerEntry(ArgsMap, ".size", true)) return false; if (!verifyIntegerEntry(ArgsMap, ".offset", true)) return false; if (!verifyScalarEntry(ArgsMap, ".value_kind", true, - msgpack::ScalarNode::SK_String, - [](msgpack::ScalarNode &SNode) { + msgpack::Type::String, + [](msgpack::DocNode &SNode) { return StringSwitch(SNode.getString()) .Case("by_value", true) .Case("global_buffer", true) @@ -128,12 +121,13 @@ bool MetadataVerifier::verifyKernelArgs(msgpack::Node &Node) { .Case("hidden_printf_buffer", true) .Case("hidden_default_queue", true) .Case("hidden_completion_action", true) + .Case("hidden_multigrid_sync_arg", true) .Default(false); })) return false; if (!verifyScalarEntry(ArgsMap, ".value_type", true, - msgpack::ScalarNode::SK_String, - [](msgpack::ScalarNode &SNode) { + msgpack::Type::String, + [](msgpack::DocNode &SNode) { return StringSwitch(SNode.getString()) .Case("struct", true) .Case("i8", true) @@ -153,8 +147,8 @@ bool MetadataVerifier::verifyKernelArgs(msgpack::Node &Node) { if (!verifyIntegerEntry(ArgsMap, ".pointee_align", false)) return false; if (!verifyScalarEntry(ArgsMap, ".address_space", false, - msgpack::ScalarNode::SK_String, - [](msgpack::ScalarNode &SNode) { + msgpack::Type::String, + [](msgpack::DocNode &SNode) { return StringSwitch(SNode.getString()) .Case("private", true) .Case("global", true) @@ -166,8 +160,8 @@ bool MetadataVerifier::verifyKernelArgs(msgpack::Node &Node) { })) return false; if (!verifyScalarEntry(ArgsMap, ".access", false, - msgpack::ScalarNode::SK_String, - [](msgpack::ScalarNode &SNode) { + msgpack::Type::String, + [](msgpack::DocNode &SNode) { return StringSwitch(SNode.getString()) .Case("read_only", true) .Case("write_only", true) @@ -176,8 +170,8 @@ bool MetadataVerifier::verifyKernelArgs(msgpack::Node &Node) { })) return false; if (!verifyScalarEntry(ArgsMap, ".actual_access", false, - msgpack::ScalarNode::SK_String, - [](msgpack::ScalarNode &SNode) { + msgpack::Type::String, + [](msgpack::DocNode &SNode) { return StringSwitch(SNode.getString()) .Case("read_only", true) .Case("write_only", true) @@ -186,36 +180,35 @@ bool MetadataVerifier::verifyKernelArgs(msgpack::Node &Node) { })) return false; if (!verifyScalarEntry(ArgsMap, ".is_const", false, - msgpack::ScalarNode::SK_Boolean)) + msgpack::Type::Boolean)) return false; if (!verifyScalarEntry(ArgsMap, ".is_restrict", false, - msgpack::ScalarNode::SK_Boolean)) + msgpack::Type::Boolean)) return false; if (!verifyScalarEntry(ArgsMap, ".is_volatile", false, - msgpack::ScalarNode::SK_Boolean)) + msgpack::Type::Boolean)) return false; if (!verifyScalarEntry(ArgsMap, ".is_pipe", false, - msgpack::ScalarNode::SK_Boolean)) + msgpack::Type::Boolean)) return false; return true; } -bool MetadataVerifier::verifyKernel(msgpack::Node &Node) { - auto KernelMapPtr = dyn_cast(&Node); - if (!KernelMapPtr) +bool MetadataVerifier::verifyKernel(msgpack::DocNode &Node) { + if (!Node.isMap()) return false; - auto &KernelMap = *KernelMapPtr; + auto &KernelMap = Node.getMap(); if (!verifyScalarEntry(KernelMap, ".name", true, - msgpack::ScalarNode::SK_String)) + msgpack::Type::String)) return false; if (!verifyScalarEntry(KernelMap, ".symbol", true, - msgpack::ScalarNode::SK_String)) + msgpack::Type::String)) return false; if (!verifyScalarEntry(KernelMap, ".language", false, - msgpack::ScalarNode::SK_String, - [](msgpack::ScalarNode &SNode) { + msgpack::Type::String, + [](msgpack::DocNode &SNode) { return StringSwitch(SNode.getString()) .Case("OpenCL C", true) .Case("OpenCL C++", true) @@ -227,41 +220,41 @@ bool MetadataVerifier::verifyKernel(msgpack::Node &Node) { })) return false; if (!verifyEntry( - KernelMap, ".language_version", false, [this](msgpack::Node &Node) { + KernelMap, ".language_version", false, [this](msgpack::DocNode &Node) { return verifyArray( Node, - [this](msgpack::Node &Node) { return verifyInteger(Node); }, 2); + [this](msgpack::DocNode &Node) { return verifyInteger(Node); }, 2); })) return false; - if (!verifyEntry(KernelMap, ".args", false, [this](msgpack::Node &Node) { - return verifyArray(Node, [this](msgpack::Node &Node) { + if (!verifyEntry(KernelMap, ".args", false, [this](msgpack::DocNode &Node) { + return verifyArray(Node, [this](msgpack::DocNode &Node) { return verifyKernelArgs(Node); }); })) return false; if (!verifyEntry(KernelMap, ".reqd_workgroup_size", false, - [this](msgpack::Node &Node) { + [this](msgpack::DocNode &Node) { return verifyArray(Node, - [this](msgpack::Node &Node) { + [this](msgpack::DocNode &Node) { return verifyInteger(Node); }, 3); })) return false; if (!verifyEntry(KernelMap, ".workgroup_size_hint", false, - [this](msgpack::Node &Node) { + [this](msgpack::DocNode &Node) { return verifyArray(Node, - [this](msgpack::Node &Node) { + [this](msgpack::DocNode &Node) { return verifyInteger(Node); }, 3); })) return false; if (!verifyScalarEntry(KernelMap, ".vec_type_hint", false, - msgpack::ScalarNode::SK_String)) + msgpack::Type::String)) return false; if (!verifyScalarEntry(KernelMap, ".device_enqueue_symbol", false, - msgpack::ScalarNode::SK_String)) + msgpack::Type::String)) return false; if (!verifyIntegerEntry(KernelMap, ".kernarg_segment_size", true)) return false; @@ -287,29 +280,28 @@ bool MetadataVerifier::verifyKernel(msgpack::Node &Node) { return true; } -bool MetadataVerifier::verify(msgpack::Node &HSAMetadataRoot) { - auto RootMapPtr = dyn_cast(&HSAMetadataRoot); - if (!RootMapPtr) +bool MetadataVerifier::verify(msgpack::DocNode &HSAMetadataRoot) { + if (!HSAMetadataRoot.isMap()) return false; - auto &RootMap = *RootMapPtr; + auto &RootMap = HSAMetadataRoot.getMap(); if (!verifyEntry( - RootMap, "amdhsa.version", true, [this](msgpack::Node &Node) { + RootMap, "amdhsa.version", true, [this](msgpack::DocNode &Node) { return verifyArray( Node, - [this](msgpack::Node &Node) { return verifyInteger(Node); }, 2); + [this](msgpack::DocNode &Node) { return verifyInteger(Node); }, 2); })) return false; if (!verifyEntry( - RootMap, "amdhsa.printf", false, [this](msgpack::Node &Node) { - return verifyArray(Node, [this](msgpack::Node &Node) { - return verifyScalar(Node, msgpack::ScalarNode::SK_String); + RootMap, "amdhsa.printf", false, [this](msgpack::DocNode &Node) { + return verifyArray(Node, [this](msgpack::DocNode &Node) { + return verifyScalar(Node, msgpack::Type::String); }); })) return false; if (!verifyEntry(RootMap, "amdhsa.kernels", true, - [this](msgpack::Node &Node) { - return verifyArray(Node, [this](msgpack::Node &Node) { + [this](msgpack::DocNode &Node) { + return verifyArray(Node, [this](msgpack::DocNode &Node) { return verifyKernel(Node); }); })) diff --git a/lib/BinaryFormat/Dwarf.cpp b/lib/BinaryFormat/Dwarf.cpp index 46f8056774b7..eb6bd33ce583 100644 --- a/lib/BinaryFormat/Dwarf.cpp +++ b/lib/BinaryFormat/Dwarf.cpp @@ -1,9 +1,8 @@ //===-- llvm/BinaryFormat/Dwarf.cpp - Dwarf Framework ------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -144,8 +143,12 @@ StringRef llvm::dwarf::OperationEncodingString(unsigned Encoding) { case DW_OP_##NAME: \ return "DW_OP_" #NAME; #include "llvm/BinaryFormat/Dwarf.def" + case DW_OP_LLVM_convert: + return "DW_OP_LLVM_convert"; case DW_OP_LLVM_fragment: return "DW_OP_LLVM_fragment"; + case DW_OP_LLVM_tag_offset: + return "DW_OP_LLVM_tag_offset"; } } @@ -154,7 +157,9 @@ unsigned llvm::dwarf::getOperationEncoding(StringRef OperationEncodingString) { #define HANDLE_DW_OP(ID, NAME, VERSION, VENDOR) \ .Case("DW_OP_" #NAME, DW_OP_##NAME) #include "llvm/BinaryFormat/Dwarf.def" + .Case("DW_OP_LLVM_convert", DW_OP_LLVM_convert) .Case("DW_OP_LLVM_fragment", DW_OP_LLVM_fragment) + .Case("DW_OP_LLVM_tag_offset", DW_OP_LLVM_tag_offset) .Default(0); } diff --git a/lib/BinaryFormat/Magic.cpp b/lib/BinaryFormat/Magic.cpp index 78efa6ec87be..7dfe23690a50 100644 --- a/lib/BinaryFormat/Magic.cpp +++ b/lib/BinaryFormat/Magic.cpp @@ -1,9 +1,8 @@ //===- llvm/BinaryFormat/Magic.cpp - File magic identification --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -62,6 +61,15 @@ file_magic llvm::identify_magic(StringRef Magic) { return file_magic::wasm_object; break; } + + case 0x01: + // XCOFF format + if (startswith(Magic, "\x01\xDF")) + return file_magic::xcoff_object_32; + if (startswith(Magic, "\x01\xF7")) + return file_magic::xcoff_object_64; + break; + case 0xDE: // 0x0B17C0DE = BC wraper if (startswith(Magic, "\xDE\xC0\x17\x0B")) return file_magic::bitcode; @@ -182,7 +190,8 @@ file_magic llvm::identify_magic(StringRef Magic) { return file_magic::coff_object; break; - case 'M': // Possible MS-DOS stub on Windows PE file or MSF/PDB file. + case 'M': // Possible MS-DOS stub on Windows PE file, MSF/PDB file or a + // Minidump file. if (startswith(Magic, "MZ") && Magic.size() >= 0x3c + 4) { uint32_t off = read32le(Magic.data() + 0x3c); // PE/COFF file, either EXE or DLL. @@ -192,6 +201,8 @@ file_magic llvm::identify_magic(StringRef Magic) { } if (Magic.startswith("Microsoft C/C++ MSF 7.00\r\n")) return file_magic::pdb; + if (startswith(Magic, "MDMP")) + return file_magic::minidump; break; case 0x64: // x86-64 or ARM64 Windows. diff --git a/lib/BinaryFormat/Minidump.cpp b/lib/BinaryFormat/Minidump.cpp new file mode 100644 index 000000000000..b618fb157012 --- /dev/null +++ b/lib/BinaryFormat/Minidump.cpp @@ -0,0 +1,14 @@ +//===-- Minidump.cpp - Minidump constants and structures ---------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/BinaryFormat/Minidump.h" + +using namespace llvm::minidump; + +constexpr uint32_t Header::MagicSignature; +constexpr uint16_t Header::MagicVersion; diff --git a/lib/BinaryFormat/MsgPackDocument.cpp b/lib/BinaryFormat/MsgPackDocument.cpp new file mode 100644 index 000000000000..e12c54a37ad0 --- /dev/null +++ b/lib/BinaryFormat/MsgPackDocument.cpp @@ -0,0 +1,245 @@ +//===-- MsgPackDocument.cpp - MsgPack Document --------------------------*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// This file implements a class that exposes a simple in-memory representation +/// of a document of MsgPack objects, that can be read from MsgPack, written to +/// MsgPack, and inspected and modified in memory. This is intended to be a +/// lighter-weight (in terms of memory allocations) replacement for +/// MsgPackTypes. +/// +//===----------------------------------------------------------------------===// + +#include "llvm/BinaryFormat/MsgPackDocument.h" +#include "llvm/BinaryFormat/MsgPackWriter.h" + +using namespace llvm; +using namespace msgpack; + +// Convert this DocNode into an empty array. +void DocNode::convertToArray() { *this = getDocument()->getArrayNode(); } + +// Convert this DocNode into an empty map. +void DocNode::convertToMap() { *this = getDocument()->getMapNode(); } + +/// Find the key in the MapDocNode. +DocNode::MapTy::iterator MapDocNode::find(StringRef S) { + return find(getDocument()->getNode(S)); +} + +/// Member access for MapDocNode. The string data must remain valid for the +/// lifetime of the Document. +DocNode &MapDocNode::operator[](StringRef S) { + return (*this)[getDocument()->getNode(S)]; +} + +/// Member access for MapDocNode. +DocNode &MapDocNode::operator[](DocNode Key) { + assert(!Key.isEmpty()); + MapTy::value_type Entry(Key, DocNode()); + auto ItAndInserted = Map->insert(Entry); + if (ItAndInserted.second) { + // Ensure a new element has its KindAndDoc initialized. + ItAndInserted.first->second = getDocument()->getNode(); + } + return ItAndInserted.first->second; +} + +/// Array element access. This extends the array if necessary. +DocNode &ArrayDocNode::operator[](size_t Index) { + if (size() <= Index) { + // Ensure new elements have their KindAndDoc initialized. + Array->resize(Index + 1, getDocument()->getNode()); + } + return (*Array)[Index]; +} + +// A level in the document reading stack. +struct StackLevel { + DocNode Node; + size_t Length; + // Points to map entry when we have just processed a map key. + DocNode *MapEntry; +}; + +// Read a document from a binary msgpack blob. +// The blob data must remain valid for the lifetime of this Document (because a +// string object in the document contains a StringRef into the original blob). +// If Multi, then this sets root to an array and adds top-level objects to it. +// If !Multi, then it only reads a single top-level object, even if there are +// more, and sets root to that. +// Returns false if failed due to illegal format. +bool Document::readFromBlob(StringRef Blob, bool Multi) { + msgpack::Reader MPReader(Blob); + SmallVector Stack; + if (Multi) { + // Create the array for multiple top-level objects. + Root = getArrayNode(); + Stack.push_back(StackLevel({Root, (size_t)-1, nullptr})); + } + do { + // On to next element (or key if doing a map key next). + // Read the value. + Object Obj; + if (!MPReader.read(Obj)) { + if (Multi && Stack.size() == 1) { + // OK to finish here as we've just done a top-level element with Multi + break; + } + return false; // Finished too early + } + // Convert it into a DocNode. + DocNode Node; + switch (Obj.Kind) { + case Type::Nil: + Node = getNode(); + break; + case Type::Int: + Node = getNode(Obj.Int); + break; + case Type::UInt: + Node = getNode(Obj.UInt); + break; + case Type::Boolean: + Node = getNode(Obj.Bool); + break; + case Type::Float: + Node = getNode(Obj.Float); + break; + case Type::String: + Node = getNode(Obj.Raw); + break; + case Type::Map: + Node = getMapNode(); + break; + case Type::Array: + Node = getArrayNode(); + break; + default: + return false; // Raw and Extension not supported + } + + // Store it. + if (Stack.empty()) + Root = Node; + else if (Stack.back().Node.getKind() == Type::Array) { + // Reading an array entry. + auto &Array = Stack.back().Node.getArray(); + Array.push_back(Node); + } else { + auto &Map = Stack.back().Node.getMap(); + if (!Stack.back().MapEntry) { + // Reading a map key. + Stack.back().MapEntry = &Map[Node]; + } else { + // Reading the value for the map key read in the last iteration. + *Stack.back().MapEntry = Node; + Stack.back().MapEntry = nullptr; + } + } + + // See if we're starting a new array or map. + switch (Node.getKind()) { + case msgpack::Type::Array: + case msgpack::Type::Map: + Stack.push_back(StackLevel({Node, Obj.Length, nullptr})); + break; + default: + break; + } + + // Pop finished stack levels. + while (!Stack.empty()) { + if (Stack.back().Node.getKind() == msgpack::Type::Array) { + if (Stack.back().Node.getArray().size() != Stack.back().Length) + break; + } else { + if (Stack.back().MapEntry || + Stack.back().Node.getMap().size() != Stack.back().Length) + break; + } + Stack.pop_back(); + } + } while (!Stack.empty()); + return true; +} + +struct WriterStackLevel { + DocNode Node; + DocNode::MapTy::iterator MapIt; + DocNode::ArrayTy::iterator ArrayIt; + bool OnKey; +}; + +/// Write a MsgPack document to a binary MsgPack blob. +void Document::writeToBlob(std::string &Blob) { + Blob.clear(); + raw_string_ostream OS(Blob); + msgpack::Writer MPWriter(OS); + SmallVector Stack; + DocNode Node = getRoot(); + for (;;) { + switch (Node.getKind()) { + case Type::Array: + MPWriter.writeArraySize(Node.getArray().size()); + Stack.push_back( + {Node, DocNode::MapTy::iterator(), Node.getArray().begin(), false}); + break; + case Type::Map: + MPWriter.writeMapSize(Node.getMap().size()); + Stack.push_back( + {Node, Node.getMap().begin(), DocNode::ArrayTy::iterator(), true}); + break; + case Type::Nil: + MPWriter.writeNil(); + break; + case Type::Boolean: + MPWriter.write(Node.getBool()); + break; + case Type::Int: + MPWriter.write(Node.getInt()); + break; + case Type::UInt: + MPWriter.write(Node.getUInt()); + break; + case Type::String: + MPWriter.write(Node.getString()); + break; + default: + llvm_unreachable("unhandled msgpack object kind"); + } + // Pop finished stack levels. + while (!Stack.empty()) { + if (Stack.back().Node.getKind() == Type::Map) { + if (Stack.back().MapIt != Stack.back().Node.getMap().end()) + break; + } else { + if (Stack.back().ArrayIt != Stack.back().Node.getArray().end()) + break; + } + Stack.pop_back(); + } + if (Stack.empty()) + break; + // Get the next value. + if (Stack.back().Node.getKind() == Type::Map) { + if (Stack.back().OnKey) { + // Do the key of a key,value pair in a map. + Node = Stack.back().MapIt->first; + Stack.back().OnKey = false; + } else { + Node = Stack.back().MapIt->second; + ++Stack.back().MapIt; + Stack.back().OnKey = true; + } + } else { + Node = *Stack.back().ArrayIt; + ++Stack.back().ArrayIt; + } + } +} + diff --git a/lib/BinaryFormat/MsgPackDocumentYAML.cpp b/lib/BinaryFormat/MsgPackDocumentYAML.cpp new file mode 100644 index 000000000000..1d9c81ef8ebc --- /dev/null +++ b/lib/BinaryFormat/MsgPackDocumentYAML.cpp @@ -0,0 +1,249 @@ +//===-- MsgPackDocumentYAML.cpp - MsgPack Document YAML interface -------*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +/// This file implements YAMLIO on a msgpack::Document. +// +//===----------------------------------------------------------------------===// + +#include "llvm/BinaryFormat/MsgPackDocument.h" +#include "llvm/Support/YAMLTraits.h" + +using namespace llvm; +using namespace msgpack; + +namespace { + +// Struct used to represent scalar node. (MapDocNode and ArrayDocNode already +// exist in MsgPackDocument.h.) +struct ScalarDocNode : DocNode { + ScalarDocNode(DocNode N) : DocNode(N) {} + + /// Get the YAML tag for this ScalarDocNode. This normally returns ""; it only + /// returns something else if the result of toString would be ambiguous, e.g. + /// a string that parses as a number or boolean. + StringRef getYAMLTag() const; +}; + +} // namespace + +/// Convert this DocNode to a string, assuming it is scalar. +std::string DocNode::toString() const { + std::string S; + raw_string_ostream OS(S); + switch (getKind()) { + case msgpack::Type::String: + OS << Raw; + break; + case msgpack::Type::Nil: + break; + case msgpack::Type::Boolean: + OS << (Bool ? "true" : "false"); + break; + case msgpack::Type::Int: + OS << Int; + break; + case msgpack::Type::UInt: + if (getDocument()->getHexMode()) + OS << format("%#llx", (unsigned long long)UInt); + else + OS << UInt; + break; + case msgpack::Type::Float: + OS << Float; + break; + default: + llvm_unreachable("not scalar"); + break; + } + return OS.str(); +} + +/// Convert the StringRef and use it to set this DocNode (assuming scalar). If +/// it is a string, copy the string into the Document's strings list so we do +/// not rely on S having a lifetime beyond this call. Tag is "" or a YAML tag. +StringRef DocNode::fromString(StringRef S, StringRef Tag) { + if (Tag == "tag:yaml.org,2002:str") + Tag = ""; + if (Tag == "!int" || Tag == "") { + // Try unsigned int then signed int. + *this = getDocument()->getNode(uint64_t(0)); + StringRef Err = yaml::ScalarTraits::input(S, nullptr, getUInt()); + if (Err != "") { + *this = getDocument()->getNode(int64_t(0)); + Err = yaml::ScalarTraits::input(S, nullptr, getInt()); + } + if (Err == "" || Tag != "") + return Err; + } + if (Tag == "!nil") { + *this = getDocument()->getNode(); + return ""; + } + if (Tag == "!bool" || Tag == "") { + *this = getDocument()->getNode(false); + StringRef Err = yaml::ScalarTraits::input(S, nullptr, getBool()); + if (Err == "" || Tag != "") + return Err; + } + if (Tag == "!float" || Tag == "") { + *this = getDocument()->getNode(0.0); + StringRef Err = yaml::ScalarTraits::input(S, nullptr, getFloat()); + if (Err == "" || Tag != "") + return Err; + } + assert((Tag == "!str" || Tag == "") && "unsupported tag"); + std::string V; + StringRef Err = yaml::ScalarTraits::input(S, nullptr, V); + if (Err == "") + *this = getDocument()->getNode(V, /*Copy=*/true); + return Err; +} + +/// Get the YAML tag for this ScalarDocNode. This normally returns ""; it only +/// returns something else if the result of toString would be ambiguous, e.g. +/// a string that parses as a number or boolean. +StringRef ScalarDocNode::getYAMLTag() const { + if (getKind() == msgpack::Type::Nil) + return "!nil"; + // Try converting both ways and see if we get the same kind. If not, we need + // a tag. + ScalarDocNode N = getDocument()->getNode(); + N.fromString(toString(), ""); + if (N.getKind() == getKind()) + return ""; + // Tolerate signedness of int changing, as tags do not differentiate between + // them anyway. + if (N.getKind() == msgpack::Type::UInt && getKind() == msgpack::Type::Int) + return ""; + if (N.getKind() == msgpack::Type::Int && getKind() == msgpack::Type::UInt) + return ""; + // We do need a tag. + switch (getKind()) { + case msgpack::Type::String: + return "!str"; + case msgpack::Type::Int: + return "!int"; + case msgpack::Type::UInt: + return "!int"; + case msgpack::Type::Boolean: + return "!bool"; + case msgpack::Type::Float: + return "!float"; + default: + llvm_unreachable("unrecognized kind"); + } +} + +namespace llvm { +namespace yaml { + +/// YAMLIO for DocNode +template <> struct PolymorphicTraits { + + static NodeKind getKind(const DocNode &N) { + switch (N.getKind()) { + case msgpack::Type::Map: + return NodeKind::Map; + case msgpack::Type::Array: + return NodeKind::Sequence; + default: + return NodeKind::Scalar; + } + } + + static MapDocNode &getAsMap(DocNode &N) { return N.getMap(/*Convert=*/true); } + + static ArrayDocNode &getAsSequence(DocNode &N) { + N.getArray(/*Convert=*/true); + return *static_cast(&N); + } + + static ScalarDocNode &getAsScalar(DocNode &N) { + return *static_cast(&N); + } +}; + +/// YAMLIO for ScalarDocNode +template <> struct TaggedScalarTraits { + + static void output(const ScalarDocNode &S, void *Ctxt, raw_ostream &OS, + raw_ostream &TagOS) { + TagOS << S.getYAMLTag(); + OS << S.toString(); + } + + static StringRef input(StringRef Str, StringRef Tag, void *Ctxt, + ScalarDocNode &S) { + return S.fromString(Str, Tag); + } + + static QuotingType mustQuote(const ScalarDocNode &S, StringRef ScalarStr) { + switch (S.getKind()) { + case Type::Int: + return ScalarTraits::mustQuote(ScalarStr); + case Type::UInt: + return ScalarTraits::mustQuote(ScalarStr); + case Type::Nil: + return ScalarTraits::mustQuote(ScalarStr); + case Type::Boolean: + return ScalarTraits::mustQuote(ScalarStr); + case Type::Float: + return ScalarTraits::mustQuote(ScalarStr); + case Type::Binary: + case Type::String: + return ScalarTraits::mustQuote(ScalarStr); + default: + llvm_unreachable("unrecognized ScalarKind"); + } + } +}; + +/// YAMLIO for MapDocNode +template <> struct CustomMappingTraits { + + static void inputOne(IO &IO, StringRef Key, MapDocNode &M) { + ScalarDocNode KeyObj = M.getDocument()->getNode(); + KeyObj.fromString(Key, ""); + IO.mapRequired(Key.str().c_str(), M.getMap()[KeyObj]); + } + + static void output(IO &IO, MapDocNode &M) { + for (auto I : M.getMap()) { + IO.mapRequired(I.first.toString().c_str(), I.second); + } + } +}; + +/// YAMLIO for ArrayNode +template <> struct SequenceTraits { + + static size_t size(IO &IO, ArrayDocNode &A) { return A.size(); } + + static DocNode &element(IO &IO, ArrayDocNode &A, size_t Index) { + return A[Index]; + } +}; + +} // namespace yaml +} // namespace llvm + +/// Convert MsgPack Document to YAML text. +void msgpack::Document::toYAML(raw_ostream &OS) { + yaml::Output Yout(OS); + Yout << getRoot(); +} + +/// Read YAML text into the MsgPack document. Returns false on failure. +bool msgpack::Document::fromYAML(StringRef S) { + clear(); + yaml::Input Yin(S); + Yin >> getRoot(); + return !Yin.error(); +} + diff --git a/lib/BinaryFormat/MsgPackReader.cpp b/lib/BinaryFormat/MsgPackReader.cpp index b510fdba9608..872a6e0e29f8 100644 --- a/lib/BinaryFormat/MsgPackReader.cpp +++ b/lib/BinaryFormat/MsgPackReader.cpp @@ -1,9 +1,8 @@ //===- MsgPackReader.cpp - Simple MsgPack reader ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/BinaryFormat/MsgPackTypes.cpp b/lib/BinaryFormat/MsgPackTypes.cpp deleted file mode 100644 index 4a8f70b10fb8..000000000000 --- a/lib/BinaryFormat/MsgPackTypes.cpp +++ /dev/null @@ -1,303 +0,0 @@ -//===- MsgPackTypes.cpp - MsgPack Types -------------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -/// \file -/// Implementation of types representing MessagePack "documents". -// -//===----------------------------------------------------------------------===// - -#include "llvm/BinaryFormat/MsgPackTypes.h" -#include "llvm/Support/Error.h" - -using namespace llvm; -using namespace msgpack; - -namespace llvm { -namespace msgpack { -void ScalarNode::anchor() {} -void ArrayNode::anchor() {} -void MapNode::anchor() {} -} -} - -Expected Node::readArray(Reader &MPReader, size_t Length) { - auto A = std::make_shared(); - for (size_t I = 0; I < Length; ++I) { - auto OptNodeOrErr = Node::read(MPReader); - if (auto Err = OptNodeOrErr.takeError()) - return std::move(Err); - if (!*OptNodeOrErr) - return make_error( - "Insufficient array elements", - std::make_error_code(std::errc::invalid_argument)); - A->push_back(std::move(**OptNodeOrErr)); - } - return OptNodePtr(std::move(A)); -} - -Expected Node::readMap(Reader &MPReader, size_t Length) { - auto M = std::make_shared(); - for (size_t I = 0; I < Length; ++I) { - auto OptKeyOrErr = Node::read(MPReader); - if (auto Err = OptKeyOrErr.takeError()) - return std::move(Err); - if (!*OptKeyOrErr) - return make_error( - "Insufficient map elements", - std::make_error_code(std::errc::invalid_argument)); - auto OptValOrErr = Node::read(MPReader); - if (auto Err = OptValOrErr.takeError()) - return std::move(Err); - if (!*OptValOrErr) - return make_error( - "Insufficient map elements", - std::make_error_code(std::errc::invalid_argument)); - auto *Key = dyn_cast((*OptKeyOrErr)->get()); - if (!Key) - return make_error( - "Only string map keys are supported", - std::make_error_code(std::errc::invalid_argument)); - if (Key->getScalarKind() != ScalarNode::SK_String) - return make_error( - "Only string map keys are supported", - std::make_error_code(std::errc::invalid_argument)); - M->try_emplace(Key->getString(), std::move(**OptValOrErr)); - } - return OptNodePtr(std::move(M)); -} - -Expected Node::read(Reader &MPReader) { - Object Obj; - - auto ContinueOrErr = MPReader.read(Obj); - if (auto Err = ContinueOrErr.takeError()) - return std::move(Err); - if (!*ContinueOrErr) - return None; - - switch (Obj.Kind) { - case Type::Int: - return OptNodePtr(std::make_shared(Obj.Int)); - case Type::UInt: - return OptNodePtr(std::make_shared(Obj.UInt)); - case Type::Nil: - return OptNodePtr(std::make_shared()); - case Type::Boolean: - return OptNodePtr(std::make_shared(Obj.Bool)); - case Type::Float: - return OptNodePtr(std::make_shared(Obj.Float)); - case Type::String: - return OptNodePtr(std::make_shared(Obj.Raw)); - case Type::Binary: - return OptNodePtr(std::make_shared(Obj.Raw)); - case Type::Array: - return Node::readArray(MPReader, Obj.Length); - case Type::Map: - return Node::readMap(MPReader, Obj.Length); - case Type::Extension: - return make_error( - "Extension types are not supported", - std::make_error_code(std::errc::invalid_argument)); - } - llvm_unreachable("msgpack::Type not handled"); -} - -void ScalarNode::destroy() { - switch (SKind) { - case SK_String: - case SK_Binary: - StringValue.~basic_string(); - break; - default: - // POD types do not require destruction - break; - } -} - -ScalarNode::ScalarNode(int64_t IntValue) - : Node(NK_Scalar), SKind(SK_Int), IntValue(IntValue) {} - -ScalarNode::ScalarNode(int32_t IntValue) - : ScalarNode(static_cast(IntValue)) {} - -ScalarNode::ScalarNode(uint64_t UIntValue) - : Node(NK_Scalar), SKind(SK_UInt), UIntValue(UIntValue) {} - -ScalarNode::ScalarNode(uint32_t IntValue) - : ScalarNode(static_cast(IntValue)) {} - -ScalarNode::ScalarNode() : Node(NK_Scalar), SKind(SK_Nil) {} - -ScalarNode::ScalarNode(bool BoolValue) - : Node(NK_Scalar), SKind(SK_Boolean), BoolValue(BoolValue) {} - -ScalarNode::ScalarNode(double FloatValue) - : Node(NK_Scalar), SKind(SK_Float), BoolValue(FloatValue) {} - -ScalarNode::ScalarNode(StringRef StringValue) - : Node(NK_Scalar), SKind(SK_String) { - new (&this->StringValue) std::string(StringValue); -} - -ScalarNode::ScalarNode(const char *StringValue) - : ScalarNode(StringRef(StringValue)) {} - -ScalarNode::ScalarNode(std::string &&StringValue) - : Node(NK_Scalar), SKind(SK_String) { - new (&this->StringValue) std::string(StringValue); -} - -ScalarNode::ScalarNode(MemoryBufferRef BinaryValue) - : Node(NK_Scalar), SKind(SK_Binary) { - new (&StringValue) std::string(BinaryValue.getBuffer()); -} - -ScalarNode::~ScalarNode() { destroy(); } - -ScalarNode &ScalarNode::operator=(ScalarNode &&RHS) { - destroy(); - switch (SKind = RHS.SKind) { - case SK_Int: - IntValue = RHS.IntValue; - break; - case SK_UInt: - UIntValue = RHS.UIntValue; - break; - case SK_Boolean: - BoolValue = RHS.BoolValue; - break; - case SK_Float: - FloatValue = RHS.FloatValue; - break; - case SK_String: - case SK_Binary: - new (&StringValue) std::string(std::move(RHS.StringValue)); - break; - case SK_Nil: - // pass - break; - } - return *this; -} - -StringRef ScalarNode::inputYAML(StringRef ScalarStr) { - switch (SKind) { - case SK_Int: - return yaml::ScalarTraits::input(ScalarStr, nullptr, IntValue); - case SK_UInt: - return yaml::ScalarTraits::input(ScalarStr, nullptr, UIntValue); - case SK_Nil: - return StringRef(); - case SK_Boolean: - return yaml::ScalarTraits::input(ScalarStr, nullptr, BoolValue); - case SK_Float: - return yaml::ScalarTraits::input(ScalarStr, nullptr, FloatValue); - case SK_Binary: - case SK_String: - return yaml::ScalarTraits::input(ScalarStr, nullptr, - StringValue); - } - llvm_unreachable("unrecognized ScalarKind"); -} - -void ScalarNode::outputYAML(raw_ostream &OS) const { - switch (SKind) { - case SK_Int: - yaml::ScalarTraits::output(IntValue, nullptr, OS); - break; - case SK_UInt: - yaml::ScalarTraits::output(UIntValue, nullptr, OS); - break; - case SK_Nil: - yaml::ScalarTraits::output("", nullptr, OS); - break; - case SK_Boolean: - yaml::ScalarTraits::output(BoolValue, nullptr, OS); - break; - case SK_Float: - yaml::ScalarTraits::output(FloatValue, nullptr, OS); - break; - case SK_Binary: - case SK_String: - yaml::ScalarTraits::output(StringValue, nullptr, OS); - break; - } -} - -yaml::QuotingType ScalarNode::mustQuoteYAML(StringRef ScalarStr) const { - switch (SKind) { - case SK_Int: - return yaml::ScalarTraits::mustQuote(ScalarStr); - case SK_UInt: - return yaml::ScalarTraits::mustQuote(ScalarStr); - case SK_Nil: - return yaml::ScalarTraits::mustQuote(ScalarStr); - case SK_Boolean: - return yaml::ScalarTraits::mustQuote(ScalarStr); - case SK_Float: - return yaml::ScalarTraits::mustQuote(ScalarStr); - case SK_Binary: - case SK_String: - return yaml::ScalarTraits::mustQuote(ScalarStr); - } - llvm_unreachable("unrecognized ScalarKind"); -} - -const char *ScalarNode::IntTag = "!int"; -const char *ScalarNode::NilTag = "!nil"; -const char *ScalarNode::BooleanTag = "!bool"; -const char *ScalarNode::FloatTag = "!float"; -const char *ScalarNode::StringTag = "!str"; -const char *ScalarNode::BinaryTag = "!bin"; - -StringRef ScalarNode::getYAMLTag() const { - switch (SKind) { - case SK_Int: - return IntTag; - case SK_UInt: - return IntTag; - case SK_Nil: - return NilTag; - case SK_Boolean: - return BooleanTag; - case SK_Float: - return FloatTag; - case SK_String: - return StringTag; - case SK_Binary: - return BinaryTag; - } - llvm_unreachable("unrecognized ScalarKind"); -} - -void ScalarNode::write(Writer &MPWriter) { - switch (SKind) { - case SK_Int: - MPWriter.write(IntValue); - break; - case SK_UInt: - MPWriter.write(UIntValue); - break; - case SK_Nil: - MPWriter.writeNil(); - break; - case SK_Boolean: - MPWriter.write(BoolValue); - break; - case SK_Float: - MPWriter.write(FloatValue); - break; - case SK_String: - MPWriter.write(StringValue); - break; - case SK_Binary: - MPWriter.write(MemoryBufferRef(StringValue, "")); - break; - } -} diff --git a/lib/BinaryFormat/MsgPackWriter.cpp b/lib/BinaryFormat/MsgPackWriter.cpp index d024bb0fcdb2..b4d70e8f78c1 100644 --- a/lib/BinaryFormat/MsgPackWriter.cpp +++ b/lib/BinaryFormat/MsgPackWriter.cpp @@ -1,9 +1,8 @@ //===- MsgPackWriter.cpp - Simple MsgPack writer ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/BinaryFormat/Wasm.cpp b/lib/BinaryFormat/Wasm.cpp index 94d40bf02a39..d46be481edb3 100644 --- a/lib/BinaryFormat/Wasm.cpp +++ b/lib/BinaryFormat/Wasm.cpp @@ -1,16 +1,15 @@ //===-- llvm/BinaryFormat/Wasm.cpp -------------------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "llvm/BinaryFormat/Wasm.h" -std::string llvm::wasm::toString(wasm::WasmSymbolType type) { - switch (type) { +std::string llvm::wasm::toString(wasm::WasmSymbolType Type) { + switch (Type) { case wasm::WASM_SYMBOL_TYPE_FUNCTION: return "WASM_SYMBOL_TYPE_FUNCTION"; case wasm::WASM_SYMBOL_TYPE_GLOBAL: @@ -25,8 +24,8 @@ std::string llvm::wasm::toString(wasm::WasmSymbolType type) { llvm_unreachable("unknown symbol type"); } -std::string llvm::wasm::relocTypetoString(uint32_t type) { - switch (type) { +std::string llvm::wasm::relocTypetoString(uint32_t Type) { + switch (Type) { #define WASM_RELOC(NAME, VALUE) \ case VALUE: \ return #NAME; @@ -36,3 +35,17 @@ std::string llvm::wasm::relocTypetoString(uint32_t type) { llvm_unreachable("unknown reloc type"); } } + +bool llvm::wasm::relocTypeHasAddend(uint32_t Type) { + switch (Type) { + case R_WASM_MEMORY_ADDR_LEB: + case R_WASM_MEMORY_ADDR_SLEB: + case R_WASM_MEMORY_ADDR_REL_SLEB: + case R_WASM_MEMORY_ADDR_I32: + case R_WASM_FUNCTION_OFFSET_I32: + case R_WASM_SECTION_OFFSET_I32: + return true; + default: + return false; + } +} diff --git a/lib/Bitcode/Reader/BitReader.cpp b/lib/Bitcode/Reader/BitReader.cpp index 3ec45956b3e5..5ac893aef14e 100644 --- a/lib/Bitcode/Reader/BitReader.cpp +++ b/lib/Bitcode/Reader/BitReader.cpp @@ -1,9 +1,8 @@ //===-- BitReader.cpp -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/Bitcode/Reader/BitcodeAnalyzer.cpp b/lib/Bitcode/Reader/BitcodeAnalyzer.cpp new file mode 100644 index 000000000000..9c30d563a314 --- /dev/null +++ b/lib/Bitcode/Reader/BitcodeAnalyzer.cpp @@ -0,0 +1,980 @@ +//===- BitcodeAnalyzer.cpp - Internal BitcodeAnalyzer implementation ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Bitcode/BitcodeAnalyzer.h" +#include "llvm/Bitcode/BitcodeReader.h" +#include "llvm/Bitcode/LLVMBitCodes.h" +#include "llvm/Bitstream/BitCodes.h" +#include "llvm/Bitstream/BitstreamReader.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/SHA1.h" + +using namespace llvm; + +static Error reportError(StringRef Message) { + return createStringError(std::errc::illegal_byte_sequence, Message.data()); +} + +/// Return a symbolic block name if known, otherwise return null. +static Optional GetBlockName(unsigned BlockID, + const BitstreamBlockInfo &BlockInfo, + CurStreamTypeType CurStreamType) { + // Standard blocks for all bitcode files. + if (BlockID < bitc::FIRST_APPLICATION_BLOCKID) { + if (BlockID == bitc::BLOCKINFO_BLOCK_ID) + return "BLOCKINFO_BLOCK"; + return None; + } + + // Check to see if we have a blockinfo record for this block, with a name. + if (const BitstreamBlockInfo::BlockInfo *Info = + BlockInfo.getBlockInfo(BlockID)) { + if (!Info->Name.empty()) + return Info->Name.c_str(); + } + + if (CurStreamType != LLVMIRBitstream) + return None; + + switch (BlockID) { + default: + return None; + case bitc::OPERAND_BUNDLE_TAGS_BLOCK_ID: + return "OPERAND_BUNDLE_TAGS_BLOCK"; + case bitc::MODULE_BLOCK_ID: + return "MODULE_BLOCK"; + case bitc::PARAMATTR_BLOCK_ID: + return "PARAMATTR_BLOCK"; + case bitc::PARAMATTR_GROUP_BLOCK_ID: + return "PARAMATTR_GROUP_BLOCK_ID"; + case bitc::TYPE_BLOCK_ID_NEW: + return "TYPE_BLOCK_ID"; + case bitc::CONSTANTS_BLOCK_ID: + return "CONSTANTS_BLOCK"; + case bitc::FUNCTION_BLOCK_ID: + return "FUNCTION_BLOCK"; + case bitc::IDENTIFICATION_BLOCK_ID: + return "IDENTIFICATION_BLOCK_ID"; + case bitc::VALUE_SYMTAB_BLOCK_ID: + return "VALUE_SYMTAB"; + case bitc::METADATA_BLOCK_ID: + return "METADATA_BLOCK"; + case bitc::METADATA_KIND_BLOCK_ID: + return "METADATA_KIND_BLOCK"; + case bitc::METADATA_ATTACHMENT_ID: + return "METADATA_ATTACHMENT_BLOCK"; + case bitc::USELIST_BLOCK_ID: + return "USELIST_BLOCK_ID"; + case bitc::GLOBALVAL_SUMMARY_BLOCK_ID: + return "GLOBALVAL_SUMMARY_BLOCK"; + case bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID: + return "FULL_LTO_GLOBALVAL_SUMMARY_BLOCK"; + case bitc::MODULE_STRTAB_BLOCK_ID: + return "MODULE_STRTAB_BLOCK"; + case bitc::STRTAB_BLOCK_ID: + return "STRTAB_BLOCK"; + case bitc::SYMTAB_BLOCK_ID: + return "SYMTAB_BLOCK"; + } +} + +/// Return a symbolic code name if known, otherwise return null. +static Optional GetCodeName(unsigned CodeID, unsigned BlockID, + const BitstreamBlockInfo &BlockInfo, + CurStreamTypeType CurStreamType) { + // Standard blocks for all bitcode files. + if (BlockID < bitc::FIRST_APPLICATION_BLOCKID) { + if (BlockID == bitc::BLOCKINFO_BLOCK_ID) { + switch (CodeID) { + default: + return None; + case bitc::BLOCKINFO_CODE_SETBID: + return "SETBID"; + case bitc::BLOCKINFO_CODE_BLOCKNAME: + return "BLOCKNAME"; + case bitc::BLOCKINFO_CODE_SETRECORDNAME: + return "SETRECORDNAME"; + } + } + return None; + } + + // Check to see if we have a blockinfo record for this record, with a name. + if (const BitstreamBlockInfo::BlockInfo *Info = + BlockInfo.getBlockInfo(BlockID)) { + for (unsigned i = 0, e = Info->RecordNames.size(); i != e; ++i) + if (Info->RecordNames[i].first == CodeID) + return Info->RecordNames[i].second.c_str(); + } + + if (CurStreamType != LLVMIRBitstream) + return None; + +#define STRINGIFY_CODE(PREFIX, CODE) \ + case bitc::PREFIX##_##CODE: \ + return #CODE; + switch (BlockID) { + default: + return None; + case bitc::MODULE_BLOCK_ID: + switch (CodeID) { + default: + return None; + STRINGIFY_CODE(MODULE_CODE, VERSION) + STRINGIFY_CODE(MODULE_CODE, TRIPLE) + STRINGIFY_CODE(MODULE_CODE, DATALAYOUT) + STRINGIFY_CODE(MODULE_CODE, ASM) + STRINGIFY_CODE(MODULE_CODE, SECTIONNAME) + STRINGIFY_CODE(MODULE_CODE, DEPLIB) // FIXME: Remove in 4.0 + STRINGIFY_CODE(MODULE_CODE, GLOBALVAR) + STRINGIFY_CODE(MODULE_CODE, FUNCTION) + STRINGIFY_CODE(MODULE_CODE, ALIAS) + STRINGIFY_CODE(MODULE_CODE, GCNAME) + STRINGIFY_CODE(MODULE_CODE, VSTOFFSET) + STRINGIFY_CODE(MODULE_CODE, METADATA_VALUES_UNUSED) + STRINGIFY_CODE(MODULE_CODE, SOURCE_FILENAME) + STRINGIFY_CODE(MODULE_CODE, HASH) + } + case bitc::IDENTIFICATION_BLOCK_ID: + switch (CodeID) { + default: + return None; + STRINGIFY_CODE(IDENTIFICATION_CODE, STRING) + STRINGIFY_CODE(IDENTIFICATION_CODE, EPOCH) + } + case bitc::PARAMATTR_BLOCK_ID: + switch (CodeID) { + default: + return None; + // FIXME: Should these be different? + case bitc::PARAMATTR_CODE_ENTRY_OLD: + return "ENTRY"; + case bitc::PARAMATTR_CODE_ENTRY: + return "ENTRY"; + } + case bitc::PARAMATTR_GROUP_BLOCK_ID: + switch (CodeID) { + default: + return None; + case bitc::PARAMATTR_GRP_CODE_ENTRY: + return "ENTRY"; + } + case bitc::TYPE_BLOCK_ID_NEW: + switch (CodeID) { + default: + return None; + STRINGIFY_CODE(TYPE_CODE, NUMENTRY) + STRINGIFY_CODE(TYPE_CODE, VOID) + STRINGIFY_CODE(TYPE_CODE, FLOAT) + STRINGIFY_CODE(TYPE_CODE, DOUBLE) + STRINGIFY_CODE(TYPE_CODE, LABEL) + STRINGIFY_CODE(TYPE_CODE, OPAQUE) + STRINGIFY_CODE(TYPE_CODE, INTEGER) + STRINGIFY_CODE(TYPE_CODE, POINTER) + STRINGIFY_CODE(TYPE_CODE, ARRAY) + STRINGIFY_CODE(TYPE_CODE, VECTOR) + STRINGIFY_CODE(TYPE_CODE, X86_FP80) + STRINGIFY_CODE(TYPE_CODE, FP128) + STRINGIFY_CODE(TYPE_CODE, PPC_FP128) + STRINGIFY_CODE(TYPE_CODE, METADATA) + STRINGIFY_CODE(TYPE_CODE, STRUCT_ANON) + STRINGIFY_CODE(TYPE_CODE, STRUCT_NAME) + STRINGIFY_CODE(TYPE_CODE, STRUCT_NAMED) + STRINGIFY_CODE(TYPE_CODE, FUNCTION) + } + + case bitc::CONSTANTS_BLOCK_ID: + switch (CodeID) { + default: + return None; + STRINGIFY_CODE(CST_CODE, SETTYPE) + STRINGIFY_CODE(CST_CODE, NULL) + STRINGIFY_CODE(CST_CODE, UNDEF) + STRINGIFY_CODE(CST_CODE, INTEGER) + STRINGIFY_CODE(CST_CODE, WIDE_INTEGER) + STRINGIFY_CODE(CST_CODE, FLOAT) + STRINGIFY_CODE(CST_CODE, AGGREGATE) + STRINGIFY_CODE(CST_CODE, STRING) + STRINGIFY_CODE(CST_CODE, CSTRING) + STRINGIFY_CODE(CST_CODE, CE_BINOP) + STRINGIFY_CODE(CST_CODE, CE_CAST) + STRINGIFY_CODE(CST_CODE, CE_GEP) + STRINGIFY_CODE(CST_CODE, CE_INBOUNDS_GEP) + STRINGIFY_CODE(CST_CODE, CE_SELECT) + STRINGIFY_CODE(CST_CODE, CE_EXTRACTELT) + STRINGIFY_CODE(CST_CODE, CE_INSERTELT) + STRINGIFY_CODE(CST_CODE, CE_SHUFFLEVEC) + STRINGIFY_CODE(CST_CODE, CE_CMP) + STRINGIFY_CODE(CST_CODE, INLINEASM) + STRINGIFY_CODE(CST_CODE, CE_SHUFVEC_EX) + STRINGIFY_CODE(CST_CODE, CE_UNOP) + case bitc::CST_CODE_BLOCKADDRESS: + return "CST_CODE_BLOCKADDRESS"; + STRINGIFY_CODE(CST_CODE, DATA) + } + case bitc::FUNCTION_BLOCK_ID: + switch (CodeID) { + default: + return None; + STRINGIFY_CODE(FUNC_CODE, DECLAREBLOCKS) + STRINGIFY_CODE(FUNC_CODE, INST_BINOP) + STRINGIFY_CODE(FUNC_CODE, INST_CAST) + STRINGIFY_CODE(FUNC_CODE, INST_GEP_OLD) + STRINGIFY_CODE(FUNC_CODE, INST_INBOUNDS_GEP_OLD) + STRINGIFY_CODE(FUNC_CODE, INST_SELECT) + STRINGIFY_CODE(FUNC_CODE, INST_EXTRACTELT) + STRINGIFY_CODE(FUNC_CODE, INST_INSERTELT) + STRINGIFY_CODE(FUNC_CODE, INST_SHUFFLEVEC) + STRINGIFY_CODE(FUNC_CODE, INST_CMP) + STRINGIFY_CODE(FUNC_CODE, INST_RET) + STRINGIFY_CODE(FUNC_CODE, INST_BR) + STRINGIFY_CODE(FUNC_CODE, INST_SWITCH) + STRINGIFY_CODE(FUNC_CODE, INST_INVOKE) + STRINGIFY_CODE(FUNC_CODE, INST_UNOP) + STRINGIFY_CODE(FUNC_CODE, INST_UNREACHABLE) + STRINGIFY_CODE(FUNC_CODE, INST_CLEANUPRET) + STRINGIFY_CODE(FUNC_CODE, INST_CATCHRET) + STRINGIFY_CODE(FUNC_CODE, INST_CATCHPAD) + STRINGIFY_CODE(FUNC_CODE, INST_PHI) + STRINGIFY_CODE(FUNC_CODE, INST_ALLOCA) + STRINGIFY_CODE(FUNC_CODE, INST_LOAD) + STRINGIFY_CODE(FUNC_CODE, INST_VAARG) + STRINGIFY_CODE(FUNC_CODE, INST_STORE) + STRINGIFY_CODE(FUNC_CODE, INST_EXTRACTVAL) + STRINGIFY_CODE(FUNC_CODE, INST_INSERTVAL) + STRINGIFY_CODE(FUNC_CODE, INST_CMP2) + STRINGIFY_CODE(FUNC_CODE, INST_VSELECT) + STRINGIFY_CODE(FUNC_CODE, DEBUG_LOC_AGAIN) + STRINGIFY_CODE(FUNC_CODE, INST_CALL) + STRINGIFY_CODE(FUNC_CODE, DEBUG_LOC) + STRINGIFY_CODE(FUNC_CODE, INST_GEP) + STRINGIFY_CODE(FUNC_CODE, OPERAND_BUNDLE) + STRINGIFY_CODE(FUNC_CODE, INST_FENCE) + STRINGIFY_CODE(FUNC_CODE, INST_ATOMICRMW) + STRINGIFY_CODE(FUNC_CODE, INST_LOADATOMIC) + STRINGIFY_CODE(FUNC_CODE, INST_STOREATOMIC) + STRINGIFY_CODE(FUNC_CODE, INST_CMPXCHG) + STRINGIFY_CODE(FUNC_CODE, INST_CALLBR) + } + case bitc::VALUE_SYMTAB_BLOCK_ID: + switch (CodeID) { + default: + return None; + STRINGIFY_CODE(VST_CODE, ENTRY) + STRINGIFY_CODE(VST_CODE, BBENTRY) + STRINGIFY_CODE(VST_CODE, FNENTRY) + STRINGIFY_CODE(VST_CODE, COMBINED_ENTRY) + } + case bitc::MODULE_STRTAB_BLOCK_ID: + switch (CodeID) { + default: + return None; + STRINGIFY_CODE(MST_CODE, ENTRY) + STRINGIFY_CODE(MST_CODE, HASH) + } + case bitc::GLOBALVAL_SUMMARY_BLOCK_ID: + case bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID: + switch (CodeID) { + default: + return None; + STRINGIFY_CODE(FS, PERMODULE) + STRINGIFY_CODE(FS, PERMODULE_PROFILE) + STRINGIFY_CODE(FS, PERMODULE_RELBF) + STRINGIFY_CODE(FS, PERMODULE_GLOBALVAR_INIT_REFS) + STRINGIFY_CODE(FS, PERMODULE_VTABLE_GLOBALVAR_INIT_REFS) + STRINGIFY_CODE(FS, COMBINED) + STRINGIFY_CODE(FS, COMBINED_PROFILE) + STRINGIFY_CODE(FS, COMBINED_GLOBALVAR_INIT_REFS) + STRINGIFY_CODE(FS, ALIAS) + STRINGIFY_CODE(FS, COMBINED_ALIAS) + STRINGIFY_CODE(FS, COMBINED_ORIGINAL_NAME) + STRINGIFY_CODE(FS, VERSION) + STRINGIFY_CODE(FS, FLAGS) + STRINGIFY_CODE(FS, TYPE_TESTS) + STRINGIFY_CODE(FS, TYPE_TEST_ASSUME_VCALLS) + STRINGIFY_CODE(FS, TYPE_CHECKED_LOAD_VCALLS) + STRINGIFY_CODE(FS, TYPE_TEST_ASSUME_CONST_VCALL) + STRINGIFY_CODE(FS, TYPE_CHECKED_LOAD_CONST_VCALL) + STRINGIFY_CODE(FS, VALUE_GUID) + STRINGIFY_CODE(FS, CFI_FUNCTION_DEFS) + STRINGIFY_CODE(FS, CFI_FUNCTION_DECLS) + STRINGIFY_CODE(FS, TYPE_ID) + STRINGIFY_CODE(FS, TYPE_ID_METADATA) + } + case bitc::METADATA_ATTACHMENT_ID: + switch (CodeID) { + default: + return None; + STRINGIFY_CODE(METADATA, ATTACHMENT) + } + case bitc::METADATA_BLOCK_ID: + switch (CodeID) { + default: + return None; + STRINGIFY_CODE(METADATA, STRING_OLD) + STRINGIFY_CODE(METADATA, VALUE) + STRINGIFY_CODE(METADATA, NODE) + STRINGIFY_CODE(METADATA, NAME) + STRINGIFY_CODE(METADATA, DISTINCT_NODE) + STRINGIFY_CODE(METADATA, KIND) // Older bitcode has it in a MODULE_BLOCK + STRINGIFY_CODE(METADATA, LOCATION) + STRINGIFY_CODE(METADATA, OLD_NODE) + STRINGIFY_CODE(METADATA, OLD_FN_NODE) + STRINGIFY_CODE(METADATA, NAMED_NODE) + STRINGIFY_CODE(METADATA, GENERIC_DEBUG) + STRINGIFY_CODE(METADATA, SUBRANGE) + STRINGIFY_CODE(METADATA, ENUMERATOR) + STRINGIFY_CODE(METADATA, BASIC_TYPE) + STRINGIFY_CODE(METADATA, FILE) + STRINGIFY_CODE(METADATA, DERIVED_TYPE) + STRINGIFY_CODE(METADATA, COMPOSITE_TYPE) + STRINGIFY_CODE(METADATA, SUBROUTINE_TYPE) + STRINGIFY_CODE(METADATA, COMPILE_UNIT) + STRINGIFY_CODE(METADATA, SUBPROGRAM) + STRINGIFY_CODE(METADATA, LEXICAL_BLOCK) + STRINGIFY_CODE(METADATA, LEXICAL_BLOCK_FILE) + STRINGIFY_CODE(METADATA, NAMESPACE) + STRINGIFY_CODE(METADATA, TEMPLATE_TYPE) + STRINGIFY_CODE(METADATA, TEMPLATE_VALUE) + STRINGIFY_CODE(METADATA, GLOBAL_VAR) + STRINGIFY_CODE(METADATA, LOCAL_VAR) + STRINGIFY_CODE(METADATA, EXPRESSION) + STRINGIFY_CODE(METADATA, OBJC_PROPERTY) + STRINGIFY_CODE(METADATA, IMPORTED_ENTITY) + STRINGIFY_CODE(METADATA, MODULE) + STRINGIFY_CODE(METADATA, MACRO) + STRINGIFY_CODE(METADATA, MACRO_FILE) + STRINGIFY_CODE(METADATA, STRINGS) + STRINGIFY_CODE(METADATA, GLOBAL_DECL_ATTACHMENT) + STRINGIFY_CODE(METADATA, GLOBAL_VAR_EXPR) + STRINGIFY_CODE(METADATA, INDEX_OFFSET) + STRINGIFY_CODE(METADATA, INDEX) + } + case bitc::METADATA_KIND_BLOCK_ID: + switch (CodeID) { + default: + return None; + STRINGIFY_CODE(METADATA, KIND) + } + case bitc::USELIST_BLOCK_ID: + switch (CodeID) { + default: + return None; + case bitc::USELIST_CODE_DEFAULT: + return "USELIST_CODE_DEFAULT"; + case bitc::USELIST_CODE_BB: + return "USELIST_CODE_BB"; + } + + case bitc::OPERAND_BUNDLE_TAGS_BLOCK_ID: + switch (CodeID) { + default: + return None; + case bitc::OPERAND_BUNDLE_TAG: + return "OPERAND_BUNDLE_TAG"; + } + case bitc::STRTAB_BLOCK_ID: + switch (CodeID) { + default: + return None; + case bitc::STRTAB_BLOB: + return "BLOB"; + } + case bitc::SYMTAB_BLOCK_ID: + switch (CodeID) { + default: + return None; + case bitc::SYMTAB_BLOB: + return "BLOB"; + } + } +#undef STRINGIFY_CODE +} + +static void printSize(raw_ostream &OS, double Bits) { + OS << format("%.2f/%.2fB/%luW", Bits, Bits / 8, (unsigned long)(Bits / 32)); +} +static void printSize(raw_ostream &OS, uint64_t Bits) { + OS << format("%lub/%.2fB/%luW", (unsigned long)Bits, (double)Bits / 8, + (unsigned long)(Bits / 32)); +} + +static Expected ReadSignature(BitstreamCursor &Stream) { + auto tryRead = [&Stream](char &Dest, size_t size) -> Error { + if (Expected MaybeWord = Stream.Read(size)) + Dest = MaybeWord.get(); + else + return MaybeWord.takeError(); + return Error::success(); + }; + + char Signature[6]; + if (Error Err = tryRead(Signature[0], 8)) + return std::move(Err); + if (Error Err = tryRead(Signature[1], 8)) + return std::move(Err); + + // Autodetect the file contents, if it is one we know. + if (Signature[0] == 'C' && Signature[1] == 'P') { + if (Error Err = tryRead(Signature[2], 8)) + return std::move(Err); + if (Error Err = tryRead(Signature[3], 8)) + return std::move(Err); + if (Signature[2] == 'C' && Signature[3] == 'H') + return ClangSerializedASTBitstream; + } else if (Signature[0] == 'D' && Signature[1] == 'I') { + if (Error Err = tryRead(Signature[2], 8)) + return std::move(Err); + if (Error Err = tryRead(Signature[3], 8)) + return std::move(Err); + if (Signature[2] == 'A' && Signature[3] == 'G') + return ClangSerializedDiagnosticsBitstream; + } else { + if (Error Err = tryRead(Signature[2], 4)) + return std::move(Err); + if (Error Err = tryRead(Signature[3], 4)) + return std::move(Err); + if (Error Err = tryRead(Signature[4], 4)) + return std::move(Err); + if (Error Err = tryRead(Signature[5], 4)) + return std::move(Err); + if (Signature[0] == 'B' && Signature[1] == 'C' && Signature[2] == 0x0 && + Signature[3] == 0xC && Signature[4] == 0xE && Signature[5] == 0xD) + return LLVMIRBitstream; + } + return UnknownBitstream; +} + +static Expected analyzeHeader(Optional O, + BitstreamCursor &Stream) { + ArrayRef Bytes = Stream.getBitcodeBytes(); + const unsigned char *BufPtr = (const unsigned char *)Bytes.data(); + const unsigned char *EndBufPtr = BufPtr + Bytes.size(); + + // If we have a wrapper header, parse it and ignore the non-bc file + // contents. The magic number is 0x0B17C0DE stored in little endian. + if (isBitcodeWrapper(BufPtr, EndBufPtr)) { + if (Bytes.size() < BWH_HeaderSize) + return reportError("Invalid bitcode wrapper header"); + + if (O) { + unsigned Magic = support::endian::read32le(&BufPtr[BWH_MagicField]); + unsigned Version = support::endian::read32le(&BufPtr[BWH_VersionField]); + unsigned Offset = support::endian::read32le(&BufPtr[BWH_OffsetField]); + unsigned Size = support::endian::read32le(&BufPtr[BWH_SizeField]); + unsigned CPUType = support::endian::read32le(&BufPtr[BWH_CPUTypeField]); + + O->OS << "\n"; + } + + if (SkipBitcodeWrapperHeader(BufPtr, EndBufPtr, true)) + return reportError("Invalid bitcode wrapper header"); + } + + // Use the cursor modified by skipping the wrapper header. + Stream = BitstreamCursor(ArrayRef(BufPtr, EndBufPtr)); + + return ReadSignature(Stream); +} + +static bool canDecodeBlob(unsigned Code, unsigned BlockID) { + return BlockID == bitc::METADATA_BLOCK_ID && Code == bitc::METADATA_STRINGS; +} + +Error BitcodeAnalyzer::decodeMetadataStringsBlob(StringRef Indent, + ArrayRef Record, + StringRef Blob, + raw_ostream &OS) { + if (Blob.empty()) + return reportError("Cannot decode empty blob."); + + if (Record.size() != 2) + return reportError( + "Decoding metadata strings blob needs two record entries."); + + unsigned NumStrings = Record[0]; + unsigned StringsOffset = Record[1]; + OS << " num-strings = " << NumStrings << " {\n"; + + StringRef Lengths = Blob.slice(0, StringsOffset); + SimpleBitstreamCursor R(Lengths); + StringRef Strings = Blob.drop_front(StringsOffset); + do { + if (R.AtEndOfStream()) + return reportError("bad length"); + + Expected MaybeSize = R.ReadVBR(6); + if (!MaybeSize) + return MaybeSize.takeError(); + uint32_t Size = MaybeSize.get(); + if (Strings.size() < Size) + return reportError("truncated chars"); + + OS << Indent << " '"; + OS.write_escaped(Strings.slice(0, Size), /*hex=*/true); + OS << "'\n"; + Strings = Strings.drop_front(Size); + } while (--NumStrings); + + OS << Indent << " }"; + return Error::success(); +} + +BitcodeAnalyzer::BitcodeAnalyzer(StringRef Buffer, + Optional BlockInfoBuffer) + : Stream(Buffer) { + if (BlockInfoBuffer) + BlockInfoStream.emplace(*BlockInfoBuffer); +} + +Error BitcodeAnalyzer::analyze(Optional O, + Optional CheckHash) { + Expected MaybeType = analyzeHeader(O, Stream); + if (!MaybeType) + return MaybeType.takeError(); + else + CurStreamType = *MaybeType; + + Stream.setBlockInfo(&BlockInfo); + + // Read block info from BlockInfoStream, if specified. + // The block info must be a top-level block. + if (BlockInfoStream) { + BitstreamCursor BlockInfoCursor(*BlockInfoStream); + Expected H = analyzeHeader(O, BlockInfoCursor); + if (!H) + return H.takeError(); + + while (!BlockInfoCursor.AtEndOfStream()) { + Expected MaybeCode = BlockInfoCursor.ReadCode(); + if (!MaybeCode) + return MaybeCode.takeError(); + if (MaybeCode.get() != bitc::ENTER_SUBBLOCK) + return reportError("Invalid record at top-level in block info file"); + + Expected MaybeBlockID = BlockInfoCursor.ReadSubBlockID(); + if (!MaybeBlockID) + return MaybeBlockID.takeError(); + if (MaybeBlockID.get() == bitc::BLOCKINFO_BLOCK_ID) { + Expected> MaybeNewBlockInfo = + BlockInfoCursor.ReadBlockInfoBlock(/*ReadBlockInfoNames=*/true); + if (!MaybeNewBlockInfo) + return MaybeNewBlockInfo.takeError(); + Optional NewBlockInfo = + std::move(MaybeNewBlockInfo.get()); + if (!NewBlockInfo) + return reportError("Malformed BlockInfoBlock in block info file"); + BlockInfo = std::move(*NewBlockInfo); + break; + } + + if (Error Err = BlockInfoCursor.SkipBlock()) + return Err; + } + } + + // Parse the top-level structure. We only allow blocks at the top-level. + while (!Stream.AtEndOfStream()) { + Expected MaybeCode = Stream.ReadCode(); + if (!MaybeCode) + return MaybeCode.takeError(); + if (MaybeCode.get() != bitc::ENTER_SUBBLOCK) + return reportError("Invalid record at top-level"); + + Expected MaybeBlockID = Stream.ReadSubBlockID(); + if (!MaybeBlockID) + return MaybeBlockID.takeError(); + + if (Error E = parseBlock(MaybeBlockID.get(), 0, O, CheckHash)) + return E; + ++NumTopBlocks; + } + + return Error::success(); +} + +void BitcodeAnalyzer::printStats(BCDumpOptions O, + Optional Filename) { + uint64_t BufferSizeBits = Stream.getBitcodeBytes().size() * CHAR_BIT; + // Print a summary of the read file. + O.OS << "Summary "; + if (Filename) + O.OS << "of " << Filename->data() << ":\n"; + O.OS << " Total size: "; + printSize(O.OS, BufferSizeBits); + O.OS << "\n"; + O.OS << " Stream type: "; + switch (CurStreamType) { + case UnknownBitstream: + O.OS << "unknown\n"; + break; + case LLVMIRBitstream: + O.OS << "LLVM IR\n"; + break; + case ClangSerializedASTBitstream: + O.OS << "Clang Serialized AST\n"; + break; + case ClangSerializedDiagnosticsBitstream: + O.OS << "Clang Serialized Diagnostics\n"; + break; + } + O.OS << " # Toplevel Blocks: " << NumTopBlocks << "\n"; + O.OS << "\n"; + + // Emit per-block stats. + O.OS << "Per-block Summary:\n"; + for (std::map::iterator I = BlockIDStats.begin(), + E = BlockIDStats.end(); + I != E; ++I) { + O.OS << " Block ID #" << I->first; + if (Optional BlockName = + GetBlockName(I->first, BlockInfo, CurStreamType)) + O.OS << " (" << *BlockName << ")"; + O.OS << ":\n"; + + const PerBlockIDStats &Stats = I->second; + O.OS << " Num Instances: " << Stats.NumInstances << "\n"; + O.OS << " Total Size: "; + printSize(O.OS, Stats.NumBits); + O.OS << "\n"; + double pct = (Stats.NumBits * 100.0) / BufferSizeBits; + O.OS << " Percent of file: " << format("%2.4f%%", pct) << "\n"; + if (Stats.NumInstances > 1) { + O.OS << " Average Size: "; + printSize(O.OS, Stats.NumBits / (double)Stats.NumInstances); + O.OS << "\n"; + O.OS << " Tot/Avg SubBlocks: " << Stats.NumSubBlocks << "/" + << Stats.NumSubBlocks / (double)Stats.NumInstances << "\n"; + O.OS << " Tot/Avg Abbrevs: " << Stats.NumAbbrevs << "/" + << Stats.NumAbbrevs / (double)Stats.NumInstances << "\n"; + O.OS << " Tot/Avg Records: " << Stats.NumRecords << "/" + << Stats.NumRecords / (double)Stats.NumInstances << "\n"; + } else { + O.OS << " Num SubBlocks: " << Stats.NumSubBlocks << "\n"; + O.OS << " Num Abbrevs: " << Stats.NumAbbrevs << "\n"; + O.OS << " Num Records: " << Stats.NumRecords << "\n"; + } + if (Stats.NumRecords) { + double pct = (Stats.NumAbbreviatedRecords * 100.0) / Stats.NumRecords; + O.OS << " Percent Abbrevs: " << format("%2.4f%%", pct) << "\n"; + } + O.OS << "\n"; + + // Print a histogram of the codes we see. + if (O.Histogram && !Stats.CodeFreq.empty()) { + std::vector> FreqPairs; // + for (unsigned i = 0, e = Stats.CodeFreq.size(); i != e; ++i) + if (unsigned Freq = Stats.CodeFreq[i].NumInstances) + FreqPairs.push_back(std::make_pair(Freq, i)); + llvm::stable_sort(FreqPairs); + std::reverse(FreqPairs.begin(), FreqPairs.end()); + + O.OS << "\tRecord Histogram:\n"; + O.OS << "\t\t Count # Bits b/Rec % Abv Record Kind\n"; + for (unsigned i = 0, e = FreqPairs.size(); i != e; ++i) { + const PerRecordStats &RecStats = Stats.CodeFreq[FreqPairs[i].second]; + + O.OS << format("\t\t%7d %9lu", RecStats.NumInstances, + (unsigned long)RecStats.TotalBits); + + if (RecStats.NumInstances > 1) + O.OS << format(" %9.1f", + (double)RecStats.TotalBits / RecStats.NumInstances); + else + O.OS << " "; + + if (RecStats.NumAbbrev) + O.OS << format(" %7.2f", (double)RecStats.NumAbbrev / + RecStats.NumInstances * 100); + else + O.OS << " "; + + O.OS << " "; + if (Optional CodeName = GetCodeName( + FreqPairs[i].second, I->first, BlockInfo, CurStreamType)) + O.OS << *CodeName << "\n"; + else + O.OS << "UnknownCode" << FreqPairs[i].second << "\n"; + } + O.OS << "\n"; + } + } +} + +Error BitcodeAnalyzer::parseBlock(unsigned BlockID, unsigned IndentLevel, + Optional O, + Optional CheckHash) { + std::string Indent(IndentLevel * 2, ' '); + uint64_t BlockBitStart = Stream.GetCurrentBitNo(); + + // Get the statistics for this BlockID. + PerBlockIDStats &BlockStats = BlockIDStats[BlockID]; + + BlockStats.NumInstances++; + + // BLOCKINFO is a special part of the stream. + bool DumpRecords = O.hasValue(); + if (BlockID == bitc::BLOCKINFO_BLOCK_ID) { + if (O) + O->OS << Indent << "\n"; + Expected> MaybeNewBlockInfo = + Stream.ReadBlockInfoBlock(/*ReadBlockInfoNames=*/true); + if (!MaybeNewBlockInfo) + return MaybeNewBlockInfo.takeError(); + Optional NewBlockInfo = + std::move(MaybeNewBlockInfo.get()); + if (!NewBlockInfo) + return reportError("Malformed BlockInfoBlock"); + BlockInfo = std::move(*NewBlockInfo); + if (Error Err = Stream.JumpToBit(BlockBitStart)) + return Err; + // It's not really interesting to dump the contents of the blockinfo + // block. + DumpRecords = false; + } + + unsigned NumWords = 0; + if (Error Err = Stream.EnterSubBlock(BlockID, &NumWords)) + return Err; + + // Keep it for later, when we see a MODULE_HASH record + uint64_t BlockEntryPos = Stream.getCurrentByteNo(); + + Optional BlockName = None; + if (DumpRecords) { + O->OS << Indent << "<"; + if ((BlockName = GetBlockName(BlockID, BlockInfo, CurStreamType))) + O->OS << *BlockName; + else + O->OS << "UnknownBlock" << BlockID; + + if (!O->Symbolic && BlockName) + O->OS << " BlockID=" << BlockID; + + O->OS << " NumWords=" << NumWords + << " BlockCodeSize=" << Stream.getAbbrevIDWidth() << ">\n"; + } + + SmallVector Record; + + // Keep the offset to the metadata index if seen. + uint64_t MetadataIndexOffset = 0; + + // Read all the records for this block. + while (1) { + if (Stream.AtEndOfStream()) + return reportError("Premature end of bitstream"); + + uint64_t RecordStartBit = Stream.GetCurrentBitNo(); + + Expected MaybeEntry = + Stream.advance(BitstreamCursor::AF_DontAutoprocessAbbrevs); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); + + switch (Entry.Kind) { + case BitstreamEntry::Error: + return reportError("malformed bitcode file"); + case BitstreamEntry::EndBlock: { + uint64_t BlockBitEnd = Stream.GetCurrentBitNo(); + BlockStats.NumBits += BlockBitEnd - BlockBitStart; + if (DumpRecords) { + O->OS << Indent << "OS << *BlockName << ">\n"; + else + O->OS << "UnknownBlock" << BlockID << ">\n"; + } + return Error::success(); + } + + case BitstreamEntry::SubBlock: { + uint64_t SubBlockBitStart = Stream.GetCurrentBitNo(); + if (Error E = parseBlock(Entry.ID, IndentLevel + 1, O, CheckHash)) + return E; + ++BlockStats.NumSubBlocks; + uint64_t SubBlockBitEnd = Stream.GetCurrentBitNo(); + + // Don't include subblock sizes in the size of this block. + BlockBitStart += SubBlockBitEnd - SubBlockBitStart; + continue; + } + case BitstreamEntry::Record: + // The interesting case. + break; + } + + if (Entry.ID == bitc::DEFINE_ABBREV) { + if (Error Err = Stream.ReadAbbrevRecord()) + return Err; + ++BlockStats.NumAbbrevs; + continue; + } + + Record.clear(); + + ++BlockStats.NumRecords; + + StringRef Blob; + uint64_t CurrentRecordPos = Stream.GetCurrentBitNo(); + Expected MaybeCode = Stream.readRecord(Entry.ID, Record, &Blob); + if (!MaybeCode) + return MaybeCode.takeError(); + unsigned Code = MaybeCode.get(); + + // Increment the # occurrences of this code. + if (BlockStats.CodeFreq.size() <= Code) + BlockStats.CodeFreq.resize(Code + 1); + BlockStats.CodeFreq[Code].NumInstances++; + BlockStats.CodeFreq[Code].TotalBits += + Stream.GetCurrentBitNo() - RecordStartBit; + if (Entry.ID != bitc::UNABBREV_RECORD) { + BlockStats.CodeFreq[Code].NumAbbrev++; + ++BlockStats.NumAbbreviatedRecords; + } + + if (DumpRecords) { + O->OS << Indent << " <"; + Optional CodeName = + GetCodeName(Code, BlockID, BlockInfo, CurStreamType); + if (CodeName) + O->OS << *CodeName; + else + O->OS << "UnknownCode" << Code; + if (!O->Symbolic && CodeName) + O->OS << " codeid=" << Code; + const BitCodeAbbrev *Abbv = nullptr; + if (Entry.ID != bitc::UNABBREV_RECORD) { + Abbv = Stream.getAbbrev(Entry.ID); + O->OS << " abbrevid=" << Entry.ID; + } + + for (unsigned i = 0, e = Record.size(); i != e; ++i) + O->OS << " op" << i << "=" << (int64_t)Record[i]; + + // If we found a metadata index, let's verify that we had an offset + // before and validate its forward reference offset was correct! + if (BlockID == bitc::METADATA_BLOCK_ID) { + if (Code == bitc::METADATA_INDEX_OFFSET) { + if (Record.size() != 2) + O->OS << "(Invalid record)"; + else { + auto Offset = Record[0] + (Record[1] << 32); + MetadataIndexOffset = Stream.GetCurrentBitNo() + Offset; + } + } + if (Code == bitc::METADATA_INDEX) { + O->OS << " (offset "; + if (MetadataIndexOffset == RecordStartBit) + O->OS << "match)"; + else + O->OS << "mismatch: " << MetadataIndexOffset << " vs " + << RecordStartBit << ")"; + } + } + + // If we found a module hash, let's verify that it matches! + if (BlockID == bitc::MODULE_BLOCK_ID && Code == bitc::MODULE_CODE_HASH && + CheckHash.hasValue()) { + if (Record.size() != 5) + O->OS << " (invalid)"; + else { + // Recompute the hash and compare it to the one in the bitcode + SHA1 Hasher; + StringRef Hash; + Hasher.update(*CheckHash); + { + int BlockSize = (CurrentRecordPos / 8) - BlockEntryPos; + auto Ptr = Stream.getPointerToByte(BlockEntryPos, BlockSize); + Hasher.update(ArrayRef(Ptr, BlockSize)); + Hash = Hasher.result(); + } + SmallString<20> RecordedHash; + RecordedHash.resize(20); + int Pos = 0; + for (auto &Val : Record) { + assert(!(Val >> 32) && "Unexpected high bits set"); + RecordedHash[Pos++] = (Val >> 24) & 0xFF; + RecordedHash[Pos++] = (Val >> 16) & 0xFF; + RecordedHash[Pos++] = (Val >> 8) & 0xFF; + RecordedHash[Pos++] = (Val >> 0) & 0xFF; + } + if (Hash == RecordedHash) + O->OS << " (match)"; + else + O->OS << " (!mismatch!)"; + } + } + + O->OS << "/>"; + + if (Abbv) { + for (unsigned i = 1, e = Abbv->getNumOperandInfos(); i != e; ++i) { + const BitCodeAbbrevOp &Op = Abbv->getOperandInfo(i); + if (!Op.isEncoding() || Op.getEncoding() != BitCodeAbbrevOp::Array) + continue; + assert(i + 2 == e && "Array op not second to last"); + std::string Str; + bool ArrayIsPrintable = true; + for (unsigned j = i - 1, je = Record.size(); j != je; ++j) { + if (!isPrint(static_cast(Record[j]))) { + ArrayIsPrintable = false; + break; + } + Str += (char)Record[j]; + } + if (ArrayIsPrintable) + O->OS << " record string = '" << Str << "'"; + break; + } + } + + if (Blob.data()) { + if (canDecodeBlob(Code, BlockID)) { + if (Error E = decodeMetadataStringsBlob(Indent, Record, Blob, O->OS)) + return E; + } else { + O->OS << " blob data = "; + if (O->ShowBinaryBlobs) { + O->OS << "'"; + O->OS.write_escaped(Blob, /*hex=*/true) << "'"; + } else { + bool BlobIsPrintable = true; + for (unsigned i = 0, e = Blob.size(); i != e; ++i) + if (!isPrint(static_cast(Blob[i]))) { + BlobIsPrintable = false; + break; + } + + if (BlobIsPrintable) + O->OS << "'" << Blob << "'"; + else + O->OS << "unprintable, " << Blob.size() << " bytes."; + } + } + } + + O->OS << "\n"; + } + + // Make sure that we can skip the current record. + if (Error Err = Stream.JumpToBit(CurrentRecordPos)) + return Err; + if (Expected Skipped = Stream.skipRecord(Entry.ID)) + ; // Do nothing. + else + return Skipped.takeError(); + } +} + diff --git a/lib/Bitcode/Reader/BitcodeReader.cpp b/lib/Bitcode/Reader/BitcodeReader.cpp index fe051e7a9125..29dc7f616392 100644 --- a/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/lib/Bitcode/Reader/BitcodeReader.cpp @@ -1,9 +1,8 @@ //===- BitcodeReader.cpp - Internal BitcodeReader implementation ----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -21,7 +20,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" #include "llvm/ADT/Twine.h" -#include "llvm/Bitcode/BitstreamReader.h" +#include "llvm/Bitstream/BitstreamReader.h" #include "llvm/Bitcode/LLVMBitCodes.h" #include "llvm/Config/llvm-config.h" #include "llvm/IR/Argument.h" @@ -106,18 +105,25 @@ static Error error(const Twine &Message) { Message, make_error_code(BitcodeError::CorruptedBitcode)); } -/// Helper to read the header common to all bitcode files. -static bool hasValidBitcodeHeader(BitstreamCursor &Stream) { - // Sniff for the signature. - if (!Stream.canSkipToPos(4) || - Stream.Read(8) != 'B' || - Stream.Read(8) != 'C' || - Stream.Read(4) != 0x0 || - Stream.Read(4) != 0xC || - Stream.Read(4) != 0xE || - Stream.Read(4) != 0xD) - return false; - return true; +static Error hasInvalidBitcodeHeader(BitstreamCursor &Stream) { + if (!Stream.canSkipToPos(4)) + return createStringError(std::errc::illegal_byte_sequence, + "file too small to contain bitcode header"); + for (unsigned C : {'B', 'C'}) + if (Expected Res = Stream.Read(8)) { + if (Res.get() != C) + return createStringError(std::errc::illegal_byte_sequence, + "file doesn't start with bitcode header"); + } else + return Res.takeError(); + for (unsigned C : {0x0, 0xC, 0xE, 0xD}) + if (Expected Res = Stream.Read(4)) { + if (Res.get() != C) + return createStringError(std::errc::illegal_byte_sequence, + "file doesn't start with bitcode header"); + } else + return Res.takeError(); + return Error::success(); } static Expected initStream(MemoryBufferRef Buffer) { @@ -134,8 +140,8 @@ static Expected initStream(MemoryBufferRef Buffer) { return error("Invalid bitcode wrapper header"); BitstreamCursor Stream(ArrayRef(BufPtr, BufEnd)); - if (!hasValidBitcodeHeader(Stream)) - return error("Invalid bitcode signature"); + if (Error Err = hasInvalidBitcodeHeader(Stream)) + return std::move(Err); return std::move(Stream); } @@ -165,8 +171,8 @@ static void stripTBAA(Module *M) { /// Read the "IDENTIFICATION_BLOCK_ID" block, do some basic enforcement on the /// "epoch" encoded in the bitcode, and return the producer name if any. static Expected readIdentificationBlock(BitstreamCursor &Stream) { - if (Stream.EnterSubBlock(bitc::IDENTIFICATION_BLOCK_ID)) - return error("Invalid record"); + if (Error Err = Stream.EnterSubBlock(bitc::IDENTIFICATION_BLOCK_ID)) + return std::move(Err); // Read all the records. SmallVector Record; @@ -174,7 +180,11 @@ static Expected readIdentificationBlock(BitstreamCursor &Stream) { std::string ProducerIdentification; while (true) { - BitstreamEntry Entry = Stream.advance(); + BitstreamEntry Entry; + if (Expected Res = Stream.advance()) + Entry = Res.get(); + else + return Res.takeError(); switch (Entry.Kind) { default: @@ -189,8 +199,10 @@ static Expected readIdentificationBlock(BitstreamCursor &Stream) { // Read a record. Record.clear(); - unsigned BitCode = Stream.readRecord(Entry.ID, Record); - switch (BitCode) { + Expected MaybeBitCode = Stream.readRecord(Entry.ID, Record); + if (!MaybeBitCode) + return MaybeBitCode.takeError(); + switch (MaybeBitCode.get()) { default: // Default behavior: reject return error("Invalid value"); case bitc::IDENTIFICATION_CODE_STRING: // IDENTIFICATION: [strchr x N] @@ -215,7 +227,12 @@ static Expected readIdentificationCode(BitstreamCursor &Stream) { if (Stream.AtEndOfStream()) return ""; - BitstreamEntry Entry = Stream.advance(); + BitstreamEntry Entry; + if (Expected Res = Stream.advance()) + Entry = std::move(Res.get()); + else + return Res.takeError(); + switch (Entry.Kind) { case BitstreamEntry::EndBlock: case BitstreamEntry::Error: @@ -226,25 +243,30 @@ static Expected readIdentificationCode(BitstreamCursor &Stream) { return readIdentificationBlock(Stream); // Ignore other sub-blocks. - if (Stream.SkipBlock()) - return error("Malformed block"); + if (Error Err = Stream.SkipBlock()) + return std::move(Err); continue; case BitstreamEntry::Record: - Stream.skipRecord(Entry.ID); - continue; + if (Expected Skipped = Stream.skipRecord(Entry.ID)) + continue; + else + return Skipped.takeError(); } } } static Expected hasObjCCategoryInModule(BitstreamCursor &Stream) { - if (Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID)) - return error("Invalid record"); + if (Error Err = Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID)) + return std::move(Err); SmallVector Record; // Read all the records for this module. while (true) { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + Expected MaybeEntry = Stream.advanceSkippingSubblocks(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); switch (Entry.Kind) { case BitstreamEntry::SubBlock: // Handled for us already. @@ -258,7 +280,10 @@ static Expected hasObjCCategoryInModule(BitstreamCursor &Stream) { } // Read a record. - switch (Stream.readRecord(Entry.ID, Record)) { + Expected MaybeRecord = Stream.readRecord(Entry.ID, Record); + if (!MaybeRecord) + return MaybeRecord.takeError(); + switch (MaybeRecord.get()) { default: break; // Default behavior, ignore unknown content. case bitc::MODULE_CODE_SECTIONNAME: { // SECTIONNAME: [strchr x N] @@ -281,7 +306,11 @@ static Expected hasObjCCategory(BitstreamCursor &Stream) { // We expect a number of well-defined blocks, though we don't necessarily // need to understand them all. while (true) { - BitstreamEntry Entry = Stream.advance(); + BitstreamEntry Entry; + if (Expected Res = Stream.advance()) + Entry = std::move(Res.get()); + else + return Res.takeError(); switch (Entry.Kind) { case BitstreamEntry::Error: @@ -294,20 +323,22 @@ static Expected hasObjCCategory(BitstreamCursor &Stream) { return hasObjCCategoryInModule(Stream); // Ignore other sub-blocks. - if (Stream.SkipBlock()) - return error("Malformed block"); + if (Error Err = Stream.SkipBlock()) + return std::move(Err); continue; case BitstreamEntry::Record: - Stream.skipRecord(Entry.ID); - continue; + if (Expected Skipped = Stream.skipRecord(Entry.ID)) + continue; + else + return Skipped.takeError(); } } } static Expected readModuleTriple(BitstreamCursor &Stream) { - if (Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID)) - return error("Invalid record"); + if (Error Err = Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID)) + return std::move(Err); SmallVector Record; @@ -315,7 +346,10 @@ static Expected readModuleTriple(BitstreamCursor &Stream) { // Read all the records for this module. while (true) { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + Expected MaybeEntry = Stream.advanceSkippingSubblocks(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); switch (Entry.Kind) { case BitstreamEntry::SubBlock: // Handled for us already. @@ -329,7 +363,10 @@ static Expected readModuleTriple(BitstreamCursor &Stream) { } // Read a record. - switch (Stream.readRecord(Entry.ID, Record)) { + Expected MaybeRecord = Stream.readRecord(Entry.ID, Record); + if (!MaybeRecord) + return MaybeRecord.takeError(); + switch (MaybeRecord.get()) { default: break; // Default behavior, ignore unknown content. case bitc::MODULE_CODE_TRIPLE: { // TRIPLE: [strchr x N] std::string S; @@ -348,7 +385,10 @@ static Expected readTriple(BitstreamCursor &Stream) { // We expect a number of well-defined blocks, though we don't necessarily // need to understand them all. while (true) { - BitstreamEntry Entry = Stream.advance(); + Expected MaybeEntry = Stream.advance(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); switch (Entry.Kind) { case BitstreamEntry::Error: @@ -361,13 +401,15 @@ static Expected readTriple(BitstreamCursor &Stream) { return readModuleTriple(Stream); // Ignore other sub-blocks. - if (Stream.SkipBlock()) - return error("Malformed block"); + if (Error Err = Stream.SkipBlock()) + return std::move(Err); continue; case BitstreamEntry::Record: - Stream.skipRecord(Entry.ID); - continue; + if (llvm::Expected Skipped = Stream.skipRecord(Entry.ID)) + continue; + else + return Skipped.takeError(); } } } @@ -452,6 +494,7 @@ class BitcodeReader : public BitcodeReaderBase, public GVMaterializer { std::vector GCTable; std::vector TypeList; + DenseMap FunctionTypes; BitcodeReaderValueList ValueList; Optional MDLoader; std::vector ComdatList; @@ -550,12 +593,42 @@ private: StructType *createIdentifiedStructType(LLVMContext &Context, StringRef Name); StructType *createIdentifiedStructType(LLVMContext &Context); - Type *getTypeByID(unsigned ID); + /// Map all pointer types within \param Ty to the opaque pointer + /// type in the same address space if opaque pointers are being + /// used, otherwise nop. This converts a bitcode-reader internal + /// type into one suitable for use in a Value. + Type *flattenPointerTypes(Type *Ty) { + return Ty; + } + + /// Given a fully structured pointer type (i.e. not opaque), return + /// the flattened form of its element, suitable for use in a Value. + Type *getPointerElementFlatType(Type *Ty) { + return flattenPointerTypes(cast(Ty)->getElementType()); + } + + /// Given a fully structured pointer type, get its element type in + /// both fully structured form, and flattened form suitable for use + /// in a Value. + std::pair getPointerElementTypes(Type *FullTy) { + Type *ElTy = cast(FullTy)->getElementType(); + return std::make_pair(ElTy, flattenPointerTypes(ElTy)); + } - Value *getFnValueByID(unsigned ID, Type *Ty) { + /// Return the flattened type (suitable for use in a Value) + /// specified by the given \param ID . + Type *getTypeByID(unsigned ID) { + return flattenPointerTypes(getFullyStructuredTypeByID(ID)); + } + + /// Return the fully structured (bitcode-reader internal) type + /// corresponding to the given \param ID . + Type *getFullyStructuredTypeByID(unsigned ID); + + Value *getFnValueByID(unsigned ID, Type *Ty, Type **FullTy = nullptr) { if (Ty && Ty->isMetadataTy()) return MetadataAsValue::get(Ty->getContext(), getFnMetadataByID(ID)); - return ValueList.getValueFwdRef(ID, Ty); + return ValueList.getValueFwdRef(ID, Ty, FullTy); } Metadata *getFnMetadataByID(unsigned ID) { @@ -577,7 +650,8 @@ private: /// Increment Slot past the number of slots used in the record. Return true on /// failure. bool getValueTypePair(SmallVectorImpl &Record, unsigned &Slot, - unsigned InstNum, Value *&ResVal) { + unsigned InstNum, Value *&ResVal, + Type **FullTy = nullptr) { if (Slot == Record.size()) return true; unsigned ValNo = (unsigned)Record[Slot++]; // Adjust the ValNo, if it was encoded relative to the InstNum. @@ -586,7 +660,7 @@ private: if (ValNo < InstNum) { // If this is not a forward reference, just return the value we already // have. - ResVal = getFnValueByID(ValNo, nullptr); + ResVal = getFnValueByID(ValNo, nullptr, FullTy); return ResVal == nullptr; } if (Slot == Record.size()) @@ -594,6 +668,8 @@ private: unsigned TypeNo = (unsigned)Record[Slot++]; ResVal = getFnValueByID(ValNo, getTypeByID(TypeNo)); + if (FullTy) + *FullTy = getFullyStructuredTypeByID(TypeNo); return ResVal == nullptr; } @@ -639,6 +715,10 @@ private: return getFnValueByID(ValNo, Ty); } + /// Upgrades old-style typeless byval attributes by adding the corresponding + /// argument's pointee type. + void propagateByValTypes(CallBase *CB, ArrayRef ArgsFullTys); + /// Converts alignment exponent (i.e. power of two (or zero)) to the /// corresponding alignment to use. If alignment is too large, returns /// a corresponding error code. @@ -748,6 +828,9 @@ private: bool HasRelBF); Error parseEntireSummary(unsigned ID); Error parseModuleStringTable(); + void parseTypeIdCompatibleVtableSummaryRecord(ArrayRef Record); + void parseTypeIdCompatibleVtableInfo(ArrayRef Record, size_t &Slot, + TypeIdCompatibleVtableInfo &TypeId); std::pair getValueInfoFromValueId(unsigned ValueId); @@ -775,7 +858,7 @@ BitcodeReader::BitcodeReader(BitstreamCursor Stream, StringRef Strtab, StringRef ProducerIdentification, LLVMContext &Context) : BitcodeReaderBase(std::move(Stream), Strtab), Context(Context), - ValueList(Context) { + ValueList(Context, Stream.SizeInBytes()) { this->ProducerIdentification = ProducerIdentification; } @@ -894,13 +977,15 @@ static GlobalValueSummary::GVFlags getDecodedGVSummaryFlags(uint64_t RawFlags, // values as live. bool Live = (RawFlags & 0x2) || Version < 3; bool Local = (RawFlags & 0x4); + bool AutoHide = (RawFlags & 0x8); - return GlobalValueSummary::GVFlags(Linkage, NotEligibleToImport, Live, Local); + return GlobalValueSummary::GVFlags(Linkage, NotEligibleToImport, Live, Local, AutoHide); } // Decode the flags for GlobalVariable in the summary static GlobalVarSummary::GVarFlags getDecodedGVarFlags(uint64_t RawFlags) { - return GlobalVarSummary::GVarFlags((RawFlags & 0x1) ? true : false); + return GlobalVarSummary::GVarFlags((RawFlags & 0x1) ? true : false, + (RawFlags & 0x2) ? true : false); } static GlobalValue::VisibilityTypes getDecodedVisibility(unsigned Val) { @@ -1035,6 +1120,8 @@ static AtomicRMWInst::BinOp getDecodedRMWOperation(unsigned Val) { case bitc::RMW_MIN: return AtomicRMWInst::Min; case bitc::RMW_UMAX: return AtomicRMWInst::UMax; case bitc::RMW_UMIN: return AtomicRMWInst::UMin; + case bitc::RMW_FADD: return AtomicRMWInst::FAdd; + case bitc::RMW_FSUB: return AtomicRMWInst::FSub; } } @@ -1095,7 +1182,7 @@ static void upgradeDLLImportExportLinkage(GlobalValue *GV, unsigned Val) { } } -Type *BitcodeReader::getTypeByID(unsigned ID) { +Type *BitcodeReader::getFullyStructuredTypeByID(unsigned ID) { // The type table size is always specified correctly. if (ID >= TypeList.size()) return nullptr; @@ -1187,6 +1274,15 @@ static uint64_t getRawAttributeMask(Attribute::AttrKind Val) { case Attribute::ShadowCallStack: return 1ULL << 59; case Attribute::SpeculativeLoadHardening: return 1ULL << 60; + case Attribute::ImmArg: + return 1ULL << 61; + case Attribute::WillReturn: + return 1ULL << 62; + case Attribute::NoFree: + return 1ULL << 63; + case Attribute::NoSync: + llvm_unreachable("nosync attribute not supported in raw format"); + break; case Attribute::Dereferenceable: llvm_unreachable("dereferenceable attribute not supported in raw format"); break; @@ -1200,6 +1296,9 @@ static uint64_t getRawAttributeMask(Attribute::AttrKind Val) { case Attribute::AllocSize: llvm_unreachable("allocsize not supported in raw format"); break; + case Attribute::SanitizeMemTag: + llvm_unreachable("sanitize_memtag attribute not supported in raw format"); + break; } llvm_unreachable("Unsupported attribute type"); } @@ -1209,10 +1308,12 @@ static void addRawAttributeValue(AttrBuilder &B, uint64_t Val) { for (Attribute::AttrKind I = Attribute::None; I != Attribute::EndAttrKinds; I = Attribute::AttrKind(I + 1)) { - if (I == Attribute::Dereferenceable || + if (I == Attribute::SanitizeMemTag || + I == Attribute::Dereferenceable || I == Attribute::DereferenceableOrNull || I == Attribute::ArgMemOnly || - I == Attribute::AllocSize) + I == Attribute::AllocSize || + I == Attribute::NoSync) continue; if (uint64_t A = (Val & getRawAttributeMask(I))) { if (I == Attribute::Alignment) @@ -1245,8 +1346,8 @@ static void decodeLLVMAttributesForBitcode(AttrBuilder &B, } Error BitcodeReader::parseAttributeBlock() { - if (Stream.EnterSubBlock(bitc::PARAMATTR_BLOCK_ID)) - return error("Invalid record"); + if (Error Err = Stream.EnterSubBlock(bitc::PARAMATTR_BLOCK_ID)) + return Err; if (!MAttributes.empty()) return error("Invalid multiple blocks"); @@ -1257,7 +1358,10 @@ Error BitcodeReader::parseAttributeBlock() { // Read all the records. while (true) { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + Expected MaybeEntry = Stream.advanceSkippingSubblocks(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); switch (Entry.Kind) { case BitstreamEntry::SubBlock: // Handled for us already. @@ -1272,7 +1376,10 @@ Error BitcodeReader::parseAttributeBlock() { // Read a record. Record.clear(); - switch (Stream.readRecord(Entry.ID, Record)) { + Expected MaybeRecord = Stream.readRecord(Entry.ID, Record); + if (!MaybeRecord) + return MaybeRecord.takeError(); + switch (MaybeRecord.get()) { default: // Default behavior: ignore. break; case bitc::PARAMATTR_CODE_ENTRY_OLD: // ENTRY: [paramidx0, attr0, ...] @@ -1345,6 +1452,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) { return Attribute::NoCapture; case bitc::ATTR_KIND_NO_DUPLICATE: return Attribute::NoDuplicate; + case bitc::ATTR_KIND_NOFREE: + return Attribute::NoFree; case bitc::ATTR_KIND_NO_IMPLICIT_FLOAT: return Attribute::NoImplicitFloat; case bitc::ATTR_KIND_NO_INLINE: @@ -1365,6 +1474,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) { return Attribute::NoRedZone; case bitc::ATTR_KIND_NO_RETURN: return Attribute::NoReturn; + case bitc::ATTR_KIND_NOSYNC: + return Attribute::NoSync; case bitc::ATTR_KIND_NOCF_CHECK: return Attribute::NoCfCheck; case bitc::ATTR_KIND_NO_UNWIND: @@ -1419,10 +1530,16 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) { return Attribute::SwiftSelf; case bitc::ATTR_KIND_UW_TABLE: return Attribute::UWTable; + case bitc::ATTR_KIND_WILLRETURN: + return Attribute::WillReturn; case bitc::ATTR_KIND_WRITEONLY: return Attribute::WriteOnly; case bitc::ATTR_KIND_Z_EXT: return Attribute::ZExt; + case bitc::ATTR_KIND_IMMARG: + return Attribute::ImmArg; + case bitc::ATTR_KIND_SANITIZE_MEMTAG: + return Attribute::SanitizeMemTag; } } @@ -1444,8 +1561,8 @@ Error BitcodeReader::parseAttrKind(uint64_t Code, Attribute::AttrKind *Kind) { } Error BitcodeReader::parseAttributeGroupBlock() { - if (Stream.EnterSubBlock(bitc::PARAMATTR_GROUP_BLOCK_ID)) - return error("Invalid record"); + if (Error Err = Stream.EnterSubBlock(bitc::PARAMATTR_GROUP_BLOCK_ID)) + return Err; if (!MAttributeGroups.empty()) return error("Invalid multiple blocks"); @@ -1454,7 +1571,10 @@ Error BitcodeReader::parseAttributeGroupBlock() { // Read all the records. while (true) { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + Expected MaybeEntry = Stream.advanceSkippingSubblocks(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); switch (Entry.Kind) { case BitstreamEntry::SubBlock: // Handled for us already. @@ -1469,7 +1589,10 @@ Error BitcodeReader::parseAttributeGroupBlock() { // Read a record. Record.clear(); - switch (Stream.readRecord(Entry.ID, Record)) { + Expected MaybeRecord = Stream.readRecord(Entry.ID, Record); + if (!MaybeRecord) + return MaybeRecord.takeError(); + switch (MaybeRecord.get()) { default: // Default behavior: ignore. break; case bitc::PARAMATTR_GRP_CODE_ENTRY: { // ENTRY: [grpid, idx, a0, a1, ...] @@ -1486,6 +1609,12 @@ Error BitcodeReader::parseAttributeGroupBlock() { if (Error Err = parseAttrKind(Record[++i], &Kind)) return Err; + // Upgrade old-style byval attribute to one with a type, even if it's + // nullptr. We will have to insert the real type when we associate + // this AttributeList with a function. + if (Kind == Attribute::ByVal) + B.addByValAttr(nullptr); + B.addAttribute(Kind); } else if (Record[i] == 1) { // Integer attribute Attribute::AttrKind Kind; @@ -1501,9 +1630,7 @@ Error BitcodeReader::parseAttributeGroupBlock() { B.addDereferenceableOrNullAttr(Record[++i]); else if (Kind == Attribute::AllocSize) B.addAllocSizeAttrFromRawRepr(Record[++i]); - } else { // String attribute - assert((Record[i] == 3 || Record[i] == 4) && - "Invalid attribute group entry"); + } else if (Record[i] == 3 || Record[i] == 4) { // String attribute bool HasValue = (Record[i++] == 4); SmallString<64> KindStr; SmallString<64> ValStr; @@ -1521,6 +1648,15 @@ Error BitcodeReader::parseAttributeGroupBlock() { } B.addAttribute(KindStr.str(), ValStr.str()); + } else { + assert((Record[i] == 5 || Record[i] == 6) && + "Invalid attribute group entry"); + bool HasType = Record[i] == 6; + Attribute::AttrKind Kind; + if (Error Err = parseAttrKind(Record[++i], &Kind)) + return Err; + if (Kind == Attribute::ByVal) + B.addByValAttr(HasType ? getTypeByID(Record[++i]) : nullptr); } } @@ -1532,8 +1668,8 @@ Error BitcodeReader::parseAttributeGroupBlock() { } Error BitcodeReader::parseTypeTable() { - if (Stream.EnterSubBlock(bitc::TYPE_BLOCK_ID_NEW)) - return error("Invalid record"); + if (Error Err = Stream.EnterSubBlock(bitc::TYPE_BLOCK_ID_NEW)) + return Err; return parseTypeTableBody(); } @@ -1549,7 +1685,10 @@ Error BitcodeReader::parseTypeTableBody() { // Read all the records for this type table. while (true) { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + Expected MaybeEntry = Stream.advanceSkippingSubblocks(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); switch (Entry.Kind) { case BitstreamEntry::SubBlock: // Handled for us already. @@ -1567,7 +1706,10 @@ Error BitcodeReader::parseTypeTableBody() { // Read a record. Record.clear(); Type *ResultTy = nullptr; - switch (Stream.readRecord(Entry.ID, Record)) { + Expected MaybeRecord = Stream.readRecord(Entry.ID, Record); + if (!MaybeRecord) + return MaybeRecord.takeError(); + switch (MaybeRecord.get()) { default: return error("Invalid value"); case bitc::TYPE_CODE_NUMENTRY: // TYPE_CODE_NUMENTRY: [numentries] @@ -1752,7 +1894,8 @@ Error BitcodeReader::parseTypeTableBody() { return error("Invalid type"); ResultTy = ArrayType::get(ResultTy, Record[0]); break; - case bitc::TYPE_CODE_VECTOR: // VECTOR: [numelts, eltty] + case bitc::TYPE_CODE_VECTOR: // VECTOR: [numelts, eltty] or + // [numelts, eltty, scalable] if (Record.size() < 2) return error("Invalid record"); if (Record[0] == 0) @@ -1760,7 +1903,8 @@ Error BitcodeReader::parseTypeTableBody() { ResultTy = getTypeByID(Record[1]); if (!ResultTy || !StructType::isValidElementType(ResultTy)) return error("Invalid type"); - ResultTy = VectorType::get(ResultTy, Record[0]); + bool Scalable = Record.size() > 2 ? Record[2] : false; + ResultTy = VectorType::get(ResultTy, Record[0], Scalable); break; } @@ -1775,8 +1919,8 @@ Error BitcodeReader::parseTypeTableBody() { } Error BitcodeReader::parseOperandBundleTags() { - if (Stream.EnterSubBlock(bitc::OPERAND_BUNDLE_TAGS_BLOCK_ID)) - return error("Invalid record"); + if (Error Err = Stream.EnterSubBlock(bitc::OPERAND_BUNDLE_TAGS_BLOCK_ID)) + return Err; if (!BundleTags.empty()) return error("Invalid multiple blocks"); @@ -1784,7 +1928,10 @@ Error BitcodeReader::parseOperandBundleTags() { SmallVector Record; while (true) { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + Expected MaybeEntry = Stream.advanceSkippingSubblocks(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); switch (Entry.Kind) { case BitstreamEntry::SubBlock: // Handled for us already. @@ -1799,7 +1946,10 @@ Error BitcodeReader::parseOperandBundleTags() { // Tags are implicitly mapped to integers by their order. - if (Stream.readRecord(Entry.ID, Record) != bitc::OPERAND_BUNDLE_TAG) + Expected MaybeRecord = Stream.readRecord(Entry.ID, Record); + if (!MaybeRecord) + return MaybeRecord.takeError(); + if (MaybeRecord.get() != bitc::OPERAND_BUNDLE_TAG) return error("Invalid record"); // OPERAND_BUNDLE_TAG: [strchr x N] @@ -1811,15 +1961,19 @@ Error BitcodeReader::parseOperandBundleTags() { } Error BitcodeReader::parseSyncScopeNames() { - if (Stream.EnterSubBlock(bitc::SYNC_SCOPE_NAMES_BLOCK_ID)) - return error("Invalid record"); + if (Error Err = Stream.EnterSubBlock(bitc::SYNC_SCOPE_NAMES_BLOCK_ID)) + return Err; if (!SSIDs.empty()) return error("Invalid multiple synchronization scope names blocks"); SmallVector Record; while (true) { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + Expected MaybeEntry = Stream.advanceSkippingSubblocks(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); + switch (Entry.Kind) { case BitstreamEntry::SubBlock: // Handled for us already. case BitstreamEntry::Error: @@ -1836,7 +1990,10 @@ Error BitcodeReader::parseSyncScopeNames() { // Synchronization scope names are implicitly mapped to synchronization // scope IDs by their order. - if (Stream.readRecord(Entry.ID, Record) != bitc::SYNC_SCOPE_NAME) + Expected MaybeRecord = Stream.readRecord(Entry.ID, Record); + if (!MaybeRecord) + return MaybeRecord.takeError(); + if (MaybeRecord.get() != bitc::SYNC_SCOPE_NAME) return error("Invalid record"); SmallString<16> SSN; @@ -1877,22 +2034,18 @@ Expected BitcodeReader::recordValue(SmallVectorImpl &Record, /// Helper to note and return the current location, and jump to the given /// offset. -static uint64_t jumpToValueSymbolTable(uint64_t Offset, - BitstreamCursor &Stream) { +static Expected jumpToValueSymbolTable(uint64_t Offset, + BitstreamCursor &Stream) { // Save the current parsing location so we can jump back at the end // of the VST read. uint64_t CurrentBit = Stream.GetCurrentBitNo(); - Stream.JumpToBit(Offset * 32); -#ifndef NDEBUG - // Do some checking if we are in debug mode. - BitstreamEntry Entry = Stream.advance(); - assert(Entry.Kind == BitstreamEntry::SubBlock); - assert(Entry.ID == bitc::VALUE_SYMTAB_BLOCK_ID); -#else - // In NDEBUG mode ignore the output so we don't get an unused variable - // warning. - Stream.advance(); -#endif + if (Error JumpFailed = Stream.JumpToBit(Offset * 32)) + return std::move(JumpFailed); + Expected MaybeEntry = Stream.advance(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + assert(MaybeEntry.get().Kind == BitstreamEntry::SubBlock); + assert(MaybeEntry.get().ID == bitc::VALUE_SYMTAB_BLOCK_ID); return CurrentBit; } @@ -1917,12 +2070,15 @@ Error BitcodeReader::parseGlobalValueSymbolTable() { unsigned FuncBitcodeOffsetDelta = Stream.getAbbrevIDWidth() + bitc::BlockIDWidth; - if (Stream.EnterSubBlock(bitc::VALUE_SYMTAB_BLOCK_ID)) - return error("Invalid record"); + if (Error Err = Stream.EnterSubBlock(bitc::VALUE_SYMTAB_BLOCK_ID)) + return Err; SmallVector Record; while (true) { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + Expected MaybeEntry = Stream.advanceSkippingSubblocks(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); switch (Entry.Kind) { case BitstreamEntry::SubBlock: @@ -1935,7 +2091,10 @@ Error BitcodeReader::parseGlobalValueSymbolTable() { } Record.clear(); - switch (Stream.readRecord(Entry.ID, Record)) { + Expected MaybeRecord = Stream.readRecord(Entry.ID, Record); + if (!MaybeRecord) + return MaybeRecord.takeError(); + switch (MaybeRecord.get()) { case bitc::VST_CODE_FNENTRY: // [valueid, offset] setDeferredFunctionInfo(FuncBitcodeOffsetDelta, cast(ValueList[Record[0]]), Record); @@ -1952,12 +2111,16 @@ Error BitcodeReader::parseValueSymbolTable(uint64_t Offset) { // VST (where we want to jump to the VST offset) and the function-level // VST (where we don't). if (Offset > 0) { - CurrentBit = jumpToValueSymbolTable(Offset, Stream); + Expected MaybeCurrentBit = jumpToValueSymbolTable(Offset, Stream); + if (!MaybeCurrentBit) + return MaybeCurrentBit.takeError(); + CurrentBit = MaybeCurrentBit.get(); // If this module uses a string table, read this as a module-level VST. if (UseStrtab) { if (Error Err = parseGlobalValueSymbolTable()) return Err; - Stream.JumpToBit(CurrentBit); + if (Error JumpFailed = Stream.JumpToBit(CurrentBit)) + return JumpFailed; return Error::success(); } // Otherwise, the VST will be in a similar format to a function-level VST, @@ -1978,8 +2141,8 @@ Error BitcodeReader::parseValueSymbolTable(uint64_t Offset) { unsigned FuncBitcodeOffsetDelta = Stream.getAbbrevIDWidth() + bitc::BlockIDWidth; - if (Stream.EnterSubBlock(bitc::VALUE_SYMTAB_BLOCK_ID)) - return error("Invalid record"); + if (Error Err = Stream.EnterSubBlock(bitc::VALUE_SYMTAB_BLOCK_ID)) + return Err; SmallVector Record; @@ -1989,7 +2152,10 @@ Error BitcodeReader::parseValueSymbolTable(uint64_t Offset) { SmallString<128> ValueName; while (true) { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + Expected MaybeEntry = Stream.advanceSkippingSubblocks(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); switch (Entry.Kind) { case BitstreamEntry::SubBlock: // Handled for us already. @@ -1997,7 +2163,8 @@ Error BitcodeReader::parseValueSymbolTable(uint64_t Offset) { return error("Malformed block"); case BitstreamEntry::EndBlock: if (Offset > 0) - Stream.JumpToBit(CurrentBit); + if (Error JumpFailed = Stream.JumpToBit(CurrentBit)) + return JumpFailed; return Error::success(); case BitstreamEntry::Record: // The interesting case. @@ -2006,7 +2173,10 @@ Error BitcodeReader::parseValueSymbolTable(uint64_t Offset) { // Read a record. Record.clear(); - switch (Stream.readRecord(Entry.ID, Record)) { + Expected MaybeRecord = Stream.readRecord(Entry.ID, Record); + if (!MaybeRecord) + return MaybeRecord.takeError(); + switch (MaybeRecord.get()) { default: // Default behavior: unknown type. break; case bitc::VST_CODE_ENTRY: { // VST_CODE_ENTRY: [valueid, namechar x N] @@ -2151,17 +2321,21 @@ static APInt readWideAPInt(ArrayRef Vals, unsigned TypeBits) { } Error BitcodeReader::parseConstants() { - if (Stream.EnterSubBlock(bitc::CONSTANTS_BLOCK_ID)) - return error("Invalid record"); + if (Error Err = Stream.EnterSubBlock(bitc::CONSTANTS_BLOCK_ID)) + return Err; SmallVector Record; // Read all the records for this value table. Type *CurTy = Type::getInt32Ty(Context); + Type *CurFullTy = Type::getInt32Ty(Context); unsigned NextCstNo = ValueList.size(); while (true) { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + Expected MaybeEntry = Stream.advanceSkippingSubblocks(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); switch (Entry.Kind) { case BitstreamEntry::SubBlock: // Handled for us already. @@ -2184,8 +2358,10 @@ Error BitcodeReader::parseConstants() { Record.clear(); Type *VoidType = Type::getVoidTy(Context); Value *V = nullptr; - unsigned BitCode = Stream.readRecord(Entry.ID, Record); - switch (BitCode) { + Expected MaybeBitCode = Stream.readRecord(Entry.ID, Record); + if (!MaybeBitCode) + return MaybeBitCode.takeError(); + switch (unsigned BitCode = MaybeBitCode.get()) { default: // Default behavior: unknown constant case bitc::CST_CODE_UNDEF: // UNDEF V = UndefValue::get(CurTy); @@ -2197,7 +2373,8 @@ Error BitcodeReader::parseConstants() { return error("Invalid record"); if (TypeList[Record[0]] == VoidType) return error("Invalid constant type"); - CurTy = TypeList[Record[0]]; + CurFullTy = TypeList[Record[0]]; + CurTy = flattenPointerTypes(CurFullTy); continue; // Skip the ValueList manipulation. case bitc::CST_CODE_NULL: // NULL V = Constant::getNullValue(CurTy); @@ -2416,23 +2593,27 @@ Error BitcodeReader::parseConstants() { InBounds = true; SmallVector Elts; + Type *Elt0FullTy = nullptr; while (OpNum != Record.size()) { + if (!Elt0FullTy) + Elt0FullTy = getFullyStructuredTypeByID(Record[OpNum]); Type *ElTy = getTypeByID(Record[OpNum++]); if (!ElTy) return error("Invalid record"); Elts.push_back(ValueList.getConstantFwdRef(Record[OpNum++], ElTy)); } - if (PointeeType && - PointeeType != - cast(Elts[0]->getType()->getScalarType()) - ->getElementType()) - return error("Explicit gep operator type does not match pointee type " - "of pointer operand"); - if (Elts.size() < 1) return error("Invalid gep with no operands"); + Type *ImplicitPointeeType = + getPointerElementFlatType(Elt0FullTy->getScalarType()); + if (!PointeeType) + PointeeType = ImplicitPointeeType; + else if (PointeeType != ImplicitPointeeType) + return error("Explicit gep operator type does not match pointee type " + "of pointer operand"); + ArrayRef Indices(Elts.begin() + 1, Elts.end()); V = ConstantExpr::getGetElementPtr(PointeeType, Elts[0], Indices, InBounds, InRangeIndex); @@ -2560,10 +2741,10 @@ Error BitcodeReader::parseConstants() { AsmStr += (char)Record[2+i]; for (unsigned i = 0; i != ConstStrSize; ++i) ConstrStr += (char)Record[3+AsmStrSize+i]; - PointerType *PTy = cast(CurTy); UpgradeInlineAsmString(&AsmStr); - V = InlineAsm::get(cast(PTy->getElementType()), - AsmStr, ConstrStr, HasSideEffects, IsAlignStack); + V = InlineAsm::get( + cast(getPointerElementFlatType(CurFullTy)), AsmStr, + ConstrStr, HasSideEffects, IsAlignStack); break; } // This version adds support for the asm dialect keywords (e.g., @@ -2586,11 +2767,11 @@ Error BitcodeReader::parseConstants() { AsmStr += (char)Record[2+i]; for (unsigned i = 0; i != ConstStrSize; ++i) ConstrStr += (char)Record[3+AsmStrSize+i]; - PointerType *PTy = cast(CurTy); UpgradeInlineAsmString(&AsmStr); - V = InlineAsm::get(cast(PTy->getElementType()), - AsmStr, ConstrStr, HasSideEffects, IsAlignStack, - InlineAsm::AsmDialect(AsmDialect)); + V = InlineAsm::get( + cast(getPointerElementFlatType(CurFullTy)), AsmStr, + ConstrStr, HasSideEffects, IsAlignStack, + InlineAsm::AsmDialect(AsmDialect)); break; } case bitc::CST_CODE_BLOCKADDRESS:{ @@ -2636,20 +2817,25 @@ Error BitcodeReader::parseConstants() { } } - ValueList.assignValue(V, NextCstNo); + assert(V->getType() == flattenPointerTypes(CurFullTy) && + "Incorrect fully structured type provided for Constant"); + ValueList.assignValue(V, NextCstNo, CurFullTy); ++NextCstNo; } } Error BitcodeReader::parseUseLists() { - if (Stream.EnterSubBlock(bitc::USELIST_BLOCK_ID)) - return error("Invalid record"); + if (Error Err = Stream.EnterSubBlock(bitc::USELIST_BLOCK_ID)) + return Err; // Read all the records. SmallVector Record; while (true) { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + Expected MaybeEntry = Stream.advanceSkippingSubblocks(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); switch (Entry.Kind) { case BitstreamEntry::SubBlock: // Handled for us already. @@ -2665,7 +2851,10 @@ Error BitcodeReader::parseUseLists() { // Read a use list record. Record.clear(); bool IsBB = false; - switch (Stream.readRecord(Entry.ID, Record)) { + Expected MaybeRecord = Stream.readRecord(Entry.ID, Record); + if (!MaybeRecord) + return MaybeRecord.takeError(); + switch (MaybeRecord.get()) { default: // Default behavior: unknown type. break; case bitc::USELIST_CODE_BB: @@ -2714,15 +2903,16 @@ Error BitcodeReader::rememberAndSkipMetadata() { DeferredMetadataInfo.push_back(CurBit); // Skip over the block for now. - if (Stream.SkipBlock()) - return error("Invalid record"); + if (Error Err = Stream.SkipBlock()) + return Err; return Error::success(); } Error BitcodeReader::materializeMetadata() { for (uint64_t BitPos : DeferredMetadataInfo) { // Move the bit stream to the saved position. - Stream.JumpToBit(BitPos); + if (Error JumpFailed = Stream.JumpToBit(BitPos)) + return JumpFailed; if (Error Err = MDLoader->parseModuleMetadata()) return Err; } @@ -2760,8 +2950,8 @@ Error BitcodeReader::rememberAndSkipFunctionBody() { DeferredFunctionInfo[Fn] = CurBit; // Skip over the function block for now. - if (Stream.SkipBlock()) - return error("Invalid record"); + if (Error Err = Stream.SkipBlock()) + return Err; return Error::success(); } @@ -2786,8 +2976,14 @@ Error BitcodeReader::globalCleanup() { } // Look for global variables which need to be renamed. + std::vector> UpgradedVariables; for (GlobalVariable &GV : TheModule->globals()) - UpgradeGlobalVariable(&GV); + if (GlobalVariable *Upgraded = UpgradeGlobalVariable(&GV)) + UpgradedVariables.emplace_back(&GV, Upgraded); + for (auto &Pair : UpgradedVariables) { + Pair.first->eraseFromParent(); + TheModule->getGlobalList().push_back(Pair.second); + } // Force deallocation of memory for these vectors to favor the client that // want lazy deserialization. @@ -2802,7 +2998,8 @@ Error BitcodeReader::globalCleanup() { /// or if we have an anonymous function being materialized, since anonymous /// functions do not have a name and are therefore not in the VST. Error BitcodeReader::rememberAndSkipFunctionBodies() { - Stream.JumpToBit(NextUnreadBit); + if (Error JumpFailed = Stream.JumpToBit(NextUnreadBit)) + return JumpFailed; if (Stream.AtEndOfStream()) return error("Could not find function in stream"); @@ -2817,7 +3014,11 @@ Error BitcodeReader::rememberAndSkipFunctionBodies() { SmallVector Record; while (true) { - BitstreamEntry Entry = Stream.advance(); + Expected MaybeEntry = Stream.advance(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + llvm::BitstreamEntry Entry = MaybeEntry.get(); + switch (Entry.Kind) { default: return error("Expect SubBlock"); @@ -2836,7 +3037,12 @@ Error BitcodeReader::rememberAndSkipFunctionBodies() { } bool BitcodeReaderBase::readBlockInfo() { - Optional NewBlockInfo = Stream.ReadBlockInfoBlock(); + Expected> MaybeNewBlockInfo = + Stream.ReadBlockInfoBlock(); + if (!MaybeNewBlockInfo) + return true; // FIXME Handle the error. + Optional NewBlockInfo = + std::move(MaybeNewBlockInfo.get()); if (!NewBlockInfo) return true; BlockInfo = std::move(*NewBlockInfo); @@ -2878,14 +3084,16 @@ static void inferDSOLocal(GlobalValue *GV) { Error BitcodeReader::parseGlobalVarRecord(ArrayRef Record) { // v1: [pointer type, isconst, initid, linkage, alignment, section, // visibility, threadlocal, unnamed_addr, externally_initialized, - // dllstorageclass, comdat, attributes, preemption specifier] (name in VST) + // dllstorageclass, comdat, attributes, preemption specifier, + // partition strtab offset, partition strtab size] (name in VST) // v2: [strtab_offset, strtab_size, v1] StringRef Name; std::tie(Name, Record) = readNameFromStrtab(Record); if (Record.size() < 6) return error("Invalid record"); - Type *Ty = getTypeByID(Record[0]); + Type *FullTy = getFullyStructuredTypeByID(Record[0]); + Type *Ty = flattenPointerTypes(FullTy); if (!Ty) return error("Invalid record"); bool isConstant = Record[1] & 1; @@ -2897,7 +3105,7 @@ Error BitcodeReader::parseGlobalVarRecord(ArrayRef Record) { if (!Ty->isPointerTy()) return error("Invalid type for value"); AddressSpace = cast(Ty)->getAddressSpace(); - Ty = cast(Ty)->getElementType(); + std::tie(FullTy, Ty) = getPointerElementTypes(FullTy); } uint64_t RawLinkage = Record[3]; @@ -2943,7 +3151,10 @@ Error BitcodeReader::parseGlobalVarRecord(ArrayRef Record) { else upgradeDLLImportExportLinkage(NewGV, RawLinkage); - ValueList.push_back(NewGV); + FullTy = PointerType::get(FullTy, AddressSpace); + assert(NewGV->getType() == flattenPointerTypes(FullTy) && + "Incorrect fully specified type for GlobalVariable"); + ValueList.push_back(NewGV, FullTy); // Remember which value to use for the global initializer. if (unsigned InitID = Record[2]) @@ -2969,6 +3180,10 @@ Error BitcodeReader::parseGlobalVarRecord(ArrayRef Record) { } inferDSOLocal(NewGV); + // Check whether we have enough values to read a partition name. + if (Record.size() > 15) + NewGV->setPartition(StringRef(Strtab.data() + Record[14], Record[15])); + return Error::success(); } @@ -2982,13 +3197,14 @@ Error BitcodeReader::parseFunctionRecord(ArrayRef Record) { if (Record.size() < 8) return error("Invalid record"); - Type *Ty = getTypeByID(Record[0]); - if (!Ty) - return error("Invalid record"); - if (auto *PTy = dyn_cast(Ty)) - Ty = PTy->getElementType(); - auto *FTy = dyn_cast(Ty); + Type *FullFTy = getFullyStructuredTypeByID(Record[0]); + Type *FTy = flattenPointerTypes(FullFTy); if (!FTy) + return error("Invalid record"); + if (isa(FTy)) + std::tie(FullFTy, FTy) = getPointerElementTypes(FullFTy); + + if (!isa(FTy)) return error("Invalid type for value"); auto CC = static_cast(Record[1]); if (CC & ~CallingConv::MaxID) @@ -2998,8 +3214,13 @@ Error BitcodeReader::parseFunctionRecord(ArrayRef Record) { if (Record.size() > 16) AddrSpace = Record[16]; - Function *Func = Function::Create(FTy, GlobalValue::ExternalLinkage, - AddrSpace, Name, TheModule); + Function *Func = + Function::Create(cast(FTy), GlobalValue::ExternalLinkage, + AddrSpace, Name, TheModule); + + assert(Func->getFunctionType() == flattenPointerTypes(FullFTy) && + "Incorrect fully specified type provided for function"); + FunctionTypes[Func] = cast(FullFTy); Func->setCallingConv(CC); bool isProto = Record[2]; @@ -3007,6 +3228,19 @@ Error BitcodeReader::parseFunctionRecord(ArrayRef Record) { Func->setLinkage(getDecodedLinkage(RawLinkage)); Func->setAttributes(getAttributes(Record[4])); + // Upgrade any old-style byval without a type by propagating the argument's + // pointee type. There should be no opaque pointers where the byval type is + // implicit. + for (unsigned i = 0; i != Func->arg_size(); ++i) { + if (!Func->hasParamAttribute(i, Attribute::ByVal)) + continue; + + Type *PTy = cast(FullFTy)->getParamType(i); + Func->removeParamAttr(i, Attribute::ByVal); + Func->addParamAttr(i, Attribute::getWithByValType( + Context, getPointerElementFlatType(PTy))); + } + unsigned Alignment; if (Error Err = parseAlignmentValue(Record[5], Alignment)) return Err; @@ -3058,7 +3292,16 @@ Error BitcodeReader::parseFunctionRecord(ArrayRef Record) { } inferDSOLocal(Func); - ValueList.push_back(Func); + // Record[16] is the address space number. + + // Check whether we have enough values to read a partition name. + if (Record.size() > 18) + Func->setPartition(StringRef(Strtab.data() + Record[17], Record[18])); + + Type *FullTy = PointerType::get(FullFTy, AddrSpace); + assert(Func->getType() == flattenPointerTypes(FullTy) && + "Incorrect fully specified type provided for Function"); + ValueList.push_back(Func, FullTy); // If this is a function with a body, remember the prototype we are // creating now, so that we can match up the body with them later. @@ -3087,7 +3330,8 @@ Error BitcodeReader::parseGlobalIndirectSymbolRecord( if (Record.size() < (3 + (unsigned)NewRecord)) return error("Invalid record"); unsigned OpNum = 0; - Type *Ty = getTypeByID(Record[OpNum++]); + Type *FullTy = getFullyStructuredTypeByID(Record[OpNum++]); + Type *Ty = flattenPointerTypes(FullTy); if (!Ty) return error("Invalid record"); @@ -3096,7 +3340,7 @@ Error BitcodeReader::parseGlobalIndirectSymbolRecord( auto *PTy = dyn_cast(Ty); if (!PTy) return error("Invalid type for value"); - Ty = PTy->getElementType(); + std::tie(FullTy, Ty) = getPointerElementTypes(FullTy); AddrSpace = PTy->getAddressSpace(); } else { AddrSpace = Record[OpNum++]; @@ -3112,6 +3356,9 @@ Error BitcodeReader::parseGlobalIndirectSymbolRecord( else NewGA = GlobalIFunc::create(Ty, AddrSpace, getDecodedLinkage(Linkage), Name, nullptr, TheModule); + + assert(NewGA->getValueType() == flattenPointerTypes(FullTy) && + "Incorrect fully structured type provided for GlobalIndirectSymbol"); // Old bitcode files didn't have visibility field. // Local linkage must have default visibility. if (OpNum != Record.size()) { @@ -3135,23 +3382,37 @@ Error BitcodeReader::parseGlobalIndirectSymbolRecord( NewGA->setDSOLocal(getDecodedDSOLocal(Record[OpNum++])); inferDSOLocal(NewGA); - ValueList.push_back(NewGA); + // Check whether we have enough values to read a partition name. + if (OpNum + 1 < Record.size()) { + NewGA->setPartition( + StringRef(Strtab.data() + Record[OpNum], Record[OpNum + 1])); + OpNum += 2; + } + + FullTy = PointerType::get(FullTy, AddrSpace); + assert(NewGA->getType() == flattenPointerTypes(FullTy) && + "Incorrect fully structured type provided for GlobalIndirectSymbol"); + ValueList.push_back(NewGA, FullTy); IndirectSymbolInits.push_back(std::make_pair(NewGA, Val)); return Error::success(); } Error BitcodeReader::parseModule(uint64_t ResumeBit, bool ShouldLazyLoadMetadata) { - if (ResumeBit) - Stream.JumpToBit(ResumeBit); - else if (Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID)) - return error("Invalid record"); + if (ResumeBit) { + if (Error JumpFailed = Stream.JumpToBit(ResumeBit)) + return JumpFailed; + } else if (Error Err = Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID)) + return Err; SmallVector Record; // Read all the records for this module. while (true) { - BitstreamEntry Entry = Stream.advance(); + Expected MaybeEntry = Stream.advance(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + llvm::BitstreamEntry Entry = MaybeEntry.get(); switch (Entry.Kind) { case BitstreamEntry::Error: @@ -3162,8 +3423,8 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit, case BitstreamEntry::SubBlock: switch (Entry.ID) { default: // Skip unknown content. - if (Stream.SkipBlock()) - return error("Invalid record"); + if (Error Err = Stream.SkipBlock()) + return Err; break; case bitc::BLOCKINFO_BLOCK_ID: if (readBlockInfo()) @@ -3196,8 +3457,8 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit, // We must have had a VST forward declaration record, which caused // the parser to jump to and parse the VST earlier. assert(VSTOffset > 0); - if (Stream.SkipBlock()) - return error("Invalid record"); + if (Error Err = Stream.SkipBlock()) + return Err; } break; case bitc::CONSTANTS_BLOCK_ID: @@ -3249,8 +3510,8 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit, // materializing functions. The ResumeBit points to the // start of the last function block recorded in the // DeferredFunctionInfo map. Skip it. - if (Stream.SkipBlock()) - return error("Invalid record"); + if (Error Err = Stream.SkipBlock()) + return Err; continue; } } @@ -3294,8 +3555,10 @@ Error BitcodeReader::parseModule(uint64_t ResumeBit, } // Read a record. - auto BitCode = Stream.readRecord(Entry.ID, Record); - switch (BitCode) { + Expected MaybeBitCode = Stream.readRecord(Entry.ID, Record); + if (!MaybeBitCode) + return MaybeBitCode.takeError(); + switch (unsigned BitCode = MaybeBitCode.get()) { default: break; // Default behavior, ignore unknown content. case bitc::MODULE_CODE_VERSION: { Expected VersionOrErr = parseVersionRecord(Record); @@ -3407,10 +3670,23 @@ Error BitcodeReader::typeCheckLoadStoreInst(Type *ValType, Type *PtrType) { return Error::success(); } +void BitcodeReader::propagateByValTypes(CallBase *CB, + ArrayRef ArgsFullTys) { + for (unsigned i = 0; i != CB->arg_size(); ++i) { + if (!CB->paramHasAttr(i, Attribute::ByVal)) + continue; + + CB->removeParamAttr(i, Attribute::ByVal); + CB->addParamAttr( + i, Attribute::getWithByValType( + Context, getPointerElementFlatType(ArgsFullTys[i]))); + } +} + /// Lazily parse the specified function body block. Error BitcodeReader::parseFunctionBody(Function *F) { - if (Stream.EnterSubBlock(bitc::FUNCTION_BLOCK_ID)) - return error("Invalid record"); + if (Error Err = Stream.EnterSubBlock(bitc::FUNCTION_BLOCK_ID)) + return Err; // Unexpected unresolved metadata when parsing function. if (MDLoader->hasFwdRefs()) @@ -3421,9 +3697,13 @@ Error BitcodeReader::parseFunctionBody(Function *F) { unsigned ModuleMDLoaderSize = MDLoader->size(); // Add all the function arguments to the value table. - for (Argument &I : F->args()) - ValueList.push_back(&I); - + unsigned ArgNo = 0; + FunctionType *FullFTy = FunctionTypes[F]; + for (Argument &I : F->args()) { + assert(I.getType() == flattenPointerTypes(FullFTy->getParamType(ArgNo)) && + "Incorrect fully specified type for Function Argument"); + ValueList.push_back(&I, FullFTy->getParamType(ArgNo++)); + } unsigned NextValueNo = ValueList.size(); BasicBlock *CurBB = nullptr; unsigned CurBBNo = 0; @@ -3444,7 +3724,10 @@ Error BitcodeReader::parseFunctionBody(Function *F) { SmallVector Record; while (true) { - BitstreamEntry Entry = Stream.advance(); + Expected MaybeEntry = Stream.advance(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + llvm::BitstreamEntry Entry = MaybeEntry.get(); switch (Entry.Kind) { case BitstreamEntry::Error: @@ -3455,8 +3738,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) { case BitstreamEntry::SubBlock: switch (Entry.ID) { default: // Skip unknown content. - if (Stream.SkipBlock()) - return error("Invalid record"); + if (Error Err = Stream.SkipBlock()) + return Err; break; case bitc::CONSTANTS_BLOCK_ID: if (Error Err = parseConstants()) @@ -3492,8 +3775,11 @@ Error BitcodeReader::parseFunctionBody(Function *F) { // Read a record. Record.clear(); Instruction *I = nullptr; - unsigned BitCode = Stream.readRecord(Entry.ID, Record); - switch (BitCode) { + Type *FullTy = nullptr; + Expected MaybeBitCode = Stream.readRecord(Entry.ID, Record); + if (!MaybeBitCode) + return MaybeBitCode.takeError(); + switch (unsigned BitCode = MaybeBitCode.get()) { default: // Default behavior: reject return error("Invalid value"); case bitc::FUNC_CODE_DECLAREBLOCKS: { // DECLAREBLOCKS: [nblocks] @@ -3634,7 +3920,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) { OpNum+2 != Record.size()) return error("Invalid record"); - Type *ResTy = getTypeByID(Record[OpNum]); + FullTy = getFullyStructuredTypeByID(Record[OpNum]); + Type *ResTy = flattenPointerTypes(FullTy); int Opc = getDecodedCastOpcode(Record[OpNum + 1]); if (Opc == -1 || !ResTy) return error("Invalid record"); @@ -3663,22 +3950,22 @@ Error BitcodeReader::parseFunctionBody(Function *F) { if (BitCode == bitc::FUNC_CODE_INST_GEP) { InBounds = Record[OpNum++]; - Ty = getTypeByID(Record[OpNum++]); + FullTy = getFullyStructuredTypeByID(Record[OpNum++]); + Ty = flattenPointerTypes(FullTy); } else { InBounds = BitCode == bitc::FUNC_CODE_INST_INBOUNDS_GEP_OLD; Ty = nullptr; } Value *BasePtr; - if (getValueTypePair(Record, OpNum, NextValueNo, BasePtr)) + Type *FullBaseTy = nullptr; + if (getValueTypePair(Record, OpNum, NextValueNo, BasePtr, &FullBaseTy)) return error("Invalid record"); - if (!Ty) - Ty = cast(BasePtr->getType()->getScalarType()) - ->getElementType(); - else if (Ty != - cast(BasePtr->getType()->getScalarType()) - ->getElementType()) + if (!Ty) { + std::tie(FullTy, Ty) = + getPointerElementTypes(FullBaseTy->getScalarType()); + } else if (Ty != getPointerElementFlatType(FullBaseTy->getScalarType())) return error( "Explicit gep type does not match pointee type of pointer operand"); @@ -3691,6 +3978,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { } I = GetElementPtrInst::Create(Ty, BasePtr, GEPIdx); + FullTy = GetElementPtrInst::getGEPReturnType(FullTy, I, GEPIdx); InstructionList.push_back(I); if (InBounds) @@ -3702,7 +3990,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { // EXTRACTVAL: [opty, opval, n x indices] unsigned OpNum = 0; Value *Agg; - if (getValueTypePair(Record, OpNum, NextValueNo, Agg)) + if (getValueTypePair(Record, OpNum, NextValueNo, Agg, &FullTy)) return error("Invalid record"); unsigned RecSize = Record.size(); @@ -3710,26 +3998,25 @@ Error BitcodeReader::parseFunctionBody(Function *F) { return error("EXTRACTVAL: Invalid instruction with 0 indices"); SmallVector EXTRACTVALIdx; - Type *CurTy = Agg->getType(); for (; OpNum != RecSize; ++OpNum) { - bool IsArray = CurTy->isArrayTy(); - bool IsStruct = CurTy->isStructTy(); + bool IsArray = FullTy->isArrayTy(); + bool IsStruct = FullTy->isStructTy(); uint64_t Index = Record[OpNum]; if (!IsStruct && !IsArray) return error("EXTRACTVAL: Invalid type"); if ((unsigned)Index != Index) return error("Invalid value"); - if (IsStruct && Index >= CurTy->getStructNumElements()) + if (IsStruct && Index >= FullTy->getStructNumElements()) return error("EXTRACTVAL: Invalid struct index"); - if (IsArray && Index >= CurTy->getArrayNumElements()) + if (IsArray && Index >= FullTy->getArrayNumElements()) return error("EXTRACTVAL: Invalid array index"); EXTRACTVALIdx.push_back((unsigned)Index); if (IsStruct) - CurTy = CurTy->getStructElementType(Index); + FullTy = FullTy->getStructElementType(Index); else - CurTy = CurTy->getArrayElementType(); + FullTy = FullTy->getArrayElementType(); } I = ExtractValueInst::Create(Agg, EXTRACTVALIdx); @@ -3741,7 +4028,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { // INSERTVAL: [opty, opval, opty, opval, n x indices] unsigned OpNum = 0; Value *Agg; - if (getValueTypePair(Record, OpNum, NextValueNo, Agg)) + if (getValueTypePair(Record, OpNum, NextValueNo, Agg, &FullTy)) return error("Invalid record"); Value *Val; if (getValueTypePair(Record, OpNum, NextValueNo, Val)) @@ -3787,7 +4074,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { // handles select i1 ... in old bitcode unsigned OpNum = 0; Value *TrueVal, *FalseVal, *Cond; - if (getValueTypePair(Record, OpNum, NextValueNo, TrueVal) || + if (getValueTypePair(Record, OpNum, NextValueNo, TrueVal, &FullTy) || popValue(Record, OpNum, NextValueNo, TrueVal->getType(), FalseVal) || popValue(Record, OpNum, NextValueNo, Type::getInt1Ty(Context), Cond)) return error("Invalid record"); @@ -3802,7 +4089,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { // handles select i1 or select [N x i1] unsigned OpNum = 0; Value *TrueVal, *FalseVal, *Cond; - if (getValueTypePair(Record, OpNum, NextValueNo, TrueVal) || + if (getValueTypePair(Record, OpNum, NextValueNo, TrueVal, &FullTy) || popValue(Record, OpNum, NextValueNo, TrueVal->getType(), FalseVal) || getValueTypePair(Record, OpNum, NextValueNo, Cond)) return error("Invalid record"); @@ -3821,18 +4108,24 @@ Error BitcodeReader::parseFunctionBody(Function *F) { I = SelectInst::Create(Cond, TrueVal, FalseVal); InstructionList.push_back(I); + if (OpNum < Record.size() && isa(I)) { + FastMathFlags FMF = getDecodedFastMathFlags(Record[OpNum]); + if (FMF.any()) + I->setFastMathFlags(FMF); + } break; } case bitc::FUNC_CODE_INST_EXTRACTELT: { // EXTRACTELT: [opty, opval, opval] unsigned OpNum = 0; Value *Vec, *Idx; - if (getValueTypePair(Record, OpNum, NextValueNo, Vec) || + if (getValueTypePair(Record, OpNum, NextValueNo, Vec, &FullTy) || getValueTypePair(Record, OpNum, NextValueNo, Idx)) return error("Invalid record"); if (!Vec->getType()->isVectorTy()) return error("Invalid type for value"); I = ExtractElementInst::Create(Vec, Idx); + FullTy = FullTy->getVectorElementType(); InstructionList.push_back(I); break; } @@ -3840,7 +4133,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { case bitc::FUNC_CODE_INST_INSERTELT: { // INSERTELT: [ty, opval,opval,opval] unsigned OpNum = 0; Value *Vec, *Elt, *Idx; - if (getValueTypePair(Record, OpNum, NextValueNo, Vec)) + if (getValueTypePair(Record, OpNum, NextValueNo, Vec, &FullTy)) return error("Invalid record"); if (!Vec->getType()->isVectorTy()) return error("Invalid type for value"); @@ -3856,7 +4149,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { case bitc::FUNC_CODE_INST_SHUFFLEVEC: {// SHUFFLEVEC: [opval,ty,opval,opval] unsigned OpNum = 0; Value *Vec1, *Vec2, *Mask; - if (getValueTypePair(Record, OpNum, NextValueNo, Vec1) || + if (getValueTypePair(Record, OpNum, NextValueNo, Vec1, &FullTy) || popValue(Record, OpNum, NextValueNo, Vec1->getType(), Vec2)) return error("Invalid record"); @@ -3865,6 +4158,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) { if (!Vec1->getType()->isVectorTy() || !Vec2->getType()->isVectorTy()) return error("Invalid type for value"); I = new ShuffleVectorInst(Vec1, Vec2, Mask); + FullTy = VectorType::get(FullTy->getVectorElementType(), + Mask->getType()->getVectorNumElements()); InstructionList.push_back(I); break; } @@ -3882,6 +4177,10 @@ Error BitcodeReader::parseFunctionBody(Function *F) { popValue(Record, OpNum, NextValueNo, LHS->getType(), RHS)) return error("Invalid record"); + if (OpNum >= Record.size()) + return error( + "Invalid record: operand number exceeded available operands"); + unsigned PredVal = Record[OpNum]; bool IsFP = LHS->getType()->isFPOrFPVectorTy(); FastMathFlags FMF; @@ -4168,31 +4467,40 @@ Error BitcodeReader::parseFunctionBody(Function *F) { BasicBlock *UnwindBB = getBasicBlock(Record[OpNum++]); FunctionType *FTy = nullptr; - if (CCInfo >> 13 & 1 && - !(FTy = dyn_cast(getTypeByID(Record[OpNum++])))) - return error("Explicit invoke type is not a function type"); + FunctionType *FullFTy = nullptr; + if ((CCInfo >> 13) & 1) { + FullFTy = + dyn_cast(getFullyStructuredTypeByID(Record[OpNum++])); + if (!FullFTy) + return error("Explicit invoke type is not a function type"); + FTy = cast(flattenPointerTypes(FullFTy)); + } Value *Callee; - if (getValueTypePair(Record, OpNum, NextValueNo, Callee)) + if (getValueTypePair(Record, OpNum, NextValueNo, Callee, &FullTy)) return error("Invalid record"); PointerType *CalleeTy = dyn_cast(Callee->getType()); if (!CalleeTy) return error("Callee is not a pointer"); if (!FTy) { - FTy = dyn_cast(CalleeTy->getElementType()); - if (!FTy) + FullFTy = + dyn_cast(cast(FullTy)->getElementType()); + if (!FullFTy) return error("Callee is not of pointer to function type"); - } else if (CalleeTy->getElementType() != FTy) + FTy = cast(flattenPointerTypes(FullFTy)); + } else if (getPointerElementFlatType(FullTy) != FTy) return error("Explicit invoke type does not match pointee type of " "callee operand"); if (Record.size() < FTy->getNumParams() + OpNum) return error("Insufficient operands to call"); SmallVector Ops; + SmallVector ArgsFullTys; for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i, ++OpNum) { Ops.push_back(getValue(Record, OpNum, NextValueNo, FTy->getParamType(i))); + ArgsFullTys.push_back(FullFTy->getParamType(i)); if (!Ops.back()) return error("Invalid record"); } @@ -4204,18 +4512,24 @@ Error BitcodeReader::parseFunctionBody(Function *F) { // Read type/value pairs for varargs params. while (OpNum != Record.size()) { Value *Op; - if (getValueTypePair(Record, OpNum, NextValueNo, Op)) + Type *FullTy; + if (getValueTypePair(Record, OpNum, NextValueNo, Op, &FullTy)) return error("Invalid record"); Ops.push_back(Op); + ArgsFullTys.push_back(FullTy); } } - I = InvokeInst::Create(Callee, NormalBB, UnwindBB, Ops, OperandBundles); + I = InvokeInst::Create(FTy, Callee, NormalBB, UnwindBB, Ops, + OperandBundles); + FullTy = FullFTy->getReturnType(); OperandBundles.clear(); InstructionList.push_back(I); cast(I)->setCallingConv( static_cast(CallingConv::MaxID & CCInfo)); cast(I)->setAttributes(PAL); + propagateByValTypes(cast(I), ArgsFullTys); + break; } case bitc::FUNC_CODE_INST_RESUME: { // RESUME: [opval] @@ -4227,6 +4541,82 @@ Error BitcodeReader::parseFunctionBody(Function *F) { InstructionList.push_back(I); break; } + case bitc::FUNC_CODE_INST_CALLBR: { + // CALLBR: [attr, cc, norm, transfs, fty, fnid, args] + unsigned OpNum = 0; + AttributeList PAL = getAttributes(Record[OpNum++]); + unsigned CCInfo = Record[OpNum++]; + + BasicBlock *DefaultDest = getBasicBlock(Record[OpNum++]); + unsigned NumIndirectDests = Record[OpNum++]; + SmallVector IndirectDests; + for (unsigned i = 0, e = NumIndirectDests; i != e; ++i) + IndirectDests.push_back(getBasicBlock(Record[OpNum++])); + + FunctionType *FTy = nullptr; + FunctionType *FullFTy = nullptr; + if ((CCInfo >> bitc::CALL_EXPLICIT_TYPE) & 1) { + FullFTy = + dyn_cast(getFullyStructuredTypeByID(Record[OpNum++])); + if (!FullFTy) + return error("Explicit call type is not a function type"); + FTy = cast(flattenPointerTypes(FullFTy)); + } + + Value *Callee; + if (getValueTypePair(Record, OpNum, NextValueNo, Callee, &FullTy)) + return error("Invalid record"); + + PointerType *OpTy = dyn_cast(Callee->getType()); + if (!OpTy) + return error("Callee is not a pointer type"); + if (!FTy) { + FullFTy = + dyn_cast(cast(FullTy)->getElementType()); + if (!FullFTy) + return error("Callee is not of pointer to function type"); + FTy = cast(flattenPointerTypes(FullFTy)); + } else if (getPointerElementFlatType(FullTy) != FTy) + return error("Explicit call type does not match pointee type of " + "callee operand"); + if (Record.size() < FTy->getNumParams() + OpNum) + return error("Insufficient operands to call"); + + SmallVector Args; + // Read the fixed params. + for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i, ++OpNum) { + if (FTy->getParamType(i)->isLabelTy()) + Args.push_back(getBasicBlock(Record[OpNum])); + else + Args.push_back(getValue(Record, OpNum, NextValueNo, + FTy->getParamType(i))); + if (!Args.back()) + return error("Invalid record"); + } + + // Read type/value pairs for varargs params. + if (!FTy->isVarArg()) { + if (OpNum != Record.size()) + return error("Invalid record"); + } else { + while (OpNum != Record.size()) { + Value *Op; + if (getValueTypePair(Record, OpNum, NextValueNo, Op)) + return error("Invalid record"); + Args.push_back(Op); + } + } + + I = CallBrInst::Create(FTy, Callee, DefaultDest, IndirectDests, Args, + OperandBundles); + FullTy = FullFTy->getReturnType(); + OperandBundles.clear(); + InstructionList.push_back(I); + cast(I)->setCallingConv( + static_cast((0x7ff & CCInfo) >> bitc::CALL_CCONV)); + cast(I)->setAttributes(PAL); + break; + } case bitc::FUNC_CODE_INST_UNREACHABLE: // UNREACHABLE I = new UnreachableInst(Context); InstructionList.push_back(I); @@ -4234,7 +4624,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) { case bitc::FUNC_CODE_INST_PHI: { // PHI: [ty, val0,bb0, ...] if (Record.size() < 1 || ((Record.size()-1)&1)) return error("Invalid record"); - Type *Ty = getTypeByID(Record[0]); + FullTy = getFullyStructuredTypeByID(Record[0]); + Type *Ty = flattenPointerTypes(FullTy); if (!Ty) return error("Invalid record"); @@ -4271,7 +4662,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) { if (Record.size() < 4) return error("Invalid record"); } - Type *Ty = getTypeByID(Record[Idx++]); + FullTy = getFullyStructuredTypeByID(Record[Idx++]); + Type *Ty = flattenPointerTypes(FullTy); if (!Ty) return error("Invalid record"); if (BitCode == bitc::FUNC_CODE_INST_LANDINGPAD_OLD) { @@ -4324,12 +4716,13 @@ Error BitcodeReader::parseFunctionBody(Function *F) { SwiftErrorMask; bool InAlloca = AlignRecord & InAllocaMask; bool SwiftError = AlignRecord & SwiftErrorMask; - Type *Ty = getTypeByID(Record[0]); + FullTy = getFullyStructuredTypeByID(Record[0]); + Type *Ty = flattenPointerTypes(FullTy); if ((AlignRecord & ExplicitTypeMask) == 0) { auto *PTy = dyn_cast_or_null(Ty); if (!PTy) return error("Old-style alloca with a non-pointer type"); - Ty = PTy->getElementType(); + std::tie(FullTy, Ty) = getPointerElementTypes(FullTy); } Type *OpTy = getTypeByID(Record[1]); Value *Size = getFnValueByID(Record[2], OpTy); @@ -4348,29 +4741,34 @@ Error BitcodeReader::parseFunctionBody(Function *F) { AI->setUsedWithInAlloca(InAlloca); AI->setSwiftError(SwiftError); I = AI; + FullTy = PointerType::get(FullTy, AS); InstructionList.push_back(I); break; } case bitc::FUNC_CODE_INST_LOAD: { // LOAD: [opty, op, align, vol] unsigned OpNum = 0; Value *Op; - if (getValueTypePair(Record, OpNum, NextValueNo, Op) || + if (getValueTypePair(Record, OpNum, NextValueNo, Op, &FullTy) || (OpNum + 2 != Record.size() && OpNum + 3 != Record.size())) return error("Invalid record"); + if (!isa(Op->getType())) + return error("Load operand is not a pointer type"); + Type *Ty = nullptr; - if (OpNum + 3 == Record.size()) - Ty = getTypeByID(Record[OpNum++]); + if (OpNum + 3 == Record.size()) { + FullTy = getFullyStructuredTypeByID(Record[OpNum++]); + Ty = flattenPointerTypes(FullTy); + } else + std::tie(FullTy, Ty) = getPointerElementTypes(FullTy); + if (Error Err = typeCheckLoadStoreInst(Ty, Op->getType())) return Err; - if (!Ty) - Ty = cast(Op->getType())->getElementType(); unsigned Align; if (Error Err = parseAlignmentValue(Record[OpNum], Align)) return Err; I = new LoadInst(Ty, Op, "", Record[OpNum + 1], Align); - InstructionList.push_back(I); break; } @@ -4378,17 +4776,22 @@ Error BitcodeReader::parseFunctionBody(Function *F) { // LOADATOMIC: [opty, op, align, vol, ordering, ssid] unsigned OpNum = 0; Value *Op; - if (getValueTypePair(Record, OpNum, NextValueNo, Op) || + if (getValueTypePair(Record, OpNum, NextValueNo, Op, &FullTy) || (OpNum + 4 != Record.size() && OpNum + 5 != Record.size())) return error("Invalid record"); + if (!isa(Op->getType())) + return error("Load operand is not a pointer type"); + Type *Ty = nullptr; - if (OpNum + 5 == Record.size()) - Ty = getTypeByID(Record[OpNum++]); + if (OpNum + 5 == Record.size()) { + FullTy = getFullyStructuredTypeByID(Record[OpNum++]); + Ty = flattenPointerTypes(FullTy); + } else + std::tie(FullTy, Ty) = getPointerElementTypes(FullTy); + if (Error Err = typeCheckLoadStoreInst(Ty, Op->getType())) return Err; - if (!Ty) - Ty = cast(Op->getType())->getElementType(); AtomicOrdering Ordering = getDecodedOrdering(Record[OpNum + 2]); if (Ordering == AtomicOrdering::NotAtomic || @@ -4402,8 +4805,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { unsigned Align; if (Error Err = parseAlignmentValue(Record[OpNum], Align)) return Err; - I = new LoadInst(Op, "", Record[OpNum+1], Align, Ordering, SSID); - + I = new LoadInst(Ty, Op, "", Record[OpNum + 1], Align, Ordering, SSID); InstructionList.push_back(I); break; } @@ -4411,12 +4813,12 @@ Error BitcodeReader::parseFunctionBody(Function *F) { case bitc::FUNC_CODE_INST_STORE_OLD: { // STORE2:[ptrty, ptr, val, align, vol] unsigned OpNum = 0; Value *Val, *Ptr; - if (getValueTypePair(Record, OpNum, NextValueNo, Ptr) || + Type *FullTy; + if (getValueTypePair(Record, OpNum, NextValueNo, Ptr, &FullTy) || (BitCode == bitc::FUNC_CODE_INST_STORE ? getValueTypePair(Record, OpNum, NextValueNo, Val) : popValue(Record, OpNum, NextValueNo, - cast(Ptr->getType())->getElementType(), - Val)) || + getPointerElementFlatType(FullTy), Val)) || OpNum + 2 != Record.size()) return error("Invalid record"); @@ -4434,13 +4836,13 @@ Error BitcodeReader::parseFunctionBody(Function *F) { // STOREATOMIC: [ptrty, ptr, val, align, vol, ordering, ssid] unsigned OpNum = 0; Value *Val, *Ptr; - if (getValueTypePair(Record, OpNum, NextValueNo, Ptr) || + Type *FullTy; + if (getValueTypePair(Record, OpNum, NextValueNo, Ptr, &FullTy) || !isa(Ptr->getType()) || (BitCode == bitc::FUNC_CODE_INST_STOREATOMIC ? getValueTypePair(Record, OpNum, NextValueNo, Val) : popValue(Record, OpNum, NextValueNo, - cast(Ptr->getType())->getElementType(), - Val)) || + getPointerElementFlatType(FullTy), Val)) || OpNum + 4 != Record.size()) return error("Invalid record"); @@ -4468,15 +4870,25 @@ Error BitcodeReader::parseFunctionBody(Function *F) { // failureordering?, isweak?] unsigned OpNum = 0; Value *Ptr, *Cmp, *New; - if (getValueTypePair(Record, OpNum, NextValueNo, Ptr) || - (BitCode == bitc::FUNC_CODE_INST_CMPXCHG - ? getValueTypePair(Record, OpNum, NextValueNo, Cmp) - : popValue(Record, OpNum, NextValueNo, - cast(Ptr->getType())->getElementType(), - Cmp)) || - popValue(Record, OpNum, NextValueNo, Cmp->getType(), New) || + if (getValueTypePair(Record, OpNum, NextValueNo, Ptr, &FullTy)) + return error("Invalid record"); + + if (!isa(Ptr->getType())) + return error("Cmpxchg operand is not a pointer type"); + + if (BitCode == bitc::FUNC_CODE_INST_CMPXCHG) { + if (getValueTypePair(Record, OpNum, NextValueNo, Cmp, &FullTy)) + return error("Invalid record"); + } else if (popValue(Record, OpNum, NextValueNo, + getPointerElementFlatType(FullTy), Cmp)) + return error("Invalid record"); + else + FullTy = cast(FullTy)->getElementType(); + + if (popValue(Record, OpNum, NextValueNo, Cmp->getType(), New) || Record.size() < OpNum + 3 || Record.size() > OpNum + 5) return error("Invalid record"); + AtomicOrdering SuccessOrdering = getDecodedOrdering(Record[OpNum + 1]); if (SuccessOrdering == AtomicOrdering::NotAtomic || SuccessOrdering == AtomicOrdering::Unordered) @@ -4494,6 +4906,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { I = new AtomicCmpXchgInst(Ptr, Cmp, New, SuccessOrdering, FailureOrdering, SSID); + FullTy = StructType::get(Context, {FullTy, Type::getInt1Ty(Context)}); cast(I)->setVolatile(Record[OpNum]); if (Record.size() < 8) { @@ -4502,6 +4915,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { // expecting the first component of a modern cmpxchg. CurBB->getInstList().push_back(I); I = ExtractValueInst::Create(I, 0); + FullTy = cast(FullTy)->getElementType(0); } else { cast(I)->setWeak(Record[OpNum+4]); } @@ -4513,11 +4927,11 @@ Error BitcodeReader::parseFunctionBody(Function *F) { // ATOMICRMW:[ptrty, ptr, val, op, vol, ordering, ssid] unsigned OpNum = 0; Value *Ptr, *Val; - if (getValueTypePair(Record, OpNum, NextValueNo, Ptr) || + if (getValueTypePair(Record, OpNum, NextValueNo, Ptr, &FullTy) || !isa(Ptr->getType()) || popValue(Record, OpNum, NextValueNo, - cast(Ptr->getType())->getElementType(), Val) || - OpNum+4 != Record.size()) + getPointerElementFlatType(FullTy), Val) || + OpNum + 4 != Record.size()) return error("Invalid record"); AtomicRMWInst::BinOp Operation = getDecodedRMWOperation(Record[OpNum]); if (Operation < AtomicRMWInst::FIRST_BINOP || @@ -4529,6 +4943,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { return error("Invalid record"); SyncScope::ID SSID = getDecodedSyncScopeID(Record[OpNum + 3]); I = new AtomicRMWInst(Operation, Ptr, Val, Ordering, SSID); + FullTy = getPointerElementFlatType(FullTy); cast(I)->setVolatile(Record[OpNum+1]); InstructionList.push_back(I); break; @@ -4563,28 +4978,36 @@ Error BitcodeReader::parseFunctionBody(Function *F) { } FunctionType *FTy = nullptr; - if (CCInfo >> bitc::CALL_EXPLICIT_TYPE & 1 && - !(FTy = dyn_cast(getTypeByID(Record[OpNum++])))) - return error("Explicit call type is not a function type"); + FunctionType *FullFTy = nullptr; + if ((CCInfo >> bitc::CALL_EXPLICIT_TYPE) & 1) { + FullFTy = + dyn_cast(getFullyStructuredTypeByID(Record[OpNum++])); + if (!FullFTy) + return error("Explicit call type is not a function type"); + FTy = cast(flattenPointerTypes(FullFTy)); + } Value *Callee; - if (getValueTypePair(Record, OpNum, NextValueNo, Callee)) + if (getValueTypePair(Record, OpNum, NextValueNo, Callee, &FullTy)) return error("Invalid record"); PointerType *OpTy = dyn_cast(Callee->getType()); if (!OpTy) return error("Callee is not a pointer type"); if (!FTy) { - FTy = dyn_cast(OpTy->getElementType()); - if (!FTy) + FullFTy = + dyn_cast(cast(FullTy)->getElementType()); + if (!FullFTy) return error("Callee is not of pointer to function type"); - } else if (OpTy->getElementType() != FTy) + FTy = cast(flattenPointerTypes(FullFTy)); + } else if (getPointerElementFlatType(FullTy) != FTy) return error("Explicit call type does not match pointee type of " "callee operand"); if (Record.size() < FTy->getNumParams() + OpNum) return error("Insufficient operands to call"); SmallVector Args; + SmallVector ArgsFullTys; // Read the fixed params. for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i, ++OpNum) { if (FTy->getParamType(i)->isLabelTy()) @@ -4592,6 +5015,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { else Args.push_back(getValue(Record, OpNum, NextValueNo, FTy->getParamType(i))); + ArgsFullTys.push_back(FullFTy->getParamType(i)); if (!Args.back()) return error("Invalid record"); } @@ -4603,13 +5027,16 @@ Error BitcodeReader::parseFunctionBody(Function *F) { } else { while (OpNum != Record.size()) { Value *Op; - if (getValueTypePair(Record, OpNum, NextValueNo, Op)) + Type *FullTy; + if (getValueTypePair(Record, OpNum, NextValueNo, Op, &FullTy)) return error("Invalid record"); Args.push_back(Op); + ArgsFullTys.push_back(FullTy); } } I = CallInst::Create(FTy, Callee, Args, OperandBundles); + FullTy = FullFTy->getReturnType(); OperandBundles.clear(); InstructionList.push_back(I); cast(I)->setCallingConv( @@ -4623,6 +5050,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { TCK = CallInst::TCK_NoTail; cast(I)->setTailCallKind(TCK); cast(I)->setAttributes(PAL); + propagateByValTypes(cast(I), ArgsFullTys); if (FMF.any()) { if (!isa(I)) return error("Fast-math-flags specified for call without " @@ -4636,7 +5064,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) { return error("Invalid record"); Type *OpTy = getTypeByID(Record[0]); Value *Op = getValue(Record, 1, NextValueNo, OpTy); - Type *ResTy = getTypeByID(Record[2]); + FullTy = getFullyStructuredTypeByID(Record[2]); + Type *ResTy = flattenPointerTypes(FullTy); if (!OpTy || !Op || !ResTy) return error("Invalid record"); I = new VAArgInst(Op, ResTy); @@ -4686,8 +5115,23 @@ Error BitcodeReader::parseFunctionBody(Function *F) { } // Non-void values get registered in the value table for future use. - if (I && !I->getType()->isVoidTy()) - ValueList.assignValue(I, NextValueNo++); + if (I && !I->getType()->isVoidTy()) { + if (!FullTy) { + FullTy = I->getType(); + assert( + !FullTy->isPointerTy() && !isa(FullTy) && + !isa(FullTy) && + (!isa(FullTy) || + FullTy->getVectorElementType()->isFloatingPointTy() || + FullTy->getVectorElementType()->isIntegerTy()) && + "Structured types must be assigned with corresponding non-opaque " + "pointer type"); + } + + assert(I->getType() == flattenPointerTypes(FullTy) && + "Incorrect fully structured type provided for Instruction"); + ValueList.assignValue(I, NextValueNo++, FullTy); + } } OutOfRecordLoop: @@ -4769,8 +5213,8 @@ Error BitcodeReader::materialize(GlobalValue *GV) { return Err; // Move the bit stream to the saved position of the deferred function body. - Stream.JumpToBit(DFII->second); - + if (Error JumpFailed = Stream.JumpToBit(DFII->second)) + return JumpFailed; if (Error Err = parseFunctionBody(F)) return Err; F->setIsMaterializable(false); @@ -4933,10 +5377,13 @@ Error ModuleSummaryIndexBitcodeReader::parseValueSymbolTable( return Error::success(); assert(Offset > 0 && "Expected non-zero VST offset"); - uint64_t CurrentBit = jumpToValueSymbolTable(Offset, Stream); + Expected MaybeCurrentBit = jumpToValueSymbolTable(Offset, Stream); + if (!MaybeCurrentBit) + return MaybeCurrentBit.takeError(); + uint64_t CurrentBit = MaybeCurrentBit.get(); - if (Stream.EnterSubBlock(bitc::VALUE_SYMTAB_BLOCK_ID)) - return error("Invalid record"); + if (Error Err = Stream.EnterSubBlock(bitc::VALUE_SYMTAB_BLOCK_ID)) + return Err; SmallVector Record; @@ -4944,7 +5391,10 @@ Error ModuleSummaryIndexBitcodeReader::parseValueSymbolTable( SmallString<128> ValueName; while (true) { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + Expected MaybeEntry = Stream.advanceSkippingSubblocks(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); switch (Entry.Kind) { case BitstreamEntry::SubBlock: // Handled for us already. @@ -4952,7 +5402,8 @@ Error ModuleSummaryIndexBitcodeReader::parseValueSymbolTable( return error("Malformed block"); case BitstreamEntry::EndBlock: // Done parsing VST, jump back to wherever we came from. - Stream.JumpToBit(CurrentBit); + if (Error JumpFailed = Stream.JumpToBit(CurrentBit)) + return JumpFailed; return Error::success(); case BitstreamEntry::Record: // The interesting case. @@ -4961,7 +5412,10 @@ Error ModuleSummaryIndexBitcodeReader::parseValueSymbolTable( // Read a record. Record.clear(); - switch (Stream.readRecord(Entry.ID, Record)) { + Expected MaybeRecord = Stream.readRecord(Entry.ID, Record); + if (!MaybeRecord) + return MaybeRecord.takeError(); + switch (MaybeRecord.get()) { default: // Default behavior: ignore (e.g. VST_CODE_BBENTRY records). break; case bitc::VST_CODE_ENTRY: { // VST_CODE_ENTRY: [valueid, namechar x N] @@ -5009,8 +5463,8 @@ Error ModuleSummaryIndexBitcodeReader::parseValueSymbolTable( // At the end of this routine the module Index is populated with a map // from global value id to GlobalValueSummary objects. Error ModuleSummaryIndexBitcodeReader::parseModule() { - if (Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID)) - return error("Invalid record"); + if (Error Err = Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID)) + return Err; SmallVector Record; DenseMap ValueIdToLinkageMap; @@ -5018,7 +5472,10 @@ Error ModuleSummaryIndexBitcodeReader::parseModule() { // Read the index for this module. while (true) { - BitstreamEntry Entry = Stream.advance(); + Expected MaybeEntry = Stream.advance(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + llvm::BitstreamEntry Entry = MaybeEntry.get(); switch (Entry.Kind) { case BitstreamEntry::Error: @@ -5029,8 +5486,8 @@ Error ModuleSummaryIndexBitcodeReader::parseModule() { case BitstreamEntry::SubBlock: switch (Entry.ID) { default: // Skip unknown content. - if (Stream.SkipBlock()) - return error("Invalid record"); + if (Error Err = Stream.SkipBlock()) + return Err; break; case bitc::BLOCKINFO_BLOCK_ID: // Need to parse these to get abbrev ids (e.g. for VST) @@ -5043,8 +5500,8 @@ Error ModuleSummaryIndexBitcodeReader::parseModule() { assert(((SeenValueSymbolTable && VSTOffset > 0) || !SeenGlobalValSummary) && "Expected early VST parse via VSTOffset record"); - if (Stream.SkipBlock()) - return error("Invalid record"); + if (Error Err = Stream.SkipBlock()) + return Err; break; case bitc::GLOBALVAL_SUMMARY_BLOCK_ID: case bitc::FULL_LTO_GLOBALVAL_SUMMARY_BLOCK_ID: @@ -5075,8 +5532,10 @@ Error ModuleSummaryIndexBitcodeReader::parseModule() { case BitstreamEntry::Record: { Record.clear(); - auto BitCode = Stream.readRecord(Entry.ID, Record); - switch (BitCode) { + Expected MaybeBitCode = Stream.readRecord(Entry.ID, Record); + if (!MaybeBitCode) + return MaybeBitCode.takeError(); + switch (MaybeBitCode.get()) { default: break; // Default behavior, ignore unknown content. case bitc::MODULE_CODE_VERSION: { @@ -5224,32 +5683,66 @@ static void parseTypeIdSummaryRecord(ArrayRef Record, parseWholeProgramDevirtResolution(Record, Strtab, Slot, TypeId); } -static void setImmutableRefs(std::vector &Refs, unsigned Count) { - // Read-only refs are in the end of the refs list. - for (unsigned RefNo = Refs.size() - Count; RefNo < Refs.size(); ++RefNo) +void ModuleSummaryIndexBitcodeReader::parseTypeIdCompatibleVtableInfo( + ArrayRef Record, size_t &Slot, + TypeIdCompatibleVtableInfo &TypeId) { + uint64_t Offset = Record[Slot++]; + ValueInfo Callee = getValueInfoFromValueId(Record[Slot++]).first; + TypeId.push_back({Offset, Callee}); +} + +void ModuleSummaryIndexBitcodeReader::parseTypeIdCompatibleVtableSummaryRecord( + ArrayRef Record) { + size_t Slot = 0; + TypeIdCompatibleVtableInfo &TypeId = + TheIndex.getOrInsertTypeIdCompatibleVtableSummary( + {Strtab.data() + Record[Slot], + static_cast(Record[Slot + 1])}); + Slot += 2; + + while (Slot < Record.size()) + parseTypeIdCompatibleVtableInfo(Record, Slot, TypeId); +} + +static void setSpecialRefs(std::vector &Refs, unsigned ROCnt, + unsigned WOCnt) { + // Readonly and writeonly refs are in the end of the refs list. + assert(ROCnt + WOCnt <= Refs.size()); + unsigned FirstWORef = Refs.size() - WOCnt; + unsigned RefNo = FirstWORef - ROCnt; + for (; RefNo < FirstWORef; ++RefNo) Refs[RefNo].setReadOnly(); + for (; RefNo < Refs.size(); ++RefNo) + Refs[RefNo].setWriteOnly(); } // Eagerly parse the entire summary block. This populates the GlobalValueSummary // objects in the index. Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { - if (Stream.EnterSubBlock(ID)) - return error("Invalid record"); + if (Error Err = Stream.EnterSubBlock(ID)) + return Err; SmallVector Record; // Parse version { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + Expected MaybeEntry = Stream.advanceSkippingSubblocks(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); + if (Entry.Kind != BitstreamEntry::Record) return error("Invalid Summary Block: record for version expected"); - if (Stream.readRecord(Entry.ID, Record) != bitc::FS_VERSION) + Expected MaybeRecord = Stream.readRecord(Entry.ID, Record); + if (!MaybeRecord) + return MaybeRecord.takeError(); + if (MaybeRecord.get() != bitc::FS_VERSION) return error("Invalid Summary Block: version expected"); } const uint64_t Version = Record[0]; const bool IsOldProfileFormat = Version == 1; - if (Version < 1 || Version > 6) + if (Version < 1 || Version > 7) return error("Invalid summary version " + Twine(Version) + - ". Version should be in the range [1-6]."); + ". Version should be in the range [1-7]."); Record.clear(); // Keep around the last seen summary to be used when we see an optional @@ -5267,7 +5760,10 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { PendingTypeCheckedLoadConstVCalls; while (true) { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + Expected MaybeEntry = Stream.advanceSkippingSubblocks(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); switch (Entry.Kind) { case BitstreamEntry::SubBlock: // Handled for us already. @@ -5288,8 +5784,10 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { // in the combined index VST entries). The records also contain // information used for ThinLTO renaming and importing. Record.clear(); - auto BitCode = Stream.readRecord(Entry.ID, Record); - switch (BitCode) { + Expected MaybeBitCode = Stream.readRecord(Entry.ID, Record); + if (!MaybeBitCode) + return MaybeBitCode.takeError(); + switch (unsigned BitCode = MaybeBitCode.get()) { default: // Default behavior: ignore. break; case bitc::FS_FLAGS: { // [flags] @@ -5343,15 +5841,19 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { unsigned InstCount = Record[2]; uint64_t RawFunFlags = 0; unsigned NumRefs = Record[3]; - unsigned NumImmutableRefs = 0; + unsigned NumRORefs = 0, NumWORefs = 0; int RefListStartIndex = 4; if (Version >= 4) { RawFunFlags = Record[3]; NumRefs = Record[4]; RefListStartIndex = 5; if (Version >= 5) { - NumImmutableRefs = Record[5]; + NumRORefs = Record[5]; RefListStartIndex = 6; + if (Version >= 7) { + NumWORefs = Record[6]; + RefListStartIndex = 7; + } } } @@ -5371,7 +5873,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { std::vector Calls = makeCallList( ArrayRef(Record).slice(CallGraphEdgeStartIndex), IsOldProfileFormat, HasProfile, HasRelBF); - setImmutableRefs(Refs, NumImmutableRefs); + setSpecialRefs(Refs, NumRORefs, NumWORefs); auto FS = llvm::make_unique( Flags, InstCount, getDecodedFFlags(RawFunFlags), /*EntryCount=*/0, std::move(Refs), std::move(Calls), std::move(PendingTypeTests), @@ -5406,14 +5908,11 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { // ownership. AS->setModulePath(getThisModule()->first()); - GlobalValue::GUID AliaseeGUID = - getValueInfoFromValueId(AliaseeID).first.getGUID(); - auto AliaseeInModule = - TheIndex.findSummaryInModule(AliaseeGUID, ModulePath); + auto AliaseeVI = getValueInfoFromValueId(AliaseeID).first; + auto AliaseeInModule = TheIndex.findSummaryInModule(AliaseeVI, ModulePath); if (!AliaseeInModule) return error("Alias expects aliasee summary to be parsed"); - AS->setAliasee(AliaseeInModule); - AS->setAliaseeGUID(AliaseeGUID); + AS->setAliasee(AliaseeVI, AliaseeInModule); auto GUID = getValueInfoFromValueId(ValueID); AS->setOriginalName(GUID.second); @@ -5425,7 +5924,8 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { unsigned ValueID = Record[0]; uint64_t RawFlags = Record[1]; unsigned RefArrayStart = 2; - GlobalVarSummary::GVarFlags GVF; + GlobalVarSummary::GVarFlags GVF(/* ReadOnly */ false, + /* WriteOnly */ false); auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); if (Version >= 5) { GVF = getDecodedGVarFlags(Record[2]); @@ -5441,6 +5941,34 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { TheIndex.addGlobalValueSummary(GUID.first, std::move(FS)); break; } + // FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS: [valueid, flags, varflags, + // numrefs, numrefs x valueid, + // n x (valueid, offset)] + case bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS: { + unsigned ValueID = Record[0]; + uint64_t RawFlags = Record[1]; + GlobalVarSummary::GVarFlags GVF = getDecodedGVarFlags(Record[2]); + unsigned NumRefs = Record[3]; + unsigned RefListStartIndex = 4; + unsigned VTableListStartIndex = RefListStartIndex + NumRefs; + auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); + std::vector Refs = makeRefList( + ArrayRef(Record).slice(RefListStartIndex, NumRefs)); + VTableFuncList VTableFuncs; + for (unsigned I = VTableListStartIndex, E = Record.size(); I != E; ++I) { + ValueInfo Callee = getValueInfoFromValueId(Record[I]).first; + uint64_t Offset = Record[++I]; + VTableFuncs.push_back({Callee, Offset}); + } + auto VS = + llvm::make_unique(Flags, GVF, std::move(Refs)); + VS->setModulePath(getThisModule()->first()); + VS->setVTableFuncs(VTableFuncs); + auto GUID = getValueInfoFromValueId(ValueID); + VS->setOriginalName(GUID.second); + TheIndex.addGlobalValueSummary(GUID.first, std::move(VS)); + break; + } // FS_COMBINED: [valueid, modid, flags, instcount, fflags, numrefs, // numrefs x valueid, n x (valueid)] // FS_COMBINED_PROFILE: [valueid, modid, flags, instcount, fflags, numrefs, @@ -5454,7 +5982,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { uint64_t RawFunFlags = 0; uint64_t EntryCount = 0; unsigned NumRefs = Record[4]; - unsigned NumImmutableRefs = 0; + unsigned NumRORefs = 0, NumWORefs = 0; int RefListStartIndex = 5; if (Version >= 4) { @@ -5462,13 +5990,19 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { RefListStartIndex = 6; size_t NumRefsIndex = 5; if (Version >= 5) { + unsigned NumRORefsOffset = 1; RefListStartIndex = 7; if (Version >= 6) { NumRefsIndex = 6; EntryCount = Record[5]; RefListStartIndex = 8; + if (Version >= 7) { + RefListStartIndex = 9; + NumWORefs = Record[8]; + NumRORefsOffset = 2; + } } - NumImmutableRefs = Record[RefListStartIndex - 1]; + NumRORefs = Record[RefListStartIndex - NumRORefsOffset]; } NumRefs = Record[NumRefsIndex]; } @@ -5484,7 +6018,7 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { ArrayRef(Record).slice(CallGraphEdgeStartIndex), IsOldProfileFormat, HasProfile, false); ValueInfo VI = getValueInfoFromValueId(ValueID).first; - setImmutableRefs(Refs, NumImmutableRefs); + setSpecialRefs(Refs, NumRORefs, NumWORefs); auto FS = llvm::make_unique( Flags, InstCount, getDecodedFFlags(RawFunFlags), EntryCount, std::move(Refs), std::move(Edges), std::move(PendingTypeTests), @@ -5516,12 +6050,9 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { LastSeenSummary = AS.get(); AS->setModulePath(ModuleIdMap[ModuleId]); - auto AliaseeGUID = - getValueInfoFromValueId(AliaseeValueId).first.getGUID(); - auto AliaseeInModule = - TheIndex.findSummaryInModule(AliaseeGUID, AS->modulePath()); - AS->setAliasee(AliaseeInModule); - AS->setAliaseeGUID(AliaseeGUID); + auto AliaseeVI = getValueInfoFromValueId(AliaseeValueId).first; + auto AliaseeInModule = TheIndex.findSummaryInModule(AliaseeVI, AS->modulePath()); + AS->setAliasee(AliaseeVI, AliaseeInModule); ValueInfo VI = getValueInfoFromValueId(ValueID).first; LastSeenGUID = VI.getGUID(); @@ -5534,7 +6065,8 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { uint64_t ModuleId = Record[1]; uint64_t RawFlags = Record[2]; unsigned RefArrayStart = 3; - GlobalVarSummary::GVarFlags GVF; + GlobalVarSummary::GVarFlags GVF(/* ReadOnly */ false, + /* WriteOnly */ false); auto Flags = getDecodedGVSummaryFlags(RawFlags, Version); if (Version >= 5) { GVF = getDecodedGVarFlags(Record[3]); @@ -5610,6 +6142,10 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { case bitc::FS_TYPE_ID: parseTypeIdSummaryRecord(Record, Strtab, TheIndex); break; + + case bitc::FS_TYPE_ID_METADATA: + parseTypeIdCompatibleVtableSummaryRecord(Record); + break; } } llvm_unreachable("Exit infinite loop"); @@ -5618,8 +6154,8 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) { // Parse the module string table block into the Index. // This populates the ModulePathStringTable map in the index. Error ModuleSummaryIndexBitcodeReader::parseModuleStringTable() { - if (Stream.EnterSubBlock(bitc::MODULE_STRTAB_BLOCK_ID)) - return error("Invalid record"); + if (Error Err = Stream.EnterSubBlock(bitc::MODULE_STRTAB_BLOCK_ID)) + return Err; SmallVector Record; @@ -5627,7 +6163,10 @@ Error ModuleSummaryIndexBitcodeReader::parseModuleStringTable() { ModuleSummaryIndex::ModuleInfo *LastSeenModule = nullptr; while (true) { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + Expected MaybeEntry = Stream.advanceSkippingSubblocks(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); switch (Entry.Kind) { case BitstreamEntry::SubBlock: // Handled for us already. @@ -5641,7 +6180,10 @@ Error ModuleSummaryIndexBitcodeReader::parseModuleStringTable() { } Record.clear(); - switch (Stream.readRecord(Entry.ID, Record)) { + Expected MaybeRecord = Stream.readRecord(Entry.ID, Record); + if (!MaybeRecord) + return MaybeRecord.takeError(); + switch (MaybeRecord.get()) { default: // Default behavior: ignore. break; case bitc::MST_CODE_ENTRY: { @@ -5707,12 +6249,16 @@ const std::error_category &llvm::BitcodeErrorCategory() { static Expected readBlobInRecord(BitstreamCursor &Stream, unsigned Block, unsigned RecordID) { - if (Stream.EnterSubBlock(Block)) - return error("Invalid record"); + if (Error Err = Stream.EnterSubBlock(Block)) + return std::move(Err); StringRef Strtab; while (true) { - BitstreamEntry Entry = Stream.advance(); + Expected MaybeEntry = Stream.advance(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + llvm::BitstreamEntry Entry = MaybeEntry.get(); + switch (Entry.Kind) { case BitstreamEntry::EndBlock: return Strtab; @@ -5721,14 +6267,18 @@ static Expected readBlobInRecord(BitstreamCursor &Stream, return error("Malformed block"); case BitstreamEntry::SubBlock: - if (Stream.SkipBlock()) - return error("Malformed block"); + if (Error Err = Stream.SkipBlock()) + return std::move(Err); break; case BitstreamEntry::Record: StringRef Blob; SmallVector Record; - if (Stream.readRecord(Entry.ID, Record, &Blob) == RecordID) + Expected MaybeRecord = + Stream.readRecord(Entry.ID, Record, &Blob); + if (!MaybeRecord) + return MaybeRecord.takeError(); + if (MaybeRecord.get() == RecordID) Strtab = Blob; break; } @@ -5764,7 +6314,11 @@ llvm::getBitcodeFileContents(MemoryBufferRef Buffer) { if (BCBegin + 8 >= Stream.getBitcodeBytes().size()) return F; - BitstreamEntry Entry = Stream.advance(); + Expected MaybeEntry = Stream.advance(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + llvm::BitstreamEntry Entry = MaybeEntry.get(); + switch (Entry.Kind) { case BitstreamEntry::EndBlock: case BitstreamEntry::Error: @@ -5774,10 +6328,16 @@ llvm::getBitcodeFileContents(MemoryBufferRef Buffer) { uint64_t IdentificationBit = -1ull; if (Entry.ID == bitc::IDENTIFICATION_BLOCK_ID) { IdentificationBit = Stream.GetCurrentBitNo() - BCBegin * 8; - if (Stream.SkipBlock()) - return error("Malformed block"); + if (Error Err = Stream.SkipBlock()) + return std::move(Err); + + { + Expected MaybeEntry = Stream.advance(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + Entry = MaybeEntry.get(); + } - Entry = Stream.advance(); if (Entry.Kind != BitstreamEntry::SubBlock || Entry.ID != bitc::MODULE_BLOCK_ID) return error("Malformed block"); @@ -5785,8 +6345,8 @@ llvm::getBitcodeFileContents(MemoryBufferRef Buffer) { if (Entry.ID == bitc::MODULE_BLOCK_ID) { uint64_t ModuleBit = Stream.GetCurrentBitNo() - BCBegin * 8; - if (Stream.SkipBlock()) - return error("Malformed block"); + if (Error Err = Stream.SkipBlock()) + return std::move(Err); F.Mods.push_back({Stream.getBitcodeBytes().slice( BCBegin, Stream.getCurrentByteNo() - BCBegin), @@ -5834,13 +6394,15 @@ llvm::getBitcodeFileContents(MemoryBufferRef Buffer) { continue; } - if (Stream.SkipBlock()) - return error("Malformed block"); + if (Error Err = Stream.SkipBlock()) + return std::move(Err); continue; } case BitstreamEntry::Record: - Stream.skipRecord(Entry.ID); - continue; + if (Expected StreamFailed = Stream.skipRecord(Entry.ID)) + continue; + else + return StreamFailed.takeError(); } } } @@ -5860,7 +6422,8 @@ BitcodeModule::getModuleImpl(LLVMContext &Context, bool MaterializeAll, std::string ProducerIdentification; if (IdentificationBit != -1ull) { - Stream.JumpToBit(IdentificationBit); + if (Error JumpFailed = Stream.JumpToBit(IdentificationBit)) + return std::move(JumpFailed); Expected ProducerIdentificationOrErr = readIdentificationBlock(Stream); if (!ProducerIdentificationOrErr) @@ -5869,7 +6432,8 @@ BitcodeModule::getModuleImpl(LLVMContext &Context, bool MaterializeAll, ProducerIdentification = *ProducerIdentificationOrErr; } - Stream.JumpToBit(ModuleBit); + if (Error JumpFailed = Stream.JumpToBit(ModuleBit)) + return std::move(JumpFailed); auto *R = new BitcodeReader(std::move(Stream), Strtab, ProducerIdentification, Context); @@ -5907,7 +6471,8 @@ BitcodeModule::getLazyModule(LLVMContext &Context, bool ShouldLazyLoadMetadata, Error BitcodeModule::readSummary(ModuleSummaryIndex &CombinedIndex, StringRef ModulePath, uint64_t ModuleId) { BitstreamCursor Stream(Buffer); - Stream.JumpToBit(ModuleBit); + if (Error JumpFailed = Stream.JumpToBit(ModuleBit)) + return JumpFailed; ModuleSummaryIndexBitcodeReader R(std::move(Stream), Strtab, CombinedIndex, ModulePath, ModuleId); @@ -5917,7 +6482,8 @@ Error BitcodeModule::readSummary(ModuleSummaryIndex &CombinedIndex, // Parse the specified bitcode buffer, returning the function info index. Expected> BitcodeModule::getSummary() { BitstreamCursor Stream(Buffer); - Stream.JumpToBit(ModuleBit); + if (Error JumpFailed = Stream.JumpToBit(ModuleBit)) + return std::move(JumpFailed); auto Index = llvm::make_unique(/*HaveGVs=*/false); ModuleSummaryIndexBitcodeReader R(std::move(Stream), Strtab, *Index, @@ -5931,12 +6497,15 @@ Expected> BitcodeModule::getSummary() { static Expected getEnableSplitLTOUnitFlag(BitstreamCursor &Stream, unsigned ID) { - if (Stream.EnterSubBlock(ID)) - return error("Invalid record"); + if (Error Err = Stream.EnterSubBlock(ID)) + return std::move(Err); SmallVector Record; while (true) { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + Expected MaybeEntry = Stream.advanceSkippingSubblocks(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); switch (Entry.Kind) { case BitstreamEntry::SubBlock: // Handled for us already. @@ -5953,8 +6522,10 @@ static Expected getEnableSplitLTOUnitFlag(BitstreamCursor &Stream, // Look for the FS_FLAGS record. Record.clear(); - auto BitCode = Stream.readRecord(Entry.ID, Record); - switch (BitCode) { + Expected MaybeBitCode = Stream.readRecord(Entry.ID, Record); + if (!MaybeBitCode) + return MaybeBitCode.takeError(); + switch (MaybeBitCode.get()) { default: // Default behavior: ignore. break; case bitc::FS_FLAGS: { // [flags] @@ -5972,13 +6543,17 @@ static Expected getEnableSplitLTOUnitFlag(BitstreamCursor &Stream, // Check if the given bitcode buffer contains a global value summary block. Expected BitcodeModule::getLTOInfo() { BitstreamCursor Stream(Buffer); - Stream.JumpToBit(ModuleBit); + if (Error JumpFailed = Stream.JumpToBit(ModuleBit)) + return std::move(JumpFailed); - if (Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID)) - return error("Invalid record"); + if (Error Err = Stream.EnterSubBlock(bitc::MODULE_BLOCK_ID)) + return std::move(Err); while (true) { - BitstreamEntry Entry = Stream.advance(); + Expected MaybeEntry = Stream.advance(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + llvm::BitstreamEntry Entry = MaybeEntry.get(); switch (Entry.Kind) { case BitstreamEntry::Error: @@ -6007,13 +6582,15 @@ Expected BitcodeModule::getLTOInfo() { } // Ignore other sub-blocks. - if (Stream.SkipBlock()) - return error("Malformed block"); + if (Error Err = Stream.SkipBlock()) + return std::move(Err); continue; case BitstreamEntry::Record: - Stream.skipRecord(Entry.ID); - continue; + if (Expected StreamFailed = Stream.skipRecord(Entry.ID)) + continue; + else + return StreamFailed.takeError(); } } } diff --git a/lib/Bitcode/Reader/BitstreamReader.cpp b/lib/Bitcode/Reader/BitstreamReader.cpp deleted file mode 100644 index 771cf3d927bc..000000000000 --- a/lib/Bitcode/Reader/BitstreamReader.cpp +++ /dev/null @@ -1,390 +0,0 @@ -//===- BitstreamReader.cpp - BitstreamReader implementation ---------------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// - -#include "llvm/Bitcode/BitstreamReader.h" -#include "llvm/ADT/StringRef.h" -#include -#include - -using namespace llvm; - -//===----------------------------------------------------------------------===// -// BitstreamCursor implementation -//===----------------------------------------------------------------------===// - -/// EnterSubBlock - Having read the ENTER_SUBBLOCK abbrevid, enter -/// the block, and return true if the block has an error. -bool BitstreamCursor::EnterSubBlock(unsigned BlockID, unsigned *NumWordsP) { - // Save the current block's state on BlockScope. - BlockScope.push_back(Block(CurCodeSize)); - BlockScope.back().PrevAbbrevs.swap(CurAbbrevs); - - // Add the abbrevs specific to this block to the CurAbbrevs list. - if (BlockInfo) { - if (const BitstreamBlockInfo::BlockInfo *Info = - BlockInfo->getBlockInfo(BlockID)) { - CurAbbrevs.insert(CurAbbrevs.end(), Info->Abbrevs.begin(), - Info->Abbrevs.end()); - } - } - - // Get the codesize of this block. - CurCodeSize = ReadVBR(bitc::CodeLenWidth); - // We can't read more than MaxChunkSize at a time - if (CurCodeSize > MaxChunkSize) - return true; - - SkipToFourByteBoundary(); - unsigned NumWords = Read(bitc::BlockSizeWidth); - if (NumWordsP) *NumWordsP = NumWords; - - // Validate that this block is sane. - return CurCodeSize == 0 || AtEndOfStream(); -} - -static uint64_t readAbbreviatedField(BitstreamCursor &Cursor, - const BitCodeAbbrevOp &Op) { - assert(!Op.isLiteral() && "Not to be used with literals!"); - - // Decode the value as we are commanded. - switch (Op.getEncoding()) { - case BitCodeAbbrevOp::Array: - case BitCodeAbbrevOp::Blob: - llvm_unreachable("Should not reach here"); - case BitCodeAbbrevOp::Fixed: - assert((unsigned)Op.getEncodingData() <= Cursor.MaxChunkSize); - return Cursor.Read((unsigned)Op.getEncodingData()); - case BitCodeAbbrevOp::VBR: - assert((unsigned)Op.getEncodingData() <= Cursor.MaxChunkSize); - return Cursor.ReadVBR64((unsigned)Op.getEncodingData()); - case BitCodeAbbrevOp::Char6: - return BitCodeAbbrevOp::DecodeChar6(Cursor.Read(6)); - } - llvm_unreachable("invalid abbreviation encoding"); -} - -static void skipAbbreviatedField(BitstreamCursor &Cursor, - const BitCodeAbbrevOp &Op) { - assert(!Op.isLiteral() && "Not to be used with literals!"); - - // Decode the value as we are commanded. - switch (Op.getEncoding()) { - case BitCodeAbbrevOp::Array: - case BitCodeAbbrevOp::Blob: - llvm_unreachable("Should not reach here"); - case BitCodeAbbrevOp::Fixed: - assert((unsigned)Op.getEncodingData() <= Cursor.MaxChunkSize); - Cursor.Read((unsigned)Op.getEncodingData()); - break; - case BitCodeAbbrevOp::VBR: - assert((unsigned)Op.getEncodingData() <= Cursor.MaxChunkSize); - Cursor.ReadVBR64((unsigned)Op.getEncodingData()); - break; - case BitCodeAbbrevOp::Char6: - Cursor.Read(6); - break; - } -} - -/// skipRecord - Read the current record and discard it. -unsigned BitstreamCursor::skipRecord(unsigned AbbrevID) { - // Skip unabbreviated records by reading past their entries. - if (AbbrevID == bitc::UNABBREV_RECORD) { - unsigned Code = ReadVBR(6); - unsigned NumElts = ReadVBR(6); - for (unsigned i = 0; i != NumElts; ++i) - (void)ReadVBR64(6); - return Code; - } - - const BitCodeAbbrev *Abbv = getAbbrev(AbbrevID); - const BitCodeAbbrevOp &CodeOp = Abbv->getOperandInfo(0); - unsigned Code; - if (CodeOp.isLiteral()) - Code = CodeOp.getLiteralValue(); - else { - if (CodeOp.getEncoding() == BitCodeAbbrevOp::Array || - CodeOp.getEncoding() == BitCodeAbbrevOp::Blob) - report_fatal_error("Abbreviation starts with an Array or a Blob"); - Code = readAbbreviatedField(*this, CodeOp); - } - - for (unsigned i = 1, e = Abbv->getNumOperandInfos(); i < e; ++i) { - const BitCodeAbbrevOp &Op = Abbv->getOperandInfo(i); - if (Op.isLiteral()) - continue; - - if (Op.getEncoding() != BitCodeAbbrevOp::Array && - Op.getEncoding() != BitCodeAbbrevOp::Blob) { - skipAbbreviatedField(*this, Op); - continue; - } - - if (Op.getEncoding() == BitCodeAbbrevOp::Array) { - // Array case. Read the number of elements as a vbr6. - unsigned NumElts = ReadVBR(6); - - // Get the element encoding. - assert(i+2 == e && "array op not second to last?"); - const BitCodeAbbrevOp &EltEnc = Abbv->getOperandInfo(++i); - - // Read all the elements. - // Decode the value as we are commanded. - switch (EltEnc.getEncoding()) { - default: - report_fatal_error("Array element type can't be an Array or a Blob"); - case BitCodeAbbrevOp::Fixed: - assert((unsigned)EltEnc.getEncodingData() <= MaxChunkSize); - JumpToBit(GetCurrentBitNo() + NumElts * EltEnc.getEncodingData()); - break; - case BitCodeAbbrevOp::VBR: - assert((unsigned)EltEnc.getEncodingData() <= MaxChunkSize); - for (; NumElts; --NumElts) - ReadVBR64((unsigned)EltEnc.getEncodingData()); - break; - case BitCodeAbbrevOp::Char6: - JumpToBit(GetCurrentBitNo() + NumElts * 6); - break; - } - continue; - } - - assert(Op.getEncoding() == BitCodeAbbrevOp::Blob); - // Blob case. Read the number of bytes as a vbr6. - unsigned NumElts = ReadVBR(6); - SkipToFourByteBoundary(); // 32-bit alignment - - // Figure out where the end of this blob will be including tail padding. - size_t NewEnd = GetCurrentBitNo()+((NumElts+3)&~3)*8; - - // If this would read off the end of the bitcode file, just set the - // record to empty and return. - if (!canSkipToPos(NewEnd/8)) { - skipToEnd(); - break; - } - - // Skip over the blob. - JumpToBit(NewEnd); - } - return Code; -} - -unsigned BitstreamCursor::readRecord(unsigned AbbrevID, - SmallVectorImpl &Vals, - StringRef *Blob) { - if (AbbrevID == bitc::UNABBREV_RECORD) { - unsigned Code = ReadVBR(6); - unsigned NumElts = ReadVBR(6); - for (unsigned i = 0; i != NumElts; ++i) - Vals.push_back(ReadVBR64(6)); - return Code; - } - - const BitCodeAbbrev *Abbv = getAbbrev(AbbrevID); - - // Read the record code first. - assert(Abbv->getNumOperandInfos() != 0 && "no record code in abbreviation?"); - const BitCodeAbbrevOp &CodeOp = Abbv->getOperandInfo(0); - unsigned Code; - if (CodeOp.isLiteral()) - Code = CodeOp.getLiteralValue(); - else { - if (CodeOp.getEncoding() == BitCodeAbbrevOp::Array || - CodeOp.getEncoding() == BitCodeAbbrevOp::Blob) - report_fatal_error("Abbreviation starts with an Array or a Blob"); - Code = readAbbreviatedField(*this, CodeOp); - } - - for (unsigned i = 1, e = Abbv->getNumOperandInfos(); i != e; ++i) { - const BitCodeAbbrevOp &Op = Abbv->getOperandInfo(i); - if (Op.isLiteral()) { - Vals.push_back(Op.getLiteralValue()); - continue; - } - - if (Op.getEncoding() != BitCodeAbbrevOp::Array && - Op.getEncoding() != BitCodeAbbrevOp::Blob) { - Vals.push_back(readAbbreviatedField(*this, Op)); - continue; - } - - if (Op.getEncoding() == BitCodeAbbrevOp::Array) { - // Array case. Read the number of elements as a vbr6. - unsigned NumElts = ReadVBR(6); - - // Get the element encoding. - if (i + 2 != e) - report_fatal_error("Array op not second to last"); - const BitCodeAbbrevOp &EltEnc = Abbv->getOperandInfo(++i); - if (!EltEnc.isEncoding()) - report_fatal_error( - "Array element type has to be an encoding of a type"); - - // Read all the elements. - switch (EltEnc.getEncoding()) { - default: - report_fatal_error("Array element type can't be an Array or a Blob"); - case BitCodeAbbrevOp::Fixed: - for (; NumElts; --NumElts) - Vals.push_back(Read((unsigned)EltEnc.getEncodingData())); - break; - case BitCodeAbbrevOp::VBR: - for (; NumElts; --NumElts) - Vals.push_back(ReadVBR64((unsigned)EltEnc.getEncodingData())); - break; - case BitCodeAbbrevOp::Char6: - for (; NumElts; --NumElts) - Vals.push_back(BitCodeAbbrevOp::DecodeChar6(Read(6))); - } - continue; - } - - assert(Op.getEncoding() == BitCodeAbbrevOp::Blob); - // Blob case. Read the number of bytes as a vbr6. - unsigned NumElts = ReadVBR(6); - SkipToFourByteBoundary(); // 32-bit alignment - - // Figure out where the end of this blob will be including tail padding. - size_t CurBitPos = GetCurrentBitNo(); - size_t NewEnd = CurBitPos+((NumElts+3)&~3)*8; - - // If this would read off the end of the bitcode file, just set the - // record to empty and return. - if (!canSkipToPos(NewEnd/8)) { - Vals.append(NumElts, 0); - skipToEnd(); - break; - } - - // Otherwise, inform the streamer that we need these bytes in memory. Skip - // over tail padding first, in case jumping to NewEnd invalidates the Blob - // pointer. - JumpToBit(NewEnd); - const char *Ptr = (const char *)getPointerToBit(CurBitPos, NumElts); - - // If we can return a reference to the data, do so to avoid copying it. - if (Blob) { - *Blob = StringRef(Ptr, NumElts); - } else { - // Otherwise, unpack into Vals with zero extension. - for (; NumElts; --NumElts) - Vals.push_back((unsigned char)*Ptr++); - } - } - - return Code; -} - -void BitstreamCursor::ReadAbbrevRecord() { - auto Abbv = std::make_shared(); - unsigned NumOpInfo = ReadVBR(5); - for (unsigned i = 0; i != NumOpInfo; ++i) { - bool IsLiteral = Read(1); - if (IsLiteral) { - Abbv->Add(BitCodeAbbrevOp(ReadVBR64(8))); - continue; - } - - BitCodeAbbrevOp::Encoding E = (BitCodeAbbrevOp::Encoding)Read(3); - if (BitCodeAbbrevOp::hasEncodingData(E)) { - uint64_t Data = ReadVBR64(5); - - // As a special case, handle fixed(0) (i.e., a fixed field with zero bits) - // and vbr(0) as a literal zero. This is decoded the same way, and avoids - // a slow path in Read() to have to handle reading zero bits. - if ((E == BitCodeAbbrevOp::Fixed || E == BitCodeAbbrevOp::VBR) && - Data == 0) { - Abbv->Add(BitCodeAbbrevOp(0)); - continue; - } - - if ((E == BitCodeAbbrevOp::Fixed || E == BitCodeAbbrevOp::VBR) && - Data > MaxChunkSize) - report_fatal_error( - "Fixed or VBR abbrev record with size > MaxChunkData"); - - Abbv->Add(BitCodeAbbrevOp(E, Data)); - } else - Abbv->Add(BitCodeAbbrevOp(E)); - } - - if (Abbv->getNumOperandInfos() == 0) - report_fatal_error("Abbrev record with no operands"); - CurAbbrevs.push_back(std::move(Abbv)); -} - -Optional -BitstreamCursor::ReadBlockInfoBlock(bool ReadBlockInfoNames) { - if (EnterSubBlock(bitc::BLOCKINFO_BLOCK_ID)) return None; - - BitstreamBlockInfo NewBlockInfo; - - SmallVector Record; - BitstreamBlockInfo::BlockInfo *CurBlockInfo = nullptr; - - // Read all the records for this module. - while (true) { - BitstreamEntry Entry = advanceSkippingSubblocks(AF_DontAutoprocessAbbrevs); - - switch (Entry.Kind) { - case llvm::BitstreamEntry::SubBlock: // Handled for us already. - case llvm::BitstreamEntry::Error: - return None; - case llvm::BitstreamEntry::EndBlock: - return std::move(NewBlockInfo); - case llvm::BitstreamEntry::Record: - // The interesting case. - break; - } - - // Read abbrev records, associate them with CurBID. - if (Entry.ID == bitc::DEFINE_ABBREV) { - if (!CurBlockInfo) return None; - ReadAbbrevRecord(); - - // ReadAbbrevRecord installs the abbrev in CurAbbrevs. Move it to the - // appropriate BlockInfo. - CurBlockInfo->Abbrevs.push_back(std::move(CurAbbrevs.back())); - CurAbbrevs.pop_back(); - continue; - } - - // Read a record. - Record.clear(); - switch (readRecord(Entry.ID, Record)) { - default: break; // Default behavior, ignore unknown content. - case bitc::BLOCKINFO_CODE_SETBID: - if (Record.size() < 1) return None; - CurBlockInfo = &NewBlockInfo.getOrCreateBlockInfo((unsigned)Record[0]); - break; - case bitc::BLOCKINFO_CODE_BLOCKNAME: { - if (!CurBlockInfo) return None; - if (!ReadBlockInfoNames) - break; // Ignore name. - std::string Name; - for (unsigned i = 0, e = Record.size(); i != e; ++i) - Name += (char)Record[i]; - CurBlockInfo->Name = Name; - break; - } - case bitc::BLOCKINFO_CODE_SETRECORDNAME: { - if (!CurBlockInfo) return None; - if (!ReadBlockInfoNames) - break; // Ignore name. - std::string Name; - for (unsigned i = 1, e = Record.size(); i != e; ++i) - Name += (char)Record[i]; - CurBlockInfo->RecordNames.push_back(std::make_pair((unsigned)Record[0], - Name)); - break; - } - } - } -} diff --git a/lib/Bitcode/Reader/MetadataLoader.cpp b/lib/Bitcode/Reader/MetadataLoader.cpp index 3289aa0acddd..108f71189585 100644 --- a/lib/Bitcode/Reader/MetadataLoader.cpp +++ b/lib/Bitcode/Reader/MetadataLoader.cpp @@ -1,9 +1,8 @@ //===- MetadataLoader.cpp - Internal BitcodeReader implementation ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -23,7 +22,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/Bitcode/BitcodeReader.h" -#include "llvm/Bitcode/BitstreamReader.h" +#include "llvm/Bitstream/BitstreamReader.h" #include "llvm/Bitcode/LLVMBitCodes.h" #include "llvm/IR/Argument.h" #include "llvm/IR/Attributes.h" @@ -104,7 +103,7 @@ static cl::opt DisableLazyLoading( namespace { -static int64_t unrotateSign(uint64_t U) { return U & 1 ? ~(U >> 1) : U >> 1; } +static int64_t unrotateSign(uint64_t U) { return (U & 1) ? ~(U >> 1) : U >> 1; } class BitcodeReaderMetadataList { /// Array of metadata references. @@ -131,8 +130,15 @@ class BitcodeReaderMetadataList { LLVMContext &Context; + /// Maximum number of valid references. Forward references exceeding the + /// maximum must be invalid. + unsigned RefsUpperBound; + public: - BitcodeReaderMetadataList(LLVMContext &C) : Context(C) {} + BitcodeReaderMetadataList(LLVMContext &C, size_t RefsUpperBound) + : Context(C), + RefsUpperBound(std::min((size_t)std::numeric_limits::max(), + RefsUpperBound)) {} // vector compatibility methods unsigned size() const { return MetadataPtrs.size(); } @@ -219,6 +225,10 @@ void BitcodeReaderMetadataList::assignValue(Metadata *MD, unsigned Idx) { } Metadata *BitcodeReaderMetadataList::getMetadataFwdRef(unsigned Idx) { + // Bail out for a clearly invalid value. + if (Idx >= RefsUpperBound) + return nullptr; + if (Idx >= size()) resize(Idx + 1); @@ -338,7 +348,7 @@ Metadata *BitcodeReaderMetadataList::resolveTypeRefArray(Metadata *MaybeTuple) { if (!Tuple || Tuple->isDistinct()) return MaybeTuple; - // Look through the DITypeRefArray, upgrading each DITypeRef. + // Look through the DITypeRefArray, upgrading each DIType *. SmallVector Ops; Ops.reserve(Tuple->getNumOperands()); for (Metadata *MD : Tuple->operands()) @@ -626,9 +636,10 @@ public: BitcodeReaderValueList &ValueList, std::function getTypeByID, bool IsImporting) - : MetadataList(TheModule.getContext()), ValueList(ValueList), - Stream(Stream), Context(TheModule.getContext()), TheModule(TheModule), - getTypeByID(std::move(getTypeByID)), IsImporting(IsImporting) {} + : MetadataList(TheModule.getContext(), Stream.SizeInBytes()), + ValueList(ValueList), Stream(Stream), Context(TheModule.getContext()), + TheModule(TheModule), getTypeByID(std::move(getTypeByID)), + IsImporting(IsImporting) {} Error parseMetadata(bool ModuleLevel); @@ -675,8 +686,12 @@ MetadataLoader::MetadataLoaderImpl::lazyLoadModuleMetadataBlock() { SmallVector Record; // Get the abbrevs, and preload record positions to make them lazy-loadable. while (true) { - BitstreamEntry Entry = IndexCursor.advanceSkippingSubblocks( + Expected MaybeEntry = IndexCursor.advanceSkippingSubblocks( BitstreamCursor::AF_DontPopBlockAtEnd); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); + switch (Entry.Kind) { case BitstreamEntry::SubBlock: // Handled for us already. case BitstreamEntry::Error: @@ -688,14 +703,22 @@ MetadataLoader::MetadataLoaderImpl::lazyLoadModuleMetadataBlock() { // The interesting case. ++NumMDRecordLoaded; uint64_t CurrentPos = IndexCursor.GetCurrentBitNo(); - auto Code = IndexCursor.skipRecord(Entry.ID); + Expected MaybeCode = IndexCursor.skipRecord(Entry.ID); + if (!MaybeCode) + return MaybeCode.takeError(); + unsigned Code = MaybeCode.get(); switch (Code) { case bitc::METADATA_STRINGS: { // Rewind and parse the strings. - IndexCursor.JumpToBit(CurrentPos); + if (Error Err = IndexCursor.JumpToBit(CurrentPos)) + return std::move(Err); StringRef Blob; Record.clear(); - IndexCursor.readRecord(Entry.ID, Record, &Blob); + if (Expected MaybeRecord = + IndexCursor.readRecord(Entry.ID, Record, &Blob)) + ; + else + return MaybeRecord.takeError(); unsigned NumStrings = Record[0]; MDStringRef.reserve(NumStrings); auto IndexNextMDString = [&](StringRef Str) { @@ -708,26 +731,37 @@ MetadataLoader::MetadataLoaderImpl::lazyLoadModuleMetadataBlock() { case bitc::METADATA_INDEX_OFFSET: { // This is the offset to the index, when we see this we skip all the // records and load only an index to these. - IndexCursor.JumpToBit(CurrentPos); + if (Error Err = IndexCursor.JumpToBit(CurrentPos)) + return std::move(Err); Record.clear(); - IndexCursor.readRecord(Entry.ID, Record); + if (Expected MaybeRecord = + IndexCursor.readRecord(Entry.ID, Record)) + ; + else + return MaybeRecord.takeError(); if (Record.size() != 2) return error("Invalid record"); auto Offset = Record[0] + (Record[1] << 32); auto BeginPos = IndexCursor.GetCurrentBitNo(); - IndexCursor.JumpToBit(BeginPos + Offset); - Entry = IndexCursor.advanceSkippingSubblocks( - BitstreamCursor::AF_DontPopBlockAtEnd); + if (Error Err = IndexCursor.JumpToBit(BeginPos + Offset)) + return std::move(Err); + Expected MaybeEntry = + IndexCursor.advanceSkippingSubblocks( + BitstreamCursor::AF_DontPopBlockAtEnd); + if (!MaybeEntry) + return MaybeEntry.takeError(); + Entry = MaybeEntry.get(); assert(Entry.Kind == BitstreamEntry::Record && "Corrupted bitcode: Expected `Record` when trying to find the " "Metadata index"); Record.clear(); - auto Code = IndexCursor.readRecord(Entry.ID, Record); - (void)Code; - assert(Code == bitc::METADATA_INDEX && "Corrupted bitcode: Expected " - "`METADATA_INDEX` when trying " - "to find the Metadata index"); - + if (Expected MaybeCode = + IndexCursor.readRecord(Entry.ID, Record)) + assert(MaybeCode.get() == bitc::METADATA_INDEX && + "Corrupted bitcode: Expected `METADATA_INDEX` when trying to " + "find the Metadata index"); + else + return MaybeCode.takeError(); // Delta unpack auto CurrentValue = BeginPos; GlobalMetadataBitPosIndex.reserve(Record.size()); @@ -743,21 +777,33 @@ MetadataLoader::MetadataLoaderImpl::lazyLoadModuleMetadataBlock() { return error("Corrupted Metadata block"); case bitc::METADATA_NAME: { // Named metadata need to be materialized now and aren't deferred. - IndexCursor.JumpToBit(CurrentPos); + if (Error Err = IndexCursor.JumpToBit(CurrentPos)) + return std::move(Err); Record.clear(); - unsigned Code = IndexCursor.readRecord(Entry.ID, Record); - assert(Code == bitc::METADATA_NAME); + + unsigned Code; + if (Expected MaybeCode = + IndexCursor.readRecord(Entry.ID, Record)) { + Code = MaybeCode.get(); + assert(Code == bitc::METADATA_NAME); + } else + return MaybeCode.takeError(); // Read name of the named metadata. SmallString<8> Name(Record.begin(), Record.end()); - Code = IndexCursor.ReadCode(); + if (Expected MaybeCode = IndexCursor.ReadCode()) + Code = MaybeCode.get(); + else + return MaybeCode.takeError(); // Named Metadata comes in two parts, we expect the name to be followed // by the node Record.clear(); - unsigned NextBitCode = IndexCursor.readRecord(Code, Record); - assert(NextBitCode == bitc::METADATA_NAMED_NODE); - (void)NextBitCode; + if (Expected MaybeNextBitCode = + IndexCursor.readRecord(Code, Record)) + assert(MaybeNextBitCode.get() == bitc::METADATA_NAMED_NODE); + else + return MaybeNextBitCode.takeError(); // Read named metadata elements. unsigned Size = Record.size(); @@ -776,9 +822,14 @@ MetadataLoader::MetadataLoaderImpl::lazyLoadModuleMetadataBlock() { case bitc::METADATA_GLOBAL_DECL_ATTACHMENT: { // FIXME: we need to do this early because we don't materialize global // value explicitly. - IndexCursor.JumpToBit(CurrentPos); + if (Error Err = IndexCursor.JumpToBit(CurrentPos)) + return std::move(Err); Record.clear(); - IndexCursor.readRecord(Entry.ID, Record); + if (Expected MaybeRecord = + IndexCursor.readRecord(Entry.ID, Record)) + ; + else + return MaybeRecord.takeError(); if (Record.size() % 2 == 0) return error("Invalid record"); unsigned ValueID = Record[0]; @@ -812,6 +863,7 @@ MetadataLoader::MetadataLoaderImpl::lazyLoadModuleMetadataBlock() { case bitc::METADATA_LEXICAL_BLOCK: case bitc::METADATA_LEXICAL_BLOCK_FILE: case bitc::METADATA_NAMESPACE: + case bitc::METADATA_COMMON_BLOCK: case bitc::METADATA_MACRO: case bitc::METADATA_MACRO_FILE: case bitc::METADATA_TEMPLATE_TYPE: @@ -845,8 +897,8 @@ Error MetadataLoader::MetadataLoaderImpl::parseMetadata(bool ModuleLevel) { // skip the whole block in case we lazy-load. auto EntryPos = Stream.GetCurrentBitNo(); - if (Stream.EnterSubBlock(bitc::METADATA_BLOCK_ID)) - return error("Invalid record"); + if (Error Err = Stream.EnterSubBlock(bitc::METADATA_BLOCK_ID)) + return Err; SmallVector Record; PlaceholderQueue Placeholders; @@ -871,9 +923,14 @@ Error MetadataLoader::MetadataLoaderImpl::parseMetadata(bool ModuleLevel) { // Return at the beginning of the block, since it is easy to skip it // entirely from there. Stream.ReadBlockEnd(); // Pop the abbrev block context. - Stream.JumpToBit(EntryPos); - if (Stream.SkipBlock()) - return error("Invalid record"); + if (Error Err = IndexCursor.JumpToBit(EntryPos)) + return Err; + if (Error Err = Stream.SkipBlock()) { + // FIXME this drops the error on the floor, which + // ThinLTO/X86/debuginfo-cu-import.ll relies on. + consumeError(std::move(Err)); + return Error::success(); + } return Error::success(); } // Couldn't load an index, fallback to loading all the block "old-style". @@ -883,7 +940,10 @@ Error MetadataLoader::MetadataLoaderImpl::parseMetadata(bool ModuleLevel) { // Read all the records. while (true) { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + Expected MaybeEntry = Stream.advanceSkippingSubblocks(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); switch (Entry.Kind) { case BitstreamEntry::SubBlock: // Handled for us already. @@ -902,10 +962,13 @@ Error MetadataLoader::MetadataLoaderImpl::parseMetadata(bool ModuleLevel) { Record.clear(); StringRef Blob; ++NumMDRecordLoaded; - unsigned Code = Stream.readRecord(Entry.ID, Record, &Blob); - if (Error Err = - parseOneMetadata(Record, Code, Placeholders, Blob, NextMetadataNo)) - return Err; + if (Expected MaybeCode = + Stream.readRecord(Entry.ID, Record, &Blob)) { + if (Error Err = parseOneMetadata(Record, MaybeCode.get(), Placeholders, + Blob, NextMetadataNo)) + return Err; + } else + return MaybeCode.takeError(); } } @@ -930,12 +993,25 @@ void MetadataLoader::MetadataLoaderImpl::lazyLoadOneMetadata( } SmallVector Record; StringRef Blob; - IndexCursor.JumpToBit(GlobalMetadataBitPosIndex[ID - MDStringRef.size()]); - auto Entry = IndexCursor.advanceSkippingSubblocks(); + if (Error Err = IndexCursor.JumpToBit( + GlobalMetadataBitPosIndex[ID - MDStringRef.size()])) + report_fatal_error("lazyLoadOneMetadata failed jumping: " + + toString(std::move(Err))); + Expected MaybeEntry = IndexCursor.advanceSkippingSubblocks(); + if (!MaybeEntry) + // FIXME this drops the error on the floor. + report_fatal_error("lazyLoadOneMetadata failed advanceSkippingSubblocks: " + + toString(MaybeEntry.takeError())); + BitstreamEntry Entry = MaybeEntry.get(); ++NumMDRecordLoaded; - unsigned Code = IndexCursor.readRecord(Entry.ID, Record, &Blob); - if (Error Err = parseOneMetadata(Record, Code, Placeholders, Blob, ID)) - report_fatal_error("Can't lazyload MD"); + if (Expected MaybeCode = + IndexCursor.readRecord(Entry.ID, Record, &Blob)) { + if (Error Err = + parseOneMetadata(Record, MaybeCode.get(), Placeholders, Blob, ID)) + report_fatal_error("Can't lazyload MD, parseOneMetadata: " + + toString(std::move(Err))); + } else + report_fatal_error("Can't lazyload MD: " + toString(MaybeCode.takeError())); } /// Ensure that all forward-references and placeholders are resolved. @@ -1032,12 +1108,17 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( // Read name of the named metadata. SmallString<8> Name(Record.begin(), Record.end()); Record.clear(); - Code = Stream.ReadCode(); + Expected MaybeCode = Stream.ReadCode(); + if (!MaybeCode) + return MaybeCode.takeError(); + Code = MaybeCode.get(); ++NumMDRecordLoaded; - unsigned NextBitCode = Stream.readRecord(Code, Record); - if (NextBitCode != bitc::METADATA_NAMED_NODE) - return error("METADATA_NAME not followed by METADATA_NAMED_NODE"); + if (Expected MaybeNextBitCode = Stream.readRecord(Code, Record)) { + if (MaybeNextBitCode.get() != bitc::METADATA_NAMED_NODE) + return error("METADATA_NAME not followed by METADATA_NAMED_NODE"); + } else + return MaybeNextBitCode.takeError(); // Read named metadata elements. unsigned Size = Record.size(); @@ -1407,12 +1488,33 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( return error("Invalid record"); bool HasSPFlags = Record[0] & 4; - DISubprogram::DISPFlags SPFlags = - HasSPFlags - ? static_cast(Record[9]) - : DISubprogram::toSPFlags( - /*IsLocalToUnit=*/Record[7], /*IsDefinition=*/Record[8], - /*IsOptimized=*/Record[14], /*Virtuality=*/Record[11]); + + DINode::DIFlags Flags; + DISubprogram::DISPFlags SPFlags; + if (!HasSPFlags) + Flags = static_cast(Record[11 + 2]); + else { + Flags = static_cast(Record[11]); + SPFlags = static_cast(Record[9]); + } + + // Support for old metadata when + // subprogram specific flags are placed in DIFlags. + const unsigned DIFlagMainSubprogram = 1 << 21; + bool HasOldMainSubprogramFlag = Flags & DIFlagMainSubprogram; + if (HasOldMainSubprogramFlag) + // Remove old DIFlagMainSubprogram from DIFlags. + // Note: This assumes that any future use of bit 21 defaults to it + // being 0. + Flags &= ~static_cast(DIFlagMainSubprogram); + + if (HasOldMainSubprogramFlag && HasSPFlags) + SPFlags |= DISubprogram::SPFlagMainSubprogram; + else if (!HasSPFlags) + SPFlags = DISubprogram::toSPFlags( + /*IsLocalToUnit=*/Record[7], /*IsDefinition=*/Record[8], + /*IsOptimized=*/Record[14], /*Virtuality=*/Record[11], + /*DIFlagMainSubprogram*/HasOldMainSubprogramFlag); // All definitions should be distinct. IsDistinct = (Record[0] & 1) || (SPFlags & DISubprogram::SPFlagDefinition); @@ -1456,7 +1558,7 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( getDITypeRefOrNull(Record[8 + OffsetA]), // containingType Record[10 + OffsetA], // virtualIndex HasThisAdj ? Record[16 + OffsetB] : 0, // thisAdjustment - static_cast(Record[11 + OffsetA]),// flags + Flags, // flags SPFlags, // SPFlags HasUnit ? CUorFn : nullptr, // unit getMDOrNull(Record[13 + OffsetB]), // templateParams @@ -1508,6 +1610,17 @@ Error MetadataLoader::MetadataLoaderImpl::parseOneMetadata( NextMetadataNo++; break; } + case bitc::METADATA_COMMON_BLOCK: { + IsDistinct = Record[0] & 1; + MetadataList.assignValue( + GET_OR_DISTINCT(DICommonBlock, + (Context, getMDOrNull(Record[1]), + getMDOrNull(Record[2]), getMDString(Record[3]), + getMDOrNull(Record[4]), Record[5])), + NextMetadataNo); + NextMetadataNo++; + break; + } case bitc::METADATA_NAMESPACE: { // Newer versions of DINamespace dropped file and line. MDString *Name; @@ -1831,7 +1944,10 @@ Error MetadataLoader::MetadataLoaderImpl::parseMetadataStrings( if (R.AtEndOfStream()) return error("Invalid record: metadata strings bad length"); - unsigned Size = R.ReadVBR(6); + Expected MaybeSize = R.ReadVBR(6); + if (!MaybeSize) + return MaybeSize.takeError(); + uint32_t Size = MaybeSize.get(); if (Strings.size() < Size) return error("Invalid record: metadata strings truncated chars"); @@ -1860,14 +1976,17 @@ Error MetadataLoader::MetadataLoaderImpl::parseGlobalObjectAttachment( /// Parse metadata attachments. Error MetadataLoader::MetadataLoaderImpl::parseMetadataAttachment( Function &F, const SmallVectorImpl &InstructionList) { - if (Stream.EnterSubBlock(bitc::METADATA_ATTACHMENT_ID)) - return error("Invalid record"); + if (Error Err = Stream.EnterSubBlock(bitc::METADATA_ATTACHMENT_ID)) + return Err; SmallVector Record; PlaceholderQueue Placeholders; while (true) { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + Expected MaybeEntry = Stream.advanceSkippingSubblocks(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); switch (Entry.Kind) { case BitstreamEntry::SubBlock: // Handled for us already. @@ -1884,7 +2003,10 @@ Error MetadataLoader::MetadataLoaderImpl::parseMetadataAttachment( // Read a metadata attachment record. Record.clear(); ++NumMDRecordLoaded; - switch (Stream.readRecord(Entry.ID, Record)) { + Expected MaybeRecord = Stream.readRecord(Entry.ID, Record); + if (!MaybeRecord) + return MaybeRecord.takeError(); + switch (MaybeRecord.get()) { default: // Default behavior: ignore. break; case bitc::METADATA_ATTACHMENT: { @@ -1958,14 +2080,17 @@ Error MetadataLoader::MetadataLoaderImpl::parseMetadataKindRecord( /// Parse the metadata kinds out of the METADATA_KIND_BLOCK. Error MetadataLoader::MetadataLoaderImpl::parseMetadataKinds() { - if (Stream.EnterSubBlock(bitc::METADATA_KIND_BLOCK_ID)) - return error("Invalid record"); + if (Error Err = Stream.EnterSubBlock(bitc::METADATA_KIND_BLOCK_ID)) + return Err; SmallVector Record; // Read all the records. while (true) { - BitstreamEntry Entry = Stream.advanceSkippingSubblocks(); + Expected MaybeEntry = Stream.advanceSkippingSubblocks(); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); switch (Entry.Kind) { case BitstreamEntry::SubBlock: // Handled for us already. @@ -1981,8 +2106,10 @@ Error MetadataLoader::MetadataLoaderImpl::parseMetadataKinds() { // Read a record. Record.clear(); ++NumMDRecordLoaded; - unsigned Code = Stream.readRecord(Entry.ID, Record); - switch (Code) { + Expected MaybeCode = Stream.readRecord(Entry.ID, Record); + if (!MaybeCode) + return MaybeCode.takeError(); + switch (MaybeCode.get()) { default: // Default behavior: ignore. break; case bitc::METADATA_KIND: { diff --git a/lib/Bitcode/Reader/MetadataLoader.h b/lib/Bitcode/Reader/MetadataLoader.h index 07a77a086f32..fe2b20273249 100644 --- a/lib/Bitcode/Reader/MetadataLoader.h +++ b/lib/Bitcode/Reader/MetadataLoader.h @@ -1,9 +1,8 @@ //===-- Bitcode/Reader/MetadataLoader.h - Load Metadatas -------*- C++ -*-====// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Bitcode/Reader/ValueList.cpp b/lib/Bitcode/Reader/ValueList.cpp index b3945a37408f..431995fd40ac 100644 --- a/lib/Bitcode/Reader/ValueList.cpp +++ b/lib/Bitcode/Reader/ValueList.cpp @@ -1,9 +1,8 @@ //===- ValueList.cpp - Internal BitcodeReader implementation --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -66,15 +65,18 @@ DEFINE_TRANSPARENT_OPERAND_ACCESSORS(ConstantPlaceHolder, Value) } // end namespace llvm -void BitcodeReaderValueList::assignValue(Value *V, unsigned Idx) { +void BitcodeReaderValueList::assignValue(Value *V, unsigned Idx, Type *FullTy) { if (Idx == size()) { - push_back(V); + push_back(V, FullTy); return; } if (Idx >= size()) resize(Idx + 1); + assert(FullTypes[Idx] == nullptr || FullTypes[Idx] == FullTy); + FullTypes[Idx] = FullTy; + WeakTrackingVH &OldV = ValuePtrs[Idx]; if (!OldV) { OldV = V; @@ -95,6 +97,10 @@ void BitcodeReaderValueList::assignValue(Value *V, unsigned Idx) { } Constant *BitcodeReaderValueList::getConstantFwdRef(unsigned Idx, Type *Ty) { + // Bail out for a clearly invalid value. + if (Idx >= RefsUpperBound) + return nullptr; + if (Idx >= size()) resize(Idx + 1); @@ -110,9 +116,10 @@ Constant *BitcodeReaderValueList::getConstantFwdRef(unsigned Idx, Type *Ty) { return C; } -Value *BitcodeReaderValueList::getValueFwdRef(unsigned Idx, Type *Ty) { - // Bail out for a clearly invalid value. This would make us call resize(0) - if (Idx == std::numeric_limits::max()) +Value *BitcodeReaderValueList::getValueFwdRef(unsigned Idx, Type *Ty, + Type **FullTy) { + // Bail out for a clearly invalid value. + if (Idx >= RefsUpperBound) return nullptr; if (Idx >= size()) @@ -122,6 +129,8 @@ Value *BitcodeReaderValueList::getValueFwdRef(unsigned Idx, Type *Ty) { // If the types don't match, it's invalid. if (Ty && Ty != V->getType()) return nullptr; + if (FullTy) + *FullTy = FullTypes[Idx]; return V; } @@ -181,8 +190,8 @@ void BitcodeReaderValueList::resolveConstantForwardRefs() { NewOp = RealVal; } else { // Otherwise, look up the placeholder in ResolveConstants. - ResolveConstantsTy::iterator It = std::lower_bound( - ResolveConstants.begin(), ResolveConstants.end(), + ResolveConstantsTy::iterator It = llvm::lower_bound( + ResolveConstants, std::pair(cast(*I), 0)); assert(It != ResolveConstants.end() && It->first == *I); NewOp = operator[](It->second); diff --git a/lib/Bitcode/Reader/ValueList.h b/lib/Bitcode/Reader/ValueList.h index 5ad7899347ad..49900498c294 100644 --- a/lib/Bitcode/Reader/ValueList.h +++ b/lib/Bitcode/Reader/ValueList.h @@ -1,9 +1,8 @@ //===-- Bitcode/Reader/ValueList.h - Number values --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -29,6 +28,13 @@ class Value; class BitcodeReaderValueList { std::vector ValuePtrs; + /// Struct containing fully-specified copies of the type of each + /// value. When pointers are opaque, this will be contain non-opaque + /// variants so that restructuring instructions can determine their + /// type correctly even if being loaded from old bitcode where some + /// types are implicit. + std::vector FullTypes; + /// As we resolve forward-referenced constants, we add information about them /// to this vector. This allows us to resolve them in bulk instead of /// resolving each reference at a time. See the code in @@ -40,8 +46,15 @@ class BitcodeReaderValueList { ResolveConstantsTy ResolveConstants; LLVMContext &Context; + /// Maximum number of valid references. Forward references exceeding the + /// maximum must be invalid. + unsigned RefsUpperBound; + public: - BitcodeReaderValueList(LLVMContext &C) : Context(C) {} + BitcodeReaderValueList(LLVMContext &C, size_t RefsUpperBound) + : Context(C), + RefsUpperBound(std::min((size_t)std::numeric_limits::max(), + RefsUpperBound)) {} ~BitcodeReaderValueList() { assert(ResolveConstants.empty() && "Constants not resolved?"); @@ -49,12 +62,19 @@ public: // vector compatibility methods unsigned size() const { return ValuePtrs.size(); } - void resize(unsigned N) { ValuePtrs.resize(N); } - void push_back(Value *V) { ValuePtrs.emplace_back(V); } + void resize(unsigned N) { + ValuePtrs.resize(N); + FullTypes.resize(N); + } + void push_back(Value *V, Type *Ty) { + ValuePtrs.emplace_back(V); + FullTypes.emplace_back(Ty); + } void clear() { assert(ResolveConstants.empty() && "Constants not resolved?"); ValuePtrs.clear(); + FullTypes.clear(); } Value *operator[](unsigned i) const { @@ -63,18 +83,22 @@ public: } Value *back() const { return ValuePtrs.back(); } - void pop_back() { ValuePtrs.pop_back(); } + void pop_back() { + ValuePtrs.pop_back(); + FullTypes.pop_back(); + } bool empty() const { return ValuePtrs.empty(); } void shrinkTo(unsigned N) { assert(N <= size() && "Invalid shrinkTo request!"); ValuePtrs.resize(N); + FullTypes.resize(N); } Constant *getConstantFwdRef(unsigned Idx, Type *Ty); - Value *getValueFwdRef(unsigned Idx, Type *Ty); + Value *getValueFwdRef(unsigned Idx, Type *Ty, Type **FullTy = nullptr); - void assignValue(Value *V, unsigned Idx); + void assignValue(Value *V, unsigned Idx, Type *FullTy); /// Once all constants are read, this method bulk resolves any forward /// references. diff --git a/lib/Bitcode/Writer/BitWriter.cpp b/lib/Bitcode/Writer/BitWriter.cpp index 763cd12aa2d7..76ca89147e52 100644 --- a/lib/Bitcode/Writer/BitWriter.cpp +++ b/lib/Bitcode/Writer/BitWriter.cpp @@ -1,9 +1,8 @@ //===-- BitWriter.cpp -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/Bitcode/Writer/BitcodeWriter.cpp b/lib/Bitcode/Writer/BitcodeWriter.cpp index ba4f932e2e6d..5c7b970a3a75 100644 --- a/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -1,9 +1,8 @@ //===- Bitcode/Writer/BitcodeWriter.cpp - Bitcode Writer ------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,8 +24,8 @@ #include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Triple.h" -#include "llvm/Bitcode/BitCodes.h" -#include "llvm/Bitcode/BitstreamWriter.h" +#include "llvm/Bitstream/BitCodes.h" +#include "llvm/Bitstream/BitstreamWriter.h" #include "llvm/Bitcode/LLVMBitCodes.h" #include "llvm/Config/llvm-config.h" #include "llvm/IR/Attributes.h" @@ -215,7 +214,8 @@ private: const Function &F); void writeModuleLevelReferences(const GlobalVariable &V, SmallVector &NameVals, - unsigned FSModRefsAbbrev); + unsigned FSModRefsAbbrev, + unsigned FSModVTableRefsAbbrev); void assignValueId(GlobalValue::GUID ValGUID) { GUIDToValueIdMap[ValGUID] = ++GlobalValueId; @@ -318,6 +318,8 @@ private: void writeDILexicalBlockFile(const DILexicalBlockFile *N, SmallVectorImpl &Record, unsigned Abbrev); + void writeDICommonBlock(const DICommonBlock *N, + SmallVectorImpl &Record, unsigned Abbrev); void writeDINamespace(const DINamespace *N, SmallVectorImpl &Record, unsigned Abbrev); void writeDIMacro(const DIMacro *N, SmallVectorImpl &Record, @@ -560,6 +562,8 @@ static unsigned getEncodedRMWOperation(AtomicRMWInst::BinOp Op) { case AtomicRMWInst::Min: return bitc::RMW_MIN; case AtomicRMWInst::UMax: return bitc::RMW_UMAX; case AtomicRMWInst::UMin: return bitc::RMW_UMIN; + case AtomicRMWInst::FAdd: return bitc::RMW_FADD; + case AtomicRMWInst::FSub: return bitc::RMW_FSUB; } } @@ -635,6 +639,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { return bitc::ATTR_KIND_NO_CAPTURE; case Attribute::NoDuplicate: return bitc::ATTR_KIND_NO_DUPLICATE; + case Attribute::NoFree: + return bitc::ATTR_KIND_NOFREE; case Attribute::NoImplicitFloat: return bitc::ATTR_KIND_NO_IMPLICIT_FLOAT; case Attribute::NoInline: @@ -653,6 +659,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { return bitc::ATTR_KIND_NO_RED_ZONE; case Attribute::NoReturn: return bitc::ATTR_KIND_NO_RETURN; + case Attribute::NoSync: + return bitc::ATTR_KIND_NOSYNC; case Attribute::NoCfCheck: return bitc::ATTR_KIND_NOCF_CHECK; case Attribute::NoUnwind: @@ -707,10 +715,16 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { return bitc::ATTR_KIND_SWIFT_SELF; case Attribute::UWTable: return bitc::ATTR_KIND_UW_TABLE; + case Attribute::WillReturn: + return bitc::ATTR_KIND_WILLRETURN; case Attribute::WriteOnly: return bitc::ATTR_KIND_WRITEONLY; case Attribute::ZExt: return bitc::ATTR_KIND_Z_EXT; + case Attribute::ImmArg: + return bitc::ATTR_KIND_IMMARG; + case Attribute::SanitizeMemTag: + return bitc::ATTR_KIND_SANITIZE_MEMTAG; case Attribute::EndAttrKinds: llvm_unreachable("Can not encode end-attribute kinds marker."); case Attribute::None: @@ -742,7 +756,7 @@ void ModuleBitcodeWriter::writeAttributeGroupTable() { Record.push_back(1); Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum())); Record.push_back(Attr.getValueAsInt()); - } else { + } else if (Attr.isStringAttribute()) { StringRef Kind = Attr.getKindAsString(); StringRef Val = Attr.getValueAsString(); @@ -753,6 +767,13 @@ void ModuleBitcodeWriter::writeAttributeGroupTable() { Record.append(Val.begin(), Val.end()); Record.push_back(0); } + } else { + assert(Attr.isTypeAttribute()); + Type *Ty = Attr.getValueAsType(); + Record.push_back(Ty ? 6 : 5); + Record.push_back(getAttrKindEncoding(Attr.getKindAsEnum())); + if (Ty) + Record.push_back(VE.getTypeID(Attr.getValueAsType())); } } @@ -926,10 +947,13 @@ void ModuleBitcodeWriter::writeTypeTable() { } case Type::VectorTyID: { VectorType *VT = cast(T); - // VECTOR [numelts, eltty] + // VECTOR [numelts, eltty] or + // [numelts, eltty, scalable] Code = bitc::TYPE_CODE_VECTOR; TypeVals.push_back(VT->getNumElements()); TypeVals.push_back(VE.getTypeID(VT->getElementType())); + if (VT->isScalable()) + TypeVals.push_back(VT->isScalable()); break; } } @@ -991,6 +1015,7 @@ static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags) { RawFlags |= Flags.NotEligibleToImport; // bool RawFlags |= (Flags.Live << 1); RawFlags |= (Flags.DSOLocal << 2); + RawFlags |= (Flags.CanAutoHide << 3); // Linkage don't need to be remapped at that time for the summary. Any future // change to the getEncodedLinkage() function will need to be taken into @@ -1001,7 +1026,7 @@ static uint64_t getEncodedGVSummaryFlags(GlobalValueSummary::GVFlags Flags) { } static uint64_t getEncodedGVarFlags(GlobalVarSummary::GVarFlags Flags) { - uint64_t RawFlags = Flags.ReadOnly; + uint64_t RawFlags = Flags.MaybeReadOnly | (Flags.MaybeWriteOnly << 1); return RawFlags; } @@ -1256,7 +1281,8 @@ void ModuleBitcodeWriter::writeModuleInfo() { GV.getDLLStorageClass() != GlobalValue::DefaultStorageClass || GV.hasComdat() || GV.hasAttributes() || - GV.isDSOLocal()) { + GV.isDSOLocal() || + GV.hasPartition()) { Vals.push_back(getEncodedVisibility(GV)); Vals.push_back(getEncodedThreadLocalMode(GV)); Vals.push_back(getEncodedUnnamedAddr(GV)); @@ -1268,6 +1294,8 @@ void ModuleBitcodeWriter::writeModuleInfo() { Vals.push_back(VE.getAttributeListID(AL)); Vals.push_back(GV.isDSOLocal()); + Vals.push_back(addToStrtab(GV.getPartition())); + Vals.push_back(GV.getPartition().size()); } else { AbbrevToUse = SimpleGVarAbbrev; } @@ -1305,6 +1333,8 @@ void ModuleBitcodeWriter::writeModuleInfo() { Vals.push_back(F.isDSOLocal()); Vals.push_back(F.getAddressSpace()); + Vals.push_back(addToStrtab(F.getPartition())); + Vals.push_back(F.getPartition().size()); unsigned AbbrevToUse = 0; Stream.EmitRecord(bitc::MODULE_CODE_FUNCTION, Vals, AbbrevToUse); @@ -1327,6 +1357,8 @@ void ModuleBitcodeWriter::writeModuleInfo() { Vals.push_back(getEncodedThreadLocalMode(A)); Vals.push_back(getEncodedUnnamedAddr(A)); Vals.push_back(A.isDSOLocal()); + Vals.push_back(addToStrtab(A.getPartition())); + Vals.push_back(A.getPartition().size()); unsigned AbbrevToUse = 0; Stream.EmitRecord(bitc::MODULE_CODE_ALIAS, Vals, AbbrevToUse); @@ -1345,6 +1377,8 @@ void ModuleBitcodeWriter::writeModuleInfo() { Vals.push_back(getEncodedLinkage(I)); Vals.push_back(getEncodedVisibility(I)); Vals.push_back(I.isDSOLocal()); + Vals.push_back(addToStrtab(I.getPartition())); + Vals.push_back(I.getPartition().size()); Stream.EmitRecord(bitc::MODULE_CODE_IFUNC, Vals); Vals.clear(); } @@ -1683,6 +1717,20 @@ void ModuleBitcodeWriter::writeDILexicalBlockFile( Record.clear(); } +void ModuleBitcodeWriter::writeDICommonBlock(const DICommonBlock *N, + SmallVectorImpl &Record, + unsigned Abbrev) { + Record.push_back(N->isDistinct()); + Record.push_back(VE.getMetadataOrNullID(N->getScope())); + Record.push_back(VE.getMetadataOrNullID(N->getDecl())); + Record.push_back(VE.getMetadataOrNullID(N->getRawName())); + Record.push_back(VE.getMetadataOrNullID(N->getFile())); + Record.push_back(N->getLineNo()); + + Stream.EmitRecord(bitc::METADATA_COMMON_BLOCK, Record, Abbrev); + Record.clear(); +} + void ModuleBitcodeWriter::writeDINamespace(const DINamespace *N, SmallVectorImpl &Record, unsigned Abbrev) { @@ -2616,12 +2664,16 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, Vals.append(IVI->idx_begin(), IVI->idx_end()); break; } - case Instruction::Select: + case Instruction::Select: { Code = bitc::FUNC_CODE_INST_VSELECT; pushValueAndType(I.getOperand(1), InstID, Vals); pushValue(I.getOperand(2), InstID, Vals); pushValueAndType(I.getOperand(0), InstID, Vals); + uint64_t Flags = getOptimizationFlags(&I); + if (Flags != 0) + Vals.push_back(Flags); break; + } case Instruction::ExtractElement: Code = bitc::FUNC_CODE_INST_EXTRACTELT; pushValueAndType(I.getOperand(0), InstID, Vals); @@ -2776,6 +2828,41 @@ void ModuleBitcodeWriter::writeInstruction(const Instruction &I, Vals.push_back(VE.getValueID(CatchSwitch.getUnwindDest())); break; } + case Instruction::CallBr: { + const CallBrInst *CBI = cast(&I); + const Value *Callee = CBI->getCalledValue(); + FunctionType *FTy = CBI->getFunctionType(); + + if (CBI->hasOperandBundles()) + writeOperandBundles(CBI, InstID); + + Code = bitc::FUNC_CODE_INST_CALLBR; + + Vals.push_back(VE.getAttributeListID(CBI->getAttributes())); + + Vals.push_back(CBI->getCallingConv() << bitc::CALL_CCONV | + 1 << bitc::CALL_EXPLICIT_TYPE); + + Vals.push_back(VE.getValueID(CBI->getDefaultDest())); + Vals.push_back(CBI->getNumIndirectDests()); + for (unsigned i = 0, e = CBI->getNumIndirectDests(); i != e; ++i) + Vals.push_back(VE.getValueID(CBI->getIndirectDest(i))); + + Vals.push_back(VE.getTypeID(FTy)); + pushValueAndType(Callee, InstID, Vals); + + // Emit value #'s for the fixed parameters. + for (unsigned i = 0, e = FTy->getNumParams(); i != e; ++i) + pushValue(I.getOperand(i), InstID, Vals); // fixed param. + + // Emit type/value pairs for varargs params. + if (FTy->isVarArg()) { + for (unsigned i = FTy->getNumParams(), e = CBI->getNumArgOperands(); + i != e; ++i) + pushValueAndType(I.getOperand(i), InstID, Vals); // vararg + } + break; + } case Instruction::Unreachable: Code = bitc::FUNC_CODE_INST_UNREACHABLE; AbbrevToUse = FUNCTION_INST_UNREACHABLE_ABBREV; @@ -3528,6 +3615,19 @@ static void writeTypeIdSummaryRecord(SmallVector &NameVals, W.second); } +static void writeTypeIdCompatibleVtableSummaryRecord( + SmallVector &NameVals, StringTableBuilder &StrtabBuilder, + const std::string &Id, const TypeIdCompatibleVtableInfo &Summary, + ValueEnumerator &VE) { + NameVals.push_back(StrtabBuilder.add(Id)); + NameVals.push_back(Id.size()); + + for (auto &P : Summary) { + NameVals.push_back(P.AddressPointOffset); + NameVals.push_back(VE.getValueID(P.VTableVI.getValue())); + } +} + // Helper to emit a single function summary record. void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord( SmallVector &NameVals, GlobalValueSummary *Summary, @@ -3538,11 +3638,13 @@ void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord( FunctionSummary *FS = cast(Summary); writeFunctionTypeMetadataRecords(Stream, FS); + auto SpecialRefCnts = FS->specialRefCounts(); NameVals.push_back(getEncodedGVSummaryFlags(FS->flags())); NameVals.push_back(FS->instCount()); NameVals.push_back(getEncodedFFlags(FS->fflags())); NameVals.push_back(FS->refs().size()); - NameVals.push_back(FS->immutableRefCount()); + NameVals.push_back(SpecialRefCnts.first); // rorefcnt + NameVals.push_back(SpecialRefCnts.second); // worefcnt for (auto &RI : FS->refs()) NameVals.push_back(VE.getValueID(RI.getValue())); @@ -3572,7 +3674,7 @@ void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord( // and emit them in a summary record. void ModuleBitcodeWriterBase::writeModuleLevelReferences( const GlobalVariable &V, SmallVector &NameVals, - unsigned FSModRefsAbbrev) { + unsigned FSModRefsAbbrev, unsigned FSModVTableRefsAbbrev) { auto VI = Index->getValueInfo(V.getGUID()); if (!VI || VI.getSummaryList().empty()) { // Only declarations should not have a summary (a declaration might however @@ -3586,6 +3688,10 @@ void ModuleBitcodeWriterBase::writeModuleLevelReferences( NameVals.push_back(getEncodedGVSummaryFlags(VS->flags())); NameVals.push_back(getEncodedGVarFlags(VS->varflags())); + auto VTableFuncs = VS->vTableFuncs(); + if (!VTableFuncs.empty()) + NameVals.push_back(VS->refs().size()); + unsigned SizeBeforeRefs = NameVals.size(); for (auto &RI : VS->refs()) NameVals.push_back(VE.getValueID(RI.getValue())); @@ -3593,15 +3699,26 @@ void ModuleBitcodeWriterBase::writeModuleLevelReferences( // been initialized from a DenseSet. llvm::sort(NameVals.begin() + SizeBeforeRefs, NameVals.end()); - Stream.EmitRecord(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS, NameVals, - FSModRefsAbbrev); + if (VTableFuncs.empty()) + Stream.EmitRecord(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS, NameVals, + FSModRefsAbbrev); + else { + // VTableFuncs pairs should already be sorted by offset. + for (auto &P : VTableFuncs) { + NameVals.push_back(VE.getValueID(P.FuncVI.getValue())); + NameVals.push_back(P.VTableOffset); + } + + Stream.EmitRecord(bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS, NameVals, + FSModVTableRefsAbbrev); + } NameVals.clear(); } // Current version for the summary. // This is bumped whenever we introduce changes in the way some record are // interpreted, like flags for instance. -static const uint64_t INDEX_VERSION = 6; +static const uint64_t INDEX_VERSION = 7; /// Emit the per-module summary section alongside the rest of /// the module's bitcode. @@ -3643,7 +3760,8 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // immutablerefcnt + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt // numrefs x valueid, n x (valueid, hotness) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); @@ -3660,7 +3778,8 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // immutablerefcnt + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt // numrefs x valueid, n x (valueid [, rel_block_freq]) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); @@ -3675,6 +3794,17 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); unsigned FSModRefsAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + // Abbrev for FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS. + Abbv = std::make_shared(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs + // numrefs x valueid, n x (valueid , offset) + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned FSModVTableRefsAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + // Abbrev for FS_ALIAS. Abbv = std::make_shared(); Abbv->Add(BitCodeAbbrevOp(bitc::FS_ALIAS)); @@ -3683,6 +3813,16 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid unsigned FSAliasAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + // Abbrev for FS_TYPE_ID_METADATA + Abbv = std::make_shared(); + Abbv->Add(BitCodeAbbrevOp(bitc::FS_TYPE_ID_METADATA)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid strtab index + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // typeid length + // n x (valueid , offset) + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); + unsigned TypeIdCompatibleVtableAbbrev = Stream.EmitAbbrev(std::move(Abbv)); + SmallVector NameVals; // Iterate over the list of functions instead of the Index to // ensure the ordering is stable. @@ -3707,7 +3847,8 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() { // Capture references from GlobalVariable initializers, which are outside // of a function scope. for (const GlobalVariable &G : M.globals()) - writeModuleLevelReferences(G, NameVals, FSModRefsAbbrev); + writeModuleLevelReferences(G, NameVals, FSModRefsAbbrev, + FSModVTableRefsAbbrev); for (const GlobalAlias &A : M.aliases()) { auto *Aliasee = A.getBaseObject(); @@ -3725,6 +3866,14 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() { NameVals.clear(); } + for (auto &S : Index->typeIdCompatibleVtableMap()) { + writeTypeIdCompatibleVtableSummaryRecord(NameVals, StrtabBuilder, S.first, + S.second, VE); + Stream.EmitRecord(bitc::FS_TYPE_ID_METADATA, NameVals, + TypeIdCompatibleVtableAbbrev); + NameVals.clear(); + } + Stream.ExitBlock(); } @@ -3762,7 +3911,8 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // entrycount Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // immutablerefcnt + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt // numrefs x valueid, n x (valueid) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); @@ -3776,8 +3926,10 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // instcount Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // fflags + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // entrycount Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs - Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // immutablerefcnt + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // rorefcnt + Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // worefcnt // numrefs x valueid, n x (valueid, hotness) Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); @@ -3825,9 +3977,13 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { NameVals.clear(); }; + std::set DefOrUseGUIDs; forEachSummary([&](GVInfo I, bool IsAliasee) { GlobalValueSummary *S = I.second; assert(S); + DefOrUseGUIDs.insert(I.first); + for (const ValueInfo &VI : S->refs()) + DefOrUseGUIDs.insert(VI.getGUID()); auto ValueId = getValueId(I.first); assert(ValueId); @@ -3879,20 +4035,24 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { // Fill in below NameVals.push_back(0); // numrefs - NameVals.push_back(0); // immutablerefcnt + NameVals.push_back(0); // rorefcnt + NameVals.push_back(0); // worefcnt - unsigned Count = 0, ImmutableRefCnt = 0; + unsigned Count = 0, RORefCnt = 0, WORefCnt = 0; for (auto &RI : FS->refs()) { auto RefValueId = getValueId(RI.getGUID()); if (!RefValueId) continue; NameVals.push_back(*RefValueId); if (RI.isReadOnly()) - ImmutableRefCnt++; + RORefCnt++; + else if (RI.isWriteOnly()) + WORefCnt++; Count++; } NameVals[6] = Count; - NameVals[7] = ImmutableRefCnt; + NameVals[7] = RORefCnt; + NameVals[8] = WORefCnt; bool HasProfileData = false; for (auto &EI : FS->calls()) { @@ -3968,20 +4128,30 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() { if (!Index.cfiFunctionDefs().empty()) { for (auto &S : Index.cfiFunctionDefs()) { - NameVals.push_back(StrtabBuilder.add(S)); - NameVals.push_back(S.size()); + if (DefOrUseGUIDs.count( + GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(S)))) { + NameVals.push_back(StrtabBuilder.add(S)); + NameVals.push_back(S.size()); + } + } + if (!NameVals.empty()) { + Stream.EmitRecord(bitc::FS_CFI_FUNCTION_DEFS, NameVals); + NameVals.clear(); } - Stream.EmitRecord(bitc::FS_CFI_FUNCTION_DEFS, NameVals); - NameVals.clear(); } if (!Index.cfiFunctionDecls().empty()) { for (auto &S : Index.cfiFunctionDecls()) { - NameVals.push_back(StrtabBuilder.add(S)); - NameVals.push_back(S.size()); + if (DefOrUseGUIDs.count( + GlobalValue::getGUID(GlobalValue::dropLLVMManglingEscape(S)))) { + NameVals.push_back(StrtabBuilder.add(S)); + NameVals.push_back(S.size()); + } + } + if (!NameVals.empty()) { + Stream.EmitRecord(bitc::FS_CFI_FUNCTION_DECLS, NameVals); + NameVals.clear(); } - Stream.EmitRecord(bitc::FS_CFI_FUNCTION_DECLS, NameVals); - NameVals.clear(); } // Walk the GUIDs that were referenced, and write the @@ -4055,15 +4225,15 @@ void ModuleBitcodeWriter::write() { // Emit blockinfo, which defines the standard abbreviations etc. writeBlockInfo(); + // Emit information describing all of the types in the module. + writeTypeTable(); + // Emit information about attribute groups. writeAttributeGroupTable(); // Emit information about parameter attributes. writeAttributeTable(); - // Emit information describing all of the types in the module. - writeTypeTable(); - writeComdats(); // Emit top-level description of module, including target triple, inline asm, diff --git a/lib/Bitcode/Writer/BitcodeWriterPass.cpp b/lib/Bitcode/Writer/BitcodeWriterPass.cpp index 41212e575f8e..6796cf8cee54 100644 --- a/lib/Bitcode/Writer/BitcodeWriterPass.cpp +++ b/lib/Bitcode/Writer/BitcodeWriterPass.cpp @@ -1,9 +1,8 @@ //===- BitcodeWriterPass.cpp - Bitcode writing pass -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Bitcode/Writer/ValueEnumerator.cpp b/lib/Bitcode/Writer/ValueEnumerator.cpp index deb04f1bb36c..f59c906c7b75 100644 --- a/lib/Bitcode/Writer/ValueEnumerator.cpp +++ b/lib/Bitcode/Writer/ValueEnumerator.cpp @@ -1,9 +1,8 @@ //===- ValueEnumerator.cpp - Number values and types for bitcode writer ---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -415,10 +414,8 @@ ValueEnumerator::ValueEnumerator(const Module &M, EnumerateMetadata(&F, MD->getMetadata()); } EnumerateType(I.getType()); - if (const CallInst *CI = dyn_cast(&I)) - EnumerateAttributes(CI->getAttributes()); - else if (const InvokeInst *II = dyn_cast(&I)) - EnumerateAttributes(II->getAttributes()); + if (const auto *Call = dyn_cast(&I)) + EnumerateAttributes(Call->getAttributes()); // Enumerate metadata attached with this instruction. MDs.clear(); @@ -752,7 +749,8 @@ void ValueEnumerator::organizeMetadata() { // Rebuild MDs, index the metadata ranges for each function in FunctionMDs, // and fix up MetadataMap. - std::vector OldMDs = std::move(MDs); + std::vector OldMDs; + MDs.swap(OldMDs); MDs.reserve(OldMDs.size()); for (unsigned I = 0, E = Order.size(); I != E && !Order[I].F; ++I) { auto *MD = Order[I].get(OldMDs); @@ -951,9 +949,11 @@ void ValueEnumerator::incorporateFunction(const Function &F) { incorporateFunctionMetadata(F); // Adding function arguments to the value table. - for (const auto &I : F.args()) + for (const auto &I : F.args()) { EnumerateValue(&I); - + if (I.hasAttribute(Attribute::ByVal)) + EnumerateType(I.getParamByValType()); + } FirstFuncConstantID = Values.size(); // Add all function-level constants to the value table. diff --git a/lib/Bitcode/Writer/ValueEnumerator.h b/lib/Bitcode/Writer/ValueEnumerator.h index 011356c32601..112f0b4a1dc4 100644 --- a/lib/Bitcode/Writer/ValueEnumerator.h +++ b/lib/Bitcode/Writer/ValueEnumerator.h @@ -1,9 +1,8 @@ //===- Bitcode/Writer/ValueEnumerator.h - Number values ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/Bitstream/Reader/BitstreamReader.cpp b/lib/Bitstream/Reader/BitstreamReader.cpp new file mode 100644 index 000000000000..a4a97ced5457 --- /dev/null +++ b/lib/Bitstream/Reader/BitstreamReader.cpp @@ -0,0 +1,510 @@ +//===- BitstreamReader.cpp - BitstreamReader implementation ---------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Bitstream/BitstreamReader.h" +#include "llvm/ADT/StringRef.h" +#include +#include + +using namespace llvm; + +//===----------------------------------------------------------------------===// +// BitstreamCursor implementation +//===----------------------------------------------------------------------===// + +/// Having read the ENTER_SUBBLOCK abbrevid, enter the block. +Error BitstreamCursor::EnterSubBlock(unsigned BlockID, unsigned *NumWordsP) { + // Save the current block's state on BlockScope. + BlockScope.push_back(Block(CurCodeSize)); + BlockScope.back().PrevAbbrevs.swap(CurAbbrevs); + + // Add the abbrevs specific to this block to the CurAbbrevs list. + if (BlockInfo) { + if (const BitstreamBlockInfo::BlockInfo *Info = + BlockInfo->getBlockInfo(BlockID)) { + CurAbbrevs.insert(CurAbbrevs.end(), Info->Abbrevs.begin(), + Info->Abbrevs.end()); + } + } + + // Get the codesize of this block. + Expected MaybeVBR = ReadVBR(bitc::CodeLenWidth); + if (!MaybeVBR) + return MaybeVBR.takeError(); + CurCodeSize = MaybeVBR.get(); + + if (CurCodeSize > MaxChunkSize) + return llvm::createStringError( + std::errc::illegal_byte_sequence, + "can't read more than %zu at a time, trying to read %u", +MaxChunkSize, + CurCodeSize); + + SkipToFourByteBoundary(); + Expected MaybeNum = Read(bitc::BlockSizeWidth); + if (!MaybeNum) + return MaybeNum.takeError(); + word_t NumWords = MaybeNum.get(); + if (NumWordsP) + *NumWordsP = NumWords; + + if (CurCodeSize == 0) + return llvm::createStringError( + std::errc::illegal_byte_sequence, + "can't enter sub-block: current code size is 0"); + if (AtEndOfStream()) + return llvm::createStringError( + std::errc::illegal_byte_sequence, + "can't enter sub block: already at end of stream"); + + return Error::success(); +} + +static Expected readAbbreviatedField(BitstreamCursor &Cursor, + const BitCodeAbbrevOp &Op) { + assert(!Op.isLiteral() && "Not to be used with literals!"); + + // Decode the value as we are commanded. + switch (Op.getEncoding()) { + case BitCodeAbbrevOp::Array: + case BitCodeAbbrevOp::Blob: + llvm_unreachable("Should not reach here"); + case BitCodeAbbrevOp::Fixed: + assert((unsigned)Op.getEncodingData() <= Cursor.MaxChunkSize); + return Cursor.Read((unsigned)Op.getEncodingData()); + case BitCodeAbbrevOp::VBR: + assert((unsigned)Op.getEncodingData() <= Cursor.MaxChunkSize); + return Cursor.ReadVBR64((unsigned)Op.getEncodingData()); + case BitCodeAbbrevOp::Char6: + if (Expected Res = Cursor.Read(6)) + return BitCodeAbbrevOp::DecodeChar6(Res.get()); + else + return Res.takeError(); + } + llvm_unreachable("invalid abbreviation encoding"); +} + +static Error skipAbbreviatedField(BitstreamCursor &Cursor, + const BitCodeAbbrevOp &Op) { + assert(!Op.isLiteral() && "Not to be used with literals!"); + + // Decode the value as we are commanded. + switch (Op.getEncoding()) { + case BitCodeAbbrevOp::Array: + case BitCodeAbbrevOp::Blob: + llvm_unreachable("Should not reach here"); + case BitCodeAbbrevOp::Fixed: + assert((unsigned)Op.getEncodingData() <= Cursor.MaxChunkSize); + if (Expected Res = Cursor.Read((unsigned)Op.getEncodingData())) + break; + else + return Res.takeError(); + case BitCodeAbbrevOp::VBR: + assert((unsigned)Op.getEncodingData() <= Cursor.MaxChunkSize); + if (Expected Res = + Cursor.ReadVBR64((unsigned)Op.getEncodingData())) + break; + else + return Res.takeError(); + case BitCodeAbbrevOp::Char6: + if (Expected Res = Cursor.Read(6)) + break; + else + return Res.takeError(); + } + return ErrorSuccess(); +} + +/// skipRecord - Read the current record and discard it. +Expected BitstreamCursor::skipRecord(unsigned AbbrevID) { + // Skip unabbreviated records by reading past their entries. + if (AbbrevID == bitc::UNABBREV_RECORD) { + Expected MaybeCode = ReadVBR(6); + if (!MaybeCode) + return MaybeCode.takeError(); + unsigned Code = MaybeCode.get(); + Expected MaybeVBR = ReadVBR(6); + if (!MaybeVBR) + return MaybeVBR.get(); + unsigned NumElts = MaybeVBR.get(); + for (unsigned i = 0; i != NumElts; ++i) + if (Expected Res = ReadVBR64(6)) + ; // Skip! + else + return Res.takeError(); + return Code; + } + + const BitCodeAbbrev *Abbv = getAbbrev(AbbrevID); + const BitCodeAbbrevOp &CodeOp = Abbv->getOperandInfo(0); + unsigned Code; + if (CodeOp.isLiteral()) + Code = CodeOp.getLiteralValue(); + else { + if (CodeOp.getEncoding() == BitCodeAbbrevOp::Array || + CodeOp.getEncoding() == BitCodeAbbrevOp::Blob) + return llvm::createStringError( + std::errc::illegal_byte_sequence, + "Abbreviation starts with an Array or a Blob"); + Expected MaybeCode = readAbbreviatedField(*this, CodeOp); + if (!MaybeCode) + return MaybeCode.takeError(); + Code = MaybeCode.get(); + } + + for (unsigned i = 1, e = Abbv->getNumOperandInfos(); i < e; ++i) { + const BitCodeAbbrevOp &Op = Abbv->getOperandInfo(i); + if (Op.isLiteral()) + continue; + + if (Op.getEncoding() != BitCodeAbbrevOp::Array && + Op.getEncoding() != BitCodeAbbrevOp::Blob) { + if (Error Err = skipAbbreviatedField(*this, Op)) + return std::move(Err); + continue; + } + + if (Op.getEncoding() == BitCodeAbbrevOp::Array) { + // Array case. Read the number of elements as a vbr6. + Expected MaybeNum = ReadVBR(6); + if (!MaybeNum) + return MaybeNum.takeError(); + unsigned NumElts = MaybeNum.get(); + + // Get the element encoding. + assert(i+2 == e && "array op not second to last?"); + const BitCodeAbbrevOp &EltEnc = Abbv->getOperandInfo(++i); + + // Read all the elements. + // Decode the value as we are commanded. + switch (EltEnc.getEncoding()) { + default: + report_fatal_error("Array element type can't be an Array or a Blob"); + case BitCodeAbbrevOp::Fixed: + assert((unsigned)EltEnc.getEncodingData() <= MaxChunkSize); + if (Error Err = JumpToBit(GetCurrentBitNo() + + NumElts * EltEnc.getEncodingData())) + return std::move(Err); + break; + case BitCodeAbbrevOp::VBR: + assert((unsigned)EltEnc.getEncodingData() <= MaxChunkSize); + for (; NumElts; --NumElts) + if (Expected Res = + ReadVBR64((unsigned)EltEnc.getEncodingData())) + ; // Skip! + else + return Res.takeError(); + break; + case BitCodeAbbrevOp::Char6: + if (Error Err = JumpToBit(GetCurrentBitNo() + NumElts * 6)) + return std::move(Err); + break; + } + continue; + } + + assert(Op.getEncoding() == BitCodeAbbrevOp::Blob); + // Blob case. Read the number of bytes as a vbr6. + Expected MaybeNum = ReadVBR(6); + if (!MaybeNum) + return MaybeNum.takeError(); + unsigned NumElts = MaybeNum.get(); + SkipToFourByteBoundary(); // 32-bit alignment + + // Figure out where the end of this blob will be including tail padding. + size_t NewEnd = GetCurrentBitNo()+((NumElts+3)&~3)*8; + + // If this would read off the end of the bitcode file, just set the + // record to empty and return. + if (!canSkipToPos(NewEnd/8)) { + skipToEnd(); + break; + } + + // Skip over the blob. + if (Error Err = JumpToBit(NewEnd)) + return std::move(Err); + } + return Code; +} + +Expected BitstreamCursor::readRecord(unsigned AbbrevID, + SmallVectorImpl &Vals, + StringRef *Blob) { + if (AbbrevID == bitc::UNABBREV_RECORD) { + Expected MaybeCode = ReadVBR(6); + if (!MaybeCode) + return MaybeCode.takeError(); + uint32_t Code = MaybeCode.get(); + Expected MaybeNumElts = ReadVBR(6); + if (!MaybeNumElts) + return MaybeNumElts.takeError(); + uint32_t NumElts = MaybeNumElts.get(); + + for (unsigned i = 0; i != NumElts; ++i) + if (Expected MaybeVal = ReadVBR64(6)) + Vals.push_back(MaybeVal.get()); + else + return MaybeVal.takeError(); + return Code; + } + + const BitCodeAbbrev *Abbv = getAbbrev(AbbrevID); + + // Read the record code first. + assert(Abbv->getNumOperandInfos() != 0 && "no record code in abbreviation?"); + const BitCodeAbbrevOp &CodeOp = Abbv->getOperandInfo(0); + unsigned Code; + if (CodeOp.isLiteral()) + Code = CodeOp.getLiteralValue(); + else { + if (CodeOp.getEncoding() == BitCodeAbbrevOp::Array || + CodeOp.getEncoding() == BitCodeAbbrevOp::Blob) + report_fatal_error("Abbreviation starts with an Array or a Blob"); + if (Expected MaybeCode = readAbbreviatedField(*this, CodeOp)) + Code = MaybeCode.get(); + else + return MaybeCode.takeError(); + } + + for (unsigned i = 1, e = Abbv->getNumOperandInfos(); i != e; ++i) { + const BitCodeAbbrevOp &Op = Abbv->getOperandInfo(i); + if (Op.isLiteral()) { + Vals.push_back(Op.getLiteralValue()); + continue; + } + + if (Op.getEncoding() != BitCodeAbbrevOp::Array && + Op.getEncoding() != BitCodeAbbrevOp::Blob) { + if (Expected MaybeVal = readAbbreviatedField(*this, Op)) + Vals.push_back(MaybeVal.get()); + else + return MaybeVal.takeError(); + continue; + } + + if (Op.getEncoding() == BitCodeAbbrevOp::Array) { + // Array case. Read the number of elements as a vbr6. + Expected MaybeNumElts = ReadVBR(6); + if (!MaybeNumElts) + return MaybeNumElts.takeError(); + uint32_t NumElts = MaybeNumElts.get(); + + // Get the element encoding. + if (i + 2 != e) + report_fatal_error("Array op not second to last"); + const BitCodeAbbrevOp &EltEnc = Abbv->getOperandInfo(++i); + if (!EltEnc.isEncoding()) + report_fatal_error( + "Array element type has to be an encoding of a type"); + + // Read all the elements. + switch (EltEnc.getEncoding()) { + default: + report_fatal_error("Array element type can't be an Array or a Blob"); + case BitCodeAbbrevOp::Fixed: + for (; NumElts; --NumElts) + if (Expected MaybeVal = + Read((unsigned)EltEnc.getEncodingData())) + Vals.push_back(MaybeVal.get()); + else + return MaybeVal.takeError(); + break; + case BitCodeAbbrevOp::VBR: + for (; NumElts; --NumElts) + if (Expected MaybeVal = + ReadVBR64((unsigned)EltEnc.getEncodingData())) + Vals.push_back(MaybeVal.get()); + else + return MaybeVal.takeError(); + break; + case BitCodeAbbrevOp::Char6: + for (; NumElts; --NumElts) + if (Expected MaybeVal = Read(6)) + Vals.push_back(BitCodeAbbrevOp::DecodeChar6(MaybeVal.get())); + else + return MaybeVal.takeError(); + } + continue; + } + + assert(Op.getEncoding() == BitCodeAbbrevOp::Blob); + // Blob case. Read the number of bytes as a vbr6. + Expected MaybeNumElts = ReadVBR(6); + if (!MaybeNumElts) + return MaybeNumElts.takeError(); + uint32_t NumElts = MaybeNumElts.get(); + SkipToFourByteBoundary(); // 32-bit alignment + + // Figure out where the end of this blob will be including tail padding. + size_t CurBitPos = GetCurrentBitNo(); + size_t NewEnd = CurBitPos+((NumElts+3)&~3)*8; + + // If this would read off the end of the bitcode file, just set the + // record to empty and return. + if (!canSkipToPos(NewEnd/8)) { + Vals.append(NumElts, 0); + skipToEnd(); + break; + } + + // Otherwise, inform the streamer that we need these bytes in memory. Skip + // over tail padding first, in case jumping to NewEnd invalidates the Blob + // pointer. + if (Error Err = JumpToBit(NewEnd)) + return std::move(Err); + const char *Ptr = (const char *)getPointerToBit(CurBitPos, NumElts); + + // If we can return a reference to the data, do so to avoid copying it. + if (Blob) { + *Blob = StringRef(Ptr, NumElts); + } else { + // Otherwise, unpack into Vals with zero extension. + for (; NumElts; --NumElts) + Vals.push_back((unsigned char)*Ptr++); + } + } + + return Code; +} + +Error BitstreamCursor::ReadAbbrevRecord() { + auto Abbv = std::make_shared(); + Expected MaybeNumOpInfo = ReadVBR(5); + if (!MaybeNumOpInfo) + return MaybeNumOpInfo.takeError(); + unsigned NumOpInfo = MaybeNumOpInfo.get(); + for (unsigned i = 0; i != NumOpInfo; ++i) { + Expected MaybeIsLiteral = Read(1); + if (!MaybeIsLiteral) + return MaybeIsLiteral.takeError(); + bool IsLiteral = MaybeIsLiteral.get(); + if (IsLiteral) { + Expected MaybeOp = ReadVBR64(8); + if (!MaybeOp) + return MaybeOp.takeError(); + Abbv->Add(BitCodeAbbrevOp(MaybeOp.get())); + continue; + } + + Expected MaybeEncoding = Read(3); + if (!MaybeEncoding) + return MaybeEncoding.takeError(); + BitCodeAbbrevOp::Encoding E = + (BitCodeAbbrevOp::Encoding)MaybeEncoding.get(); + if (BitCodeAbbrevOp::hasEncodingData(E)) { + Expected MaybeData = ReadVBR64(5); + if (!MaybeData) + return MaybeData.takeError(); + uint64_t Data = MaybeData.get(); + + // As a special case, handle fixed(0) (i.e., a fixed field with zero bits) + // and vbr(0) as a literal zero. This is decoded the same way, and avoids + // a slow path in Read() to have to handle reading zero bits. + if ((E == BitCodeAbbrevOp::Fixed || E == BitCodeAbbrevOp::VBR) && + Data == 0) { + Abbv->Add(BitCodeAbbrevOp(0)); + continue; + } + + if ((E == BitCodeAbbrevOp::Fixed || E == BitCodeAbbrevOp::VBR) && + Data > MaxChunkSize) + report_fatal_error( + "Fixed or VBR abbrev record with size > MaxChunkData"); + + Abbv->Add(BitCodeAbbrevOp(E, Data)); + } else + Abbv->Add(BitCodeAbbrevOp(E)); + } + + if (Abbv->getNumOperandInfos() == 0) + report_fatal_error("Abbrev record with no operands"); + CurAbbrevs.push_back(std::move(Abbv)); + + return Error::success(); +} + +Expected> +BitstreamCursor::ReadBlockInfoBlock(bool ReadBlockInfoNames) { + if (llvm::Error Err = EnterSubBlock(bitc::BLOCKINFO_BLOCK_ID)) + return std::move(Err); + + BitstreamBlockInfo NewBlockInfo; + + SmallVector Record; + BitstreamBlockInfo::BlockInfo *CurBlockInfo = nullptr; + + // Read all the records for this module. + while (true) { + Expected MaybeEntry = + advanceSkippingSubblocks(AF_DontAutoprocessAbbrevs); + if (!MaybeEntry) + return MaybeEntry.takeError(); + BitstreamEntry Entry = MaybeEntry.get(); + + switch (Entry.Kind) { + case llvm::BitstreamEntry::SubBlock: // Handled for us already. + case llvm::BitstreamEntry::Error: + return None; + case llvm::BitstreamEntry::EndBlock: + return std::move(NewBlockInfo); + case llvm::BitstreamEntry::Record: + // The interesting case. + break; + } + + // Read abbrev records, associate them with CurBID. + if (Entry.ID == bitc::DEFINE_ABBREV) { + if (!CurBlockInfo) return None; + if (Error Err = ReadAbbrevRecord()) + return std::move(Err); + + // ReadAbbrevRecord installs the abbrev in CurAbbrevs. Move it to the + // appropriate BlockInfo. + CurBlockInfo->Abbrevs.push_back(std::move(CurAbbrevs.back())); + CurAbbrevs.pop_back(); + continue; + } + + // Read a record. + Record.clear(); + Expected MaybeBlockInfo = readRecord(Entry.ID, Record); + if (!MaybeBlockInfo) + return MaybeBlockInfo.takeError(); + switch (MaybeBlockInfo.get()) { + default: + break; // Default behavior, ignore unknown content. + case bitc::BLOCKINFO_CODE_SETBID: + if (Record.size() < 1) + return None; + CurBlockInfo = &NewBlockInfo.getOrCreateBlockInfo((unsigned)Record[0]); + break; + case bitc::BLOCKINFO_CODE_BLOCKNAME: { + if (!CurBlockInfo) + return None; + if (!ReadBlockInfoNames) + break; // Ignore name. + std::string Name; + for (unsigned i = 0, e = Record.size(); i != e; ++i) + Name += (char)Record[i]; + CurBlockInfo->Name = Name; + break; + } + case bitc::BLOCKINFO_CODE_SETRECORDNAME: { + if (!CurBlockInfo) return None; + if (!ReadBlockInfoNames) + break; // Ignore name. + std::string Name; + for (unsigned i = 1, e = Record.size(); i != e; ++i) + Name += (char)Record[i]; + CurBlockInfo->RecordNames.push_back(std::make_pair((unsigned)Record[0], + Name)); + break; + } + } + } +} diff --git a/lib/CodeGen/AggressiveAntiDepBreaker.cpp b/lib/CodeGen/AggressiveAntiDepBreaker.cpp index 632ea8e9cdc4..444f618d8b8c 100644 --- a/lib/CodeGen/AggressiveAntiDepBreaker.cpp +++ b/lib/CodeGen/AggressiveAntiDepBreaker.cpp @@ -1,9 +1,8 @@ //===- AggressiveAntiDepBreaker.cpp - Anti-dep breaker --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/AggressiveAntiDepBreaker.h b/lib/CodeGen/AggressiveAntiDepBreaker.h index 5dce3c2499e5..0cf2e6d78f7f 100644 --- a/lib/CodeGen/AggressiveAntiDepBreaker.h +++ b/lib/CodeGen/AggressiveAntiDepBreaker.h @@ -1,9 +1,8 @@ //==- llvm/CodeGen/AggressiveAntiDepBreaker.h - Anti-Dep Support -*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/AllocationOrder.cpp b/lib/CodeGen/AllocationOrder.cpp index 37dcb0be824e..c99800659bfd 100644 --- a/lib/CodeGen/AllocationOrder.cpp +++ b/lib/CodeGen/AllocationOrder.cpp @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/AllocationOrder.cpp - Allocation Order ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/AllocationOrder.h b/lib/CodeGen/AllocationOrder.h index 467bcc2edc6f..9247dd844936 100644 --- a/lib/CodeGen/AllocationOrder.h +++ b/lib/CodeGen/AllocationOrder.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/AllocationOrder.h - Allocation Order -*- C++ -*-------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/Analysis.cpp b/lib/CodeGen/Analysis.cpp index 797f05ee5cf3..d158e70b86ac 100644 --- a/lib/CodeGen/Analysis.cpp +++ b/lib/CodeGen/Analysis.cpp @@ -1,9 +1,8 @@ //===-- Analysis.cpp - CodeGen LLVM IR Analysis Utilities -----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -83,6 +82,7 @@ unsigned llvm::ComputeLinearIndex(Type *Ty, /// void llvm::ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *Ty, SmallVectorImpl &ValueVTs, + SmallVectorImpl *MemVTs, SmallVectorImpl *Offsets, uint64_t StartingOffset) { // Given a struct type, recursively traverse the elements. @@ -92,7 +92,7 @@ void llvm::ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, EI = EB, EE = STy->element_end(); EI != EE; ++EI) - ComputeValueVTs(TLI, DL, *EI, ValueVTs, Offsets, + ComputeValueVTs(TLI, DL, *EI, ValueVTs, MemVTs, Offsets, StartingOffset + SL->getElementOffset(EI - EB)); return; } @@ -101,7 +101,7 @@ void llvm::ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, Type *EltTy = ATy->getElementType(); uint64_t EltSize = DL.getTypeAllocSize(EltTy); for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) - ComputeValueVTs(TLI, DL, EltTy, ValueVTs, Offsets, + ComputeValueVTs(TLI, DL, EltTy, ValueVTs, MemVTs, Offsets, StartingOffset + i * EltSize); return; } @@ -110,10 +110,50 @@ void llvm::ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, return; // Base case: we can get an EVT for this LLVM IR type. ValueVTs.push_back(TLI.getValueType(DL, Ty)); + if (MemVTs) + MemVTs->push_back(TLI.getMemValueType(DL, Ty)); if (Offsets) Offsets->push_back(StartingOffset); } +void llvm::ComputeValueVTs(const TargetLowering &TLI, const DataLayout &DL, + Type *Ty, SmallVectorImpl &ValueVTs, + SmallVectorImpl *Offsets, + uint64_t StartingOffset) { + return ComputeValueVTs(TLI, DL, Ty, ValueVTs, /*MemVTs=*/nullptr, Offsets, + StartingOffset); +} + +void llvm::computeValueLLTs(const DataLayout &DL, Type &Ty, + SmallVectorImpl &ValueTys, + SmallVectorImpl *Offsets, + uint64_t StartingOffset) { + // Given a struct type, recursively traverse the elements. + if (StructType *STy = dyn_cast(&Ty)) { + const StructLayout *SL = DL.getStructLayout(STy); + for (unsigned I = 0, E = STy->getNumElements(); I != E; ++I) + computeValueLLTs(DL, *STy->getElementType(I), ValueTys, Offsets, + StartingOffset + SL->getElementOffset(I)); + return; + } + // Given an array type, recursively traverse the elements. + if (ArrayType *ATy = dyn_cast(&Ty)) { + Type *EltTy = ATy->getElementType(); + uint64_t EltSize = DL.getTypeAllocSize(EltTy); + for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) + computeValueLLTs(DL, *EltTy, ValueTys, Offsets, + StartingOffset + i * EltSize); + return; + } + // Interpret void as zero return values. + if (Ty.isVoidTy()) + return; + // Base case: we can get an LLT for this LLVM IR type. + ValueTys.push_back(getLLTForType(Ty, DL)); + if (Offsets != nullptr) + Offsets->push_back(StartingOffset * 8); +} + /// ExtractTypeInfo - Returns the type info, possibly bitcast, encoded in V. GlobalValue *llvm::ExtractTypeInfo(Value *V) { V = V->stripPointerCasts(); diff --git a/lib/CodeGen/AntiDepBreaker.h b/lib/CodeGen/AntiDepBreaker.h index d93716287981..b11148595136 100644 --- a/lib/CodeGen/AntiDepBreaker.h +++ b/lib/CodeGen/AntiDepBreaker.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/AntiDepBreaker.h - Anti-Dependence Breaking -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/AsmPrinter/ARMException.cpp b/lib/CodeGen/AsmPrinter/ARMException.cpp index 9011f025f595..f6ef85a5b78f 100644 --- a/lib/CodeGen/AsmPrinter/ARMException.cpp +++ b/lib/CodeGen/AsmPrinter/ARMException.cpp @@ -1,9 +1,8 @@ //===-- CodeGen/AsmPrinter/ARMException.cpp - ARM EHABI Exception Impl ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/AsmPrinter/AccelTable.cpp b/lib/CodeGen/AsmPrinter/AccelTable.cpp index 95875ccb8a0b..b1b7921ea976 100644 --- a/lib/CodeGen/AsmPrinter/AccelTable.cpp +++ b/lib/CodeGen/AsmPrinter/AccelTable.cpp @@ -1,9 +1,8 @@ //===- llvm/CodeGen/AsmPrinter/AccelTable.cpp - Accelerator Tables --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -56,10 +55,10 @@ void AccelTableBase::finalize(AsmPrinter *Asm, StringRef Prefix) { // Create the individual hash data outputs. for (auto &E : Entries) { // Unique the entries. - std::stable_sort(E.second.Values.begin(), E.second.Values.end(), - [](const AccelTableData *A, const AccelTableData *B) { - return *A < *B; - }); + llvm::stable_sort(E.second.Values, + [](const AccelTableData *A, const AccelTableData *B) { + return *A < *B; + }); E.second.Values.erase( std::unique(E.second.Values.begin(), E.second.Values.end()), E.second.Values.end()); @@ -82,10 +81,9 @@ void AccelTableBase::finalize(AsmPrinter *Asm, StringRef Prefix) { // Sort the contents of the buckets by hash value so that hash collisions end // up together. Stable sort makes testing easier and doesn't cost much more. for (auto &Bucket : Buckets) - std::stable_sort(Bucket.begin(), Bucket.end(), - [](HashData *LHS, HashData *RHS) { - return LHS->HashValue < RHS->HashValue; - }); + llvm::stable_sort(Bucket, [](HashData *LHS, HashData *RHS) { + return LHS->HashValue < RHS->HashValue; + }); } namespace { @@ -557,8 +555,8 @@ void llvm::emitDWARF5AccelTable( SmallVector CUIndex(CUs.size()); int Count = 0; for (const auto &CU : enumerate(CUs)) { - if (CU.value()->getCUNode()->getNameTableKind() == - DICompileUnit::DebugNameTableKind::None) + if (CU.value()->getCUNode()->getNameTableKind() != + DICompileUnit::DebugNameTableKind::Default) continue; CUIndex[CU.index()] = Count++; assert(CU.index() == CU.value()->getUniqueID()); @@ -616,30 +614,10 @@ void AppleAccelTableStaticTypeData::emit(AsmPrinter *Asm) const { Asm->emitInt32(QualifiedNameHash); } -#ifndef _MSC_VER -// The lines below are rejected by older versions (TBD) of MSVC. constexpr AppleAccelTableData::Atom AppleAccelTableTypeData::Atoms[]; constexpr AppleAccelTableData::Atom AppleAccelTableOffsetData::Atoms[]; constexpr AppleAccelTableData::Atom AppleAccelTableStaticOffsetData::Atoms[]; constexpr AppleAccelTableData::Atom AppleAccelTableStaticTypeData::Atoms[]; -#else -// FIXME: Erase this path once the minimum MSCV version has been bumped. -const SmallVector - AppleAccelTableOffsetData::Atoms = { - Atom(dwarf::DW_ATOM_die_offset, dwarf::DW_FORM_data4)}; -const SmallVector AppleAccelTableTypeData::Atoms = - {Atom(dwarf::DW_ATOM_die_offset, dwarf::DW_FORM_data4), - Atom(dwarf::DW_ATOM_die_tag, dwarf::DW_FORM_data2), - Atom(dwarf::DW_ATOM_type_flags, dwarf::DW_FORM_data1)}; -const SmallVector - AppleAccelTableStaticOffsetData::Atoms = { - Atom(dwarf::DW_ATOM_die_offset, dwarf::DW_FORM_data4)}; -const SmallVector - AppleAccelTableStaticTypeData::Atoms = { - Atom(dwarf::DW_ATOM_die_offset, dwarf::DW_FORM_data4), - Atom(dwarf::DW_ATOM_die_tag, dwarf::DW_FORM_data2), - Atom(5, dwarf::DW_FORM_data1), Atom(6, dwarf::DW_FORM_data4)}; -#endif #ifndef NDEBUG void AppleAccelTableWriter::Header::print(raw_ostream &OS) const { diff --git a/lib/CodeGen/AsmPrinter/AddressPool.cpp b/lib/CodeGen/AsmPrinter/AddressPool.cpp index 042243b79259..f11c7de5ed8a 100644 --- a/lib/CodeGen/AsmPrinter/AddressPool.cpp +++ b/lib/CodeGen/AsmPrinter/AddressPool.cpp @@ -1,9 +1,8 @@ //===- llvm/CodeGen/AddressPool.cpp - Dwarf Debug Framework ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -24,21 +23,24 @@ unsigned AddressPool::getIndex(const MCSymbol *Sym, bool TLS) { return IterBool.first->second.Number; } - -void AddressPool::emitHeader(AsmPrinter &Asm, MCSection *Section) { +MCSymbol *AddressPool::emitHeader(AsmPrinter &Asm, MCSection *Section) { static const uint8_t AddrSize = Asm.getDataLayout().getPointerSize(); - uint64_t Length = sizeof(uint16_t) // version - + sizeof(uint8_t) // address_size - + sizeof(uint8_t) // segment_selector_size - + AddrSize * Pool.size(); // entries + StringRef Prefix = "debug_addr_"; + MCSymbol *BeginLabel = Asm.createTempSymbol(Prefix + "start"); + MCSymbol *EndLabel = Asm.createTempSymbol(Prefix + "end"); + Asm.OutStreamer->AddComment("Length of contribution"); - Asm.emitInt32(Length); // TODO: Support DWARF64 format. + Asm.EmitLabelDifference(EndLabel, BeginLabel, + 4); // TODO: Support DWARF64 format. + Asm.OutStreamer->EmitLabel(BeginLabel); Asm.OutStreamer->AddComment("DWARF version number"); Asm.emitInt16(Asm.getDwarfVersion()); Asm.OutStreamer->AddComment("Address size"); Asm.emitInt8(AddrSize); Asm.OutStreamer->AddComment("Segment selector size"); Asm.emitInt8(0); // TODO: Support non-zero segment_selector_size. + + return EndLabel; } // Emit addresses into the section given. @@ -49,8 +51,10 @@ void AddressPool::emit(AsmPrinter &Asm, MCSection *AddrSection) { // Start the dwarf addr section. Asm.OutStreamer->SwitchSection(AddrSection); + MCSymbol *EndLabel = nullptr; + if (Asm.getDwarfVersion() >= 5) - emitHeader(Asm, AddrSection); + EndLabel = emitHeader(Asm, AddrSection); // Define the symbol that marks the start of the contribution. // It is referenced via DW_AT_addr_base. @@ -67,4 +71,7 @@ void AddressPool::emit(AsmPrinter &Asm, MCSection *AddrSection) { for (const MCExpr *Entry : Entries) Asm.OutStreamer->EmitValue(Entry, Asm.getDataLayout().getPointerSize()); + + if (EndLabel) + Asm.OutStreamer->EmitLabel(EndLabel); } diff --git a/lib/CodeGen/AsmPrinter/AddressPool.h b/lib/CodeGen/AsmPrinter/AddressPool.h index 2209c7eb50ed..f92cf72093ca 100644 --- a/lib/CodeGen/AsmPrinter/AddressPool.h +++ b/lib/CodeGen/AsmPrinter/AddressPool.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/AddressPool.h - Dwarf Debug Framework -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -55,7 +54,7 @@ public: void setLabel(MCSymbol *Sym) { AddressTableBaseSym = Sym; } private: - void emitHeader(AsmPrinter &Asm, MCSection *Section); + MCSymbol *emitHeader(AsmPrinter &Asm, MCSection *Section); /// Symbol designates the start of the contribution to the address table. MCSymbol *AddressTableBaseSym = nullptr; diff --git a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp index 7070451e3330..54f6cc2d5571 100644 --- a/lib/CodeGen/AsmPrinter/AsmPrinter.cpp +++ b/lib/CodeGen/AsmPrinter/AsmPrinter.cpp @@ -1,9 +1,8 @@ //===- AsmPrinter.cpp - Common AsmPrinter code ----------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -35,7 +34,6 @@ #include "llvm/BinaryFormat/COFF.h" #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/BinaryFormat/ELF.h" -#include "llvm/CodeGen/AsmPrinterHandler.h" #include "llvm/CodeGen/GCMetadata.h" #include "llvm/CodeGen/GCMetadataPrinter.h" #include "llvm/CodeGen/GCStrategy.h" @@ -60,7 +58,6 @@ #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/CodeGen/TargetRegisterInfo.h" -#include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Comdat.h" #include "llvm/IR/Constant.h" @@ -80,6 +77,7 @@ #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/Operator.h" +#include "llvm/IR/RemarkStreamer.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" #include "llvm/MC/MCAsmInfo.h" @@ -101,6 +99,9 @@ #include "llvm/MC/MCValue.h" #include "llvm/MC/SectionKind.h" #include "llvm/Pass.h" +#include "llvm/Remarks/Remark.h" +#include "llvm/Remarks/RemarkFormat.h" +#include "llvm/Remarks/RemarkStringTable.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" @@ -143,9 +144,10 @@ static const char *const CodeViewLineTablesGroupDescription = STATISTIC(EmittedInsts, "Number of machine instrs printed"); -static cl::opt - PrintSchedule("print-schedule", cl::Hidden, cl::init(false), - cl::desc("Print 'sched: [latency:throughput]' in .s output")); +static cl::opt EnableRemarksSection( + "remarks-section", + cl::desc("Emit a section containing remark diagnostics metadata"), + cl::init(false)); char AsmPrinter::ID = 0; @@ -232,6 +234,12 @@ void AsmPrinter::EmitToStreamer(MCStreamer &S, const MCInst &Inst) { S.EmitInstruction(Inst, getSubtargetInfo()); } +void AsmPrinter::emitInitialRawDwarfLocDirective(const MachineFunction &MF) { + assert(DD && "Dwarf debug file is not defined."); + assert(OutStreamer->hasRawTextSupport() && "Expected assembly output mode."); + (void)DD->emitInitialLocDirective(MF, /*CUID=*/0); +} + /// getCurrentSection() - Return the current section we are emitting to. const MCSection *AsmPrinter::getCurrentSection() const { return OutStreamer->getCurrentSectionOnly(); @@ -252,6 +260,9 @@ bool AsmPrinter::doInitialization(Module &M) { const_cast(getObjFileLowering()) .Initialize(OutContext, TM); + const_cast(getObjFileLowering()) + .getModuleMetadata(M); + OutStreamer->InitSections(false); // Emit the version-min deployment target directive if needed. @@ -300,16 +311,17 @@ bool AsmPrinter::doInitialization(Module &M) { if (MAI->doesSupportDebugInformation()) { bool EmitCodeView = MMI->getModule()->getCodeViewFlag(); if (EmitCodeView && TM.getTargetTriple().isOSWindows()) { - Handlers.push_back(HandlerInfo(new CodeViewDebug(this), - DbgTimerName, DbgTimerDescription, - CodeViewLineTablesGroupName, - CodeViewLineTablesGroupDescription)); + Handlers.emplace_back(llvm::make_unique(this), + DbgTimerName, DbgTimerDescription, + CodeViewLineTablesGroupName, + CodeViewLineTablesGroupDescription); } if (!EmitCodeView || MMI->getModule()->getDwarfVersion()) { DD = new DwarfDebug(this, &M); DD->beginModule(); - Handlers.push_back(HandlerInfo(DD, DbgTimerName, DbgTimerDescription, - DWARFGroupName, DWARFGroupDescription)); + Handlers.emplace_back(std::unique_ptr(DD), DbgTimerName, + DbgTimerDescription, DWARFGroupName, + DWARFGroupDescription); } } @@ -362,14 +374,15 @@ bool AsmPrinter::doInitialization(Module &M) { break; } if (ES) - Handlers.push_back(HandlerInfo(ES, EHTimerName, EHTimerDescription, - DWARFGroupName, DWARFGroupDescription)); + Handlers.emplace_back(std::unique_ptr(ES), EHTimerName, + EHTimerDescription, DWARFGroupName, + DWARFGroupDescription); if (mdconst::extract_or_null( MMI->getModule()->getModuleFlag("cfguardtable"))) - Handlers.push_back(HandlerInfo(new WinCFGuard(this), CFGuardName, - CFGuardDescription, DWARFGroupName, - DWARFGroupDescription)); + Handlers.emplace_back(llvm::make_unique(this), CFGuardName, + CFGuardDescription, DWARFGroupName, + DWARFGroupDescription); return false; } @@ -483,7 +496,7 @@ void AsmPrinter::EmitGlobalVariable(const GlobalVariable *GV) { SectionKind GVKind = TargetLoweringObjectFile::getKindForGlobal(GV, TM); const DataLayout &DL = GV->getParent()->getDataLayout(); - uint64_t Size = DL.getTypeAllocSize(GV->getType()->getElementType()); + uint64_t Size = DL.getTypeAllocSize(GV->getValueType()); // If the alignment is specified, we *must* obey it. Overaligning a global // with a specified alignment is a prompt way to break globals emitted to @@ -658,6 +671,9 @@ void AsmPrinter::EmitFunctionHeader() { if (MAI->hasDotTypeDotSizeDirective()) OutStreamer->EmitSymbolAttribute(CurrentFnSym, MCSA_ELF_TypeFunction); + if (F.hasFnAttribute(Attribute::Cold)) + OutStreamer->EmitSymbolAttribute(CurrentFnSym, MCSA_Cold); + if (isVerbose()) { F.printAsOperand(OutStreamer->GetCommentOS(), /*PrintType=*/false, F.getParent()); @@ -738,74 +754,30 @@ void AsmPrinter::EmitFunctionEntryLabel() { } /// emitComments - Pretty-print comments for instructions. -/// It returns true iff the sched comment was emitted. -/// Otherwise it returns false. -static bool emitComments(const MachineInstr &MI, raw_ostream &CommentOS, - AsmPrinter *AP) { +static void emitComments(const MachineInstr &MI, raw_ostream &CommentOS) { const MachineFunction *MF = MI.getMF(); const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); // Check for spills and reloads - int FI; - - const MachineFrameInfo &MFI = MF->getFrameInfo(); - bool Commented = false; - - auto getSize = - [&MFI](const SmallVectorImpl &Accesses) { - unsigned Size = 0; - for (auto A : Accesses) - if (MFI.isSpillSlotObjectIndex( - cast(A->getPseudoValue()) - ->getFrameIndex())) - Size += A->getSize(); - return Size; - }; // We assume a single instruction only has a spill or reload, not // both. - const MachineMemOperand *MMO; - SmallVector Accesses; - if (TII->isLoadFromStackSlotPostFE(MI, FI)) { - if (MFI.isSpillSlotObjectIndex(FI)) { - MMO = *MI.memoperands_begin(); - CommentOS << MMO->getSize() << "-byte Reload"; - Commented = true; - } - } else if (TII->hasLoadFromStackSlot(MI, Accesses)) { - if (auto Size = getSize(Accesses)) { - CommentOS << Size << "-byte Folded Reload"; - Commented = true; - } - } else if (TII->isStoreToStackSlotPostFE(MI, FI)) { - if (MFI.isSpillSlotObjectIndex(FI)) { - MMO = *MI.memoperands_begin(); - CommentOS << MMO->getSize() << "-byte Spill"; - Commented = true; - } - } else if (TII->hasStoreToStackSlot(MI, Accesses)) { - if (auto Size = getSize(Accesses)) { - CommentOS << Size << "-byte Folded Spill"; - Commented = true; - } + Optional Size; + if ((Size = MI.getRestoreSize(TII))) { + CommentOS << *Size << "-byte Reload\n"; + } else if ((Size = MI.getFoldedRestoreSize(TII))) { + if (*Size) + CommentOS << *Size << "-byte Folded Reload\n"; + } else if ((Size = MI.getSpillSize(TII))) { + CommentOS << *Size << "-byte Spill\n"; + } else if ((Size = MI.getFoldedSpillSize(TII))) { + if (*Size) + CommentOS << *Size << "-byte Folded Spill\n"; } // Check for spill-induced copies - if (MI.getAsmPrinterFlag(MachineInstr::ReloadReuse)) { - Commented = true; - CommentOS << " Reload Reuse"; - } - - if (Commented) { - if (AP->EnablePrintSchedInfo) { - // If any comment was added above and we need sched info comment then add - // this new comment just after the above comment w/o "\n" between them. - CommentOS << " " << MF->getSubtarget().getSchedInfoStr(MI) << "\n"; - return true; - } - CommentOS << "\n"; - } - return false; + if (MI.getAsmPrinterFlag(MachineInstr::ReloadReuse)) + CommentOS << " Reload Reuse\n"; } /// emitImplicitDef - This method emits the specified machine instruction @@ -1093,10 +1065,8 @@ void AsmPrinter::EmitFunctionBody() { } } - if (isVerbose() && emitComments(MI, OutStreamer->GetCommentOS(), this)) { - MachineInstr *MIP = const_cast(&MI); - MIP->setAsmPrinterFlag(MachineInstr::NoSchedComment); - } + if (isVerbose()) + emitComments(MI, OutStreamer->GetCommentOS()); switch (MI.getOpcode()) { case TargetOpcode::CFI_INSTRUCTION: @@ -1105,11 +1075,13 @@ void AsmPrinter::EmitFunctionBody() { case TargetOpcode::LOCAL_ESCAPE: emitFrameAlloc(MI); break; + case TargetOpcode::ANNOTATION_LABEL: case TargetOpcode::EH_LABEL: case TargetOpcode::GC_LABEL: OutStreamer->EmitLabel(MI.getOperand(0).getMCSymbol()); break; case TargetOpcode::INLINEASM: + case TargetOpcode::INLINEASM_BR: EmitInlineAsm(&MI); break; case TargetOpcode::DBG_VALUE: @@ -1266,7 +1238,7 @@ static bool isGOTEquivalentCandidate(const GlobalVariable *GV, // GlobalVariable or Function, i.e., as GlobalValue. if (!GV->hasGlobalUnnamedAddr() || !GV->hasInitializer() || !GV->isConstant() || !GV->isDiscardableIfUnused() || - !dyn_cast(GV->getOperand(0))) + !isa(GV->getOperand(0))) return false; // To be a got equivalent, at least one of its users need to be a constant @@ -1329,9 +1301,19 @@ void AsmPrinter::emitGlobalIndirectSymbol(Module &M, else assert(GIS.hasLocalLinkage() && "Invalid alias or ifunc linkage"); + bool IsFunction = GIS.getValueType()->isFunctionTy(); + + // Treat bitcasts of functions as functions also. This is important at least + // on WebAssembly where object and function addresses can't alias each other. + if (!IsFunction) + if (auto *CE = dyn_cast(GIS.getIndirectSymbol())) + if (CE->getOpcode() == Instruction::BitCast) + IsFunction = + CE->getOperand(0)->getType()->getPointerElementType()->isFunctionTy(); + // Set the symbol type to function if the alias has a function type. // This affects codegen when the aliasee is not a function. - if (GIS.getType()->getPointerElementType()->isFunctionTy()) { + if (IsFunction) { OutStreamer->EmitSymbolAttribute(Name, MCSA_ELF_TypeFunction); if (isa(GIS)) OutStreamer->EmitSymbolAttribute(Name, MCSA_ELF_TypeIndFunction); @@ -1363,6 +1345,66 @@ void AsmPrinter::emitGlobalIndirectSymbol(Module &M, } } +void AsmPrinter::emitRemarksSection(Module &M) { + RemarkStreamer *RS = M.getContext().getRemarkStreamer(); + if (!RS) + return; + const remarks::Serializer &Serializer = RS->getSerializer(); + + // Switch to the right section: .remarks/__remarks. + MCSection *RemarksSection = + OutContext.getObjectFileInfo()->getRemarksSection(); + OutStreamer->SwitchSection(RemarksSection); + + // Emit the magic number. + OutStreamer->EmitBytes(remarks::Magic); + // Explicitly emit a '\0'. + OutStreamer->EmitIntValue(/*Value=*/0, /*Size=*/1); + + // Emit the version number: little-endian uint64_t. + // The version number is located at the offset 0x0 in the section. + std::array Version; + support::endian::write64le(Version.data(), remarks::Version); + OutStreamer->EmitBinaryData(StringRef(Version.data(), Version.size())); + + // Emit the string table in the section. + // Note: we need to use the streamer here to emit it in the section. We can't + // just use the serialize function with a raw_ostream because of the way + // MCStreamers work. + uint64_t StrTabSize = + Serializer.StrTab ? Serializer.StrTab->SerializedSize : 0; + // Emit the total size of the string table (the size itself excluded): + // little-endian uint64_t. + // The total size is located after the version number. + // Note: even if no string table is used, emit 0. + std::array StrTabSizeBuf; + support::endian::write64le(StrTabSizeBuf.data(), StrTabSize); + OutStreamer->EmitBinaryData( + StringRef(StrTabSizeBuf.data(), StrTabSizeBuf.size())); + + if (const Optional &StrTab = Serializer.StrTab) { + std::vector StrTabStrings = StrTab->serialize(); + // Emit a list of null-terminated strings. + // Note: the order is important here: the ID used in the remarks corresponds + // to the position of the string in the section. + for (StringRef Str : StrTabStrings) { + OutStreamer->EmitBytes(Str); + // Explicitly emit a '\0'. + OutStreamer->EmitIntValue(/*Value=*/0, /*Size=*/1); + } + } + + // Emit the null-terminated absolute path to the remark file. + // The path is located at the offset 0x4 in the section. + StringRef FilenameRef = RS->getFilename(); + SmallString<128> Filename = FilenameRef; + sys::fs::make_absolute(Filename); + assert(!Filename.empty() && "The filename can't be empty."); + OutStreamer->EmitBytes(Filename); + // Explicitly emit a '\0'. + OutStreamer->EmitIntValue(/*Value=*/0, /*Size=*/1); +} + bool AsmPrinter::doFinalization(Module &M) { // Set the MachineFunction to nullptr so that we can catch attempted // accesses to MF specific features at the module level and so that @@ -1394,6 +1436,12 @@ bool AsmPrinter::doFinalization(Module &M) { EmitVisibility(Name, V, false); } + // Emit the remarks section contents. + // FIXME: Figure out when is the safest time to emit this section. It should + // not come after debug info. + if (EnableRemarksSection) + emitRemarksSection(M); + const TargetLoweringObjectFile &TLOF = getObjFileLowering(); TLOF.emitModuleMetadata(*OutStreamer, M); @@ -1448,7 +1496,6 @@ bool AsmPrinter::doFinalization(Module &M) { NamedRegionTimer T(HI.TimerName, HI.TimerDescription, HI.TimerGroupName, HI.TimerGroupDescription, TimePassesIsEnabled); HI.Handler->endModule(); - delete HI.Handler; } Handlers.clear(); DD = nullptr; @@ -1592,6 +1639,24 @@ bool AsmPrinter::doFinalization(Module &M) { OutStreamer->EmitAddrsigSym(getSymbol(&GV)); } + // Emit symbol partition specifications (ELF only). + if (TM.getTargetTriple().isOSBinFormatELF()) { + unsigned UniqueID = 0; + for (const GlobalValue &GV : M.global_values()) { + if (!GV.hasPartition() || GV.isDeclarationForLinker() || + GV.getVisibility() != GlobalValue::DefaultVisibility) + continue; + + OutStreamer->SwitchSection(OutContext.getELFSection( + ".llvm_sympart", ELF::SHT_LLVM_SYMPART, 0, 0, "", ++UniqueID)); + OutStreamer->EmitBytes(GV.getPartition()); + OutStreamer->EmitZeros(1); + OutStreamer->EmitValue( + MCSymbolRefExpr::create(getSymbol(&GV), OutContext), + MAI->getCodePointerSize()); + } + } + // Allow the target to emit any magic that it wants at the end of the file, // after everything else has gone out. EmitEndOfAsmFile(M); @@ -1628,11 +1693,6 @@ void AsmPrinter::SetupMachineFunction(MachineFunction &MF) { } ORE = &getAnalysis().getORE(); - - const TargetSubtargetInfo &STI = MF.getSubtarget(); - EnablePrintSchedInfo = PrintSchedule.getNumOccurrences() - ? PrintSchedule - : STI.supportPrintSchedInfo(); } namespace { @@ -1905,8 +1965,7 @@ bool AsmPrinter::EmitSpecialLLVMGlobal(const GlobalVariable *GV) { } /// EmitLLVMUsedList - For targets that define a MAI::UsedDirective, mark each -/// global in the specified llvm.used list for which emitUsedDirectiveFor -/// is true, as being used with this directive. +/// global in the specified llvm.used list. void AsmPrinter::EmitLLVMUsedList(const ConstantArray *InitList) { // Should be an array of 'i8*'. for (unsigned i = 0, e = InitList->getNumOperands(); i != e; ++i) { @@ -1933,7 +1992,7 @@ struct Structor { /// priority. void AsmPrinter::EmitXXStructorList(const DataLayout &DL, const Constant *List, bool isCtor) { - // Should be an array of '{ int, void ()* }' structs. The first value is the + // Should be an array of '{ i32, void ()*, i8* }' structs. The first value is the // init priority. if (!isa(List)) return; @@ -1941,12 +2000,10 @@ void AsmPrinter::EmitXXStructorList(const DataLayout &DL, const Constant *List, const ConstantArray *InitList = dyn_cast(List); if (!InitList) return; // Not an array! StructType *ETy = dyn_cast(InitList->getType()->getElementType()); - // FIXME: Only allow the 3-field form in LLVM 4.0. - if (!ETy || ETy->getNumElements() < 2 || ETy->getNumElements() > 3) - return; // Not an array of two or three elements! - if (!isa(ETy->getTypeAtIndex(0U)) || - !isa(ETy->getTypeAtIndex(1U))) return; // Not (int, ptr). - if (ETy->getNumElements() == 3 && !isa(ETy->getTypeAtIndex(2U))) + if (!ETy || ETy->getNumElements() != 3 || + !isa(ETy->getTypeAtIndex(0U)) || + !isa(ETy->getTypeAtIndex(1U)) || + !isa(ETy->getTypeAtIndex(2U))) return; // Not (int, ptr, ptr). // Gather the structors in a form that's convenient for sorting by priority. @@ -1962,16 +2019,16 @@ void AsmPrinter::EmitXXStructorList(const DataLayout &DL, const Constant *List, Structor &S = Structors.back(); S.Priority = Priority->getLimitedValue(65535); S.Func = CS->getOperand(1); - if (ETy->getNumElements() == 3 && !CS->getOperand(2)->isNullValue()) + if (!CS->getOperand(2)->isNullValue()) S.ComdatKey = dyn_cast(CS->getOperand(2)->stripPointerCasts()); } // Emit the function pointers in the target-specific order unsigned Align = Log2_32(DL.getPointerPrefAlignment()); - std::stable_sort(Structors.begin(), Structors.end(), - [](const Structor &L, - const Structor &R) { return L.Priority < R.Priority; }); + llvm::stable_sort(Structors, [](const Structor &L, const Structor &R) { + return L.Priority < R.Priority; + }); for (Structor &S : Structors) { const TargetLoweringObjectFile &Obj = getObjFileLowering(); const MCSymbol *KeySym = nullptr; @@ -2199,7 +2256,10 @@ const MCExpr *AsmPrinter::lowerConstant(const Constant *CV) { // We can emit the pointer value into this slot if the slot is an // integer slot equal to the size of the pointer. - if (DL.getTypeAllocSize(Ty) == DL.getTypeAllocSize(Op->getType())) + // + // If the pointer is larger than the resultant integer, then + // as with Trunc just depend on the assembler to truncate it. + if (DL.getTypeAllocSize(Ty) <= DL.getTypeAllocSize(Op->getType())) return OpExpr; // Otherwise the pointer is smaller than the resultant integer, mask off @@ -2740,7 +2800,7 @@ MCSymbol *AsmPrinter::GetBlockAddressSymbol(const BasicBlock *BB) const { /// GetCPISymbol - Return the symbol for the specified constant pool entry. MCSymbol *AsmPrinter::GetCPISymbol(unsigned CPID) const { - if (getSubtargetInfo().getTargetTriple().isKnownWindowsMSVCEnvironment()) { + if (getSubtargetInfo().getTargetTriple().isWindowsMSVCEnvironment()) { const MachineConstantPoolEntry &CPE = MF->getConstantPool()->getConstants()[CPID]; if (!CPE.isMachineConstantPoolEntry()) { @@ -2858,7 +2918,7 @@ void AsmPrinter::setupCodePaddingContext(const MachineBasicBlock &MBB, MCCodePaddingContext &Context) const { assert(MF != nullptr && "Machine function must be valid"); Context.IsPaddingActive = !MF->hasInlineAsm() && - !MF->getFunction().optForSize() && + !MF->getFunction().hasOptSize() && TM.getOptLevel() != CodeGenOpt::None; Context.IsBasicBlockReachableViaFallthrough = std::find(MBB.pred_begin(), MBB.pred_end(), MBB.getPrevNode()) != @@ -2918,13 +2978,16 @@ void AsmPrinter::EmitBasicBlockStart(const MachineBasicBlock &MBB) const { // Print the main label for the block. if (MBB.pred_empty() || - (isBlockOnlyReachableByFallthrough(&MBB) && !MBB.isEHFuncletEntry())) { + (isBlockOnlyReachableByFallthrough(&MBB) && !MBB.isEHFuncletEntry() && + !MBB.hasLabelMustBeEmitted())) { if (isVerbose()) { // NOTE: Want this comment at start of line, don't emit with AddComment. OutStreamer->emitRawComment(" %bb." + Twine(MBB.getNumber()) + ":", false); } } else { + if (isVerbose() && MBB.hasLabelMustBeEmitted()) + OutStreamer->AddComment("Label of block must be emitted"); OutStreamer->EmitLabel(MBB.getSymbol()); } } diff --git a/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp b/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp index afce3ad3133b..992e44d95306 100644 --- a/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp +++ b/lib/CodeGen/AsmPrinter/AsmPrinterDwarf.cpp @@ -1,9 +1,8 @@ //===-- AsmPrinterDwarf.cpp - AsmPrinter Dwarf Support --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,6 +18,7 @@ #include "llvm/CodeGen/MachineFunction.h" #include "llvm/IR/DataLayout.h" #include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCDwarf.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCSection.h" #include "llvm/MC/MCStreamer.h" @@ -43,11 +43,11 @@ void AsmPrinter::EmitSLEB128(int64_t Value, const char *Desc) const { OutStreamer->EmitSLEB128IntValue(Value); } -void AsmPrinter::EmitULEB128(uint64_t Value, const char *Desc) const { +void AsmPrinter::EmitULEB128(uint64_t Value, const char *Desc, unsigned PadTo) const { if (isVerbose() && Desc) OutStreamer->AddComment(Desc); - OutStreamer->EmitULEB128IntValue(Value); + OutStreamer->EmitULEB128IntValue(Value, PadTo); } /// Emit something like ".uleb128 Hi-Lo". @@ -183,6 +183,25 @@ void AsmPrinter::EmitDwarfOffset(const MCSymbol *Label, uint64_t Offset) const { EmitLabelPlusOffset(Label, Offset, MAI->getCodePointerSize()); } +void AsmPrinter::EmitCallSiteOffset(const MCSymbol *Hi, + const MCSymbol *Lo, + unsigned Encoding) const { + // The least significant 3 bits specify the width of the encoding + if ((Encoding & 0x7) == dwarf::DW_EH_PE_uleb128) + EmitLabelDifferenceAsULEB128(Hi, Lo); + else + EmitLabelDifference(Hi, Lo, GetSizeOfEncodedValue(Encoding)); +} + +void AsmPrinter::EmitCallSiteValue(uint64_t Value, + unsigned Encoding) const { + // The least significant 3 bits specify the width of the encoding + if ((Encoding & 0x7) == dwarf::DW_EH_PE_uleb128) + EmitULEB128(Value); + else + OutStreamer->EmitIntValue(Value, GetSizeOfEncodedValue(Encoding)); +} + //===----------------------------------------------------------------------===// // Dwarf Lowering Routines //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp b/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp index 62103e3107c0..7721e996aca5 100644 --- a/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp +++ b/lib/CodeGen/AsmPrinter/AsmPrinterInlineAsm.cpp @@ -1,9 +1,8 @@ //===-- AsmPrinterInlineAsm.cpp - AsmPrinter Inline Asm Handling ----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,7 +18,6 @@ #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetRegisterInfo.h" -#include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/InlineAsm.h" @@ -155,15 +153,10 @@ void AsmPrinter::EmitInlineAsm(StringRef Str, const MCSubtargetInfo &STI, " we don't have an asm parser for this target\n"); Parser->setAssemblerDialect(Dialect); Parser->setTargetParser(*TAP.get()); - Parser->setEnablePrintSchedInfo(EnablePrintSchedInfo); // Enable lexing Masm binary and hex integer literals in intel inline // assembly. if (Dialect == InlineAsm::AD_Intel) Parser->getLexer().setLexMasmIntegers(true); - if (MF) { - const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo(); - TAP->SetFrameRegister(TRI->getFrameRegister(*MF)); - } emitInlineAsmStart(); // Don't implicitly switch to the text section before the asm. @@ -176,9 +169,8 @@ void AsmPrinter::EmitInlineAsm(StringRef Str, const MCSubtargetInfo &STI, } static void EmitMSInlineAsmStr(const char *AsmStr, const MachineInstr *MI, - MachineModuleInfo *MMI, int InlineAsmVariant, - AsmPrinter *AP, unsigned LocCookie, - raw_ostream &OS) { + MachineModuleInfo *MMI, AsmPrinter *AP, + unsigned LocCookie, raw_ostream &OS) { // Switch to the inline assembly variant. OS << "\t.intel_syntax\n\t"; @@ -270,11 +262,9 @@ static void EmitMSInlineAsmStr(const char *AsmStr, const MachineInstr *MI, ++OpNo; // Skip over the ID number. if (InlineAsm::isMemKind(OpFlags)) { - Error = AP->PrintAsmMemoryOperand(MI, OpNo, InlineAsmVariant, - /*Modifier*/ nullptr, OS); + Error = AP->PrintAsmMemoryOperand(MI, OpNo, /*Modifier*/ nullptr, OS); } else { - Error = AP->PrintAsmOperand(MI, OpNo, InlineAsmVariant, - /*Modifier*/ nullptr, OS); + Error = AP->PrintAsmOperand(MI, OpNo, /*Modifier*/ nullptr, OS); } } if (Error) { @@ -291,9 +281,9 @@ static void EmitMSInlineAsmStr(const char *AsmStr, const MachineInstr *MI, } static void EmitGCCInlineAsmStr(const char *AsmStr, const MachineInstr *MI, - MachineModuleInfo *MMI, int InlineAsmVariant, - int AsmPrinterVariant, AsmPrinter *AP, - unsigned LocCookie, raw_ostream &OS) { + MachineModuleInfo *MMI, int AsmPrinterVariant, + AsmPrinter *AP, unsigned LocCookie, + raw_ostream &OS) { int CurVariant = -1; // The number of the {.|.|.} region we are in. const char *LastEmitted = AsmStr; // One past the last character emitted. unsigned NumOperands = MI->getNumOperands(); @@ -435,17 +425,25 @@ static void EmitGCCInlineAsmStr(const char *AsmStr, const MachineInstr *MI, unsigned OpFlags = MI->getOperand(OpNo).getImm(); ++OpNo; // Skip over the ID number. + // FIXME: Shouldn't arch-independent output template handling go into + // PrintAsmOperand? if (Modifier[0] == 'l') { // Labels are target independent. - // FIXME: What if the operand isn't an MBB, report error? - const MCSymbol *Sym = MI->getOperand(OpNo).getMBB()->getSymbol(); - Sym->print(OS, AP->MAI); + if (MI->getOperand(OpNo).isBlockAddress()) { + const BlockAddress *BA = MI->getOperand(OpNo).getBlockAddress(); + MCSymbol *Sym = AP->GetBlockAddressSymbol(BA); + Sym->print(OS, AP->MAI); + } else if (MI->getOperand(OpNo).isMBB()) { + const MCSymbol *Sym = MI->getOperand(OpNo).getMBB()->getSymbol(); + Sym->print(OS, AP->MAI); + } else { + Error = true; + } } else { if (InlineAsm::isMemKind(OpFlags)) { - Error = AP->PrintAsmMemoryOperand(MI, OpNo, InlineAsmVariant, - Modifier[0] ? Modifier : nullptr, - OS); + Error = AP->PrintAsmMemoryOperand( + MI, OpNo, Modifier[0] ? Modifier : nullptr, OS); } else { - Error = AP->PrintAsmOperand(MI, OpNo, InlineAsmVariant, + Error = AP->PrintAsmOperand(MI, OpNo, Modifier[0] ? Modifier : nullptr, OS); } } @@ -515,18 +513,11 @@ void AsmPrinter::EmitInlineAsm(const MachineInstr *MI) const { // The variant of the current asmprinter. int AsmPrinterVariant = MAI->getAssemblerDialect(); - InlineAsm::AsmDialect InlineAsmVariant = MI->getInlineAsmDialect(); AsmPrinter *AP = const_cast(this); - if (InlineAsmVariant == InlineAsm::AD_ATT) - EmitGCCInlineAsmStr(AsmStr, MI, MMI, InlineAsmVariant, AsmPrinterVariant, - AP, LocCookie, OS); + if (MI->getInlineAsmDialect() == InlineAsm::AD_ATT) + EmitGCCInlineAsmStr(AsmStr, MI, MMI, AsmPrinterVariant, AP, LocCookie, OS); else - EmitMSInlineAsmStr(AsmStr, MI, MMI, InlineAsmVariant, AP, LocCookie, OS); - - // Reset SanitizeAddress based on the function's attribute. - MCTargetOptions MCOptions = TM.Options.MCOptions; - MCOptions.SanitizeAddress = - MF->getFunction().hasFnAttribute(Attribute::SanitizeAddress); + EmitMSInlineAsmStr(AsmStr, MI, MMI, AP, LocCookie, OS); // Emit warnings if we use reserved registers on the clobber list, as // that might give surprising results. @@ -566,7 +557,7 @@ void AsmPrinter::EmitInlineAsm(const MachineInstr *MI) const { SrcMgr.PrintMessage(Loc, SourceMgr::DK_Note, Note); } - EmitInlineAsm(OS.str(), getSubtargetInfo(), MCOptions, LocMD, + EmitInlineAsm(OS.str(), getSubtargetInfo(), TM.Options.MCOptions, LocMD, MI->getInlineAsmDialect()); // Emit the #NOAPP end marker. This has to happen even if verbose-asm isn't @@ -608,32 +599,50 @@ void AsmPrinter::PrintSpecial(const MachineInstr *MI, raw_ostream &OS, } } +void AsmPrinter::PrintSymbolOperand(const MachineOperand &MO, raw_ostream &OS) { + assert(MO.isGlobal() && "caller should check MO.isGlobal"); + getSymbol(MO.getGlobal())->print(OS, MAI); + printOffset(MO.getOffset(), OS); +} + /// PrintAsmOperand - Print the specified operand of MI, an INLINEASM /// instruction, using the specified assembler variant. Targets should -/// override this to format as appropriate. +/// override this to format as appropriate for machine specific ExtraCodes +/// or when the arch-independent handling would be too complex otherwise. bool AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, - unsigned AsmVariant, const char *ExtraCode, - raw_ostream &O) { + const char *ExtraCode, raw_ostream &O) { // Does this asm operand have a single letter operand modifier? if (ExtraCode && ExtraCode[0]) { if (ExtraCode[1] != 0) return true; // Unknown modifier. + // https://gcc.gnu.org/onlinedocs/gccint/Output-Template.html const MachineOperand &MO = MI->getOperand(OpNo); switch (ExtraCode[0]) { default: return true; // Unknown modifier. + case 'a': // Print as memory address. + if (MO.isReg()) { + PrintAsmMemoryOperand(MI, OpNo, nullptr, O); + return false; + } + LLVM_FALLTHROUGH; // GCC allows '%a' to behave like '%c' with immediates. case 'c': // Substitute immediate value without immediate syntax - if (MO.getType() != MachineOperand::MO_Immediate) - return true; - O << MO.getImm(); - return false; + if (MO.isImm()) { + O << MO.getImm(); + return false; + } + if (MO.isGlobal()) { + PrintSymbolOperand(MO, O); + return false; + } + return true; case 'n': // Negate the immediate constant. - if (MO.getType() != MachineOperand::MO_Immediate) + if (!MO.isImm()) return true; O << -MO.getImm(); return false; case 's': // The GCC deprecated s modifier - if (MO.getType() != MachineOperand::MO_Immediate) + if (!MO.isImm()) return true; O << ((32 - MO.getImm()) & 31); return false; @@ -643,7 +652,6 @@ bool AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, } bool AsmPrinter::PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNo, - unsigned AsmVariant, const char *ExtraCode, raw_ostream &O) { // Target doesn't support this yet! return true; diff --git a/lib/CodeGen/AsmPrinter/ByteStreamer.h b/lib/CodeGen/AsmPrinter/ByteStreamer.h index 2163cc7e3e11..db2ff458eb2e 100644 --- a/lib/CodeGen/AsmPrinter/ByteStreamer.h +++ b/lib/CodeGen/AsmPrinter/ByteStreamer.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/ByteStreamer.h - ByteStreamer class --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -32,7 +31,7 @@ class ByteStreamer { // For now we're just handling the calls we need for dwarf emission/hashing. virtual void EmitInt8(uint8_t Byte, const Twine &Comment = "") = 0; virtual void EmitSLEB128(uint64_t DWord, const Twine &Comment = "") = 0; - virtual void EmitULEB128(uint64_t DWord, const Twine &Comment = "") = 0; + virtual void EmitULEB128(uint64_t DWord, const Twine &Comment = "", unsigned PadTo = 0) = 0; }; class APByteStreamer final : public ByteStreamer { @@ -49,7 +48,7 @@ public: AP.OutStreamer->AddComment(Comment); AP.EmitSLEB128(DWord); } - void EmitULEB128(uint64_t DWord, const Twine &Comment) override { + void EmitULEB128(uint64_t DWord, const Twine &Comment, unsigned PadTo) override { AP.OutStreamer->AddComment(Comment); AP.EmitULEB128(DWord); } @@ -66,7 +65,7 @@ class HashingByteStreamer final : public ByteStreamer { void EmitSLEB128(uint64_t DWord, const Twine &Comment) override { Hash.addSLEB128(DWord); } - void EmitULEB128(uint64_t DWord, const Twine &Comment) override { + void EmitULEB128(uint64_t DWord, const Twine &Comment, unsigned PadTo) override { Hash.addULEB128(DWord); } }; @@ -103,9 +102,9 @@ public: } } - void EmitULEB128(uint64_t DWord, const Twine &Comment) override { + void EmitULEB128(uint64_t DWord, const Twine &Comment, unsigned PadTo) override { raw_svector_ostream OSE(Buffer); - unsigned Length = encodeULEB128(DWord, OSE); + unsigned Length = encodeULEB128(DWord, OSE, PadTo); if (GenerateComments) { Comments.push_back(Comment.str()); // Add some empty comments to keep the Buffer and Comments vectors aligned diff --git a/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp b/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp index 8cabad4ad312..932959c311fa 100644 --- a/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp +++ b/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp @@ -1,9 +1,8 @@ //===- llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.cpp ----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -42,6 +41,7 @@ #include "llvm/Config/llvm-config.h" #include "llvm/DebugInfo/CodeView/CVTypeVisitor.h" #include "llvm/DebugInfo/CodeView/CodeView.h" +#include "llvm/DebugInfo/CodeView/CodeViewRecordIO.h" #include "llvm/DebugInfo/CodeView/ContinuationRecordBuilder.h" #include "llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h" #include "llvm/DebugInfo/CodeView/EnumTables.h" @@ -51,6 +51,7 @@ #include "llvm/DebugInfo/CodeView/TypeIndex.h" #include "llvm/DebugInfo/CodeView/TypeRecord.h" #include "llvm/DebugInfo/CodeView/TypeTableCollection.h" +#include "llvm/DebugInfo/CodeView/TypeVisitorCallbackPipeline.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DataLayout.h" #include "llvm/IR/DebugInfoMetadata.h" @@ -67,6 +68,7 @@ #include "llvm/MC/MCSymbol.h" #include "llvm/Support/BinaryByteStream.h" #include "llvm/Support/BinaryStreamReader.h" +#include "llvm/Support/BinaryStreamWriter.h" #include "llvm/Support/Casting.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Compiler.h" @@ -93,6 +95,26 @@ using namespace llvm; using namespace llvm::codeview; +namespace { +class CVMCAdapter : public CodeViewRecordStreamer { +public: + CVMCAdapter(MCStreamer &OS) : OS(&OS) {} + + void EmitBytes(StringRef Data) { OS->EmitBytes(Data); } + + void EmitIntValue(uint64_t Value, unsigned Size) { + OS->EmitIntValueInHex(Value, Size); + } + + void EmitBinaryData(StringRef Data) { OS->EmitBinaryData(Data); } + + void AddComment(const Twine &T) { OS->AddComment(T); } + +private: + MCStreamer *OS = nullptr; +}; +} // namespace + static CPUType mapArchToCVCPUType(Triple::ArchType Type) { switch (Type) { case Triple::ArchType::x86: @@ -273,7 +295,7 @@ static const DISubprogram *getQualifiedNameComponents( StringRef ScopeName = getPrettyScopeName(Scope); if (!ScopeName.empty()) QualifiedNameComponents.push_back(ScopeName); - Scope = Scope->getScope().resolve(); + Scope = Scope->getScope(); } return ClosestSubprogram; } @@ -309,7 +331,7 @@ struct CodeViewDebug::TypeLoweringScope { }; static std::string getFullyQualifiedName(const DIScope *Ty) { - const DIScope *Scope = Ty->getScope().resolve(); + const DIScope *Scope = Ty->getScope(); return getFullyQualifiedName(Scope, getPrettyScopeName(Ty)); } @@ -344,7 +366,7 @@ TypeIndex CodeViewDebug::getFuncIdForSubprogram(const DISubprogram *SP) { // MSVC. StringRef DisplayName = SP->getName().split('<').first; - const DIScope *Scope = SP->getScope().resolve(); + const DIScope *Scope = SP->getScope(); TypeIndex TI; if (const auto *Class = dyn_cast_or_null(Scope)) { // If the scope is a DICompositeType, then this must be a method. Member @@ -364,8 +386,8 @@ TypeIndex CodeViewDebug::getFuncIdForSubprogram(const DISubprogram *SP) { return recordTypeIndexForDINode(SP, TI); } -static bool isTrivial(const DICompositeType *DCTy) { - return ((DCTy->getFlags() & DINode::FlagTrivial) == DINode::FlagTrivial); +static bool isNonTrivial(const DICompositeType *DCTy) { + return ((DCTy->getFlags() & DINode::FlagNonTrivial) == DINode::FlagNonTrivial); } static FunctionOptions @@ -376,16 +398,16 @@ getFunctionOptions(const DISubroutineType *Ty, const DIType *ReturnTy = nullptr; if (auto TypeArray = Ty->getTypeArray()) { if (TypeArray.size()) - ReturnTy = TypeArray[0].resolve(); + ReturnTy = TypeArray[0]; } if (auto *ReturnDCTy = dyn_cast_or_null(ReturnTy)) { - if (!isTrivial(ReturnDCTy)) + if (isNonTrivial(ReturnDCTy)) FO |= FunctionOptions::CxxReturnUdt; } // DISubroutineType is unnamed. Use DISubprogram's i.e. SPName in comparison. - if (ClassTy && !isTrivial(ClassTy) && SPName == ClassTy->getName()) { + if (ClassTy && isNonTrivial(ClassTy) && SPName == ClassTy->getName()) { FO |= FunctionOptions::Constructor; // TODO: put the FunctionOptions::ConstructorWithVirtualBases flag. @@ -582,8 +604,9 @@ void CodeViewDebug::endModule() { clear(); } -static void emitNullTerminatedSymbolName(MCStreamer &OS, StringRef S, - unsigned MaxFixedRecordLength = 0xF00) { +static void +emitNullTerminatedSymbolName(MCStreamer &OS, StringRef S, + unsigned MaxFixedRecordLength = 0xF00) { // The maximum CV record length is 0xFF00. Most of the strings we emit appear // after a fixed length portion of the record. The fixed length portion should // always be less than 0xF00 (3840) bytes, so truncate the string so that the @@ -594,6 +617,13 @@ static void emitNullTerminatedSymbolName(MCStreamer &OS, StringRef S, OS.EmitBytes(NullTerminatedString); } +static StringRef getTypeLeafName(TypeLeafKind TypeKind) { + for (const EnumEntry &EE : getTypeLeafNames()) + if (EE.Value == TypeKind) + return EE.Name; + return ""; +} + void CodeViewDebug::emitTypeInformation() { if (TypeTable.empty()) return; @@ -610,31 +640,55 @@ void CodeViewDebug::emitTypeInformation() { } TypeTableCollection Table(TypeTable.records()); + SmallString<512> CommentBlock; + raw_svector_ostream CommentOS(CommentBlock); + std::unique_ptr SP; + std::unique_ptr TDV; + TypeVisitorCallbackPipeline Pipeline; + + if (OS.isVerboseAsm()) { + // To construct block comment describing the type record for readability. + SP = llvm::make_unique(CommentOS); + SP->setPrefix(CommentPrefix); + TDV = llvm::make_unique(Table, SP.get(), false); + Pipeline.addCallbackToPipeline(*TDV); + } + + // To emit type record using Codeview MCStreamer adapter + CVMCAdapter CVMCOS(OS); + TypeRecordMapping typeMapping(CVMCOS); + Pipeline.addCallbackToPipeline(typeMapping); + Optional B = Table.getFirst(); while (B) { // This will fail if the record data is invalid. CVType Record = Table.getType(*B); + CommentBlock.clear(); + + auto RecordLen = Record.length(); + auto RecordKind = Record.kind(); + if (OS.isVerboseAsm()) + CVMCOS.AddComment("Record length"); + CVMCOS.EmitIntValue(RecordLen - 2, 2); + if (OS.isVerboseAsm()) + CVMCOS.AddComment("Record kind: " + getTypeLeafName(RecordKind)); + CVMCOS.EmitIntValue(RecordKind, sizeof(RecordKind)); + + Error E = codeview::visitTypeRecord(Record, *B, Pipeline); + + if (E) { + logAllUnhandledErrors(std::move(E), errs(), "error: "); + llvm_unreachable("produced malformed type record"); + } + if (OS.isVerboseAsm()) { - // Emit a block comment describing the type record for readability. - SmallString<512> CommentBlock; - raw_svector_ostream CommentOS(CommentBlock); - ScopedPrinter SP(CommentOS); - SP.setPrefix(CommentPrefix); - TypeDumpVisitor TDV(Table, &SP, false); - - Error E = codeview::visitTypeRecord(Record, *B, TDV); - if (E) { - logAllUnhandledErrors(std::move(E), errs(), "error: "); - llvm_unreachable("produced malformed type record"); - } // emitRawComment will insert its own tab and comment string before // the first line, so strip off our first one. It also prints its own // newline. OS.emitRawComment( CommentOS.str().drop_front(CommentPrefix.size() - 1).rtrim()); } - OS.EmitBinaryData(Record.str_data()); B = Table.getNext(*B); } } @@ -700,6 +754,8 @@ static SourceLanguage MapDWLangToCVLang(unsigned DWLang) { return SourceLanguage::Java; case dwarf::DW_LANG_D: return SourceLanguage::D; + case dwarf::DW_LANG_Swift: + return SourceLanguage::Swift; default: // There's no CodeView representation for this language, and CV doesn't // have an "unknown" option for the language field, so we'll use MASM, @@ -973,8 +1029,7 @@ void CodeViewDebug::emitDebugInfoForFunction(const Function *GV, // If we have a display name, build the fully qualified name by walking the // chain of scopes. if (!SP->getName().empty()) - FuncName = - getFullyQualifiedName(SP->getScope().resolve(), SP->getName()); + FuncName = getFullyQualifiedName(SP->getScope(), SP->getName()); // If our DISubprogram name is empty, use the mangled name. if (FuncName.empty()) @@ -1071,6 +1126,28 @@ void CodeViewDebug::emitDebugInfoForFunction(const Function *GV, endSymbolRecord(AnnotEnd); } + for (auto HeapAllocSite : FI.HeapAllocSites) { + MCSymbol *BeginLabel = std::get<0>(HeapAllocSite); + MCSymbol *EndLabel = std::get<1>(HeapAllocSite); + + // The labels might not be defined if the instruction was replaced + // somewhere in the codegen pipeline. + if (!BeginLabel->isDefined() || !EndLabel->isDefined()) + continue; + + DIType *DITy = std::get<2>(HeapAllocSite); + MCSymbol *HeapAllocEnd = beginSymbolRecord(SymbolKind::S_HEAPALLOCSITE); + OS.AddComment("Call site offset"); + OS.EmitCOFFSecRel32(BeginLabel, /*Offset=*/0); + OS.AddComment("Call site section index"); + OS.EmitCOFFSectionIndex(BeginLabel); + OS.AddComment("Call instruction length"); + OS.emitAbsoluteSymbolDiff(EndLabel, BeginLabel, 2); + OS.AddComment("Type index"); + OS.EmitIntValue(getCompleteTypeIndex(DITy).getIndex(), 4); + endSymbolRecord(HeapAllocEnd); + } + if (SP != nullptr) emitDebugInfoForUDTs(LocalUDTs); @@ -1118,9 +1195,15 @@ void CodeViewDebug::collectVariableInfoFromMFTable( // If the variable has an attached offset expression, extract it. // FIXME: Try to handle DW_OP_deref as well. int64_t ExprOffset = 0; - if (VI.Expr) - if (!VI.Expr->extractIfOffset(ExprOffset)) + bool Deref = false; + if (VI.Expr) { + // If there is one DW_OP_deref element, use offset of 0 and keep going. + if (VI.Expr->getNumElements() == 1 && + VI.Expr->getElement(0) == llvm::dwarf::DW_OP_deref) + Deref = true; + else if (!VI.Expr->extractIfOffset(ExprOffset)) continue; + } // Get the frame register used and the offset. unsigned FrameReg = 0; @@ -1130,6 +1213,7 @@ void CodeViewDebug::collectVariableInfoFromMFTable( // Calculate the label ranges. LocalVarDefRange DefRange = createDefRangeMem(CVReg, FrameOffset + ExprOffset); + for (const InsnRange &Range : Scope->getRanges()) { const MCSymbol *Begin = getLabelBeforeInsn(Range.first); const MCSymbol *End = getLabelAfterInsn(Range.second); @@ -1140,6 +1224,9 @@ void CodeViewDebug::collectVariableInfoFromMFTable( LocalVariable Var; Var.DIVar = VI.Var; Var.DefRanges.emplace_back(std::move(DefRange)); + if (Deref) + Var.UseReferenceType = true; + recordLocalVariable(std::move(Var), Scope); } } @@ -1153,13 +1240,15 @@ static bool needsReferenceType(const DbgVariableLocation &Loc) { } void CodeViewDebug::calculateRanges( - LocalVariable &Var, const DbgValueHistoryMap::InstrRanges &Ranges) { + LocalVariable &Var, const DbgValueHistoryMap::Entries &Entries) { const TargetRegisterInfo *TRI = Asm->MF->getSubtarget().getRegisterInfo(); // Calculate the definition ranges. - for (auto I = Ranges.begin(), E = Ranges.end(); I != E; ++I) { - const InsnRange &Range = *I; - const MachineInstr *DVInst = Range.first; + for (auto I = Entries.begin(), E = Entries.end(); I != E; ++I) { + const auto &Entry = *I; + if (!Entry.isDbgValue()) + continue; + const MachineInstr *DVInst = Entry.getInstr(); assert(DVInst->isDebugValue() && "Invalid History entry"); // FIXME: Find a way to represent constant variables, since they are // relatively common. @@ -1186,7 +1275,7 @@ void CodeViewDebug::calculateRanges( // Start over using that. Var.UseReferenceType = true; Var.DefRanges.clear(); - calculateRanges(Var, Ranges); + calculateRanges(Var, Entries); return; } @@ -1214,21 +1303,15 @@ void CodeViewDebug::calculateRanges( } // Compute the label range. - const MCSymbol *Begin = getLabelBeforeInsn(Range.first); - const MCSymbol *End = getLabelAfterInsn(Range.second); - if (!End) { - // This range is valid until the next overlapping bitpiece. In the - // common case, ranges will not be bitpieces, so they will overlap. - auto J = std::next(I); - const DIExpression *DIExpr = DVInst->getDebugExpression(); - while (J != E && - !DIExpr->fragmentsOverlap(J->first->getDebugExpression())) - ++J; - if (J != E) - End = getLabelBeforeInsn(J->first); - else - End = Asm->getFunctionEnd(); - } + const MCSymbol *Begin = getLabelBeforeInsn(Entry.getInstr()); + const MCSymbol *End; + if (Entry.getEndIndex() != DbgValueHistoryMap::NoEntry) { + auto &EndingEntry = Entries[Entry.getEndIndex()]; + End = EndingEntry.isDbgValue() + ? getLabelBeforeInsn(EndingEntry.getInstr()) + : getLabelAfterInsn(EndingEntry.getInstr()); + } else + End = Asm->getFunctionEnd(); // If the last range end is our begin, just extend the last range. // Otherwise make a new range. @@ -1256,7 +1339,7 @@ void CodeViewDebug::collectVariableInfo(const DISubprogram *SP) { const DILocation *InlinedAt = IV.second; // Instruction ranges, specifying where IV is accessible. - const auto &Ranges = I.second; + const auto &Entries = I.second; LexicalScope *Scope = nullptr; if (InlinedAt) @@ -1270,7 +1353,7 @@ void CodeViewDebug::collectVariableInfo(const DISubprogram *SP) { LocalVariable Var; Var.DIVar = DIVar; - calculateRanges(Var, Ranges); + calculateRanges(Var, Entries); recordLocalVariable(std::move(Var), Scope); } } @@ -1340,8 +1423,8 @@ void CodeViewDebug::beginFunctionImpl(const MachineFunction *MF) { FPO |= FrameProcedureOptions::SecurityChecks; FPO |= FrameProcedureOptions(uint32_t(CurFn->EncodedLocalFramePtrReg) << 14U); FPO |= FrameProcedureOptions(uint32_t(CurFn->EncodedParamFramePtrReg) << 16U); - if (Asm->TM.getOptLevel() != CodeGenOpt::None && !GV.optForSize() && - !GV.hasFnAttribute(Attribute::OptimizeNone)) + if (Asm->TM.getOptLevel() != CodeGenOpt::None && + !GV.hasOptSize() && !GV.hasOptNone()) FPO |= FrameProcedureOptions::OptimizedForSpeed; // FIXME: Set GuardCfg when it is implemented. CurFn->FrameProcOpts = FPO; @@ -1379,7 +1462,7 @@ static bool shouldEmitUdt(const DIType *T) { // MSVC does not emit UDTs for typedefs that are scoped to classes. if (T->getTag() == dwarf::DW_TAG_typedef) { - if (DIScope *Scope = T->getScope().resolve()) { + if (DIScope *Scope = T->getScope()) { switch (Scope->getTag()) { case dwarf::DW_TAG_structure_type: case dwarf::DW_TAG_class_type: @@ -1396,7 +1479,7 @@ static bool shouldEmitUdt(const DIType *T) { const DIDerivedType *DT = dyn_cast(T); if (!DT) return true; - T = DT->getBaseType().resolve(); + T = DT->getBaseType(); } return true; } @@ -1409,8 +1492,8 @@ void CodeViewDebug::addToUDTs(const DIType *Ty) { return; SmallVector QualifiedNameComponents; - const DISubprogram *ClosestSubprogram = getQualifiedNameComponents( - Ty->getScope().resolve(), QualifiedNameComponents); + const DISubprogram *ClosestSubprogram = + getQualifiedNameComponents(Ty->getScope(), QualifiedNameComponents); std::string FullyQualifiedName = getQualifiedName(QualifiedNameComponents, getPrettyScopeName(Ty)); @@ -1479,8 +1562,7 @@ TypeIndex CodeViewDebug::lowerType(const DIType *Ty, const DIType *ClassTy) { } TypeIndex CodeViewDebug::lowerTypeAlias(const DIDerivedType *Ty) { - DITypeRef UnderlyingTypeRef = Ty->getBaseType(); - TypeIndex UnderlyingTypeIndex = getTypeIndex(UnderlyingTypeRef); + TypeIndex UnderlyingTypeIndex = getTypeIndex(Ty->getBaseType()); StringRef TypeName = Ty->getName(); addToUDTs(Ty); @@ -1496,14 +1578,14 @@ TypeIndex CodeViewDebug::lowerTypeAlias(const DIDerivedType *Ty) { } TypeIndex CodeViewDebug::lowerTypeArray(const DICompositeType *Ty) { - DITypeRef ElementTypeRef = Ty->getBaseType(); - TypeIndex ElementTypeIndex = getTypeIndex(ElementTypeRef); + const DIType *ElementType = Ty->getBaseType(); + TypeIndex ElementTypeIndex = getTypeIndex(ElementType); // IndexType is size_t, which depends on the bitness of the target. TypeIndex IndexType = getPointerSizeInBytes() == 8 ? TypeIndex(SimpleTypeKind::UInt64Quad) : TypeIndex(SimpleTypeKind::UInt32Long); - uint64_t ElementSize = getBaseTypeSize(ElementTypeRef) / 8; + uint64_t ElementSize = getBaseTypeSize(ElementType) / 8; // Add subranges to array type. DINodeArray Elements = Ty->getElements(); @@ -1764,7 +1846,7 @@ TypeIndex CodeViewDebug::lowerTypeModifier(const DIDerivedType *Ty) { break; } if (IsModifier) - BaseTy = cast(BaseTy)->getBaseType().resolve(); + BaseTy = cast(BaseTy)->getBaseType(); } // Check if the inner type will use an LF_POINTER record. If so, the @@ -1797,8 +1879,8 @@ TypeIndex CodeViewDebug::lowerTypeModifier(const DIDerivedType *Ty) { TypeIndex CodeViewDebug::lowerTypeFunction(const DISubroutineType *Ty) { SmallVector ReturnAndArgTypeIndices; - for (DITypeRef ArgTypeRef : Ty->getTypeArray()) - ReturnAndArgTypeIndices.push_back(getTypeIndex(ArgTypeRef)); + for (const DIType *ArgType : Ty->getTypeArray()) + ReturnAndArgTypeIndices.push_back(getTypeIndex(ArgType)); // MSVC uses type none for variadic argument. if (ReturnAndArgTypeIndices.size() > 1 && @@ -1836,7 +1918,10 @@ TypeIndex CodeViewDebug::lowerTypeMemberFunction(const DISubroutineType *Ty, unsigned Index = 0; SmallVector ArgTypeIndices; - TypeIndex ReturnTypeIndex = getTypeIndex(ReturnAndArgs[Index++]); + TypeIndex ReturnTypeIndex = TypeIndex::Void(); + if (ReturnAndArgs.size() > Index) { + ReturnTypeIndex = getTypeIndex(ReturnAndArgs[Index++]); + } // If the first argument is a pointer type and this isn't a static method, // treat it as the special 'this' parameter, which is encoded separately from @@ -1844,7 +1929,7 @@ TypeIndex CodeViewDebug::lowerTypeMemberFunction(const DISubroutineType *Ty, TypeIndex ThisTypeIndex; if (!IsStaticMethod && ReturnAndArgs.size() > Index) { if (const DIDerivedType *PtrTy = - dyn_cast_or_null(ReturnAndArgs[Index].resolve())) { + dyn_cast_or_null(ReturnAndArgs[Index])) { if (PtrTy->getTag() == dwarf::DW_TAG_pointer_type) { ThisTypeIndex = getTypeIndexForThisPtr(PtrTy, Ty); Index++; @@ -1942,7 +2027,7 @@ static ClassOptions getCommonClassOptions(const DICompositeType *Ty) { // Put the Nested flag on a type if it appears immediately inside a tag type. // Do not walk the scope chain. Do not attempt to compute ContainsNestedClass // here. That flag is only set on definitions, and not forward declarations. - const DIScope *ImmediateScope = Ty->getScope().resolve(); + const DIScope *ImmediateScope = Ty->getScope(); if (ImmediateScope && isa(ImmediateScope)) CO |= ClassOptions::Nested; @@ -1955,7 +2040,7 @@ static ClassOptions getCommonClassOptions(const DICompositeType *Ty) { CO |= ClassOptions::Scoped; } else { for (const DIScope *Scope = ImmediateScope; Scope != nullptr; - Scope = Scope->getScope().resolve()) { + Scope = Scope->getScope()) { if (isa(Scope)) { CO |= ClassOptions::Scoped; break; @@ -2075,7 +2160,7 @@ void CodeViewDebug::collectMemberInfo(ClassInfo &Info, // succeeds, and drop the member if that fails. assert((DDTy->getOffsetInBits() % 8) == 0 && "Unnamed bitfield member!"); uint64_t Offset = DDTy->getOffsetInBits(); - const DIType *Ty = DDTy->getBaseType().resolve(); + const DIType *Ty = DDTy->getBaseType(); bool FullyResolved = false; while (!FullyResolved) { switch (Ty->getTag()) { @@ -2083,7 +2168,7 @@ void CodeViewDebug::collectMemberInfo(ClassInfo &Info, case dwarf::DW_TAG_volatile_type: // FIXME: we should apply the qualifier types to the indirect fields // rather than dropping them. - Ty = cast(Ty)->getBaseType().resolve(); + Ty = cast(Ty)->getBaseType(); break; default: FullyResolved = true; @@ -2184,6 +2269,14 @@ TypeIndex CodeViewDebug::lowerCompleteTypeClass(const DICompositeType *Ty) { if (ContainsNestedClass) CO |= ClassOptions::ContainsNestedClass; + // MSVC appears to set this flag by searching any destructor or method with + // FunctionOptions::Constructor among the emitted members. Clang AST has all + // the members, however special member functions are not yet emitted into + // debug information. For now checking a class's non-triviality seems enough. + // FIXME: not true for a nested unnamed struct. + if (isNonTrivial(Ty)) + CO |= ClassOptions::HasConstructorOrDestructor; + std::string FullName = getFullyQualifiedName(Ty); uint64_t SizeInBytes = Ty->getSizeInBits() / 8; @@ -2358,7 +2451,7 @@ CodeViewDebug::lowerRecordFieldList(const DICompositeType *Ty) { // Create nested classes. for (const DIType *Nested : Info.NestedTypes) { - NestedTypeRecord R(getTypeIndex(DITypeRef(Nested)), Nested->getName()); + NestedTypeRecord R(getTypeIndex(Nested), Nested->getName()); ContinuationBuilder.writeMemberType(R); MemberCount++; } @@ -2385,10 +2478,7 @@ TypeIndex CodeViewDebug::getVBPTypeIndex() { return VBPType; } -TypeIndex CodeViewDebug::getTypeIndex(DITypeRef TypeRef, DITypeRef ClassTyRef) { - const DIType *Ty = TypeRef.resolve(); - const DIType *ClassTy = ClassTyRef.resolve(); - +TypeIndex CodeViewDebug::getTypeIndex(const DIType *Ty, const DIType *ClassTy) { // The null DIType is the void type. Don't try to hash it. if (!Ty) return TypeIndex::Void(); @@ -2431,8 +2521,7 @@ CodeViewDebug::getTypeIndexForThisPtr(const DIDerivedType *PtrTy, return recordTypeIndexForDINode(PtrTy, TI, SubroutineTy); } -TypeIndex CodeViewDebug::getTypeIndexForReferenceTo(DITypeRef TypeRef) { - DIType *Ty = TypeRef.resolve(); +TypeIndex CodeViewDebug::getTypeIndexForReferenceTo(const DIType *Ty) { PointerRecord PR(getTypeIndex(Ty), getPointerSizeInBytes() == 8 ? PointerKind::Near64 : PointerKind::Near32, @@ -2441,9 +2530,7 @@ TypeIndex CodeViewDebug::getTypeIndexForReferenceTo(DITypeRef TypeRef) { return TypeTable.writeLeafType(PR); } -TypeIndex CodeViewDebug::getCompleteTypeIndex(DITypeRef TypeRef) { - const DIType *Ty = TypeRef.resolve(); - +TypeIndex CodeViewDebug::getCompleteTypeIndex(const DIType *Ty) { // The null DIType is the void type. Don't try to hash it. if (!Ty) return TypeIndex::Void(); @@ -2454,7 +2541,7 @@ TypeIndex CodeViewDebug::getCompleteTypeIndex(DITypeRef TypeRef) { if (Ty->getTag() == dwarf::DW_TAG_typedef) (void)getTypeIndex(Ty); while (Ty->getTag() == dwarf::DW_TAG_typedef) - Ty = cast(Ty)->getBaseType().resolve(); + Ty = cast(Ty)->getBaseType(); // If this is a non-record type, the complete type index is the same as the // normal type index. Just call getTypeIndex. @@ -2467,11 +2554,7 @@ TypeIndex CodeViewDebug::getCompleteTypeIndex(DITypeRef TypeRef) { return getTypeIndex(Ty); } - // Check if we've already translated the complete record type. const auto *CTy = cast(Ty); - auto InsertResult = CompleteTypeIndices.insert({CTy, TypeIndex()}); - if (!InsertResult.second) - return InsertResult.first->second; TypeLoweringScope S(*this); @@ -2489,6 +2572,13 @@ TypeIndex CodeViewDebug::getCompleteTypeIndex(DITypeRef TypeRef) { return FwdDeclTI; } + // Check if we've already translated the complete record type. + // Insert the type with a null TypeIndex to signify that the type is currently + // being lowered. + auto InsertResult = CompleteTypeIndices.insert({CTy, TypeIndex()}); + if (!InsertResult.second) + return InsertResult.first->second; + TypeIndex TI; switch (CTy->getTag()) { case dwarf::DW_TAG_class_type: @@ -2799,6 +2889,7 @@ void CodeViewDebug::endFunctionImpl(const MachineFunction *MF) { } CurFn->Annotations = MF->getCodeViewAnnotations(); + CurFn->HeapAllocSites = MF->getCodeViewHeapAllocSites(); CurFn->End = Asm->getFunctionEnd(); @@ -2914,10 +3005,19 @@ void CodeViewDebug::collectGlobalVariableInfo() { for (const MDNode *Node : CUs->operands()) { const auto *CU = cast(Node); for (const auto *GVE : CU->getGlobalVariables()) { + const DIGlobalVariable *DIGV = GVE->getVariable(); + const DIExpression *DIE = GVE->getExpression(); + + // Emit constant global variables in a global symbol section. + if (GlobalMap.count(GVE) == 0 && DIE->isConstant()) { + CVGlobalVariable CVGV = {DIGV, DIE}; + GlobalVariables.emplace_back(std::move(CVGV)); + } + const auto *GV = GlobalMap.lookup(GVE); if (!GV || GV->isDeclarationForLinker()) continue; - const DIGlobalVariable *DIGV = GVE->getVariable(); + DIScope *Scope = DIGV->getScope(); SmallVector *VariableList; if (Scope && isa(Scope)) { @@ -2932,7 +3032,7 @@ void CodeViewDebug::collectGlobalVariableInfo() { // Emit this global variable into a COMDAT section. VariableList = &ComdatVariables; else - // Emit this globla variable in a single global symbol section. + // Emit this global variable in a single global symbol section. VariableList = &GlobalVariables; CVGlobalVariable CVGV = {DIGV, GV}; VariableList->emplace_back(std::move(CVGV)); @@ -2955,13 +3055,14 @@ void CodeViewDebug::emitDebugInfoForGlobals() { // Second, emit each global that is in a comdat into its own .debug$S // section along with its own symbol substream. for (const CVGlobalVariable &CVGV : ComdatVariables) { - MCSymbol *GVSym = Asm->getSymbol(CVGV.GV); + const GlobalVariable *GV = CVGV.GVInfo.get(); + MCSymbol *GVSym = Asm->getSymbol(GV); OS.AddComment("Symbol subsection for " + - Twine(GlobalValue::dropLLVMManglingEscape(CVGV.GV->getName()))); + Twine(GlobalValue::dropLLVMManglingEscape(GV->getName()))); switchToDebugSectionForSymbol(GVSym); MCSymbol *EndLabel = beginCVSubsection(DebugSubsectionKind::Symbols); // FIXME: emitDebugInfoForGlobal() doesn't handle DIExpressions. - emitDebugInfoForGlobal(CVGV.DIGV, CVGV.GV, GVSym); + emitDebugInfoForGlobal(CVGV); endCVSubsection(EndLabel); } } @@ -2981,31 +3082,63 @@ void CodeViewDebug::emitDebugInfoForRetainedTypes() { // Emit each global variable in the specified array. void CodeViewDebug::emitGlobalVariableList(ArrayRef Globals) { for (const CVGlobalVariable &CVGV : Globals) { - MCSymbol *GVSym = Asm->getSymbol(CVGV.GV); // FIXME: emitDebugInfoForGlobal() doesn't handle DIExpressions. - emitDebugInfoForGlobal(CVGV.DIGV, CVGV.GV, GVSym); - } -} - -void CodeViewDebug::emitDebugInfoForGlobal(const DIGlobalVariable *DIGV, - const GlobalVariable *GV, - MCSymbol *GVSym) { - // DataSym record, see SymbolRecord.h for more info. Thread local data - // happens to have the same format as global data. - SymbolKind DataSym = GV->isThreadLocal() - ? (DIGV->isLocalToUnit() ? SymbolKind::S_LTHREAD32 - : SymbolKind::S_GTHREAD32) - : (DIGV->isLocalToUnit() ? SymbolKind::S_LDATA32 - : SymbolKind::S_GDATA32); - MCSymbol *DataEnd = beginSymbolRecord(DataSym); - OS.AddComment("Type"); - OS.EmitIntValue(getCompleteTypeIndex(DIGV->getType()).getIndex(), 4); - OS.AddComment("DataOffset"); - OS.EmitCOFFSecRel32(GVSym, /*Offset=*/0); - OS.AddComment("Segment"); - OS.EmitCOFFSectionIndex(GVSym); - OS.AddComment("Name"); - const unsigned LengthOfDataRecord = 12; - emitNullTerminatedSymbolName(OS, DIGV->getName(), LengthOfDataRecord); - endSymbolRecord(DataEnd); + emitDebugInfoForGlobal(CVGV); + } +} + +void CodeViewDebug::emitDebugInfoForGlobal(const CVGlobalVariable &CVGV) { + const DIGlobalVariable *DIGV = CVGV.DIGV; + if (const GlobalVariable *GV = + CVGV.GVInfo.dyn_cast()) { + // DataSym record, see SymbolRecord.h for more info. Thread local data + // happens to have the same format as global data. + MCSymbol *GVSym = Asm->getSymbol(GV); + SymbolKind DataSym = GV->isThreadLocal() + ? (DIGV->isLocalToUnit() ? SymbolKind::S_LTHREAD32 + : SymbolKind::S_GTHREAD32) + : (DIGV->isLocalToUnit() ? SymbolKind::S_LDATA32 + : SymbolKind::S_GDATA32); + MCSymbol *DataEnd = beginSymbolRecord(DataSym); + OS.AddComment("Type"); + OS.EmitIntValue(getCompleteTypeIndex(DIGV->getType()).getIndex(), 4); + OS.AddComment("DataOffset"); + OS.EmitCOFFSecRel32(GVSym, /*Offset=*/0); + OS.AddComment("Segment"); + OS.EmitCOFFSectionIndex(GVSym); + OS.AddComment("Name"); + const unsigned LengthOfDataRecord = 12; + emitNullTerminatedSymbolName(OS, DIGV->getName(), LengthOfDataRecord); + endSymbolRecord(DataEnd); + } else { + // FIXME: Currently this only emits the global variables in the IR metadata. + // This should also emit enums and static data members. + const DIExpression *DIE = CVGV.GVInfo.get(); + assert(DIE->isConstant() && + "Global constant variables must contain a constant expression."); + uint64_t Val = DIE->getElement(1); + + MCSymbol *SConstantEnd = beginSymbolRecord(SymbolKind::S_CONSTANT); + OS.AddComment("Type"); + OS.EmitIntValue(getTypeIndex(DIGV->getType()).getIndex(), 4); + OS.AddComment("Value"); + + // Encoded integers shouldn't need more than 10 bytes. + uint8_t data[10]; + BinaryStreamWriter Writer(data, llvm::support::endianness::little); + CodeViewRecordIO IO(Writer); + cantFail(IO.mapEncodedInteger(Val)); + StringRef SRef((char *)data, Writer.getOffset()); + OS.EmitBinaryData(SRef); + + OS.AddComment("Name"); + const DIScope *Scope = DIGV->getScope(); + // For static data members, get the scope from the declaration. + if (const auto *MemberDecl = dyn_cast_or_null( + DIGV->getRawStaticDataMemberDeclaration())) + Scope = MemberDecl->getScope(); + emitNullTerminatedSymbolName(OS, + getFullyQualifiedName(Scope, DIGV->getName())); + endSymbolRecord(SConstantEnd); + } } diff --git a/lib/CodeGen/AsmPrinter/CodeViewDebug.h b/lib/CodeGen/AsmPrinter/CodeViewDebug.h index 21557ed1be35..ce57b789d7fa 100644 --- a/lib/CodeGen/AsmPrinter/CodeViewDebug.h +++ b/lib/CodeGen/AsmPrinter/CodeViewDebug.h @@ -1,9 +1,8 @@ //===- llvm/lib/CodeGen/AsmPrinter/CodeViewDebug.h --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,6 +17,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseSet.h" #include "llvm/ADT/MapVector.h" +#include "llvm/ADT/PointerUnion.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/DbgEntityHistoryCalculator.h" @@ -101,7 +101,7 @@ class LLVM_LIBRARY_VISIBILITY CodeViewDebug : public DebugHandlerBase { struct CVGlobalVariable { const DIGlobalVariable *DIGV; - const GlobalVariable *GV; + PointerUnion GVInfo; }; struct InlineSite { @@ -148,6 +148,7 @@ class LLVM_LIBRARY_VISIBILITY CodeViewDebug : public DebugHandlerBase { SmallVector ChildBlocks; std::vector> Annotations; + std::vector> HeapAllocSites; const MCSymbol *Begin = nullptr; const MCSymbol *End = nullptr; @@ -223,7 +224,7 @@ class LLVM_LIBRARY_VISIBILITY CodeViewDebug : public DebugHandlerBase { codeview::TypeIndex getFuncIdForSubprogram(const DISubprogram *SP); void calculateRanges(LocalVariable &Var, - const DbgValueHistoryMap::InstrRanges &Ranges); + const DbgValueHistoryMap::Entries &Entries); static void collectInlineSiteChildren(SmallVectorImpl &Children, const FunctionInfo &FI, @@ -313,8 +314,7 @@ class LLVM_LIBRARY_VISIBILITY CodeViewDebug : public DebugHandlerBase { void emitDebugInfoForGlobals(); void emitGlobalVariableList(ArrayRef Globals); - void emitDebugInfoForGlobal(const DIGlobalVariable *DIGV, - const GlobalVariable *GV, MCSymbol *GVSym); + void emitDebugInfoForGlobal(const CVGlobalVariable &CVGV); /// Opens a subsection of the given kind in a .debug$S codeview section. /// Returns an end label for use with endCVSubsection when the subsection is @@ -373,14 +373,14 @@ class LLVM_LIBRARY_VISIBILITY CodeViewDebug : public DebugHandlerBase { /// Translates the DIType to codeview if necessary and returns a type index /// for it. - codeview::TypeIndex getTypeIndex(DITypeRef TypeRef, - DITypeRef ClassTyRef = DITypeRef()); + codeview::TypeIndex getTypeIndex(const DIType *Ty, + const DIType *ClassTy = nullptr); codeview::TypeIndex getTypeIndexForThisPtr(const DIDerivedType *PtrTy, const DISubroutineType *SubroutineTy); - codeview::TypeIndex getTypeIndexForReferenceTo(DITypeRef TypeRef); + codeview::TypeIndex getTypeIndexForReferenceTo(const DIType *Ty); codeview::TypeIndex getMemberFunctionType(const DISubprogram *SP, const DICompositeType *Class); @@ -419,7 +419,7 @@ class LLVM_LIBRARY_VISIBILITY CodeViewDebug : public DebugHandlerBase { /// use this entry point when generating symbol records. The complete and /// incomplete type indices only differ for record types. All other types use /// the same index. - codeview::TypeIndex getCompleteTypeIndex(DITypeRef TypeRef); + codeview::TypeIndex getCompleteTypeIndex(const DIType *Ty); codeview::TypeIndex lowerCompleteTypeClass(const DICompositeType *Ty); codeview::TypeIndex lowerCompleteTypeUnion(const DICompositeType *Ty); diff --git a/lib/CodeGen/AsmPrinter/DIE.cpp b/lib/CodeGen/AsmPrinter/DIE.cpp index e27659494f08..f4134da48caa 100644 --- a/lib/CodeGen/AsmPrinter/DIE.cpp +++ b/lib/CodeGen/AsmPrinter/DIE.cpp @@ -1,9 +1,8 @@ //===--- lib/CodeGen/DIE.cpp - DWARF Info Entries -------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -212,7 +211,7 @@ const DIE *DIE::getUnitDie() const { return nullptr; } -const DIEUnit *DIE::getUnit() const { +DIEUnit *DIE::getUnit() const { const DIE *UnitDie = getUnitDie(); if (UnitDie) return UnitDie->Owner.dyn_cast(); @@ -506,6 +505,23 @@ unsigned DIELabel::SizeOf(const AsmPrinter *AP, dwarf::Form Form) const { LLVM_DUMP_METHOD void DIELabel::print(raw_ostream &O) const { O << "Lbl: " << Label->getName(); } +//===----------------------------------------------------------------------===// +// DIEBaseTypeRef Implementation +//===----------------------------------------------------------------------===// + +void DIEBaseTypeRef::EmitValue(const AsmPrinter *AP, dwarf::Form Form) const { + uint64_t Offset = CU->ExprRefedBaseTypes[Index].Die->getOffset(); + assert(Offset < (1ULL << (ULEB128PadSize * 7)) && "Offset wont fit"); + AP->EmitULEB128(Offset, nullptr, ULEB128PadSize); +} + +unsigned DIEBaseTypeRef::SizeOf(const AsmPrinter *AP, dwarf::Form Form) const { + return ULEB128PadSize; +} + +LLVM_DUMP_METHOD +void DIEBaseTypeRef::print(raw_ostream &O) const { O << "BaseTypeRef: " << Index; } + //===----------------------------------------------------------------------===// // DIEDelta Implementation //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/AsmPrinter/DIEHash.cpp b/lib/CodeGen/AsmPrinter/DIEHash.cpp index b8f1202494d7..bfac8850a2a6 100644 --- a/lib/CodeGen/AsmPrinter/DIEHash.cpp +++ b/lib/CodeGen/AsmPrinter/DIEHash.cpp @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/DIEHash.cpp - Dwarf Hashing Framework ----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -226,7 +225,7 @@ void DIEHash::hashLocList(const DIELocList &LocList) { DwarfDebug &DD = *AP->getDwarfDebug(); const DebugLocStream &Locs = DD.getDebugLocs(); for (const auto &Entry : Locs.getEntries(Locs.getList(LocList.getValue()))) - DD.emitDebugLocEntry(Streamer, Entry); + DD.emitDebugLocEntry(Streamer, Entry, nullptr); } // Hash an individual attribute \param Attr based on the type of attribute and @@ -310,6 +309,7 @@ void DIEHash::hashAttribute(const DIEValue &Value, dwarf::Tag Tag) { // FIXME: It's uncertain whether or not we should handle this at the moment. case DIEValue::isExpr: case DIEValue::isLabel: + case DIEValue::isBaseTypeRef: case DIEValue::isDelta: llvm_unreachable("Add support for additional value types."); } diff --git a/lib/CodeGen/AsmPrinter/DIEHash.h b/lib/CodeGen/AsmPrinter/DIEHash.h index dae517ab2c29..2e49514c98be 100644 --- a/lib/CodeGen/AsmPrinter/DIEHash.h +++ b/lib/CodeGen/AsmPrinter/DIEHash.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/DIEHash.h - Dwarf Hashing Framework -------*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp b/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp index 09867822c30a..ddd60575b6c0 100644 --- a/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp +++ b/lib/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp @@ -1,15 +1,15 @@ //===- llvm/CodeGen/AsmPrinter/DbgEntityHistoryCalculator.cpp -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "llvm/CodeGen/DbgEntityHistoryCalculator.h" #include "llvm/ADT/BitVector.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunction.h" @@ -31,51 +31,62 @@ using namespace llvm; #define DEBUG_TYPE "dwarfdebug" +namespace { +using EntryIndex = DbgValueHistoryMap::EntryIndex; +} + // If @MI is a DBG_VALUE with debug value described by a // defined register, returns the number of this register. // In the other case, returns 0. -static unsigned isDescribedByReg(const MachineInstr &MI) { +static Register isDescribedByReg(const MachineInstr &MI) { assert(MI.isDebugValue()); assert(MI.getNumOperands() == 4); + // If the location of variable is an entry value (DW_OP_entry_value) + // do not consider it as a register location. + if (MI.getDebugExpression()->isEntryValue()) + return 0; // If location of variable is described using a register (directly or // indirectly), this register is always a first operand. - return MI.getOperand(0).isReg() ? MI.getOperand(0).getReg() : 0; + return MI.getOperand(0).isReg() ? MI.getOperand(0).getReg() : Register(); } -void DbgValueHistoryMap::startInstrRange(InlinedEntity Var, - const MachineInstr &MI) { +bool DbgValueHistoryMap::startDbgValue(InlinedEntity Var, + const MachineInstr &MI, + EntryIndex &NewIndex) { // Instruction range should start with a DBG_VALUE instruction for the // variable. assert(MI.isDebugValue() && "not a DBG_VALUE"); - auto &Ranges = VarInstrRanges[Var]; - if (!Ranges.empty() && Ranges.back().second == nullptr && - Ranges.back().first->isIdenticalTo(MI)) { + auto &Entries = VarEntries[Var]; + if (!Entries.empty() && Entries.back().isDbgValue() && + !Entries.back().isClosed() && + Entries.back().getInstr()->isIdenticalTo(MI)) { LLVM_DEBUG(dbgs() << "Coalescing identical DBG_VALUE entries:\n" - << "\t" << Ranges.back().first << "\t" << MI << "\n"); - return; + << "\t" << Entries.back().getInstr() << "\t" << MI + << "\n"); + return false; } - Ranges.push_back(std::make_pair(&MI, nullptr)); + Entries.emplace_back(&MI, Entry::DbgValue); + NewIndex = Entries.size() - 1; + return true; } -void DbgValueHistoryMap::endInstrRange(InlinedEntity Var, - const MachineInstr &MI) { - auto &Ranges = VarInstrRanges[Var]; - // Verify that the current instruction range is not yet closed. - assert(!Ranges.empty() && Ranges.back().second == nullptr); - // For now, instruction ranges are not allowed to cross basic block - // boundaries. - assert(Ranges.back().first->getParent() == MI.getParent()); - Ranges.back().second = &MI; +EntryIndex DbgValueHistoryMap::startClobber(InlinedEntity Var, + const MachineInstr &MI) { + auto &Entries = VarEntries[Var]; + // If an instruction clobbers multiple registers that the variable is + // described by, then we may have already created a clobbering instruction. + if (Entries.back().isClobber() && Entries.back().getInstr() == &MI) + return Entries.size() - 1; + Entries.emplace_back(&MI, Entry::Clobber); + return Entries.size() - 1; } -unsigned DbgValueHistoryMap::getRegisterForVar(InlinedEntity Var) const { - const auto &I = VarInstrRanges.find(Var); - if (I == VarInstrRanges.end()) - return 0; - const auto &Ranges = I->second; - if (Ranges.empty() || Ranges.back().second != nullptr) - return 0; - return isDescribedByReg(*Ranges.back().first); +void DbgValueHistoryMap::Entry::endEntry(EntryIndex Index) { + // For now, instruction ranges are not allowed to cross basic block + // boundaries. + assert(isDbgValue() && "Setting end index for non-debug value"); + assert(!isClosed() && "End index has already been set"); + EndIndex = Index; } void DbgLabelInstrMap::addInstr(InlinedEntity Label, const MachineInstr &MI) { @@ -89,6 +100,12 @@ namespace { using InlinedEntity = DbgValueHistoryMap::InlinedEntity; using RegDescribedVarsMap = std::map>; +// Keeps track of the debug value entries that are currently live for each +// inlined entity. As the history map entries are stored in a SmallVector, they +// may be moved at insertion of new entries, so store indices rather than +// pointers. +using DbgValueEntriesMap = std::map>; + } // end anonymous namespace // Claim that @Var is not described by @RegNo anymore. @@ -114,16 +131,88 @@ static void addRegDescribedVar(RegDescribedVarsMap &RegVars, unsigned RegNo, VarSet.push_back(Var); } +/// Create a clobbering entry and end all open debug value entries +/// for \p Var that are described by \p RegNo using that entry. +static void clobberRegEntries(InlinedEntity Var, unsigned RegNo, + const MachineInstr &ClobberingInstr, + DbgValueEntriesMap &LiveEntries, + DbgValueHistoryMap &HistMap) { + EntryIndex ClobberIndex = HistMap.startClobber(Var, ClobberingInstr); + + // Close all entries whose values are described by the register. + SmallVector IndicesToErase; + for (auto Index : LiveEntries[Var]) { + auto &Entry = HistMap.getEntry(Var, Index); + assert(Entry.isDbgValue() && "Not a DBG_VALUE in LiveEntries"); + if (isDescribedByReg(*Entry.getInstr()) == RegNo) { + IndicesToErase.push_back(Index); + Entry.endEntry(ClobberIndex); + } + } + + // Drop all entries that have ended. + for (auto Index : IndicesToErase) + LiveEntries[Var].erase(Index); +} + +/// Add a new debug value for \p Var. Closes all overlapping debug values. +static void handleNewDebugValue(InlinedEntity Var, const MachineInstr &DV, + RegDescribedVarsMap &RegVars, + DbgValueEntriesMap &LiveEntries, + DbgValueHistoryMap &HistMap) { + EntryIndex NewIndex; + if (HistMap.startDbgValue(Var, DV, NewIndex)) { + SmallDenseMap TrackedRegs; + + // If we have created a new debug value entry, close all preceding + // live entries that overlap. + SmallVector IndicesToErase; + const DIExpression *DIExpr = DV.getDebugExpression(); + for (auto Index : LiveEntries[Var]) { + auto &Entry = HistMap.getEntry(Var, Index); + assert(Entry.isDbgValue() && "Not a DBG_VALUE in LiveEntries"); + const MachineInstr &DV = *Entry.getInstr(); + bool Overlaps = DIExpr->fragmentsOverlap(DV.getDebugExpression()); + if (Overlaps) { + IndicesToErase.push_back(Index); + Entry.endEntry(NewIndex); + } + if (unsigned Reg = isDescribedByReg(DV)) + TrackedRegs[Reg] |= !Overlaps; + } + + // If the new debug value is described by a register, add tracking of + // that register if it is not already tracked. + if (unsigned NewReg = isDescribedByReg(DV)) { + if (!TrackedRegs.count(NewReg)) + addRegDescribedVar(RegVars, NewReg, Var); + LiveEntries[Var].insert(NewIndex); + TrackedRegs[NewReg] = true; + } + + // Drop tracking of registers that are no longer used. + for (auto I : TrackedRegs) + if (!I.second) + dropRegDescribedVar(RegVars, I.first, Var); + + // Drop all entries that have ended, and mark the new entry as live. + for (auto Index : IndicesToErase) + LiveEntries[Var].erase(Index); + LiveEntries[Var].insert(NewIndex); + } +} + // Terminate the location range for variables described by register at // @I by inserting @ClobberingInstr to their history. static void clobberRegisterUses(RegDescribedVarsMap &RegVars, RegDescribedVarsMap::iterator I, DbgValueHistoryMap &HistMap, + DbgValueEntriesMap &LiveEntries, const MachineInstr &ClobberingInstr) { // Iterate over all variables described by this register and add this // instruction to their history, clobbering it. for (const auto &Var : I->second) - HistMap.endInstrRange(Var, ClobberingInstr); + clobberRegEntries(Var, I->first, ClobberingInstr, LiveEntries, HistMap); RegVars.erase(I); } @@ -131,115 +220,25 @@ static void clobberRegisterUses(RegDescribedVarsMap &RegVars, // @RegNo by inserting @ClobberingInstr to their history. static void clobberRegisterUses(RegDescribedVarsMap &RegVars, unsigned RegNo, DbgValueHistoryMap &HistMap, + DbgValueEntriesMap &LiveEntries, const MachineInstr &ClobberingInstr) { const auto &I = RegVars.find(RegNo); if (I == RegVars.end()) return; - clobberRegisterUses(RegVars, I, HistMap, ClobberingInstr); -} - -// Returns the first instruction in @MBB which corresponds to -// the function epilogue, or nullptr if @MBB doesn't contain an epilogue. -static const MachineInstr *getFirstEpilogueInst(const MachineBasicBlock &MBB) { - auto LastMI = MBB.getLastNonDebugInstr(); - if (LastMI == MBB.end() || !LastMI->isReturn()) - return nullptr; - // Assume that epilogue starts with instruction having the same debug location - // as the return instruction. - DebugLoc LastLoc = LastMI->getDebugLoc(); - auto Res = LastMI; - for (MachineBasicBlock::const_reverse_iterator I = LastMI.getReverse(), - E = MBB.rend(); - I != E; ++I) { - if (I->getDebugLoc() != LastLoc) - return &*Res; - Res = &*I; - } - // If all instructions have the same debug location, assume whole MBB is - // an epilogue. - return &*MBB.begin(); -} - -// Collect registers that are modified in the function body (their -// contents is changed outside of the prologue and epilogue). -static void collectChangingRegs(const MachineFunction *MF, - const TargetRegisterInfo *TRI, - BitVector &Regs) { - for (const auto &MBB : *MF) { - auto FirstEpilogueInst = getFirstEpilogueInst(MBB); - - for (const auto &MI : MBB) { - // Avoid looking at prologue or epilogue instructions. - if (&MI == FirstEpilogueInst) - break; - if (MI.getFlag(MachineInstr::FrameSetup)) - continue; - - // Look for register defs and register masks. Register masks are - // typically on calls and they clobber everything not in the mask. - for (const MachineOperand &MO : MI.operands()) { - // Skip virtual registers since they are handled by the parent. - if (MO.isReg() && MO.isDef() && MO.getReg() && - !TRI->isVirtualRegister(MO.getReg())) { - for (MCRegAliasIterator AI(MO.getReg(), TRI, true); AI.isValid(); - ++AI) - Regs.set(*AI); - } else if (MO.isRegMask()) { - Regs.setBitsNotInMask(MO.getRegMask()); - } - } - } - } + clobberRegisterUses(RegVars, I, HistMap, LiveEntries, ClobberingInstr); } void llvm::calculateDbgEntityHistory(const MachineFunction *MF, const TargetRegisterInfo *TRI, DbgValueHistoryMap &DbgValues, DbgLabelInstrMap &DbgLabels) { - BitVector ChangingRegs(TRI->getNumRegs()); - collectChangingRegs(MF, TRI, ChangingRegs); - const TargetLowering *TLI = MF->getSubtarget().getTargetLowering(); unsigned SP = TLI->getStackPointerRegisterToSaveRestore(); + unsigned FrameReg = TRI->getFrameRegister(*MF); RegDescribedVarsMap RegVars; + DbgValueEntriesMap LiveEntries; for (const auto &MBB : *MF) { for (const auto &MI : MBB) { - if (!MI.isDebugInstr()) { - // Not a DBG_VALUE instruction. It may clobber registers which describe - // some variables. - for (const MachineOperand &MO : MI.operands()) { - if (MO.isReg() && MO.isDef() && MO.getReg()) { - // Ignore call instructions that claim to clobber SP. The AArch64 - // backend does this for aggregate function arguments. - if (MI.isCall() && MO.getReg() == SP) - continue; - // If this is a virtual register, only clobber it since it doesn't - // have aliases. - if (TRI->isVirtualRegister(MO.getReg())) - clobberRegisterUses(RegVars, MO.getReg(), DbgValues, MI); - // If this is a register def operand, it may end a debug value - // range. - else { - for (MCRegAliasIterator AI(MO.getReg(), TRI, true); AI.isValid(); - ++AI) - if (ChangingRegs.test(*AI)) - clobberRegisterUses(RegVars, *AI, DbgValues, MI); - } - } else if (MO.isRegMask()) { - // If this is a register mask operand, clobber all debug values in - // non-CSRs. - for (unsigned I : ChangingRegs.set_bits()) { - // Don't consider SP to be clobbered by register masks. - if (unsigned(I) != SP && TRI->isPhysicalRegister(I) && - MO.clobbersPhysReg(I)) { - clobberRegisterUses(RegVars, I, DbgValues, MI); - } - } - } - } - continue; - } - if (MI.isDebugValue()) { assert(MI.getNumOperands() > 1 && "Invalid DBG_VALUE instruction!"); // Use the base variable (without any DW_OP_piece expressions) @@ -250,13 +249,7 @@ void llvm::calculateDbgEntityHistory(const MachineFunction *MF, "Expected inlined-at fields to agree"); InlinedEntity Var(RawVar, MI.getDebugLoc()->getInlinedAt()); - if (unsigned PrevReg = DbgValues.getRegisterForVar(Var)) - dropRegDescribedVar(RegVars, PrevReg, Var); - - DbgValues.startInstrRange(Var, MI); - - if (unsigned NewReg = isDescribedByReg(MI)) - addRegDescribedVar(RegVars, NewReg, Var); + handleNewDebugValue(Var, MI, RegVars, LiveEntries, DbgValues); } else if (MI.isDebugLabel()) { assert(MI.getNumOperands() == 1 && "Invalid DBG_LABEL instruction!"); const DILabel *RawLabel = MI.getDebugLabel(); @@ -268,18 +261,75 @@ void llvm::calculateDbgEntityHistory(const MachineFunction *MF, InlinedEntity L(RawLabel, MI.getDebugLoc()->getInlinedAt()); DbgLabels.addInstr(L, MI); } - } - // Make sure locations for register-described variables are valid only - // until the end of the basic block (unless it's the last basic block, in - // which case let their liveness run off to the end of the function). + if (MI.isDebugInstr()) + continue; + + // Not a DBG_VALUE instruction. It may clobber registers which describe + // some variables. + for (const MachineOperand &MO : MI.operands()) { + if (MO.isReg() && MO.isDef() && MO.getReg()) { + // Ignore call instructions that claim to clobber SP. The AArch64 + // backend does this for aggregate function arguments. + if (MI.isCall() && MO.getReg() == SP) + continue; + // If this is a virtual register, only clobber it since it doesn't + // have aliases. + if (TRI->isVirtualRegister(MO.getReg())) + clobberRegisterUses(RegVars, MO.getReg(), DbgValues, LiveEntries, + MI); + // If this is a register def operand, it may end a debug value + // range. Ignore frame-register defs in the epilogue and prologue, + // we expect debuggers to understand that stack-locations are + // invalid outside of the function body. + else if (MO.getReg() != FrameReg || + (!MI.getFlag(MachineInstr::FrameDestroy) && + !MI.getFlag(MachineInstr::FrameSetup))) { + for (MCRegAliasIterator AI(MO.getReg(), TRI, true); AI.isValid(); + ++AI) + clobberRegisterUses(RegVars, *AI, DbgValues, LiveEntries, MI); + } + } else if (MO.isRegMask()) { + // If this is a register mask operand, clobber all debug values in + // non-CSRs. + SmallVector RegsToClobber; + // Don't consider SP to be clobbered by register masks. + for (auto It : RegVars) { + unsigned int Reg = It.first; + if (Reg != SP && TRI->isPhysicalRegister(Reg) && + MO.clobbersPhysReg(Reg)) + RegsToClobber.push_back(Reg); + } + + for (unsigned Reg : RegsToClobber) { + clobberRegisterUses(RegVars, Reg, DbgValues, LiveEntries, MI); + } + } + } // End MO loop. + } // End instr loop. + + // Make sure locations for all variables are valid only until the end of + // the basic block (unless it's the last basic block, in which case let + // their liveness run off to the end of the function). if (!MBB.empty() && &MBB != &MF->back()) { - for (auto I = RegVars.begin(), E = RegVars.end(); I != E;) { - auto CurElem = I++; // CurElem can be erased below. - if (TRI->isVirtualRegister(CurElem->first) || - ChangingRegs.test(CurElem->first)) - clobberRegisterUses(RegVars, CurElem, DbgValues, MBB.back()); + // Iterate over all variables that have open debug values. + for (auto &Pair : LiveEntries) { + if (Pair.second.empty()) + continue; + + // Create a clobbering entry. + EntryIndex ClobIdx = DbgValues.startClobber(Pair.first, MBB.back()); + + // End all entries. + for (EntryIndex Idx : Pair.second) { + DbgValueHistoryMap::Entry &Ent = DbgValues.getEntry(Pair.first, Idx); + assert(Ent.isDbgValue() && !Ent.isClosed()); + Ent.endEntry(ClobIdx); + } } + + LiveEntries.clear(); + RegVars.clear(); } } } @@ -289,7 +339,7 @@ LLVM_DUMP_METHOD void DbgValueHistoryMap::dump() const { dbgs() << "DbgValueHistoryMap:\n"; for (const auto &VarRangePair : *this) { const InlinedEntity &Var = VarRangePair.first; - const InstrRanges &Ranges = VarRangePair.second; + const Entries &Entries = VarRangePair.second; const DILocalVariable *LocalVar = cast(Var.first); const DILocation *Location = Var.second; @@ -304,10 +354,20 @@ LLVM_DUMP_METHOD void DbgValueHistoryMap::dump() const { dbgs() << " --\n"; - for (const InstrRange &Range : Ranges) { - dbgs() << " Begin: " << *Range.first; - if (Range.second) - dbgs() << " End : " << *Range.second; + for (const auto &E : enumerate(Entries)) { + const auto &Entry = E.value(); + dbgs() << " Entry[" << E.index() << "]: "; + if (Entry.isDbgValue()) + dbgs() << "Debug value\n"; + else + dbgs() << "Clobber\n"; + dbgs() << " Instr: " << *Entry.getInstr(); + if (Entry.isDbgValue()) { + if (Entry.getEndIndex() == NoEntry) + dbgs() << " - Valid until end of function\n"; + else + dbgs() << " - Closed by Entry[" << Entry.getEndIndex() << "]\n"; + } dbgs() << "\n"; } } diff --git a/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp b/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp index 551cd36d1984..22f458e4b03e 100644 --- a/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp +++ b/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp @@ -1,9 +1,8 @@ //===-- llvm/lib/CodeGen/AsmPrinter/DebugHandlerBase.cpp -------*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -141,10 +140,9 @@ DebugHandlerBase::getFunctionLocalOffsetAfterInsn(const MachineInstr *MI) { } /// If this type is derived from a base type then return base type size. -uint64_t DebugHandlerBase::getBaseTypeSize(const DITypeRef TyRef) { - DIType *Ty = TyRef.resolve(); +uint64_t DebugHandlerBase::getBaseTypeSize(const DIType *Ty) { assert(Ty); - DIDerivedType *DDTy = dyn_cast(Ty); + const DIDerivedType *DDTy = dyn_cast(Ty); if (!DDTy) return Ty->getSizeInBits(); @@ -155,7 +153,7 @@ uint64_t DebugHandlerBase::getBaseTypeSize(const DITypeRef TyRef) { Tag != dwarf::DW_TAG_restrict_type && Tag != dwarf::DW_TAG_atomic_type) return DDTy->getSizeInBits(); - DIType *BaseType = DDTy->getBaseType().resolve(); + DIType *BaseType = DDTy->getBaseType(); if (!BaseType) return 0; @@ -212,36 +210,58 @@ void DebugHandlerBase::beginFunction(const MachineFunction *MF) { // Request labels for the full history. for (const auto &I : DbgValues) { - const auto &Ranges = I.second; - if (Ranges.empty()) + const auto &Entries = I.second; + if (Entries.empty()) continue; - // The first mention of a function argument gets the CurrentFnBegin - // label, so arguments are visible when breaking at function entry. - const DILocalVariable *DIVar = Ranges.front().first->getDebugVariable(); + auto IsDescribedByReg = [](const MachineInstr *MI) { + return MI->getOperand(0).isReg() && MI->getOperand(0).getReg(); + }; + + // The first mention of a function argument gets the CurrentFnBegin label, + // so arguments are visible when breaking at function entry. + // + // We do not change the label for values that are described by registers, + // as that could place them above their defining instructions. We should + // ideally not change the labels for constant debug values either, since + // doing that violates the ranges that are calculated in the history map. + // However, we currently do not emit debug values for constant arguments + // directly at the start of the function, so this code is still useful. + const DILocalVariable *DIVar = + Entries.front().getInstr()->getDebugVariable(); if (DIVar->isParameter() && getDISubprogram(DIVar->getScope())->describes(&MF->getFunction())) { - LabelsBeforeInsn[Ranges.front().first] = Asm->getFunctionBegin(); - if (Ranges.front().first->getDebugExpression()->isFragment()) { + if (!IsDescribedByReg(Entries.front().getInstr())) + LabelsBeforeInsn[Entries.front().getInstr()] = Asm->getFunctionBegin(); + if (Entries.front().getInstr()->getDebugExpression()->isFragment()) { // Mark all non-overlapping initial fragments. - for (auto I = Ranges.begin(); I != Ranges.end(); ++I) { - const DIExpression *Fragment = I->first->getDebugExpression(); - if (std::all_of(Ranges.begin(), I, - [&](DbgValueHistoryMap::InstrRange Pred) { - return !Fragment->fragmentsOverlap( - Pred.first->getDebugExpression()); + for (auto I = Entries.begin(); I != Entries.end(); ++I) { + if (!I->isDbgValue()) + continue; + const DIExpression *Fragment = I->getInstr()->getDebugExpression(); + if (std::any_of(Entries.begin(), I, + [&](DbgValueHistoryMap::Entry Pred) { + return Pred.isDbgValue() && + Fragment->fragmentsOverlap( + Pred.getInstr()->getDebugExpression()); })) - LabelsBeforeInsn[I->first] = Asm->getFunctionBegin(); - else break; + // The code that generates location lists for DWARF assumes that the + // entries' start labels are monotonically increasing, and since we + // don't change the label for fragments that are described by + // registers, we must bail out when encountering such a fragment. + if (IsDescribedByReg(I->getInstr())) + break; + LabelsBeforeInsn[I->getInstr()] = Asm->getFunctionBegin(); } } } - for (const auto &Range : Ranges) { - requestLabelBeforeInsn(Range.first); - if (Range.second) - requestLabelAfterInsn(Range.second); + for (const auto &Entry : Entries) { + if (Entry.isDbgValue()) + requestLabelBeforeInsn(Entry.getInstr()); + else + requestLabelAfterInsn(Entry.getInstr()); } } diff --git a/lib/CodeGen/AsmPrinter/DebugLocEntry.h b/lib/CodeGen/AsmPrinter/DebugLocEntry.h index befa4b941c8d..17e39b3d3268 100644 --- a/lib/CodeGen/AsmPrinter/DebugLocEntry.h +++ b/lib/CodeGen/AsmPrinter/DebugLocEntry.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/DebugLocEntry.h - Entry in debug_loc list -*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -21,6 +20,73 @@ namespace llvm { class AsmPrinter; +/// A single location or constant. +class DbgValueLoc { + /// Any complex address location expression for this DbgValueLoc. + const DIExpression *Expression; + + /// Type of entry that this represents. + enum EntryType { E_Location, E_Integer, E_ConstantFP, E_ConstantInt }; + enum EntryType EntryKind; + + /// Either a constant, + union { + int64_t Int; + const ConstantFP *CFP; + const ConstantInt *CIP; + } Constant; + + /// Or a location in the machine frame. + MachineLocation Loc; + +public: + DbgValueLoc(const DIExpression *Expr, int64_t i) + : Expression(Expr), EntryKind(E_Integer) { + Constant.Int = i; + } + DbgValueLoc(const DIExpression *Expr, const ConstantFP *CFP) + : Expression(Expr), EntryKind(E_ConstantFP) { + Constant.CFP = CFP; + } + DbgValueLoc(const DIExpression *Expr, const ConstantInt *CIP) + : Expression(Expr), EntryKind(E_ConstantInt) { + Constant.CIP = CIP; + } + DbgValueLoc(const DIExpression *Expr, MachineLocation Loc) + : Expression(Expr), EntryKind(E_Location), Loc(Loc) { + assert(cast(Expr)->isValid()); + } + + bool isLocation() const { return EntryKind == E_Location; } + bool isInt() const { return EntryKind == E_Integer; } + bool isConstantFP() const { return EntryKind == E_ConstantFP; } + bool isConstantInt() const { return EntryKind == E_ConstantInt; } + int64_t getInt() const { return Constant.Int; } + const ConstantFP *getConstantFP() const { return Constant.CFP; } + const ConstantInt *getConstantInt() const { return Constant.CIP; } + MachineLocation getLoc() const { return Loc; } + bool isFragment() const { return getExpression()->isFragment(); } + bool isEntryVal() const { return getExpression()->isEntryValue(); } + const DIExpression *getExpression() const { return Expression; } + friend bool operator==(const DbgValueLoc &, const DbgValueLoc &); + friend bool operator<(const DbgValueLoc &, const DbgValueLoc &); +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + LLVM_DUMP_METHOD void dump() const { + if (isLocation()) { + llvm::dbgs() << "Loc = { reg=" << Loc.getReg() << " "; + if (Loc.isIndirect()) + llvm::dbgs() << "+0"; + llvm::dbgs() << "} "; + } else if (isConstantInt()) + Constant.CIP->dump(); + else if (isConstantFP()) + Constant.CFP->dump(); + if (Expression) + Expression->dump(); + } +#endif +}; + /// This struct describes location entries emitted in the .debug_loc /// section. class DebugLocEntry { @@ -28,90 +94,20 @@ class DebugLocEntry { const MCSymbol *Begin; const MCSymbol *End; -public: - /// A single location or constant. - struct Value { - Value(const DIExpression *Expr, int64_t i) - : Expression(Expr), EntryKind(E_Integer) { - Constant.Int = i; - } - Value(const DIExpression *Expr, const ConstantFP *CFP) - : Expression(Expr), EntryKind(E_ConstantFP) { - Constant.CFP = CFP; - } - Value(const DIExpression *Expr, const ConstantInt *CIP) - : Expression(Expr), EntryKind(E_ConstantInt) { - Constant.CIP = CIP; - } - Value(const DIExpression *Expr, MachineLocation Loc) - : Expression(Expr), EntryKind(E_Location), Loc(Loc) { - assert(cast(Expr)->isValid()); - } - - /// Any complex address location expression for this Value. - const DIExpression *Expression; - - /// Type of entry that this represents. - enum EntryType { E_Location, E_Integer, E_ConstantFP, E_ConstantInt }; - enum EntryType EntryKind; - - /// Either a constant, - union { - int64_t Int; - const ConstantFP *CFP; - const ConstantInt *CIP; - } Constant; - - // Or a location in the machine frame. - MachineLocation Loc; - - bool isLocation() const { return EntryKind == E_Location; } - bool isInt() const { return EntryKind == E_Integer; } - bool isConstantFP() const { return EntryKind == E_ConstantFP; } - bool isConstantInt() const { return EntryKind == E_ConstantInt; } - int64_t getInt() const { return Constant.Int; } - const ConstantFP *getConstantFP() const { return Constant.CFP; } - const ConstantInt *getConstantInt() const { return Constant.CIP; } - MachineLocation getLoc() const { return Loc; } - bool isFragment() const { return getExpression()->isFragment(); } - const DIExpression *getExpression() const { return Expression; } - friend bool operator==(const Value &, const Value &); - friend bool operator<(const Value &, const Value &); -#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) - LLVM_DUMP_METHOD void dump() const { - if (isLocation()) { - llvm::dbgs() << "Loc = { reg=" << Loc.getReg() << " "; - if (Loc.isIndirect()) - llvm::dbgs() << "+0"; - llvm::dbgs() << "} "; - } - else if (isConstantInt()) - Constant.CIP->dump(); - else if (isConstantFP()) - Constant.CFP->dump(); - if (Expression) - Expression->dump(); - } -#endif - }; - -private: /// A nonempty list of locations/constants belonging to this entry, /// sorted by offset. - SmallVector Values; + SmallVector Values; public: - DebugLocEntry(const MCSymbol *B, const MCSymbol *E, Value Val) - : Begin(B), End(E) { - Values.push_back(std::move(Val)); + /// Create a location list entry for the range [\p Begin, \p End). + /// + /// \param Vals One or more values describing (parts of) the variable. + DebugLocEntry(const MCSymbol *Begin, const MCSymbol *End, + ArrayRef Vals) + : Begin(Begin), End(End) { + addValues(Vals); } - /// If this and Next are describing different pieces of the same - /// variable, merge them by appending Next's values to the current - /// list of values. - /// Return true if the merge was successful. - bool MergeValues(const DebugLocEntry &Next); - /// Attempt to merge this DebugLocEntry with Next and return /// true if the merge was successful. Entries can be merged if they /// share the same Loc/Constant and if Next immediately follows this @@ -127,35 +123,36 @@ public: const MCSymbol *getBeginSym() const { return Begin; } const MCSymbol *getEndSym() const { return End; } - ArrayRef getValues() const { return Values; } - void addValues(ArrayRef Vals) { + ArrayRef getValues() const { return Values; } + void addValues(ArrayRef Vals) { Values.append(Vals.begin(), Vals.end()); sortUniqueValues(); - assert(all_of(Values, [](DebugLocEntry::Value V) { - return V.isFragment(); - }) && "value must be a piece"); + assert((Values.size() == 1 || all_of(Values, [](DbgValueLoc V) { + return V.isFragment(); + })) && "must either have a single value or multiple pieces"); } // Sort the pieces by offset. // Remove any duplicate entries by dropping all but the first. void sortUniqueValues() { llvm::sort(Values); - Values.erase( - std::unique( - Values.begin(), Values.end(), [](const Value &A, const Value &B) { - return A.getExpression() == B.getExpression(); - }), - Values.end()); + Values.erase(std::unique(Values.begin(), Values.end(), + [](const DbgValueLoc &A, const DbgValueLoc &B) { + return A.getExpression() == B.getExpression(); + }), + Values.end()); } /// Lower this entry into a DWARF expression. - void finalize(const AsmPrinter &AP, DebugLocStream::ListBuilder &List, - const DIBasicType *BT); + void finalize(const AsmPrinter &AP, + DebugLocStream::ListBuilder &List, + const DIBasicType *BT, + DwarfCompileUnit &TheCU); }; -/// Compare two Values for equality. -inline bool operator==(const DebugLocEntry::Value &A, - const DebugLocEntry::Value &B) { +/// Compare two DbgValueLocs for equality. +inline bool operator==(const DbgValueLoc &A, + const DbgValueLoc &B) { if (A.EntryKind != B.EntryKind) return false; @@ -163,21 +160,21 @@ inline bool operator==(const DebugLocEntry::Value &A, return false; switch (A.EntryKind) { - case DebugLocEntry::Value::E_Location: + case DbgValueLoc::E_Location: return A.Loc == B.Loc; - case DebugLocEntry::Value::E_Integer: + case DbgValueLoc::E_Integer: return A.Constant.Int == B.Constant.Int; - case DebugLocEntry::Value::E_ConstantFP: + case DbgValueLoc::E_ConstantFP: return A.Constant.CFP == B.Constant.CFP; - case DebugLocEntry::Value::E_ConstantInt: + case DbgValueLoc::E_ConstantInt: return A.Constant.CIP == B.Constant.CIP; } llvm_unreachable("unhandled EntryKind"); } /// Compare two fragments based on their offset. -inline bool operator<(const DebugLocEntry::Value &A, - const DebugLocEntry::Value &B) { +inline bool operator<(const DbgValueLoc &A, + const DbgValueLoc &B) { return A.getExpression()->getFragmentInfo()->OffsetInBits < B.getExpression()->getFragmentInfo()->OffsetInBits; } diff --git a/lib/CodeGen/AsmPrinter/DebugLocStream.cpp b/lib/CodeGen/AsmPrinter/DebugLocStream.cpp index 7e8ed7104af3..f483d532ff07 100644 --- a/lib/CodeGen/AsmPrinter/DebugLocStream.cpp +++ b/lib/CodeGen/AsmPrinter/DebugLocStream.cpp @@ -1,9 +1,8 @@ //===- DebugLocStream.cpp - DWARF debug_loc stream --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/AsmPrinter/DebugLocStream.h b/lib/CodeGen/AsmPrinter/DebugLocStream.h index 8dcf5cbc1889..789291771b5a 100644 --- a/lib/CodeGen/AsmPrinter/DebugLocStream.h +++ b/lib/CodeGen/AsmPrinter/DebugLocStream.h @@ -1,9 +1,8 @@ //===--- lib/CodeGen/DebugLocStream.h - DWARF debug_loc stream --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp b/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp index 1990456cc555..207a7284dafa 100644 --- a/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfCFIException.cpp @@ -1,9 +1,8 @@ //===-- CodeGen/AsmPrinter/DwarfException.cpp - Dwarf Exception Impl ------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp index 1dca3f0fce5b..9548ad9918c1 100644 --- a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp @@ -1,9 +1,8 @@ //===- llvm/CodeGen/DwarfCompileUnit.cpp - Dwarf Compile Units ------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,6 +17,7 @@ #include "DwarfUnit.h" #include "llvm/ADT/None.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/SmallString.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/Dwarf.h" @@ -104,7 +104,7 @@ unsigned DwarfCompileUnit::getOrCreateSourceID(const DIFile *File) { // extend .file to support this. unsigned CUID = Asm->OutStreamer->hasRawTextSupport() ? 0 : getUniqueID(); if (!File) - return Asm->OutStreamer->EmitDwarfFileDirective(0, "", "", nullptr, None, CUID); + return Asm->OutStreamer->EmitDwarfFileDirective(0, "", "", None, None, CUID); return Asm->OutStreamer->EmitDwarfFileDirective( 0, File->getDirectory(), File->getFilename(), getMD5AsBytes(File), File->getSource(), CUID); @@ -119,17 +119,19 @@ DIE *DwarfCompileUnit::getOrCreateGlobalVariableDIE( assert(GV); auto *GVContext = GV->getScope(); - auto *GTy = DD->resolve(GV->getType()); + const DIType *GTy = GV->getType(); // Construct the context before querying for the existence of the DIE in // case such construction creates the DIE. - DIE *ContextDIE = getOrCreateContextDIE(GVContext); + auto *CB = GVContext ? dyn_cast(GVContext) : nullptr; + DIE *ContextDIE = CB ? getOrCreateCommonBlock(CB, GlobalExprs) + : getOrCreateContextDIE(GVContext); // Add to map. DIE *VariableDIE = &createAndAddDIE(GV->getTag(), *ContextDIE, GV); DIScope *DeclContext; if (auto *SDMDecl = GV->getStaticDataMemberDeclaration()) { - DeclContext = resolve(SDMDecl->getScope()); + DeclContext = SDMDecl->getScope(); assert(SDMDecl->isStaticMember() && "Expected static member decl"); assert(GV->isDefinition()); // We need the declaration DIE that is in the static member's class. @@ -137,7 +139,7 @@ DIE *DwarfCompileUnit::getOrCreateGlobalVariableDIE( addDIEEntry(*VariableDIE, dwarf::DW_AT_specification, *VariableSpecDIE); // If the global variable's type is different from the one in the class // member type, assume that it's more specific and also emit it. - if (GTy != DD->resolve(SDMDecl->getBaseType())) + if (GTy != SDMDecl->getBaseType()) addType(*VariableDIE, GTy); } else { DeclContext = GV->getScope(); @@ -166,8 +168,16 @@ DIE *DwarfCompileUnit::getOrCreateGlobalVariableDIE( addTemplateParams(*VariableDIE, DINodeArray(TP)); // Add location. + addLocationAttribute(VariableDIE, GV, GlobalExprs); + + return VariableDIE; +} + +void DwarfCompileUnit::addLocationAttribute( + DIE *VariableDIE, const DIGlobalVariable *GV, ArrayRef GlobalExprs) { bool addToAccelTable = false; DIELoc *Loc = nullptr; + Optional NVPTXAddressSpace; std::unique_ptr DwarfExpr; for (const auto &GE : GlobalExprs) { const GlobalVariable *Global = GE.Var; @@ -201,8 +211,24 @@ DIE *DwarfCompileUnit::getOrCreateGlobalVariableDIE( DwarfExpr = llvm::make_unique(*Asm, *this, *Loc); } - if (Expr) + if (Expr) { + // According to + // https://docs.nvidia.com/cuda/archive/10.0/ptx-writers-guide-to-interoperability/index.html#cuda-specific-dwarf + // cuda-gdb requires DW_AT_address_class for all variables to be able to + // correctly interpret address space of the variable address. + // Decode DW_OP_constu DW_OP_swap DW_OP_xderef + // sequence for the NVPTX + gdb target. + unsigned LocalNVPTXAddressSpace; + if (Asm->TM.getTargetTriple().isNVPTX() && DD->tuneForGDB()) { + const DIExpression *NewExpr = + DIExpression::extractAddressClass(Expr, LocalNVPTXAddressSpace); + if (NewExpr != Expr) { + Expr = NewExpr; + NVPTXAddressSpace = LocalNVPTXAddressSpace; + } + } DwarfExpr->addFragmentOffset(Expr); + } if (Global) { const MCSymbol *Sym = Asm->getSymbol(Global); @@ -247,6 +273,15 @@ DIE *DwarfCompileUnit::getOrCreateGlobalVariableDIE( DwarfExpr->setMemoryLocationKind(); DwarfExpr->addExpression(Expr); } + if (Asm->TM.getTargetTriple().isNVPTX() && DD->tuneForGDB()) { + // According to + // https://docs.nvidia.com/cuda/archive/10.0/ptx-writers-guide-to-interoperability/index.html#cuda-specific-dwarf + // cuda-gdb requires DW_AT_address_class for all variables to be able to + // correctly interpret address space of the variable address. + const unsigned NVPTX_ADDR_global_space = 5; + addUInt(*VariableDIE, dwarf::DW_AT_address_class, dwarf::DW_FORM_data1, + NVPTXAddressSpace ? *NVPTXAddressSpace : NVPTX_ADDR_global_space); + } if (Loc) addBlock(*VariableDIE, dwarf::DW_AT_location, DwarfExpr->finalize()); @@ -262,8 +297,25 @@ DIE *DwarfCompileUnit::getOrCreateGlobalVariableDIE( DD->useAllLinkageNames()) DD->addAccelName(*CUNode, GV->getLinkageName(), *VariableDIE); } +} - return VariableDIE; +DIE *DwarfCompileUnit::getOrCreateCommonBlock( + const DICommonBlock *CB, ArrayRef GlobalExprs) { + // Construct the context before querying for the existence of the DIE in case + // such construction creates the DIE. + DIE *ContextDIE = getOrCreateContextDIE(CB->getScope()); + + if (DIE *NDie = getDIE(CB)) + return NDie; + DIE &NDie = createAndAddDIE(dwarf::DW_TAG_common_block, *ContextDIE, CB); + StringRef Name = CB->getName().empty() ? "_BLNK_" : CB->getName(); + addString(NDie, dwarf::DW_AT_name, Name); + addGlobalName(Name, NDie, CB->getScope()); + if (CB->getFile()) + addSourceLine(NDie, CB->getLineNo(), CB->getFile()); + if (DIGlobalVariable *V = CB->getDecl()) + getCU().addLocationAttribute(&NDie, V, GlobalExprs); + return &NDie; } void DwarfCompileUnit::addRange(RangeSpan Range) { @@ -491,6 +543,8 @@ DIE *DwarfCompileUnit::constructInlinedScopeDIE(LexicalScope *Scope) { addUInt(*ScopeDIE, dwarf::DW_AT_call_file, None, getOrCreateSourceID(IA->getFile())); addUInt(*ScopeDIE, dwarf::DW_AT_call_line, None, IA->getLine()); + if (IA->getColumn()) + addUInt(*ScopeDIE, dwarf::DW_AT_call_column, None, IA->getColumn()); if (IA->getDiscriminator() && DD->getDwarfVersion() >= 4) addUInt(*ScopeDIE, dwarf::DW_AT_GNU_discriminator, None, IA->getDiscriminator()); @@ -555,36 +609,27 @@ DIE *DwarfCompileUnit::constructVariableDIEImpl(const DbgVariable &DV, return VariableDie; } - // Check if variable is described by a DBG_VALUE instruction. - if (const MachineInstr *DVInsn = DV.getMInsn()) { - assert(DVInsn->getNumOperands() == 4); - if (DVInsn->getOperand(0).isReg()) { - auto RegOp = DVInsn->getOperand(0); - auto Op1 = DVInsn->getOperand(1); - // If the second operand is an immediate, this is an indirect value. - assert((!Op1.isImm() || (Op1.getImm() == 0)) && "unexpected offset"); - MachineLocation Location(RegOp.getReg(), Op1.isImm()); - addVariableAddress(DV, *VariableDie, Location); - } else if (DVInsn->getOperand(0).isImm()) { - // This variable is described by a single constant. - // Check whether it has a DIExpression. + // Check if variable has a single location description. + if (auto *DVal = DV.getValueLoc()) { + if (DVal->isLocation()) + addVariableAddress(DV, *VariableDie, DVal->getLoc()); + else if (DVal->isInt()) { auto *Expr = DV.getSingleExpression(); if (Expr && Expr->getNumElements()) { DIELoc *Loc = new (DIEValueAllocator) DIELoc; DIEDwarfExpression DwarfExpr(*Asm, *this, *Loc); // If there is an expression, emit raw unsigned bytes. DwarfExpr.addFragmentOffset(Expr); - DwarfExpr.addUnsignedConstant(DVInsn->getOperand(0).getImm()); + DwarfExpr.addUnsignedConstant(DVal->getInt()); DwarfExpr.addExpression(Expr); addBlock(*VariableDie, dwarf::DW_AT_location, DwarfExpr.finalize()); } else - addConstantValue(*VariableDie, DVInsn->getOperand(0), DV.getType()); - } else if (DVInsn->getOperand(0).isFPImm()) - addConstantFPValue(*VariableDie, DVInsn->getOperand(0)); - else if (DVInsn->getOperand(0).isCImm()) - addConstantValue(*VariableDie, DVInsn->getOperand(0).getCImm(), - DV.getType()); - + addConstantValue(*VariableDie, DVal->getInt(), DV.getType()); + } else if (DVal->isConstantFP()) { + addConstantFPValue(*VariableDie, DVal->getConstantFP()); + } else if (DVal->isConstantInt()) { + addConstantValue(*VariableDie, DVal->getConstantInt(), DV.getType()); + } return VariableDie; } @@ -592,6 +637,7 @@ DIE *DwarfCompileUnit::constructVariableDIEImpl(const DbgVariable &DV, if (!DV.hasFrameIndexExprs()) return VariableDie; + Optional NVPTXAddressSpace; DIELoc *Loc = new (DIEValueAllocator) DIELoc; DIEDwarfExpression DwarfExpr(*Asm, *this, *Loc); for (auto &Fragment : DV.getFrameIndexExprs()) { @@ -603,7 +649,23 @@ DIE *DwarfCompileUnit::constructVariableDIEImpl(const DbgVariable &DV, SmallVector Ops; Ops.push_back(dwarf::DW_OP_plus_uconst); Ops.push_back(Offset); - Ops.append(Expr->elements_begin(), Expr->elements_end()); + // According to + // https://docs.nvidia.com/cuda/archive/10.0/ptx-writers-guide-to-interoperability/index.html#cuda-specific-dwarf + // cuda-gdb requires DW_AT_address_class for all variables to be able to + // correctly interpret address space of the variable address. + // Decode DW_OP_constu DW_OP_swap DW_OP_xderef + // sequence for the NVPTX + gdb target. + unsigned LocalNVPTXAddressSpace; + if (Asm->TM.getTargetTriple().isNVPTX() && DD->tuneForGDB()) { + const DIExpression *NewExpr = + DIExpression::extractAddressClass(Expr, LocalNVPTXAddressSpace); + if (NewExpr != Expr) { + Expr = NewExpr; + NVPTXAddressSpace = LocalNVPTXAddressSpace; + } + } + if (Expr) + Ops.append(Expr->elements_begin(), Expr->elements_end()); DIExpressionCursor Cursor(Ops); DwarfExpr.setMemoryLocationKind(); if (const MCSymbol *FrameSymbol = Asm->getFunctionFrameSymbol()) @@ -613,7 +675,19 @@ DIE *DwarfCompileUnit::constructVariableDIEImpl(const DbgVariable &DV, *Asm->MF->getSubtarget().getRegisterInfo(), Cursor, FrameReg); DwarfExpr.addExpression(std::move(Cursor)); } + if (Asm->TM.getTargetTriple().isNVPTX() && DD->tuneForGDB()) { + // According to + // https://docs.nvidia.com/cuda/archive/10.0/ptx-writers-guide-to-interoperability/index.html#cuda-specific-dwarf + // cuda-gdb requires DW_AT_address_class for all variables to be able to + // correctly interpret address space of the variable address. + const unsigned NVPTX_ADDR_local_space = 6; + addUInt(*VariableDie, dwarf::DW_AT_address_class, dwarf::DW_FORM_data1, + NVPTXAddressSpace ? *NVPTXAddressSpace : NVPTX_ADDR_local_space); + } addBlock(*VariableDie, dwarf::DW_AT_location, DwarfExpr.finalize()); + if (DwarfExpr.TagOffset) + addUInt(*VariableDie, dwarf::DW_AT_LLVM_tag_offset, dwarf::DW_FORM_data1, + *DwarfExpr.TagOffset); return VariableDie; } @@ -800,7 +874,7 @@ void DwarfCompileUnit::constructAbstractSubprogramScopeDIE( ContextDIE = &getUnitDie(); getOrCreateSubprogramDIE(SPDecl); } else { - ContextDIE = getOrCreateContextDIE(resolve(SP->getScope())); + ContextDIE = getOrCreateContextDIE(SP->getScope()); // The scope may be shared with a subprogram that has already been // constructed in another CU, in which case we need to construct this // subprogram in the same CU. @@ -849,7 +923,7 @@ DIE *DwarfCompileUnit::constructImportedEntityDIE( DIE *IMDie = DIE::get(DIEValueAllocator, (dwarf::Tag)Module->getTag()); insertDIE(Module, IMDie); DIE *EntityDie; - auto *Entity = resolve(Module->getEntity()); + auto *Entity = Module->getEntity(); if (auto *NS = dyn_cast(Entity)) EntityDie = getOrCreateNameSpace(NS); else if (auto *M = dyn_cast(Entity)) @@ -958,7 +1032,9 @@ bool DwarfCompileUnit::hasDwarfPubSections() const { return true; case DICompileUnit::DebugNameTableKind::Default: return DD->tuneForGDB() && !includeMinimalInlineScopes() && - !CUNode->isDebugDirectivesOnly(); + !CUNode->isDebugDirectivesOnly() && + DD->getAccelTableKind() != AccelTableKind::Apple && + DD->getDwarfVersion() < 5; } llvm_unreachable("Unhandled DICompileUnit::DebugNameTableKind enum"); } @@ -1054,6 +1130,12 @@ void DwarfCompileUnit::addComplexAddress(const DbgVariable &DV, DIE &Die, DwarfExpr.setMemoryLocationKind(); DIExpressionCursor Cursor(DIExpr); + + if (DIExpr->isEntryValue()) { + DwarfExpr.setEntryValueFlag(); + DwarfExpr.addEntryValueExpression(Cursor); + } + const TargetRegisterInfo &TRI = *Asm->MF->getSubtarget().getRegisterInfo(); if (!DwarfExpr.addMachineRegExpression(TRI, Cursor, Location.getReg())) return; @@ -1112,7 +1194,7 @@ void DwarfCompileUnit::addAddressExpr(DIE &Die, dwarf::Attribute Attribute, void DwarfCompileUnit::applySubprogramAttributesToDefinition( const DISubprogram *SP, DIE &SPDie) { auto *SPDecl = SP->getDeclaration(); - auto *Context = resolve(SPDecl ? SPDecl->getScope() : SP->getScope()); + auto *Context = SPDecl ? SPDecl->getScope() : SP->getScope(); applySubprogramAttributes(SP, SPDie, includeMinimalInlineScopes()); addGlobalName(SP->getName(), SPDie, Context); } @@ -1121,6 +1203,10 @@ bool DwarfCompileUnit::isDwoUnit() const { return DD->useSplitDwarf() && Skeleton; } +void DwarfCompileUnit::finishNonUnitTypeDIE(DIE& D, const DICompositeType *CTy) { + constructTypeDIE(D, CTy); +} + bool DwarfCompileUnit::includeMinimalInlineScopes() const { return getCUNode()->getEmissionKind() == DICompileUnit::LineTablesOnly || (DD->useSplitDwarf() && !Skeleton); @@ -1134,3 +1220,27 @@ void DwarfCompileUnit::addAddrTableBase() { : dwarf::DW_AT_GNU_addr_base, Label, TLOF.getDwarfAddrSection()->getBeginSymbol()); } + +void DwarfCompileUnit::addBaseTypeRef(DIEValueList &Die, int64_t Idx) { + Die.addValue(DIEValueAllocator, (dwarf::Attribute)0, dwarf::DW_FORM_udata, + new (DIEValueAllocator) DIEBaseTypeRef(this, Idx)); +} + +void DwarfCompileUnit::createBaseTypeDIEs() { + // Insert the base_type DIEs directly after the CU so that their offsets will + // fit in the fixed size ULEB128 used inside the location expressions. + // Maintain order by iterating backwards and inserting to the front of CU + // child list. + for (auto &Btr : reverse(ExprRefedBaseTypes)) { + DIE &Die = getUnitDie().addChildFront( + DIE::get(DIEValueAllocator, dwarf::DW_TAG_base_type)); + SmallString<32> Str; + addString(Die, dwarf::DW_AT_name, + Twine(dwarf::AttributeEncodingString(Btr.Encoding) + + "_" + Twine(Btr.BitSize)).toStringRef(Str)); + addUInt(Die, dwarf::DW_AT_encoding, dwarf::DW_FORM_data1, Btr.Encoding); + addUInt(Die, dwarf::DW_AT_byte_size, None, Btr.BitSize / 8); + + Btr.Die = &Die; + } +} diff --git a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h index 9ec22f68c12f..ea980dfda17e 100644 --- a/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h +++ b/lib/CodeGen/AsmPrinter/DwarfCompileUnit.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/DwarfCompileUnit.h - Dwarf Compile Unit -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -101,6 +100,8 @@ class DwarfCompileUnit final : public DwarfUnit { return DU->getAbstractEntities(); } + void finishNonUnitTypeDIE(DIE& D, const DICompositeType *CTy) override; + public: DwarfCompileUnit(unsigned UID, const DICompileUnit *Node, AsmPrinter *A, DwarfDebug *DW, DwarfFile *DWU); @@ -125,11 +126,27 @@ public: const DIExpression *Expr; }; + struct BaseTypeRef { + BaseTypeRef(unsigned BitSize, dwarf::TypeKind Encoding) : + BitSize(BitSize), Encoding(Encoding) {} + unsigned BitSize; + dwarf::TypeKind Encoding; + DIE *Die = nullptr; + }; + + std::vector ExprRefedBaseTypes; + /// Get or create global variable DIE. DIE * getOrCreateGlobalVariableDIE(const DIGlobalVariable *GV, ArrayRef GlobalExprs); + DIE *getOrCreateCommonBlock(const DICommonBlock *CB, + ArrayRef GlobalExprs); + + void addLocationAttribute(DIE *ToDIE, const DIGlobalVariable *GV, + ArrayRef GlobalExprs); + /// addLabelAddress - Add a dwarf label attribute data and value using /// either DW_FORM_addr or DW_FORM_GNU_addr_index. void addLabelAddress(DIE &Die, dwarf::Attribute Attribute, @@ -200,6 +217,8 @@ public: SmallVectorImpl &Children, bool *HasNonScopeChildren = nullptr); + void createBaseTypeDIEs(); + /// Construct a DIE for this subprogram scope. DIE &constructSubprogramScopeDIE(const DISubprogram *Sub, LexicalScope *Scope); @@ -314,6 +333,8 @@ public: void setDWOId(uint64_t DwoId) { DWOId = DwoId; } bool hasDwarfPubSections() const; + + void addBaseTypeRef(DIEValueList &Die, int64_t Idx); }; } // end namespace llvm diff --git a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp index 1de2ffb6cfa1..71bb2b0858cc 100644 --- a/lib/CodeGen/AsmPrinter/DwarfDebug.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfDebug.cpp @@ -1,9 +1,8 @@ //===- llvm/CodeGen/DwarfDebug.cpp - Dwarf Debug Framework ----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -42,6 +41,8 @@ #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" +#include "llvm/DebugInfo/DWARF/DWARFExpression.h" +#include "llvm/DebugInfo/DWARF/DWARFDataExtractor.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/DebugLoc.h" @@ -162,6 +163,7 @@ static const char *const DWARFGroupName = "dwarf"; static const char *const DWARFGroupDescription = "DWARF Emission"; static const char *const DbgTimerName = "writer"; static const char *const DbgTimerDescription = "DWARF Debug Writer"; +static constexpr unsigned ULEB128PadSize = 4; void DebugLocDwarfExpression::emitOp(uint8_t Op, const char *Comment) { BS.EmitInt8( @@ -177,6 +179,15 @@ void DebugLocDwarfExpression::emitUnsigned(uint64_t Value) { BS.EmitULEB128(Value, Twine(Value)); } +void DebugLocDwarfExpression::emitData1(uint8_t Value) { + BS.EmitInt8(Value, Twine(Value)); +} + +void DebugLocDwarfExpression::emitBaseTypeRef(uint64_t Idx) { + assert(Idx < (1ULL << (ULEB128PadSize * 7)) && "Idx wont fit"); + BS.EmitULEB128(Idx, Twine(Idx), ULEB128PadSize); +} + bool DebugLocDwarfExpression::isFrameRegister(const TargetRegisterInfo &TRI, unsigned MachineReg) { // This information is not available while emitting .debug_loc entries. @@ -185,11 +196,11 @@ bool DebugLocDwarfExpression::isFrameRegister(const TargetRegisterInfo &TRI, bool DbgVariable::isBlockByrefVariable() const { assert(getVariable() && "Invalid complex DbgVariable!"); - return getVariable()->getType().resolve()->isBlockByrefStruct(); + return getVariable()->getType()->isBlockByrefStruct(); } const DIType *DbgVariable::getType() const { - DIType *Ty = getVariable()->getType().resolve(); + DIType *Ty = getVariable()->getType(); // FIXME: isBlockByrefVariable should be reformulated in terms of complex // addresses instead. if (Ty->isBlockByrefStruct()) { @@ -221,18 +232,55 @@ const DIType *DbgVariable::getType() const { uint16_t tag = Ty->getTag(); if (tag == dwarf::DW_TAG_pointer_type) - subType = resolve(cast(Ty)->getBaseType()); + subType = cast(Ty)->getBaseType(); auto Elements = cast(subType)->getElements(); for (unsigned i = 0, N = Elements.size(); i < N; ++i) { auto *DT = cast(Elements[i]); if (getName() == DT->getName()) - return resolve(DT->getBaseType()); + return DT->getBaseType(); } } return Ty; } +/// Get .debug_loc entry for the instruction range starting at MI. +static DbgValueLoc getDebugLocValue(const MachineInstr *MI) { + const DIExpression *Expr = MI->getDebugExpression(); + assert(MI->getNumOperands() == 4); + if (MI->getOperand(0).isReg()) { + auto RegOp = MI->getOperand(0); + auto Op1 = MI->getOperand(1); + // If the second operand is an immediate, this is a + // register-indirect address. + assert((!Op1.isImm() || (Op1.getImm() == 0)) && "unexpected offset"); + MachineLocation MLoc(RegOp.getReg(), Op1.isImm()); + return DbgValueLoc(Expr, MLoc); + } + if (MI->getOperand(0).isImm()) + return DbgValueLoc(Expr, MI->getOperand(0).getImm()); + if (MI->getOperand(0).isFPImm()) + return DbgValueLoc(Expr, MI->getOperand(0).getFPImm()); + if (MI->getOperand(0).isCImm()) + return DbgValueLoc(Expr, MI->getOperand(0).getCImm()); + + llvm_unreachable("Unexpected 4-operand DBG_VALUE instruction!"); +} + +void DbgVariable::initializeDbgValue(const MachineInstr *DbgValue) { + assert(FrameIndexExprs.empty() && "Already initialized?"); + assert(!ValueLoc.get() && "Already initialized?"); + + assert(getVariable() == DbgValue->getDebugVariable() && "Wrong variable"); + assert(getInlinedAt() == DbgValue->getDebugLoc()->getInlinedAt() && + "Wrong inlined-at"); + + ValueLoc = llvm::make_unique(getDebugLocValue(DbgValue)); + if (auto *E = DbgValue->getDebugExpression()) + if (E->getNumElements()) + FrameIndexExprs.push_back({0, E}); +} + ArrayRef DbgVariable::getFrameIndexExprs() const { if (FrameIndexExprs.size() == 1) return FrameIndexExprs; @@ -252,8 +300,8 @@ ArrayRef DbgVariable::getFrameIndexExprs() const { } void DbgVariable::addMMIEntry(const DbgVariable &V) { - assert(DebugLocListIndex == ~0U && !MInsn && "not an MMI entry"); - assert(V.DebugLocListIndex == ~0U && !V.MInsn && "not an MMI entry"); + assert(DebugLocListIndex == ~0U && !ValueLoc.get() && "not an MMI entry"); + assert(V.DebugLocListIndex == ~0U && !V.ValueLoc.get() && "not an MMI entry"); assert(V.getVariable() == getVariable() && "conflicting variable"); assert(V.getInlinedAt() == getInlinedAt() && "conflicting inlined-at location"); @@ -315,7 +363,7 @@ DwarfDebug::DwarfDebug(AsmPrinter *A, Module *M) IsDarwin(A->TM.getTargetTriple().isOSDarwin()) { const Triple &TT = Asm->TM.getTargetTriple(); - // Make sure we know our "debugger tuning." The target option takes + // Make sure we know our "debugger tuning". The target option takes // precedence; fall back to triple-based defaults. if (Asm->TM.Options.DebuggerTuning != DebuggerKind::Default) DebuggerTuning = Asm->TM.Options.DebuggerTuning; @@ -658,6 +706,11 @@ DwarfDebug::getOrCreateDwarfCompileUnit(const DICompileUnit *DIUnit) { NewCU.setSection(Asm->getObjFileLowering().getDwarfInfoSection()); } + // Create DIEs for function declarations used for call site debug info. + for (auto Scope : DIUnit->getRetainedTypes()) + if (auto *SP = dyn_cast_or_null(Scope)) + NewCU.getOrCreateSubprogramDIE(SP); + CUMap.insert({DIUnit, &NewCU}); CUDieMap.insert({&NewCU.getUnitDie(), &NewCU}); return NewCU; @@ -890,13 +943,6 @@ void DwarfDebug::finalizeModuleInfo() { // ranges for all subprogram DIEs for mach-o. DwarfCompileUnit &U = SkCU ? *SkCU : TheCU; - // We don't keep track of which addresses are used in which CU so this - // is a bit pessimistic under LTO. - if (!AddrPool.isEmpty() && - (getDwarfVersion() >= 5 || - (SkCU && !empty(TheCU.getUnitDie().children())))) - U.addAddrTableBase(); - if (unsigned NumRanges = TheCU.getRanges().size()) { if (NumRanges > 1 && useRangesSection()) // A DW_AT_low_pc attribute may also be specified in combination with @@ -909,6 +955,13 @@ void DwarfDebug::finalizeModuleInfo() { U.attachRangesOrLowHighPC(U.getUnitDie(), TheCU.takeRanges()); } + // We don't keep track of which addresses are used in which CU so this + // is a bit pessimistic under LTO. + if (!AddrPool.isEmpty() && + (getDwarfVersion() >= 5 || + (SkCU && !empty(TheCU.getUnitDie().children())))) + U.addAddrTableBase(); + if (getDwarfVersion() >= 5) { if (U.hasRangeLists()) U.addRnglistsBase(); @@ -941,6 +994,11 @@ void DwarfDebug::endModule() { assert(CurFn == nullptr); assert(CurMI == nullptr); + for (const auto &P : CUMap) { + auto &CU = *P.second; + CU.createBaseTypeDIEs(); + } + // If we aren't actually generating debug info (check beginModule - // conditionalized on !DisableDebugInfoPrinting and the presence of the // llvm.dbg.cu metadata node) @@ -1059,161 +1117,177 @@ void DwarfDebug::collectVariableInfoFromMFTable( } } -// Get .debug_loc entry for the instruction range starting at MI. -static DebugLocEntry::Value getDebugLocValue(const MachineInstr *MI) { - const DIExpression *Expr = MI->getDebugExpression(); - assert(MI->getNumOperands() == 4); - if (MI->getOperand(0).isReg()) { - auto RegOp = MI->getOperand(0); - auto Op1 = MI->getOperand(1); - // If the second operand is an immediate, this is a - // register-indirect address. - assert((!Op1.isImm() || (Op1.getImm() == 0)) && "unexpected offset"); - MachineLocation MLoc(RegOp.getReg(), Op1.isImm()); - return DebugLocEntry::Value(Expr, MLoc); - } - if (MI->getOperand(0).isImm()) - return DebugLocEntry::Value(Expr, MI->getOperand(0).getImm()); - if (MI->getOperand(0).isFPImm()) - return DebugLocEntry::Value(Expr, MI->getOperand(0).getFPImm()); - if (MI->getOperand(0).isCImm()) - return DebugLocEntry::Value(Expr, MI->getOperand(0).getCImm()); - - llvm_unreachable("Unexpected 4-operand DBG_VALUE instruction!"); -} +/// Determine whether a *singular* DBG_VALUE is valid for the entirety of its +/// enclosing lexical scope. The check ensures there are no other instructions +/// in the same lexical scope preceding the DBG_VALUE and that its range is +/// either open or otherwise rolls off the end of the scope. +static bool validThroughout(LexicalScopes &LScopes, + const MachineInstr *DbgValue, + const MachineInstr *RangeEnd) { + assert(DbgValue->getDebugLoc() && "DBG_VALUE without a debug location"); + auto MBB = DbgValue->getParent(); + auto DL = DbgValue->getDebugLoc(); + auto *LScope = LScopes.findLexicalScope(DL); + // Scope doesn't exist; this is a dead DBG_VALUE. + if (!LScope) + return false; + auto &LSRange = LScope->getRanges(); + if (LSRange.size() == 0) + return false; -/// If this and Next are describing different fragments of the same -/// variable, merge them by appending Next's values to the current -/// list of values. -/// Return true if the merge was successful. -bool DebugLocEntry::MergeValues(const DebugLocEntry &Next) { - if (Begin == Next.Begin) { - auto *FirstExpr = cast(Values[0].Expression); - auto *FirstNextExpr = cast(Next.Values[0].Expression); - if (!FirstExpr->isFragment() || !FirstNextExpr->isFragment()) + // Determine if the DBG_VALUE is valid at the beginning of its lexical block. + const MachineInstr *LScopeBegin = LSRange.front().first; + // Early exit if the lexical scope begins outside of the current block. + if (LScopeBegin->getParent() != MBB) + return false; + MachineBasicBlock::const_reverse_iterator Pred(DbgValue); + for (++Pred; Pred != MBB->rend(); ++Pred) { + if (Pred->getFlag(MachineInstr::FrameSetup)) + break; + auto PredDL = Pred->getDebugLoc(); + if (!PredDL || Pred->isMetaInstruction()) + continue; + // Check whether the instruction preceding the DBG_VALUE is in the same + // (sub)scope as the DBG_VALUE. + if (DL->getScope() == PredDL->getScope()) + return false; + auto *PredScope = LScopes.findLexicalScope(PredDL); + if (!PredScope || LScope->dominates(PredScope)) return false; + } - // We can only merge entries if none of the fragments overlap any others. - // In doing so, we can take advantage of the fact that both lists are - // sorted. - for (unsigned i = 0, j = 0; i < Values.size(); ++i) { - for (; j < Next.Values.size(); ++j) { - int res = cast(Values[i].Expression)->fragmentCmp( - cast(Next.Values[j].Expression)); - if (res == 0) // The two expressions overlap, we can't merge. - return false; - // Values[i] is entirely before Next.Values[j], - // so go back to the next entry of Values. - else if (res == -1) - break; - // Next.Values[j] is entirely before Values[i], so go on to the - // next entry of Next.Values. - } - } + // If the range of the DBG_VALUE is open-ended, report success. + if (!RangeEnd) + return true; - addValues(Next.Values); - End = Next.End; + // Fail if there are instructions belonging to our scope in another block. + const MachineInstr *LScopeEnd = LSRange.back().second; + if (LScopeEnd->getParent() != MBB) + return false; + + // Single, constant DBG_VALUEs in the prologue are promoted to be live + // throughout the function. This is a hack, presumably for DWARF v2 and not + // necessarily correct. It would be much better to use a dbg.declare instead + // if we know the constant is live throughout the scope. + if (DbgValue->getOperand(0).isImm() && MBB->pred_empty()) return true; - } + return false; } /// Build the location list for all DBG_VALUEs in the function that -/// describe the same variable. If the ranges of several independent -/// fragments of the same variable overlap partially, split them up and -/// combine the ranges. The resulting DebugLocEntries are will have +/// describe the same variable. The resulting DebugLocEntries will have /// strict monotonically increasing begin addresses and will never -/// overlap. +/// overlap. If the resulting list has only one entry that is valid +/// throughout variable's scope return true. +// +// See the definition of DbgValueHistoryMap::Entry for an explanation of the +// different kinds of history map entries. One thing to be aware of is that if +// a debug value is ended by another entry (rather than being valid until the +// end of the function), that entry's instruction may or may not be included in +// the range, depending on if the entry is a clobbering entry (it has an +// instruction that clobbers one or more preceding locations), or if it is an +// (overlapping) debug value entry. This distinction can be seen in the example +// below. The first debug value is ended by the clobbering entry 2, and the +// second and third debug values are ended by the overlapping debug value entry +// 4. // // Input: // -// Ranges History [var, loc, fragment ofs size] -// 0 | [x, (reg0, fragment 0, 32)] -// 1 | | [x, (reg1, fragment 32, 32)] <- IsFragmentOfPrevEntry -// 2 | | ... -// 3 | [clobber reg0] -// 4 [x, (mem, fragment 0, 64)] <- overlapping with both previous fragments of -// x. +// History map entries [type, end index, mi] // -// Output: +// 0 | [DbgValue, 2, DBG_VALUE $reg0, [...] (fragment 0, 32)] +// 1 | | [DbgValue, 4, DBG_VALUE $reg1, [...] (fragment 32, 32)] +// 2 | | [Clobber, $reg0 = [...], -, -] +// 3 | | [DbgValue, 4, DBG_VALUE 123, [...] (fragment 64, 32)] +// 4 [DbgValue, ~0, DBG_VALUE @g, [...] (fragment 0, 96)] // -// [0-1] [x, (reg0, fragment 0, 32)] -// [1-3] [x, (reg0, fragment 0, 32), (reg1, fragment 32, 32)] -// [3-4] [x, (reg1, fragment 32, 32)] -// [4- ] [x, (mem, fragment 0, 64)] -void -DwarfDebug::buildLocationList(SmallVectorImpl &DebugLoc, - const DbgValueHistoryMap::InstrRanges &Ranges) { - SmallVector OpenRanges; - - for (auto I = Ranges.begin(), E = Ranges.end(); I != E; ++I) { - const MachineInstr *Begin = I->first; - const MachineInstr *End = I->second; - assert(Begin->isDebugValue() && "Invalid History entry"); - - // Check if a variable is inaccessible in this range. - if (Begin->getNumOperands() > 1 && - Begin->getOperand(0).isReg() && !Begin->getOperand(0).getReg()) { - OpenRanges.clear(); - continue; - } - - // If this fragment overlaps with any open ranges, truncate them. - const DIExpression *DIExpr = Begin->getDebugExpression(); - auto Last = remove_if(OpenRanges, [&](DebugLocEntry::Value R) { - return DIExpr->fragmentsOverlap(R.getExpression()); - }); +// Output [start, end) [Value...]: +// +// [0-1) [(reg0, fragment 0, 32)] +// [1-3) [(reg0, fragment 0, 32), (reg1, fragment 32, 32)] +// [3-4) [(reg1, fragment 32, 32), (123, fragment 64, 32)] +// [4-) [(@g, fragment 0, 96)] +bool DwarfDebug::buildLocationList(SmallVectorImpl &DebugLoc, + const DbgValueHistoryMap::Entries &Entries) { + using OpenRange = + std::pair; + SmallVector OpenRanges; + bool isSafeForSingleLocation = true; + const MachineInstr *StartDebugMI = nullptr; + const MachineInstr *EndMI = nullptr; + + for (auto EB = Entries.begin(), EI = EB, EE = Entries.end(); EI != EE; ++EI) { + const MachineInstr *Instr = EI->getInstr(); + + // Remove all values that are no longer live. + size_t Index = std::distance(EB, EI); + auto Last = + remove_if(OpenRanges, [&](OpenRange &R) { return R.first <= Index; }); OpenRanges.erase(Last, OpenRanges.end()); - const MCSymbol *StartLabel = getLabelBeforeInsn(Begin); - assert(StartLabel && "Forgot label before DBG_VALUE starting a range!"); + // If we are dealing with a clobbering entry, this iteration will result in + // a location list entry starting after the clobbering instruction. + const MCSymbol *StartLabel = + EI->isClobber() ? getLabelAfterInsn(Instr) : getLabelBeforeInsn(Instr); + assert(StartLabel && + "Forgot label before/after instruction starting a range!"); const MCSymbol *EndLabel; - if (End != nullptr) - EndLabel = getLabelAfterInsn(End); - else if (std::next(I) == Ranges.end()) + if (std::next(EI) == Entries.end()) { EndLabel = Asm->getFunctionEnd(); + if (EI->isClobber()) + EndMI = EI->getInstr(); + } + else if (std::next(EI)->isClobber()) + EndLabel = getLabelAfterInsn(std::next(EI)->getInstr()); else - EndLabel = getLabelBeforeInsn(std::next(I)->first); + EndLabel = getLabelBeforeInsn(std::next(EI)->getInstr()); assert(EndLabel && "Forgot label after instruction ending a range!"); - LLVM_DEBUG(dbgs() << "DotDebugLoc: " << *Begin << "\n"); + if (EI->isDbgValue()) + LLVM_DEBUG(dbgs() << "DotDebugLoc: " << *Instr << "\n"); + + // If this history map entry has a debug value, add that to the list of + // open ranges and check if its location is valid for a single value + // location. + if (EI->isDbgValue()) { + // Do not add undef debug values, as they are redundant information in + // the location list entries. An undef debug results in an empty location + // description. If there are any non-undef fragments then padding pieces + // with empty location descriptions will automatically be inserted, and if + // all fragments are undef then the whole location list entry is + // redundant. + if (!Instr->isUndefDebugValue()) { + auto Value = getDebugLocValue(Instr); + OpenRanges.emplace_back(EI->getEndIndex(), Value); + + // TODO: Add support for single value fragment locations. + if (Instr->getDebugExpression()->isFragment()) + isSafeForSingleLocation = false; + + if (!StartDebugMI) + StartDebugMI = Instr; + } else { + isSafeForSingleLocation = false; + } + } - auto Value = getDebugLocValue(Begin); + // Location list entries with empty location descriptions are redundant + // information in DWARF, so do not emit those. + if (OpenRanges.empty()) + continue; // Omit entries with empty ranges as they do not have any effect in DWARF. if (StartLabel == EndLabel) { - // If this is a fragment, we must still add the value to the list of - // open ranges, since it may describe non-overlapping parts of the - // variable. - if (DIExpr->isFragment()) - OpenRanges.push_back(Value); LLVM_DEBUG(dbgs() << "Omitting location list entry with empty range.\n"); continue; } - DebugLocEntry Loc(StartLabel, EndLabel, Value); - bool couldMerge = false; - - // If this is a fragment, it may belong to the current DebugLocEntry. - if (DIExpr->isFragment()) { - // Add this value to the list of open ranges. - OpenRanges.push_back(Value); - - // Attempt to add the fragment to the last entry. - if (!DebugLoc.empty()) - if (DebugLoc.back().MergeValues(Loc)) - couldMerge = true; - } - - if (!couldMerge) { - // Need to add a new DebugLocEntry. Add all values from still - // valid non-overlapping fragments. - if (OpenRanges.size()) - Loc.addValues(OpenRanges); - - DebugLoc.push_back(std::move(Loc)); - } + SmallVector Values; + for (auto &R : OpenRanges) + Values.push_back(R.second); + DebugLoc.emplace_back(StartLabel, EndLabel, Values); // Attempt to coalesce the ranges of two otherwise identical // DebugLocEntries. @@ -1229,6 +1303,9 @@ DwarfDebug::buildLocationList(SmallVectorImpl &DebugLoc, if (PrevEntry != DebugLoc.rend() && PrevEntry->MergeRanges(*CurEntry)) DebugLoc.pop_back(); } + + return DebugLoc.size() == 1 && isSafeForSingleLocation && + validThroughout(LScopes, StartDebugMI, EndMI); } DbgEntity *DwarfDebug::createConcreteEntity(DwarfCompileUnit &TheCU, @@ -1253,64 +1330,6 @@ DbgEntity *DwarfDebug::createConcreteEntity(DwarfCompileUnit &TheCU, return ConcreteEntities.back().get(); } -/// Determine whether a *singular* DBG_VALUE is valid for the entirety of its -/// enclosing lexical scope. The check ensures there are no other instructions -/// in the same lexical scope preceding the DBG_VALUE and that its range is -/// either open or otherwise rolls off the end of the scope. -static bool validThroughout(LexicalScopes &LScopes, - const MachineInstr *DbgValue, - const MachineInstr *RangeEnd) { - assert(DbgValue->getDebugLoc() && "DBG_VALUE without a debug location"); - auto MBB = DbgValue->getParent(); - auto DL = DbgValue->getDebugLoc(); - auto *LScope = LScopes.findLexicalScope(DL); - // Scope doesn't exist; this is a dead DBG_VALUE. - if (!LScope) - return false; - auto &LSRange = LScope->getRanges(); - if (LSRange.size() == 0) - return false; - - // Determine if the DBG_VALUE is valid at the beginning of its lexical block. - const MachineInstr *LScopeBegin = LSRange.front().first; - // Early exit if the lexical scope begins outside of the current block. - if (LScopeBegin->getParent() != MBB) - return false; - MachineBasicBlock::const_reverse_iterator Pred(DbgValue); - for (++Pred; Pred != MBB->rend(); ++Pred) { - if (Pred->getFlag(MachineInstr::FrameSetup)) - break; - auto PredDL = Pred->getDebugLoc(); - if (!PredDL || Pred->isMetaInstruction()) - continue; - // Check whether the instruction preceding the DBG_VALUE is in the same - // (sub)scope as the DBG_VALUE. - if (DL->getScope() == PredDL->getScope()) - return false; - auto *PredScope = LScopes.findLexicalScope(PredDL); - if (!PredScope || LScope->dominates(PredScope)) - return false; - } - - // If the range of the DBG_VALUE is open-ended, report success. - if (!RangeEnd) - return true; - - // Fail if there are instructions belonging to our scope in another block. - const MachineInstr *LScopeEnd = LSRange.back().second; - if (LScopeEnd->getParent() != MBB) - return false; - - // Single, constant DBG_VALUEs in the prologue are promoted to be live - // throughout the function. This is a hack, presumably for DWARF v2 and not - // necessarily correct. It would be much better to use a dbg.declare instead - // if we know the constant is live throughout the scope. - if (DbgValue->getOperand(0).isImm() && MBB->pred_empty()) - return true; - - return false; -} - // Find variables for each lexical scope. void DwarfDebug::collectEntityInfo(DwarfCompileUnit &TheCU, const DISubprogram *SP, @@ -1324,8 +1343,8 @@ void DwarfDebug::collectEntityInfo(DwarfCompileUnit &TheCU, continue; // Instruction ranges, specifying where IV is accessible. - const auto &Ranges = I.second; - if (Ranges.empty()) + const auto &HistoryMapEntries = I.second; + if (HistoryMapEntries.empty()) continue; LexicalScope *Scope = nullptr; @@ -1342,15 +1361,24 @@ void DwarfDebug::collectEntityInfo(DwarfCompileUnit &TheCU, DbgVariable *RegVar = cast(createConcreteEntity(TheCU, *Scope, LocalVar, IV.second)); - const MachineInstr *MInsn = Ranges.front().first; + const MachineInstr *MInsn = HistoryMapEntries.front().getInstr(); assert(MInsn->isDebugValue() && "History must begin with debug value"); // Check if there is a single DBG_VALUE, valid throughout the var's scope. - if (Ranges.size() == 1 && - validThroughout(LScopes, MInsn, Ranges.front().second)) { - RegVar->initializeDbgValue(MInsn); - continue; + // If the history map contains a single debug value, there may be an + // additional entry which clobbers the debug value. + size_t HistSize = HistoryMapEntries.size(); + bool SingleValueWithClobber = + HistSize == 2 && HistoryMapEntries[1].isClobber(); + if (HistSize == 1 || SingleValueWithClobber) { + const auto *End = + SingleValueWithClobber ? HistoryMapEntries[1].getInstr() : nullptr; + if (validThroughout(LScopes, MInsn, End)) { + RegVar->initializeDbgValue(MInsn); + continue; + } } + // Do not emit location lists if .debug_loc secton is disabled. if (!useLocSection()) continue; @@ -1360,7 +1388,15 @@ void DwarfDebug::collectEntityInfo(DwarfCompileUnit &TheCU, // Build the location list for this variable. SmallVector Entries; - buildLocationList(Entries, Ranges); + bool isValidSingleLocation = buildLocationList(Entries, HistoryMapEntries); + + // Check whether buildLocationList managed to merge all locations to one + // that is valid throughout the variable's scope. If so, produce single + // value location. + if (isValidSingleLocation) { + RegVar->initializeDbgValue(Entries[0].getValues()[0]); + continue; + } // If the variable has a DIBasicType, extract it. Basic types cannot have // unique identifiers, so don't bother resolving the type with the @@ -1370,7 +1406,7 @@ void DwarfDebug::collectEntityInfo(DwarfCompileUnit &TheCU, // Finalize the entry by lowering it into a DWARF bytestream. for (auto &Entry : Entries) - Entry.finalize(*Asm, List, BT); + Entry.finalize(*Asm, List, BT, TheCU); } // For each InlinedEntity collected from DBG_LABEL instructions, convert to @@ -1489,7 +1525,7 @@ void DwarfDebug::beginInstruction(const MachineInstr *MI) { // We have an explicit location, different from the previous location. // Don't repeat a line-0 record, but otherwise emit the new location. // (The new location might be an explicit line 0, which we do emit.) - if (PrevInstLoc && DL.getLine() == 0 && LastAsmLine == 0) + if (DL.getLine() == 0 && LastAsmLine == 0) return; unsigned Flags = 0; if (DL == PrologEndLoc) { @@ -1521,6 +1557,46 @@ static DebugLoc findPrologueEndLoc(const MachineFunction *MF) { return DebugLoc(); } +/// Register a source line with debug info. Returns the unique label that was +/// emitted and which provides correspondence to the source line list. +static void recordSourceLine(AsmPrinter &Asm, unsigned Line, unsigned Col, + const MDNode *S, unsigned Flags, unsigned CUID, + uint16_t DwarfVersion, + ArrayRef> DCUs) { + StringRef Fn; + unsigned FileNo = 1; + unsigned Discriminator = 0; + if (auto *Scope = cast_or_null(S)) { + Fn = Scope->getFilename(); + if (Line != 0 && DwarfVersion >= 4) + if (auto *LBF = dyn_cast(Scope)) + Discriminator = LBF->getDiscriminator(); + + FileNo = static_cast(*DCUs[CUID]) + .getOrCreateSourceID(Scope->getFile()); + } + Asm.OutStreamer->EmitDwarfLocDirective(FileNo, Line, Col, Flags, 0, + Discriminator, Fn); +} + +DebugLoc DwarfDebug::emitInitialLocDirective(const MachineFunction &MF, + unsigned CUID) { + // Get beginning of function. + if (DebugLoc PrologEndLoc = findPrologueEndLoc(&MF)) { + // Ensure the compile unit is created if the function is called before + // beginFunction(). + (void)getOrCreateDwarfCompileUnit( + MF.getFunction().getSubprogram()->getUnit()); + // We'd like to list the prologue as "not statements" but GDB behaves + // poorly if we do that. Revisit this with caution/GDB (7.5+) testing. + const DISubprogram *SP = PrologEndLoc->getInlinedAtScope()->getSubprogram(); + ::recordSourceLine(*Asm, SP->getScopeLine(), 0, SP, DWARF2_FLAG_IS_STMT, + CUID, getDwarfVersion(), getUnits()); + return PrologEndLoc; + } + return DebugLoc(); +} + // Gather pre-function debug information. Assumes being called immediately // after the function entry point has been emitted. void DwarfDebug::beginFunctionImpl(const MachineFunction *MF) { @@ -1543,13 +1619,8 @@ void DwarfDebug::beginFunctionImpl(const MachineFunction *MF) { Asm->OutStreamer->getContext().setDwarfCompileUnitID(CU.getUniqueID()); // Record beginning of function. - PrologEndLoc = findPrologueEndLoc(MF); - if (PrologEndLoc) { - // We'd like to list the prologue as "not statements" but GDB behaves - // poorly if we do that. Revisit this with caution/GDB (7.5+) testing. - auto *SP = PrologEndLoc->getInlinedAtScope()->getSubprogram(); - recordSourceLine(SP->getScopeLine(), 0, SP, DWARF2_FLAG_IS_STMT); - } + PrologEndLoc = emitInitialLocDirective( + *MF, Asm->OutStreamer->getContext().getDwarfCompileUnitID()); } void DwarfDebug::skippedNonDebugFunction() { @@ -1647,21 +1718,9 @@ void DwarfDebug::endFunctionImpl(const MachineFunction *MF) { // emitted and which provides correspondence to the source line list. void DwarfDebug::recordSourceLine(unsigned Line, unsigned Col, const MDNode *S, unsigned Flags) { - StringRef Fn; - unsigned FileNo = 1; - unsigned Discriminator = 0; - if (auto *Scope = cast_or_null(S)) { - Fn = Scope->getFilename(); - if (Line != 0 && getDwarfVersion() >= 4) - if (auto *LBF = dyn_cast(Scope)) - Discriminator = LBF->getDiscriminator(); - - unsigned CUID = Asm->OutStreamer->getContext().getDwarfCompileUnitID(); - FileNo = static_cast(*InfoHolder.getUnits()[CUID]) - .getOrCreateSourceID(Scope->getFile()); - } - Asm->OutStreamer->EmitDwarfLocDirective(FileNo, Line, Col, Flags, 0, - Discriminator, Fn); + ::recordSourceLine(*Asm, Line, Col, S, Flags, + Asm->OutStreamer->getContext().getDwarfCompileUnitID(), + getDwarfVersion(), getUnits()); } //===----------------------------------------------------------------------===// @@ -1890,17 +1949,59 @@ void DwarfDebug::emitDebugStr() { } void DwarfDebug::emitDebugLocEntry(ByteStreamer &Streamer, - const DebugLocStream::Entry &Entry) { + const DebugLocStream::Entry &Entry, + const DwarfCompileUnit *CU) { auto &&Comments = DebugLocs.getComments(Entry); auto Comment = Comments.begin(); auto End = Comments.end(); - for (uint8_t Byte : DebugLocs.getBytes(Entry)) - Streamer.EmitInt8(Byte, Comment != End ? *(Comment++) : ""); + + // The expressions are inserted into a byte stream rather early (see + // DwarfExpression::addExpression) so for those ops (e.g. DW_OP_convert) that + // need to reference a base_type DIE the offset of that DIE is not yet known. + // To deal with this we instead insert a placeholder early and then extract + // it here and replace it with the real reference. + unsigned PtrSize = Asm->MAI->getCodePointerSize(); + DWARFDataExtractor Data(StringRef(DebugLocs.getBytes(Entry).data(), + DebugLocs.getBytes(Entry).size()), + Asm->getDataLayout().isLittleEndian(), PtrSize); + DWARFExpression Expr(Data, getDwarfVersion(), PtrSize); + + using Encoding = DWARFExpression::Operation::Encoding; + uint32_t Offset = 0; + for (auto &Op : Expr) { + assert(Op.getCode() != dwarf::DW_OP_const_type && + "3 operand ops not yet supported"); + Streamer.EmitInt8(Op.getCode(), Comment != End ? *(Comment++) : ""); + Offset++; + for (unsigned I = 0; I < 2; ++I) { + if (Op.getDescription().Op[I] == Encoding::SizeNA) + continue; + if (Op.getDescription().Op[I] == Encoding::BaseTypeRef) { + if (CU) { + uint64_t Offset = CU->ExprRefedBaseTypes[Op.getRawOperand(I)].Die->getOffset(); + assert(Offset < (1ULL << (ULEB128PadSize * 7)) && "Offset wont fit"); + Asm->EmitULEB128(Offset, nullptr, ULEB128PadSize); + } else { + // Emit a reference to the 'generic type'. + Asm->EmitULEB128(0, nullptr, ULEB128PadSize); + } + // Make sure comments stay aligned. + for (unsigned J = 0; J < ULEB128PadSize; ++J) + if (Comment != End) + Comment++; + } else { + for (uint32_t J = Offset; J < Op.getOperandEndOffset(I); ++J) + Streamer.EmitInt8(Data.getData()[J], Comment != End ? *(Comment++) : ""); + } + Offset = Op.getOperandEndOffset(I); + } + assert(Offset == Op.getEndOffset()); + } } -static void emitDebugLocValue(const AsmPrinter &AP, const DIBasicType *BT, - const DebugLocEntry::Value &Value, - DwarfExpression &DwarfExpr) { +void DwarfDebug::emitDebugLocValue(const AsmPrinter &AP, const DIBasicType *BT, + const DbgValueLoc &Value, + DwarfExpression &DwarfExpr) { auto *DIExpr = Value.getExpression(); DIExpressionCursor ExprCursor(DIExpr); DwarfExpr.addFragmentOffset(DIExpr); @@ -1916,6 +2017,12 @@ static void emitDebugLocValue(const AsmPrinter &AP, const DIBasicType *BT, if (Location.isIndirect()) DwarfExpr.setMemoryLocationKind(); DIExpressionCursor Cursor(DIExpr); + + if (DIExpr->isEntryValue()) { + DwarfExpr.setEntryValueFlag(); + DwarfExpr.addEntryValueExpression(Cursor); + } + const TargetRegisterInfo &TRI = *AP.MF->getSubtarget().getRegisterInfo(); if (!DwarfExpr.addMachineRegExpression(TRI, Cursor, Location.getReg())) return; @@ -1929,38 +2036,50 @@ static void emitDebugLocValue(const AsmPrinter &AP, const DIBasicType *BT, void DebugLocEntry::finalize(const AsmPrinter &AP, DebugLocStream::ListBuilder &List, - const DIBasicType *BT) { + const DIBasicType *BT, + DwarfCompileUnit &TheCU) { + assert(!Values.empty() && + "location list entries without values are redundant"); assert(Begin != End && "unexpected location list entry with empty range"); DebugLocStream::EntryBuilder Entry(List, Begin, End); BufferByteStreamer Streamer = Entry.getStreamer(); - DebugLocDwarfExpression DwarfExpr(AP.getDwarfVersion(), Streamer); - const DebugLocEntry::Value &Value = Values[0]; + DebugLocDwarfExpression DwarfExpr(AP.getDwarfVersion(), Streamer, TheCU); + const DbgValueLoc &Value = Values[0]; if (Value.isFragment()) { // Emit all fragments that belong to the same variable and range. - assert(llvm::all_of(Values, [](DebugLocEntry::Value P) { + assert(llvm::all_of(Values, [](DbgValueLoc P) { return P.isFragment(); }) && "all values are expected to be fragments"); assert(std::is_sorted(Values.begin(), Values.end()) && "fragments are expected to be sorted"); for (auto Fragment : Values) - emitDebugLocValue(AP, BT, Fragment, DwarfExpr); + DwarfDebug::emitDebugLocValue(AP, BT, Fragment, DwarfExpr); } else { assert(Values.size() == 1 && "only fragments may have >1 value"); - emitDebugLocValue(AP, BT, Value, DwarfExpr); + DwarfDebug::emitDebugLocValue(AP, BT, Value, DwarfExpr); } DwarfExpr.finalize(); } -void DwarfDebug::emitDebugLocEntryLocation(const DebugLocStream::Entry &Entry) { +void DwarfDebug::emitDebugLocEntryLocation(const DebugLocStream::Entry &Entry, + const DwarfCompileUnit *CU) { // Emit the size. Asm->OutStreamer->AddComment("Loc expr size"); - Asm->emitInt16(DebugLocs.getBytes(Entry).size()); - + if (getDwarfVersion() >= 5) + Asm->EmitULEB128(DebugLocs.getBytes(Entry).size()); + else if (DebugLocs.getBytes(Entry).size() <= std::numeric_limits::max()) + Asm->emitInt16(DebugLocs.getBytes(Entry).size()); + else { + // The entry is too big to fit into 16 bit, drop it as there is nothing we + // can do. + Asm->emitInt16(0); + return; + } // Emit the entry. APByteStreamer Streamer(*Asm); - emitDebugLocEntry(Streamer, Entry); + emitDebugLocEntry(Streamer, Entry, CU); } // Emit the common part of the DWARF 5 range/locations list tables header. @@ -2060,7 +2179,7 @@ void DwarfDebug::emitDebugLoc() { Asm->EmitLabelDifference(Entry.EndSym, Base, Size); } - emitDebugLocEntryLocation(Entry); + emitDebugLocEntryLocation(Entry, CU); continue; } @@ -2081,7 +2200,7 @@ void DwarfDebug::emitDebugLoc() { Asm->OutStreamer->EmitSymbolValue(Entry.EndSym, Size); } - emitDebugLocEntryLocation(Entry); + emitDebugLocEntryLocation(Entry, CU); } if (IsLocLists) { @@ -2100,9 +2219,9 @@ void DwarfDebug::emitDebugLoc() { } void DwarfDebug::emitDebugLocDWO() { - Asm->OutStreamer->SwitchSection( - Asm->getObjFileLowering().getDwarfLocDWOSection()); for (const auto &List : DebugLocs.getLists()) { + Asm->OutStreamer->SwitchSection( + Asm->getObjFileLowering().getDwarfLocDWOSection()); Asm->OutStreamer->EmitLabel(List.Label); for (const auto &Entry : DebugLocs.getEntries(List)) { // GDB only supports startx_length in pre-standard split-DWARF. @@ -2117,7 +2236,7 @@ void DwarfDebug::emitDebugLocDWO() { Asm->EmitULEB128(idx); Asm->EmitLabelDifference(Entry.EndSym, Entry.BeginSym, 4); - emitDebugLocEntryLocation(Entry); + emitDebugLocEntryLocation(Entry, List.CU); } Asm->emitInt8(dwarf::DW_LLE_end_of_list); } @@ -2170,19 +2289,18 @@ void DwarfDebug::emitDebugARanges() { } // Sort the symbols by offset within the section. - std::stable_sort( - List.begin(), List.end(), [&](const SymbolCU &A, const SymbolCU &B) { - unsigned IA = A.Sym ? Asm->OutStreamer->GetSymbolOrder(A.Sym) : 0; - unsigned IB = B.Sym ? Asm->OutStreamer->GetSymbolOrder(B.Sym) : 0; - - // Symbols with no order assigned should be placed at the end. - // (e.g. section end labels) - if (IA == 0) - return false; - if (IB == 0) - return true; - return IA < IB; - }); + llvm::stable_sort(List, [&](const SymbolCU &A, const SymbolCU &B) { + unsigned IA = A.Sym ? Asm->OutStreamer->GetSymbolOrder(A.Sym) : 0; + unsigned IB = B.Sym ? Asm->OutStreamer->GetSymbolOrder(B.Sym) : 0; + + // Symbols with no order assigned should be placed at the end. + // (e.g. section end labels) + if (IA == 0) + return false; + if (IB == 0) + return true; + return IA < IB; + }); // Insert a final terminator. List.push_back(SymbolCU(nullptr, Asm->OutStreamer->endSection(Section))); @@ -2687,6 +2805,22 @@ void DwarfDebug::addDwarfTypeUnitType(DwarfCompileUnit &CU, CU.addDIETypeSignature(RefDie, Signature); } +DwarfDebug::NonTypeUnitContext::NonTypeUnitContext(DwarfDebug *DD) + : DD(DD), + TypeUnitsUnderConstruction(std::move(DD->TypeUnitsUnderConstruction)) { + DD->TypeUnitsUnderConstruction.clear(); + assert(TypeUnitsUnderConstruction.empty() || !DD->AddrPool.hasBeenUsed()); +} + +DwarfDebug::NonTypeUnitContext::~NonTypeUnitContext() { + DD->TypeUnitsUnderConstruction = std::move(TypeUnitsUnderConstruction); + DD->AddrPool.resetUsedFlag(); +} + +DwarfDebug::NonTypeUnitContext DwarfDebug::enterNonTypeUnitContext() { + return NonTypeUnitContext(this); +} + // Add the Name along with its companion DIE to the appropriate accelerator // table (for AccelTableKind::Dwarf it's always AccelDebugNames, for // AccelTableKind::Apple, we use the table we got as an argument). If @@ -2699,7 +2833,7 @@ void DwarfDebug::addAccelNameImpl(const DICompileUnit &CU, return; if (getAccelTableKind() != AccelTableKind::Apple && - CU.getNameTableKind() == DICompileUnit::DebugNameTableKind::None) + CU.getNameTableKind() != DICompileUnit::DebugNameTableKind::Default) return; DwarfFile &Holder = useSplitDwarf() ? SkeletonHolder : InfoHolder; diff --git a/lib/CodeGen/AsmPrinter/DwarfDebug.h b/lib/CodeGen/AsmPrinter/DwarfDebug.h index 8a31e989b289..3ac474e2bdda 100644 --- a/lib/CodeGen/AsmPrinter/DwarfDebug.h +++ b/lib/CodeGen/AsmPrinter/DwarfDebug.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/DwarfDebug.h - Dwarf Debug Framework --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -16,6 +15,7 @@ #include "AddressPool.h" #include "DebugLocStream.h" +#include "DebugLocEntry.h" #include "DwarfFile.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" @@ -52,6 +52,7 @@ class ByteStreamer; class DebugLocEntry; class DIE; class DwarfCompileUnit; +class DwarfExpression; class DwarfTypeUnit; class DwarfUnit; class LexicalScope; @@ -111,12 +112,14 @@ public: /// /// Variables can be created from \c DBG_VALUE instructions. Those whose /// location changes over time use \a DebugLocListIndex, while those with a -/// single instruction use \a MInsn and (optionally) a single entry of \a Expr. +/// single location use \a ValueLoc and (optionally) a single entry of \a Expr. /// /// Variables that have been optimized out use none of these fields. class DbgVariable : public DbgEntity { - unsigned DebugLocListIndex = ~0u; /// Offset in DebugLocs. - const MachineInstr *MInsn = nullptr; /// DBG_VALUE instruction. + /// Offset in DebugLocs. + unsigned DebugLocListIndex = ~0u; + /// Single value location description. + std::unique_ptr ValueLoc = nullptr; struct FrameIndexExpr { int FI; @@ -136,7 +139,7 @@ public: /// Initialize from the MMI table. void initializeMMI(const DIExpression *E, int FI) { assert(FrameIndexExprs.empty() && "Already initialized?"); - assert(!MInsn && "Already initialized?"); + assert(!ValueLoc.get() && "Already initialized?"); assert((!E || E->isValid()) && "Expected valid expression"); assert(FI != std::numeric_limits::max() && "Expected valid index"); @@ -144,35 +147,35 @@ public: FrameIndexExprs.push_back({FI, E}); } - /// Initialize from a DBG_VALUE instruction. - void initializeDbgValue(const MachineInstr *DbgValue) { + // Initialize variable's location. + void initializeDbgValue(DbgValueLoc Value) { assert(FrameIndexExprs.empty() && "Already initialized?"); - assert(!MInsn && "Already initialized?"); + assert(!ValueLoc && "Already initialized?"); + assert(!Value.getExpression()->isFragment() && "Fragments not supported."); - assert(getVariable() == DbgValue->getDebugVariable() && "Wrong variable"); - assert(getInlinedAt() == DbgValue->getDebugLoc()->getInlinedAt() && - "Wrong inlined-at"); - - MInsn = DbgValue; - if (auto *E = DbgValue->getDebugExpression()) + ValueLoc = llvm::make_unique(Value); + if (auto *E = ValueLoc->getExpression()) if (E->getNumElements()) FrameIndexExprs.push_back({0, E}); } + /// Initialize from a DBG_VALUE instruction. + void initializeDbgValue(const MachineInstr *DbgValue); + // Accessors. const DILocalVariable *getVariable() const { return cast(getEntity()); } const DIExpression *getSingleExpression() const { - assert(MInsn && FrameIndexExprs.size() <= 1); + assert(ValueLoc.get() && FrameIndexExprs.size() <= 1); return FrameIndexExprs.size() ? FrameIndexExprs[0].Expr : nullptr; } void setDebugLocListIndex(unsigned O) { DebugLocListIndex = O; } unsigned getDebugLocListIndex() const { return DebugLocListIndex; } StringRef getName() const { return getVariable()->getName(); } - const MachineInstr *getMInsn() const { return MInsn; } + const DbgValueLoc *getValueLoc() const { return ValueLoc.get(); } /// Get the FI entries, sorted by fragment offset. ArrayRef getFrameIndexExprs() const; bool hasFrameIndexExprs() const { return !FrameIndexExprs.empty(); } @@ -205,7 +208,7 @@ public: } bool hasComplexAddress() const { - assert(MInsn && "Expected DBG_VALUE, not MMI variable"); + assert(ValueLoc.get() && "Expected DBG_VALUE, not MMI variable"); assert((FrameIndexExprs.empty() || (FrameIndexExprs.size() == 1 && FrameIndexExprs[0].Expr->getNumElements())) && @@ -219,11 +222,6 @@ public: static bool classof(const DbgEntity *N) { return N->getDbgEntityID() == DbgVariableKind; } - -private: - template T *resolve(TypedDINodeRef Ref) const { - return Ref.resolve(); - } }; //===----------------------------------------------------------------------===// @@ -254,11 +252,6 @@ public: static bool classof(const DbgEntity *N) { return N->getDbgEntityID() == DbgLabelKind; } - -private: - template T *resolve(TypedDINodeRef Ref) const { - return Ref.resolve(); - } }; /// Helper used to pair up a symbol and its DWARF compile unit. @@ -558,9 +551,11 @@ class DwarfDebug : public DebugHandlerBase { DenseSet &ProcessedVars); /// Build the location list for all DBG_VALUEs in the - /// function that describe the same variable. - void buildLocationList(SmallVectorImpl &DebugLoc, - const DbgValueHistoryMap::InstrRanges &Ranges); + /// function that describe the same variable. If the resulting + /// list has only one entry that is valid for entire variable's + /// scope return true. + bool buildLocationList(SmallVectorImpl &DebugLoc, + const DbgValueHistoryMap::Entries &Entries); /// Collect variable information from the side table maintained by MF. void collectVariableInfoFromMFTable(DwarfCompileUnit &TheCU, @@ -593,6 +588,9 @@ public: /// Emit all Dwarf sections that should come after the content. void endModule() override; + /// Emits inital debug location directive. + DebugLoc emitInitialLocDirective(const MachineFunction &MF, unsigned CUID); + /// Process beginning of an instruction. void beginInstruction(const MachineInstr *MI) override; @@ -604,6 +602,19 @@ public: void addDwarfTypeUnitType(DwarfCompileUnit &CU, StringRef Identifier, DIE &Die, const DICompositeType *CTy); + friend class NonTypeUnitContext; + class NonTypeUnitContext { + DwarfDebug *DD; + decltype(DwarfDebug::TypeUnitsUnderConstruction) TypeUnitsUnderConstruction; + friend class DwarfDebug; + NonTypeUnitContext(DwarfDebug *DD); + public: + NonTypeUnitContext(NonTypeUnitContext&&) = default; + ~NonTypeUnitContext(); + }; + + NonTypeUnitContext enterNonTypeUnitContext(); + /// Add a label so that arange data can be generated for it. void addArangeLabel(SymbolCU SCU) { ArangeLabels.push_back(SCU); } @@ -680,15 +691,12 @@ public: /// Emit an entry for the debug loc section. This can be used to /// handle an entry that's going to be emitted into the debug loc section. void emitDebugLocEntry(ByteStreamer &Streamer, - const DebugLocStream::Entry &Entry); + const DebugLocStream::Entry &Entry, + const DwarfCompileUnit *CU); /// Emit the location for a debug loc entry, including the size header. - void emitDebugLocEntryLocation(const DebugLocStream::Entry &Entry); - - /// Find the MDNode for the given reference. - template T *resolve(TypedDINodeRef Ref) const { - return Ref.resolve(); - } + void emitDebugLocEntryLocation(const DebugLocStream::Entry &Entry, + const DwarfCompileUnit *CU); void addSubprogramNames(const DICompileUnit &CU, const DISubprogram *SP, DIE &Die); @@ -728,6 +736,10 @@ public: void addSectionLabel(const MCSymbol *Sym); const MCSymbol *getSectionLabel(const MCSection *S); + + static void emitDebugLocValue(const AsmPrinter &AP, const DIBasicType *BT, + const DbgValueLoc &Value, + DwarfExpression &DwarfExpr); }; } // end namespace llvm diff --git a/lib/CodeGen/AsmPrinter/DwarfException.h b/lib/CodeGen/AsmPrinter/DwarfException.h index b57ea8fc6322..24bbf58b91ec 100644 --- a/lib/CodeGen/AsmPrinter/DwarfException.h +++ b/lib/CodeGen/AsmPrinter/DwarfException.h @@ -1,9 +1,8 @@ //===-- DwarfException.h - Dwarf Exception Framework -----------*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/AsmPrinter/DwarfExpression.cpp b/lib/CodeGen/AsmPrinter/DwarfExpression.cpp index 19c350afbf17..2858afaa1cf1 100644 --- a/lib/CodeGen/AsmPrinter/DwarfExpression.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfExpression.cpp @@ -1,9 +1,8 @@ //===- llvm/CodeGen/DwarfExpression.cpp - Dwarf Debug Framework -----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -12,6 +11,7 @@ //===----------------------------------------------------------------------===// #include "DwarfExpression.h" +#include "DwarfCompileUnit.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/SmallBitVector.h" #include "llvm/BinaryFormat/Dwarf.h" @@ -40,7 +40,7 @@ void DwarfExpression::emitConstu(uint64_t Value) { void DwarfExpression::addReg(int DwarfReg, const char *Comment) { assert(DwarfReg >= 0 && "invalid negative dwarf register number"); - assert((LocationKind == Unknown || LocationKind == Register) && + assert((isUnknownLocation() || isRegisterLocation()) && "location description already locked down"); LocationKind = Register; if (DwarfReg < 32) { @@ -53,7 +53,7 @@ void DwarfExpression::addReg(int DwarfReg, const char *Comment) { void DwarfExpression::addBReg(int DwarfReg, int Offset) { assert(DwarfReg >= 0 && "invalid negative dwarf register number"); - assert(LocationKind != Register && "location description already locked down"); + assert(!isRegisterLocation() && "location description already locked down"); if (DwarfReg < 32) { emitOp(dwarf::DW_OP_breg0 + DwarfReg); } else { @@ -184,20 +184,20 @@ void DwarfExpression::addStackValue() { } void DwarfExpression::addSignedConstant(int64_t Value) { - assert(LocationKind == Implicit || LocationKind == Unknown); + assert(isImplicitLocation() || isUnknownLocation()); LocationKind = Implicit; emitOp(dwarf::DW_OP_consts); emitSigned(Value); } void DwarfExpression::addUnsignedConstant(uint64_t Value) { - assert(LocationKind == Implicit || LocationKind == Unknown); + assert(isImplicitLocation() || isUnknownLocation()); LocationKind = Implicit; emitConstu(Value); } void DwarfExpression::addUnsignedConstant(const APInt &Value) { - assert(LocationKind == Implicit || LocationKind == Unknown); + assert(isImplicitLocation() || isUnknownLocation()); LocationKind = Implicit; unsigned Size = Value.getBitWidth(); @@ -242,12 +242,16 @@ bool DwarfExpression::addMachineRegExpression(const TargetRegisterInfo &TRI, } // Handle simple register locations. - if (LocationKind != Memory && !HasComplexExpression) { + if (!isMemoryLocation() && !HasComplexExpression) { for (auto &Reg : DwarfRegs) { if (Reg.DwarfRegNo >= 0) addReg(Reg.DwarfRegNo, Reg.Comment); addOpPiece(Reg.Size); } + + if (isEntryValue() && DwarfVersion >= 4) + emitOp(dwarf::DW_OP_stack_value); + DwarfRegs.clear(); return true; } @@ -296,6 +300,19 @@ bool DwarfExpression::addMachineRegExpression(const TargetRegisterInfo &TRI, return true; } +void DwarfExpression::addEntryValueExpression(DIExpressionCursor &ExprCursor) { + auto Op = ExprCursor.take(); + assert(Op && Op->getOp() == dwarf::DW_OP_entry_value); + assert(!isMemoryLocation() && + "We don't support entry values of memory locations yet"); + + if (DwarfVersion >= 5) + emitOp(dwarf::DW_OP_entry_value); + else + emitOp(dwarf::DW_OP_GNU_entry_value); + emitUnsigned(Op->getArg(0)); +} + /// Assuming a well-formed expression, match "DW_OP_deref* DW_OP_LLVM_fragment?". static bool isMemoryLocation(DIExpressionCursor ExprCursor) { while (ExprCursor) { @@ -319,6 +336,8 @@ void DwarfExpression::addExpression(DIExpressionCursor &&ExprCursor, if (SubRegisterSizeInBits && N && (N->getOp() != dwarf::DW_OP_LLVM_fragment)) maskSubRegister(); + Optional PrevConvertOp = None; + while (ExprCursor) { auto Op = ExprCursor.take(); switch (Op->getOp()) { @@ -341,7 +360,7 @@ void DwarfExpression::addExpression(DIExpressionCursor &&ExprCursor, SizeInBits = std::min(SizeInBits, SubRegisterSizeInBits); // Emit a DW_OP_stack_value for implicit location descriptions. - if (LocationKind == Implicit) + if (isImplicitLocation()) addStackValue(); // Emit the DW_OP_piece. @@ -352,7 +371,7 @@ void DwarfExpression::addExpression(DIExpressionCursor &&ExprCursor, return; } case dwarf::DW_OP_plus_uconst: - assert(LocationKind != Register); + assert(!isRegisterLocation()); emitOp(dwarf::DW_OP_plus_uconst); emitUnsigned(Op->getArg(0)); break; @@ -373,8 +392,8 @@ void DwarfExpression::addExpression(DIExpressionCursor &&ExprCursor, emitOp(Op->getOp()); break; case dwarf::DW_OP_deref: - assert(LocationKind != Register); - if (LocationKind != Memory && ::isMemoryLocation(ExprCursor)) + assert(!isRegisterLocation()); + if (!isMemoryLocation() && ::isMemoryLocation(ExprCursor)) // Turning this into a memory location description makes the deref // implicit. LocationKind = Memory; @@ -382,26 +401,69 @@ void DwarfExpression::addExpression(DIExpressionCursor &&ExprCursor, emitOp(dwarf::DW_OP_deref); break; case dwarf::DW_OP_constu: - assert(LocationKind != Register); + assert(!isRegisterLocation()); emitConstu(Op->getArg(0)); break; + case dwarf::DW_OP_LLVM_convert: { + unsigned BitSize = Op->getArg(0); + dwarf::TypeKind Encoding = static_cast(Op->getArg(1)); + if (DwarfVersion >= 5) { + emitOp(dwarf::DW_OP_convert); + // Reuse the base_type if we already have one in this CU otherwise we + // create a new one. + unsigned I = 0, E = CU.ExprRefedBaseTypes.size(); + for (; I != E; ++I) + if (CU.ExprRefedBaseTypes[I].BitSize == BitSize && + CU.ExprRefedBaseTypes[I].Encoding == Encoding) + break; + + if (I == E) + CU.ExprRefedBaseTypes.emplace_back(BitSize, Encoding); + + // If targeting a location-list; simply emit the index into the raw + // byte stream as ULEB128, DwarfDebug::emitDebugLocEntry has been + // fitted with means to extract it later. + // If targeting a inlined DW_AT_location; insert a DIEBaseTypeRef + // (containing the index and a resolve mechanism during emit) into the + // DIE value list. + emitBaseTypeRef(I); + } else { + if (PrevConvertOp && PrevConvertOp->getArg(0) < BitSize) { + if (Encoding == dwarf::DW_ATE_signed) + emitLegacySExt(PrevConvertOp->getArg(0)); + else if (Encoding == dwarf::DW_ATE_unsigned) + emitLegacyZExt(PrevConvertOp->getArg(0)); + PrevConvertOp = None; + } else { + PrevConvertOp = Op; + } + } + break; + } case dwarf::DW_OP_stack_value: LocationKind = Implicit; break; case dwarf::DW_OP_swap: - assert(LocationKind != Register); + assert(!isRegisterLocation()); emitOp(dwarf::DW_OP_swap); break; case dwarf::DW_OP_xderef: - assert(LocationKind != Register); + assert(!isRegisterLocation()); emitOp(dwarf::DW_OP_xderef); break; + case dwarf::DW_OP_deref_size: + emitOp(dwarf::DW_OP_deref_size); + emitData1(Op->getArg(0)); + break; + case dwarf::DW_OP_LLVM_tag_offset: + TagOffset = Op->getArg(0); + break; default: llvm_unreachable("unhandled opcode found in expression"); } } - if (LocationKind == Implicit) + if (isImplicitLocation()) // Turn this into an implicit location description. addStackValue(); } @@ -437,3 +499,25 @@ void DwarfExpression::addFragmentOffset(const DIExpression *Expr) { addOpPiece(FragmentOffset - OffsetInBits); OffsetInBits = FragmentOffset; } + +void DwarfExpression::emitLegacySExt(unsigned FromBits) { + // (((X >> (FromBits - 1)) * (~0)) << FromBits) | X + emitOp(dwarf::DW_OP_dup); + emitOp(dwarf::DW_OP_constu); + emitUnsigned(FromBits - 1); + emitOp(dwarf::DW_OP_shr); + emitOp(dwarf::DW_OP_lit0); + emitOp(dwarf::DW_OP_not); + emitOp(dwarf::DW_OP_mul); + emitOp(dwarf::DW_OP_constu); + emitUnsigned(FromBits); + emitOp(dwarf::DW_OP_shl); + emitOp(dwarf::DW_OP_or); +} + +void DwarfExpression::emitLegacyZExt(unsigned FromBits) { + // (X & (1 << FromBits - 1)) + emitOp(dwarf::DW_OP_constu); + emitUnsigned((1ULL << FromBits) - 1); + emitOp(dwarf::DW_OP_and); +} diff --git a/lib/CodeGen/AsmPrinter/DwarfExpression.h b/lib/CodeGen/AsmPrinter/DwarfExpression.h index 91568ba6d107..ec2ef6e575f7 100644 --- a/lib/CodeGen/AsmPrinter/DwarfExpression.h +++ b/lib/CodeGen/AsmPrinter/DwarfExpression.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/DwarfExpression.h - Dwarf Compile Unit ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -28,7 +27,7 @@ namespace llvm { class AsmPrinter; class APInt; class ByteStreamer; -class DwarfUnit; +class DwarfCompileUnit; class DIELoc; class TargetRegisterInfo; @@ -105,23 +104,56 @@ protected: const char *Comment; }; + DwarfCompileUnit &CU; + /// The register location, if any. SmallVector DwarfRegs; /// Current Fragment Offset in Bits. uint64_t OffsetInBits = 0; - unsigned DwarfVersion; /// Sometimes we need to add a DW_OP_bit_piece to describe a subregister. - unsigned SubRegisterSizeInBits = 0; - unsigned SubRegisterOffsetInBits = 0; + unsigned SubRegisterSizeInBits : 16; + unsigned SubRegisterOffsetInBits : 16; /// The kind of location description being produced. - enum { Unknown = 0, Register, Memory, Implicit } LocationKind = Unknown; + enum { Unknown = 0, Register, Memory, Implicit }; + + /// The flags of location description being produced. + enum { EntryValue = 1 }; + + unsigned LocationKind : 3; + unsigned LocationFlags : 2; + unsigned DwarfVersion : 4; + +public: + bool isUnknownLocation() const { + return LocationKind == Unknown; + } + + bool isMemoryLocation() const { + return LocationKind == Memory; + } + + bool isRegisterLocation() const { + return LocationKind == Register; + } + + bool isImplicitLocation() const { + return LocationKind == Implicit; + } + + bool isEntryValue() const { + return LocationFlags & EntryValue; + } + Optional TagOffset; + +protected: /// Push a DW_OP_piece / DW_OP_bit_piece for emitting later, if one is needed /// to represent a subregister. void setSubRegisterPiece(unsigned SizeInBits, unsigned OffsetInBits) { + assert(SizeInBits < 65536 && OffsetInBits < 65536); SubRegisterSizeInBits = SizeInBits; SubRegisterOffsetInBits = OffsetInBits; } @@ -138,6 +170,10 @@ protected: /// Emit a raw unsigned value. virtual void emitUnsigned(uint64_t Value) = 0; + virtual void emitData1(uint8_t Value) = 0; + + virtual void emitBaseTypeRef(uint64_t Idx) = 0; + /// Emit a normalized unsigned constant. void emitConstu(uint64_t Value); @@ -200,7 +236,10 @@ protected: ~DwarfExpression() = default; public: - DwarfExpression(unsigned DwarfVersion) : DwarfVersion(DwarfVersion) {} + DwarfExpression(unsigned DwarfVersion, DwarfCompileUnit &CU) + : CU(CU), SubRegisterSizeInBits(0), SubRegisterOffsetInBits(0), + LocationKind(Unknown), LocationFlags(Unknown), + DwarfVersion(DwarfVersion) {} /// This needs to be called last to commit any pending changes. void finalize(); @@ -214,15 +253,17 @@ public: /// Emit an unsigned constant. void addUnsignedConstant(const APInt &Value); - bool isMemoryLocation() const { return LocationKind == Memory; } - bool isUnknownLocation() const { return LocationKind == Unknown; } - /// Lock this down to become a memory location description. void setMemoryLocationKind() { - assert(LocationKind == Unknown); + assert(isUnknownLocation()); LocationKind = Memory; } + /// Lock this down to become an entry value location. + void setEntryValueFlag() { + LocationFlags |= EntryValue; + } + /// Emit a machine register location. As an optimization this may also consume /// the prefix of a DwarfExpression if a more efficient representation for /// combining the register location and the first operation exists. @@ -237,6 +278,9 @@ public: DIExpressionCursor &Expr, unsigned MachineReg, unsigned FragmentOffsetInBits = 0); + /// Emit entry value dwarf operation. + void addEntryValueExpression(DIExpressionCursor &ExprCursor); + /// Emit all remaining operations in the DIExpressionCursor. /// /// \param FragmentOffsetInBits If this is one fragment out of multiple @@ -248,6 +292,9 @@ public: /// If applicable, emit an empty DW_OP_piece / DW_OP_bit_piece to advance to /// the fragment described by \c Expr. void addFragmentOffset(const DIExpression *Expr); + + void emitLegacySExt(unsigned FromBits); + void emitLegacyZExt(unsigned FromBits); }; /// DwarfExpression implementation for .debug_loc entries. @@ -257,27 +304,30 @@ class DebugLocDwarfExpression final : public DwarfExpression { void emitOp(uint8_t Op, const char *Comment = nullptr) override; void emitSigned(int64_t Value) override; void emitUnsigned(uint64_t Value) override; + void emitData1(uint8_t Value) override; + void emitBaseTypeRef(uint64_t Idx) override; bool isFrameRegister(const TargetRegisterInfo &TRI, unsigned MachineReg) override; public: - DebugLocDwarfExpression(unsigned DwarfVersion, ByteStreamer &BS) - : DwarfExpression(DwarfVersion), BS(BS) {} + DebugLocDwarfExpression(unsigned DwarfVersion, ByteStreamer &BS, DwarfCompileUnit &CU) + : DwarfExpression(DwarfVersion, CU), BS(BS) {} }; /// DwarfExpression implementation for singular DW_AT_location. class DIEDwarfExpression final : public DwarfExpression { const AsmPrinter &AP; - DwarfUnit &DU; DIELoc &DIE; void emitOp(uint8_t Op, const char *Comment = nullptr) override; void emitSigned(int64_t Value) override; void emitUnsigned(uint64_t Value) override; + void emitData1(uint8_t Value) override; + void emitBaseTypeRef(uint64_t Idx) override; bool isFrameRegister(const TargetRegisterInfo &TRI, unsigned MachineReg) override; public: - DIEDwarfExpression(const AsmPrinter &AP, DwarfUnit &DU, DIELoc &DIE); + DIEDwarfExpression(const AsmPrinter &AP, DwarfCompileUnit &CU, DIELoc &DIE); DIELoc *finalize() { DwarfExpression::finalize(); diff --git a/lib/CodeGen/AsmPrinter/DwarfFile.cpp b/lib/CodeGen/AsmPrinter/DwarfFile.cpp index 78ccad481411..e3c9095d1343 100644 --- a/lib/CodeGen/AsmPrinter/DwarfFile.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfFile.cpp @@ -1,9 +1,8 @@ //===- llvm/CodeGen/DwarfFile.cpp - Dwarf Debug Framework -----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -44,6 +43,11 @@ void DwarfFile::emitUnit(DwarfUnit *TheU, bool UseOffsets) { if (!S) return; + // Skip CUs that ended up not being needed (split CUs that were abandoned + // because they added no information beyond the non-split CU) + if (llvm::empty(TheU->getUnitDie().values())) + return; + Asm->OutStreamer->SwitchSection(S); TheU->emitHeader(UseOffsets); Asm->emitDwarfDIE(TheU->getUnitDie()); @@ -63,6 +67,11 @@ void DwarfFile::computeSizeAndOffsets() { if (TheU->getCUNode()->isDebugDirectivesOnly()) continue; + // Skip CUs that ended up not being needed (split CUs that were abandoned + // because they added no information beyond the non-split CU) + if (llvm::empty(TheU->getUnitDie().values())) + return; + TheU->setDebugSectionOffset(SecOffset); SecOffset += computeSizeAndOffsetsForUnit(TheU.get()); } diff --git a/lib/CodeGen/AsmPrinter/DwarfFile.h b/lib/CodeGen/AsmPrinter/DwarfFile.h index 51acca8c1e53..244678ce9dc1 100644 --- a/lib/CodeGen/AsmPrinter/DwarfFile.h +++ b/lib/CodeGen/AsmPrinter/DwarfFile.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/DwarfFile.h - Dwarf Debug Framework ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -59,7 +58,6 @@ public: MCSymbol *getSym() const { return RangeSym; } const DwarfCompileUnit &getCU() const { return *CU; } const SmallVectorImpl &getRanges() const { return Ranges; } - void addRange(RangeSpan Range) { Ranges.push_back(Range); } }; class DwarfFile { @@ -148,7 +146,7 @@ public: void emitUnits(bool UseOffsets); /// Emit the given unit to its section. - void emitUnit(DwarfUnit *U, bool UseOffsets); + void emitUnit(DwarfUnit *TheU, bool UseOffsets); /// Emit a set of abbreviations to the specific section. void emitAbbrevs(MCSection *); diff --git a/lib/CodeGen/AsmPrinter/DwarfStringPool.cpp b/lib/CodeGen/AsmPrinter/DwarfStringPool.cpp index 02016534a774..2a76dcb1b082 100644 --- a/lib/CodeGen/AsmPrinter/DwarfStringPool.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfStringPool.cpp @@ -1,9 +1,8 @@ //===- llvm/CodeGen/DwarfStringPool.cpp - Dwarf Debug Framework -----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/AsmPrinter/DwarfStringPool.h b/lib/CodeGen/AsmPrinter/DwarfStringPool.h index f484540d8d37..c5f5637fdae3 100644 --- a/lib/CodeGen/AsmPrinter/DwarfStringPool.h +++ b/lib/CodeGen/AsmPrinter/DwarfStringPool.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/DwarfStringPool.h - Dwarf Debug Framework ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/AsmPrinter/DwarfUnit.cpp b/lib/CodeGen/AsmPrinter/DwarfUnit.cpp index 80b365f1aa43..991ab94b50ab 100644 --- a/lib/CodeGen/AsmPrinter/DwarfUnit.cpp +++ b/lib/CodeGen/AsmPrinter/DwarfUnit.cpp @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/DwarfUnit.cpp - Dwarf Type and Compile Units ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -47,21 +46,30 @@ using namespace llvm; #define DEBUG_TYPE "dwarfdebug" -DIEDwarfExpression::DIEDwarfExpression(const AsmPrinter &AP, DwarfUnit &DU, +DIEDwarfExpression::DIEDwarfExpression(const AsmPrinter &AP, + DwarfCompileUnit &CU, DIELoc &DIE) - : DwarfExpression(AP.getDwarfVersion()), AP(AP), DU(DU), + : DwarfExpression(AP.getDwarfVersion(), CU), AP(AP), DIE(DIE) {} void DIEDwarfExpression::emitOp(uint8_t Op, const char* Comment) { - DU.addUInt(DIE, dwarf::DW_FORM_data1, Op); + CU.addUInt(DIE, dwarf::DW_FORM_data1, Op); } void DIEDwarfExpression::emitSigned(int64_t Value) { - DU.addSInt(DIE, dwarf::DW_FORM_sdata, Value); + CU.addSInt(DIE, dwarf::DW_FORM_sdata, Value); } void DIEDwarfExpression::emitUnsigned(uint64_t Value) { - DU.addUInt(DIE, dwarf::DW_FORM_udata, Value); + CU.addUInt(DIE, dwarf::DW_FORM_udata, Value); +} + +void DIEDwarfExpression::emitData1(uint8_t Value) { + CU.addUInt(DIE, dwarf::DW_FORM_data1, Value); +} + +void DIEDwarfExpression::emitBaseTypeRef(uint64_t Idx) { + CU.addBaseTypeRef(DIE, Idx); } bool DIEDwarfExpression::isFrameRegister(const TargetRegisterInfo &TRI, @@ -285,21 +293,21 @@ void DwarfUnit::addSectionOffset(DIE &Die, dwarf::Attribute Attribute, addUInt(Die, Attribute, dwarf::DW_FORM_data4, Integer); } -MD5::MD5Result *DwarfUnit::getMD5AsBytes(const DIFile *File) const { +Optional DwarfUnit::getMD5AsBytes(const DIFile *File) const { assert(File); if (DD->getDwarfVersion() < 5) - return nullptr; + return None; Optional> Checksum = File->getChecksum(); if (!Checksum || Checksum->Kind != DIFile::CSK_MD5) - return nullptr; + return None; // Convert the string checksum to an MD5Result for the streamer. // The verifier validates the checksum so we assume it's okay. // An MD5 checksum is 16 bytes. std::string ChecksumString = fromHex(Checksum->Value); - void *CKMem = Asm->OutStreamer->getContext().allocate(16, 1); - memcpy(CKMem, ChecksumString.data(), 16); - return reinterpret_cast(CKMem); + MD5::MD5Result CKMem; + std::copy(ChecksumString.begin(), ChecksumString.end(), CKMem.Bytes.data()); + return CKMem; } unsigned DwarfTypeUnit::getOrCreateSourceID(const DIFile *File) { @@ -311,7 +319,9 @@ unsigned DwarfTypeUnit::getOrCreateSourceID(const DIFile *File) { addSectionOffset(getUnitDie(), dwarf::DW_AT_stmt_list, 0); } return SplitLineTable->getFile(File->getDirectory(), File->getFilename(), - getMD5AsBytes(File), File->getSource()); + getMD5AsBytes(File), + Asm->OutContext.getDwarfVersion(), + File->getSource()); } void DwarfUnit::addOpAddress(DIELoc &Die, const MCSymbol *Sym) { @@ -393,7 +403,6 @@ void DwarfUnit::addSourceLine(DIE &Die, unsigned Line, const DIFile *File) { return; unsigned FileID = getOrCreateSourceID(File); - assert(FileID && "Invalid file id"); addUInt(Die, dwarf::DW_AT_decl_file, None, FileID); addUInt(Die, dwarf::DW_AT_decl_line, None, Line); } @@ -462,9 +471,8 @@ static bool isUnsignedDIType(DwarfDebug *DD, const DIType *Ty) { assert(T == dwarf::DW_TAG_typedef || T == dwarf::DW_TAG_const_type || T == dwarf::DW_TAG_volatile_type || T == dwarf::DW_TAG_restrict_type || T == dwarf::DW_TAG_atomic_type); - DITypeRef Deriv = DTy->getBaseType(); - assert(Deriv && "Expected valid base type"); - return isUnsignedDIType(DD, DD->resolve(Deriv)); + assert(DTy->getBaseType() && "Expected valid base type"); + return isUnsignedDIType(DD, DTy->getBaseType()); } auto *BTy = cast(Ty); @@ -523,6 +531,10 @@ void DwarfUnit::addConstantValue(DIE &Die, const MachineOperand &MO, addConstantValue(Die, isUnsignedDIType(DD, Ty), MO.getImm()); } +void DwarfUnit::addConstantValue(DIE &Die, uint64_t Val, const DIType *Ty) { + addConstantValue(Die, isUnsignedDIType(DD, Ty), Val); +} + void DwarfUnit::addConstantValue(DIE &Die, bool Unsigned, uint64_t Val) { // FIXME: This is a bit conservative/simple - it emits negative values always // sign extended to 64 bits rather than minimizing the number of bytes. @@ -603,8 +615,8 @@ DIE *DwarfUnit::getOrCreateContextDIE(const DIScope *Context) { return getDIE(Context); } -DIE *DwarfTypeUnit::createTypeDIE(const DICompositeType *Ty) { - auto *Context = resolve(Ty->getScope()); +DIE *DwarfUnit::createTypeDIE(const DICompositeType *Ty) { + auto *Context = Ty->getScope(); DIE *ContextDIE = getOrCreateContextDIE(Context); if (DIE *TyDIE = getDIE(Ty)) @@ -619,6 +631,37 @@ DIE *DwarfTypeUnit::createTypeDIE(const DICompositeType *Ty) { return &TyDIE; } +DIE *DwarfUnit::createTypeDIE(const DIScope *Context, DIE &ContextDIE, + const DIType *Ty) { + // Create new type. + DIE &TyDIE = createAndAddDIE(Ty->getTag(), ContextDIE, Ty); + + updateAcceleratorTables(Context, Ty, TyDIE); + + if (auto *BT = dyn_cast(Ty)) + constructTypeDIE(TyDIE, BT); + else if (auto *STy = dyn_cast(Ty)) + constructTypeDIE(TyDIE, STy); + else if (auto *CTy = dyn_cast(Ty)) { + if (DD->generateTypeUnits() && !Ty->isForwardDecl() && + (Ty->getRawName() || CTy->getRawIdentifier())) { + // Skip updating the accelerator tables since this is not the full type. + if (MDString *TypeId = CTy->getRawIdentifier()) + DD->addDwarfTypeUnitType(getCU(), TypeId->getString(), TyDIE, CTy); + else { + auto X = DD->enterNonTypeUnitContext(); + finishNonUnitTypeDIE(TyDIE, CTy); + } + return &TyDIE; + } + constructTypeDIE(TyDIE, CTy); + } else { + constructTypeDIE(TyDIE, cast(Ty)); + } + + return &TyDIE; +} + DIE *DwarfUnit::getOrCreateTypeDIE(const MDNode *TyNode) { if (!TyNode) return nullptr; @@ -627,43 +670,23 @@ DIE *DwarfUnit::getOrCreateTypeDIE(const MDNode *TyNode) { // DW_TAG_restrict_type is not supported in DWARF2 if (Ty->getTag() == dwarf::DW_TAG_restrict_type && DD->getDwarfVersion() <= 2) - return getOrCreateTypeDIE(resolve(cast(Ty)->getBaseType())); + return getOrCreateTypeDIE(cast(Ty)->getBaseType()); // DW_TAG_atomic_type is not supported in DWARF < 5 if (Ty->getTag() == dwarf::DW_TAG_atomic_type && DD->getDwarfVersion() < 5) - return getOrCreateTypeDIE(resolve(cast(Ty)->getBaseType())); + return getOrCreateTypeDIE(cast(Ty)->getBaseType()); // Construct the context before querying for the existence of the DIE in case // such construction creates the DIE. - auto *Context = resolve(Ty->getScope()); + auto *Context = Ty->getScope(); DIE *ContextDIE = getOrCreateContextDIE(Context); assert(ContextDIE); if (DIE *TyDIE = getDIE(Ty)) return TyDIE; - // Create new type. - DIE &TyDIE = createAndAddDIE(Ty->getTag(), *ContextDIE, Ty); - - updateAcceleratorTables(Context, Ty, TyDIE); - - if (auto *BT = dyn_cast(Ty)) - constructTypeDIE(TyDIE, BT); - else if (auto *STy = dyn_cast(Ty)) - constructTypeDIE(TyDIE, STy); - else if (auto *CTy = dyn_cast(Ty)) { - if (DD->generateTypeUnits() && !Ty->isForwardDecl()) - if (MDString *TypeId = CTy->getRawIdentifier()) { - DD->addDwarfTypeUnitType(getCU(), TypeId->getString(), TyDIE, CTy); - // Skip updating the accelerator tables since this is not the full type. - return &TyDIE; - } - constructTypeDIE(TyDIE, CTy); - } else { - constructTypeDIE(TyDIE, cast(Ty)); - } - - return &TyDIE; + return static_cast(ContextDIE->getUnit()) + ->createTypeDIE(Context, *ContextDIE, Ty); } void DwarfUnit::updateAcceleratorTables(const DIScope *Context, @@ -679,7 +702,7 @@ void DwarfUnit::updateAcceleratorTables(const DIScope *Context, DD->addAccelType(*CUNode, Ty->getName(), TyDIE, Flags); if (!Context || isa(Context) || isa(Context) || - isa(Context)) + isa(Context) || isa(Context)) addGlobalType(Ty, TyDIE, Context); } } @@ -702,8 +725,8 @@ std::string DwarfUnit::getParentContextString(const DIScope *Context) const { SmallVector Parents; while (!isa(Context)) { Parents.push_back(Context); - if (Context->getScope()) - Context = resolve(Context->getScope()); + if (const DIScope *S = Context->getScope()) + Context = S; else // Structure, etc types will have a NULL context if they're at the top // level. @@ -754,7 +777,7 @@ void DwarfUnit::constructTypeDIE(DIE &Buffer, const DIDerivedType *DTy) { uint16_t Tag = Buffer.getTag(); // Map to main type, void will not have a type. - const DIType *FromTy = resolve(DTy->getBaseType()); + const DIType *FromTy = DTy->getBaseType(); if (FromTy) addType(Buffer, FromTy); @@ -770,24 +793,23 @@ void DwarfUnit::constructTypeDIE(DIE &Buffer, const DIDerivedType *DTy) { addUInt(Buffer, dwarf::DW_AT_byte_size, None, Size); if (Tag == dwarf::DW_TAG_ptr_to_member_type) - addDIEEntry( - Buffer, dwarf::DW_AT_containing_type, - *getOrCreateTypeDIE(resolve(cast(DTy)->getClassType()))); + addDIEEntry(Buffer, dwarf::DW_AT_containing_type, + *getOrCreateTypeDIE(cast(DTy)->getClassType())); // Add source line info if available and TyDesc is not a forward declaration. if (!DTy->isForwardDecl()) addSourceLine(Buffer, DTy); - // If DWARF address space value is other than None, add it for pointer and - // reference types as DW_AT_address_class. - if (DTy->getDWARFAddressSpace() && (Tag == dwarf::DW_TAG_pointer_type || - Tag == dwarf::DW_TAG_reference_type)) + // If DWARF address space value is other than None, add it. The IR + // verifier checks that DWARF address space only exists for pointer + // or reference types. + if (DTy->getDWARFAddressSpace()) addUInt(Buffer, dwarf::DW_AT_address_class, dwarf::DW_FORM_data4, DTy->getDWARFAddressSpace().getValue()); } void DwarfUnit::constructSubprogramArguments(DIE &Buffer, DITypeRefArray Args) { for (unsigned i = 1, N = Args.size(); i < N; ++i) { - const DIType *Ty = resolve(Args[i]); + const DIType *Ty = Args[i]; if (!Ty) { assert(i == N-1 && "Unspecified parameter must be the last argument"); createAndAddDIE(dwarf::DW_TAG_unspecified_parameters, Buffer); @@ -804,7 +826,7 @@ void DwarfUnit::constructTypeDIE(DIE &Buffer, const DISubroutineType *CTy) { // Add return type. A void return won't have a type. auto Elements = cast(CTy)->getTypeArray(); if (Elements.size()) - if (auto RTy = resolve(Elements[0])) + if (auto RTy = Elements[0]) addType(Buffer, RTy); bool isPrototyped = true; @@ -875,7 +897,7 @@ void DwarfUnit::constructTypeDIE(DIE &Buffer, const DICompositeType *CTy) { else if (auto *DDTy = dyn_cast(Element)) { if (DDTy->getTag() == dwarf::DW_TAG_friend) { DIE &ElemDie = createAndAddDIE(dwarf::DW_TAG_friend, Buffer); - addType(ElemDie, resolve(DDTy->getBaseType()), dwarf::DW_AT_friend); + addType(ElemDie, DDTy->getBaseType(), dwarf::DW_AT_friend); } else if (DDTy->isStaticMember()) { getOrCreateStaticMemberDIE(DDTy); } else if (Tag == dwarf::DW_TAG_variant_part) { @@ -884,7 +906,7 @@ void DwarfUnit::constructTypeDIE(DIE &Buffer, const DICompositeType *CTy) { DIE &Variant = createAndAddDIE(dwarf::DW_TAG_variant, Buffer); if (const ConstantInt *CI = dyn_cast_or_null(DDTy->getDiscriminantValue())) { - if (isUnsignedDIType(DD, resolve(Discriminator->getBaseType()))) + if (isUnsignedDIType(DD, Discriminator->getBaseType())) addUInt(Variant, dwarf::DW_AT_discr_value, None, CI->getZExtValue()); else addSInt(Variant, dwarf::DW_AT_discr_value, None, CI->getSExtValue()); @@ -898,7 +920,7 @@ void DwarfUnit::constructTypeDIE(DIE &Buffer, const DICompositeType *CTy) { StringRef PropertyName = Property->getName(); addString(ElemDie, dwarf::DW_AT_APPLE_property_name, PropertyName); if (Property->getType()) - addType(ElemDie, resolve(Property->getType())); + addType(ElemDie, Property->getType()); addSourceLine(ElemDie, Property); StringRef GetterName = Property->getGetterName(); if (!GetterName.empty()) @@ -924,7 +946,7 @@ void DwarfUnit::constructTypeDIE(DIE &Buffer, const DICompositeType *CTy) { // inside C++ composite types to point to the base class with the vtable. // Rust uses DW_AT_containing_type to link a vtable to the type // for which it was created. - if (auto *ContainingType = resolve(CTy->getVTableHolder())) + if (auto *ContainingType = CTy->getVTableHolder()) addDIEEntry(Buffer, dwarf::DW_AT_containing_type, *getOrCreateTypeDIE(ContainingType)); @@ -994,7 +1016,7 @@ void DwarfUnit::constructTemplateTypeParameterDIE( createAndAddDIE(dwarf::DW_TAG_template_type_parameter, Buffer); // Add the type if it exists, it could be void and therefore no type. if (TP->getType()) - addType(ParamDIE, resolve(TP->getType())); + addType(ParamDIE, TP->getType()); if (!TP->getName().empty()) addString(ParamDIE, dwarf::DW_AT_name, TP->getName()); } @@ -1006,12 +1028,12 @@ void DwarfUnit::constructTemplateValueParameterDIE( // Add the type if there is one, template template and template parameter // packs will not have a type. if (VP->getTag() == dwarf::DW_TAG_template_value_parameter) - addType(ParamDIE, resolve(VP->getType())); + addType(ParamDIE, VP->getType()); if (!VP->getName().empty()) addString(ParamDIE, dwarf::DW_AT_name, VP->getName()); if (Metadata *Val = VP->getValue()) { if (ConstantInt *CI = mdconst::dyn_extract(Val)) - addConstantValue(ParamDIE, CI, resolve(VP->getType())); + addConstantValue(ParamDIE, CI, VP->getType()); else if (GlobalValue *GV = mdconst::dyn_extract(Val)) { // We cannot describe the location of dllimport'd entities: the // computation of their address requires loads from the IAT. @@ -1085,7 +1107,7 @@ DIE *DwarfUnit::getOrCreateSubprogramDIE(const DISubprogram *SP, bool Minimal) { // such construction creates the DIE (as is the case for member function // declarations). DIE *ContextDIE = - Minimal ? &getUnitDie() : getOrCreateContextDIE(resolve(SP->getScope())); + Minimal ? &getUnitDie() : getOrCreateContextDIE(SP->getScope()); if (DIE *SPDie = getDIE(SP)) return SPDie; @@ -1107,7 +1129,8 @@ DIE *DwarfUnit::getOrCreateSubprogramDIE(const DISubprogram *SP, bool Minimal) { if (SP->isDefinition()) return &SPDie; - applySubprogramAttributes(SP, SPDie); + static_cast(SPDie.getUnit()) + ->applySubprogramAttributes(SP, SPDie); return &SPDie; } @@ -1197,7 +1220,7 @@ void DwarfUnit::applySubprogramAttributes(const DISubprogram *SP, DIE &SPDie, // Add a return type. If this is a type like a C/C++ void type we don't add a // return type. if (Args.size()) - if (auto Ty = resolve(Args[0])) + if (auto Ty = Args[0]) addType(SPDie, Ty); unsigned VK = SP->getVirtuality(); @@ -1209,8 +1232,7 @@ void DwarfUnit::applySubprogramAttributes(const DISubprogram *SP, DIE &SPDie, addUInt(*Block, dwarf::DW_FORM_udata, SP->getVirtualIndex()); addBlock(SPDie, dwarf::DW_AT_vtable_elem_location, Block); } - ContainingTypeMap.insert( - std::make_pair(&SPDie, resolve(SP->getContainingType()))); + ContainingTypeMap.insert(std::make_pair(&SPDie, SP->getContainingType())); } if (!SP->isDefinition()) { @@ -1261,6 +1283,12 @@ void DwarfUnit::applySubprogramAttributes(const DISubprogram *SP, DIE &SPDie, if (SP->isMainSubprogram()) addFlag(SPDie, dwarf::DW_AT_main_subprogram); + if (SP->isPure()) + addFlag(SPDie, dwarf::DW_AT_pure); + if (SP->isElemental()) + addFlag(SPDie, dwarf::DW_AT_elemental); + if (SP->isRecursive()) + addFlag(SPDie, dwarf::DW_AT_recursive); } void DwarfUnit::constructSubrangeDIE(DIE &Buffer, const DISubrange *SR, @@ -1310,7 +1338,7 @@ static bool hasVectorBeenPadded(const DICompositeType *CTy) { const uint64_t ActualSize = CTy->getSizeInBits(); // Obtain the size of each element in the vector. - DIType *BaseTy = CTy->getBaseType().resolve(); + DIType *BaseTy = CTy->getBaseType(); assert(BaseTy && "Unknown vector element type."); const uint64_t ElementSize = BaseTy->getSizeInBits(); @@ -1338,7 +1366,7 @@ void DwarfUnit::constructArrayTypeDIE(DIE &Buffer, const DICompositeType *CTy) { } // Emit the element type. - addType(Buffer, resolve(CTy->getBaseType())); + addType(Buffer, CTy->getBaseType()); // Get an anonymous type for index type. // FIXME: This type should be passed down from the front end @@ -1356,7 +1384,7 @@ void DwarfUnit::constructArrayTypeDIE(DIE &Buffer, const DICompositeType *CTy) { } void DwarfUnit::constructEnumTypeDIE(DIE &Buffer, const DICompositeType *CTy) { - const DIType *DTy = resolve(CTy->getBaseType()); + const DIType *DTy = CTy->getBaseType(); bool IsUnsigned = DTy && isUnsignedDIType(DD, DTy); if (DTy) { if (DD->getDwarfVersion() >= 3) @@ -1365,6 +1393,9 @@ void DwarfUnit::constructEnumTypeDIE(DIE &Buffer, const DICompositeType *CTy) { addFlag(Buffer, dwarf::DW_AT_enum_class); } + auto *Context = CTy->getScope(); + bool IndexEnumerators = !Context || isa(Context) || isa(Context) || + isa(Context) || isa(Context); DINodeArray Elements = CTy->getElements(); // Add enumerators to enumeration type. @@ -1376,6 +1407,8 @@ void DwarfUnit::constructEnumTypeDIE(DIE &Buffer, const DICompositeType *CTy) { addString(Enumerator, dwarf::DW_AT_name, Name); auto Value = static_cast(Enum->getValue()); addConstantValue(Enumerator, IsUnsigned, Value); + if (IndexEnumerators) + addGlobalName(Name, Enumerator, Context); } } } @@ -1400,7 +1433,7 @@ DIE &DwarfUnit::constructMemberDIE(DIE &Buffer, const DIDerivedType *DT) { if (!Name.empty()) addString(MemberDie, dwarf::DW_AT_name, Name); - if (DIType *Resolved = resolve(DT->getBaseType())) + if (DIType *Resolved = DT->getBaseType()) addType(MemberDie, Resolved); addSourceLine(MemberDie, DT); @@ -1509,7 +1542,7 @@ DIE *DwarfUnit::getOrCreateStaticMemberDIE(const DIDerivedType *DT) { // Construct the context before querying for the existence of the DIE in case // such construction creates the DIE. - DIE *ContextDIE = getOrCreateContextDIE(resolve(DT->getScope())); + DIE *ContextDIE = getOrCreateContextDIE(DT->getScope()); assert(dwarf::isType(ContextDIE->getTag()) && "Static member should belong to a type."); @@ -1518,7 +1551,7 @@ DIE *DwarfUnit::getOrCreateStaticMemberDIE(const DIDerivedType *DT) { DIE &StaticMemberDIE = createAndAddDIE(DT->getTag(), *ContextDIE, DT); - const DIType *Ty = resolve(DT->getBaseType()); + const DIType *Ty = DT->getBaseType(); addString(StaticMemberDIE, dwarf::DW_AT_name, DT->getName()); addType(StaticMemberDIE, Ty); @@ -1671,3 +1704,11 @@ void DwarfUnit::addLoclistsBase() { DU->getLoclistsTableBaseSym(), TLOF.getDwarfLoclistsSection()->getBeginSymbol()); } + +void DwarfTypeUnit::finishNonUnitTypeDIE(DIE& D, const DICompositeType *CTy) { + addFlag(D, dwarf::DW_AT_declaration); + StringRef Name = CTy->getName(); + if (!Name.empty()) + addString(D, dwarf::DW_AT_name, Name); + getCU().createTypeDIE(CTy); +} diff --git a/lib/CodeGen/AsmPrinter/DwarfUnit.h b/lib/CodeGen/AsmPrinter/DwarfUnit.h index a59ebb7c1465..56c934a35ae8 100644 --- a/lib/CodeGen/AsmPrinter/DwarfUnit.h +++ b/lib/CodeGen/AsmPrinter/DwarfUnit.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/DwarfUnit.h - Dwarf Compile Unit ---*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -199,6 +198,7 @@ public: void addConstantValue(DIE &Die, const ConstantInt *CI, const DIType *Ty); void addConstantValue(DIE &Die, const APInt &Val, const DIType *Ty); void addConstantValue(DIE &Die, const APInt &Val, bool Unsigned); + void addConstantValue(DIE &Die, uint64_t Val, const DIType *Ty); void addConstantValue(DIE &Die, bool Unsigned, uint64_t Val); /// Add constant value entry in variable DIE. @@ -237,6 +237,9 @@ public: void applySubprogramAttributes(const DISubprogram *SP, DIE &SPDie, bool SkipSPAttributes = false); + /// Creates type DIE with specific context. + DIE *createTypeDIE(const DIScope *Context, DIE &ContextDIE, const DIType *Ty); + /// Find existing DIE or create new DIE for the given type. DIE *getOrCreateTypeDIE(const MDNode *TyNode); @@ -294,7 +297,10 @@ public: /// If the \p File has an MD5 checksum, return it as an MD5Result /// allocated in the MCContext. - MD5::MD5Result *getMD5AsBytes(const DIFile *File) const; + Optional getMD5AsBytes(const DIFile *File) const; + + /// Get context owner's DIE. + DIE *createTypeDIE(const DICompositeType *Ty); protected: ~DwarfUnit(); @@ -306,17 +312,6 @@ protected: /// create a new ID and insert it in the line table. virtual unsigned getOrCreateSourceID(const DIFile *File) = 0; - /// Look in the DwarfDebug map for the MDNode that corresponds to the - /// reference. - template T *resolve(TypedDINodeRef Ref) const { - return Ref.resolve(); - } - - /// If this is a named finished type then include it in the list of types for - /// the accelerator tables. - void updateAcceleratorTables(const DIScope *Context, const DIType *Ty, - const DIE &TyDIE); - /// Emit the common part of the header for this unit. void emitCommonHeader(bool UseOffsets, dwarf::UnitType UT); @@ -344,6 +339,13 @@ private: /// Set D as anonymous type for index which can be reused later. void setIndexTyDie(DIE *D) { IndexTyDie = D; } + virtual void finishNonUnitTypeDIE(DIE& D, const DICompositeType *CTy) = 0; + + /// If this is a named finished type then include it in the list of types for + /// the accelerator tables. + void updateAcceleratorTables(const DIScope *Context, const DIType *Ty, + const DIE &TyDIE); + virtual bool isDwoUnit() const = 0; const MCSymbol *getCrossSectionRelativeBaseAddress() const override; }; @@ -356,6 +358,7 @@ class DwarfTypeUnit final : public DwarfUnit { bool UsedLineTable = false; unsigned getOrCreateSourceID(const DIFile *File) override; + void finishNonUnitTypeDIE(DIE& D, const DICompositeType *CTy) override; bool isDwoUnit() const override; public: @@ -365,9 +368,6 @@ public: void setTypeSignature(uint64_t Signature) { TypeSignature = Signature; } void setType(const DIE *Ty) { this->Ty = Ty; } - /// Get context owner's DIE. - DIE *createTypeDIE(const DICompositeType *Ty); - /// Emit the header for this unit, not including the initial length field. void emitHeader(bool UseOffsets) override; unsigned getHeaderSize() const override { diff --git a/lib/CodeGen/AsmPrinter/EHStreamer.cpp b/lib/CodeGen/AsmPrinter/EHStreamer.cpp index 7599121de2b0..99e3687b36b8 100644 --- a/lib/CodeGen/AsmPrinter/EHStreamer.cpp +++ b/lib/CodeGen/AsmPrinter/EHStreamer.cpp @@ -1,9 +1,8 @@ //===- CodeGen/AsmPrinter/EHStreamer.cpp - Exception Directive Streamer ---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -379,7 +378,8 @@ MCSymbol *EHStreamer::emitExceptionTable() { bool IsSJLJ = Asm->MAI->getExceptionHandlingType() == ExceptionHandling::SjLj; bool IsWasm = Asm->MAI->getExceptionHandlingType() == ExceptionHandling::Wasm; unsigned CallSiteEncoding = - IsSJLJ ? dwarf::DW_EH_PE_udata4 : dwarf::DW_EH_PE_uleb128; + IsSJLJ ? static_cast(dwarf::DW_EH_PE_udata4) : + Asm->getObjFileLowering().getCallSiteEncoding(); bool HaveTTData = !TypeInfos.empty() || !FilterIds.empty(); // Type infos. @@ -524,24 +524,24 @@ MCSymbol *EHStreamer::emitExceptionTable() { // Offset of the call site relative to the start of the procedure. if (VerboseAsm) Asm->OutStreamer->AddComment(">> Call Site " + Twine(++Entry) + " <<"); - Asm->EmitLabelDifferenceAsULEB128(BeginLabel, EHFuncBeginSym); + Asm->EmitCallSiteOffset(BeginLabel, EHFuncBeginSym, CallSiteEncoding); if (VerboseAsm) Asm->OutStreamer->AddComment(Twine(" Call between ") + BeginLabel->getName() + " and " + EndLabel->getName()); - Asm->EmitLabelDifferenceAsULEB128(EndLabel, BeginLabel); + Asm->EmitCallSiteOffset(EndLabel, BeginLabel, CallSiteEncoding); // Offset of the landing pad relative to the start of the procedure. if (!S.LPad) { if (VerboseAsm) Asm->OutStreamer->AddComment(" has no landing pad"); - Asm->EmitULEB128(0); + Asm->EmitCallSiteValue(0, CallSiteEncoding); } else { if (VerboseAsm) Asm->OutStreamer->AddComment(Twine(" jumps to ") + S.LPad->LandingPadLabel->getName()); - Asm->EmitLabelDifferenceAsULEB128(S.LPad->LandingPadLabel, - EHFuncBeginSym); + Asm->EmitCallSiteOffset(S.LPad->LandingPadLabel, EHFuncBeginSym, + CallSiteEncoding); } // Offset of the first associated action record, relative to the start of diff --git a/lib/CodeGen/AsmPrinter/EHStreamer.h b/lib/CodeGen/AsmPrinter/EHStreamer.h index ce912d032c6d..e62cf17a05d4 100644 --- a/lib/CodeGen/AsmPrinter/EHStreamer.h +++ b/lib/CodeGen/AsmPrinter/EHStreamer.h @@ -1,9 +1,8 @@ //===- EHStreamer.h - Exception Handling Directive Streamer -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/AsmPrinter/ErlangGCPrinter.cpp b/lib/CodeGen/AsmPrinter/ErlangGCPrinter.cpp index 34677ecc9e69..39392b79e960 100644 --- a/lib/CodeGen/AsmPrinter/ErlangGCPrinter.cpp +++ b/lib/CodeGen/AsmPrinter/ErlangGCPrinter.cpp @@ -1,9 +1,8 @@ //===- ErlangGCPrinter.cpp - Erlang/OTP frametable emitter ----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp b/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp index 3479a00def23..3145cc90dc73 100644 --- a/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp +++ b/lib/CodeGen/AsmPrinter/OcamlGCPrinter.cpp @@ -1,9 +1,8 @@ //===- OcamlGCPrinter.cpp - Ocaml frametable emitter ----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/AsmPrinter/WasmException.cpp b/lib/CodeGen/AsmPrinter/WasmException.cpp index 527e5ae50146..444b0ed17b6d 100644 --- a/lib/CodeGen/AsmPrinter/WasmException.cpp +++ b/lib/CodeGen/AsmPrinter/WasmException.cpp @@ -1,9 +1,8 @@ //===-- CodeGen/AsmPrinter/WasmException.cpp - Wasm Exception Impl --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,10 +18,10 @@ using namespace llvm; void WasmException::endModule() { - // This is the symbol used in 'throw' and 'if_except' instruction to denote + // This is the symbol used in 'throw' and 'br_on_exn' instruction to denote // this is a C++ exception. This symbol has to be emitted somewhere once in // the module. Check if the symbol has already been created, i.e., we have at - // least one 'throw' or 'if_except' instruction in the module, and emit the + // least one 'throw' or 'br_on_exn' instruction in the module, and emit the // symbol only if so. SmallString<60> NameStr; Mangler::getNameWithPrefix(NameStr, "__cpp_exception", Asm->getDataLayout()); diff --git a/lib/CodeGen/AsmPrinter/WasmException.h b/lib/CodeGen/AsmPrinter/WasmException.h index cbdb42457cf8..1893b6b2df43 100644 --- a/lib/CodeGen/AsmPrinter/WasmException.h +++ b/lib/CodeGen/AsmPrinter/WasmException.h @@ -1,9 +1,8 @@ //===-- WasmException.h - Wasm Exception Framework -------------*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/AsmPrinter/WinCFGuard.cpp b/lib/CodeGen/AsmPrinter/WinCFGuard.cpp index 18d37caf57ee..290be81c6baa 100644 --- a/lib/CodeGen/AsmPrinter/WinCFGuard.cpp +++ b/lib/CodeGen/AsmPrinter/WinCFGuard.cpp @@ -1,9 +1,8 @@ //===-- CodeGen/AsmPrinter/WinCFGuard.cpp - Control Flow Guard Impl ------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/AsmPrinter/WinCFGuard.h b/lib/CodeGen/AsmPrinter/WinCFGuard.h index 28f119e35966..def0a59ab007 100644 --- a/lib/CodeGen/AsmPrinter/WinCFGuard.h +++ b/lib/CodeGen/AsmPrinter/WinCFGuard.h @@ -1,9 +1,8 @@ //===-- WinCFGuard.h - Windows Control Flow Guard Handling ----*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/AsmPrinter/WinException.cpp b/lib/CodeGen/AsmPrinter/WinException.cpp index cf8e8c69bc2a..155e91ce61a1 100644 --- a/lib/CodeGen/AsmPrinter/WinException.cpp +++ b/lib/CodeGen/AsmPrinter/WinException.cpp @@ -1,9 +1,8 @@ //===-- CodeGen/AsmPrinter/WinException.cpp - Dwarf Exception Impl ------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -110,6 +109,12 @@ void WinException::beginFunction(const MachineFunction *MF) { beginFunclet(MF->front(), Asm->CurrentFnSym); } +void WinException::markFunctionEnd() { + if (isAArch64 && CurrentFuncletEntry && + (shouldEmitMoves || shouldEmitPersonality)) + Asm->OutStreamer->EmitWinCFIFuncletOrFuncEnd(); +} + /// endFunction - Gather and emit post-function exception information. /// void WinException::endFunction(const MachineFunction *MF) { @@ -129,7 +134,7 @@ void WinException::endFunction(const MachineFunction *MF) { NonConstMF->tidyLandingPads(); } - endFunclet(); + endFuncletImpl(); // endFunclet will emit the necessary .xdata tables for x64 SEH. if (Per == EHPersonality::MSVC_Win64SEH && MF->hasEHFunclets()) @@ -232,6 +237,15 @@ void WinException::beginFunclet(const MachineBasicBlock &MBB, } void WinException::endFunclet() { + if (isAArch64 && CurrentFuncletEntry && + (shouldEmitMoves || shouldEmitPersonality)) { + Asm->OutStreamer->SwitchSection(CurrentFuncletTextSection); + Asm->OutStreamer->EmitWinCFIFuncletOrFuncEnd(); + } + endFuncletImpl(); +} + +void WinException::endFuncletImpl() { // No funclet to process? Great, we have nothing to do. if (!CurrentFuncletEntry) return; @@ -247,8 +261,6 @@ void WinException::endFunclet() { // to EmitWinEHHandlerData below can calculate the size of the funclet or // function. if (isAArch64) { - Asm->OutStreamer->SwitchSection(CurrentFuncletTextSection); - Asm->OutStreamer->EmitWinCFIFuncletOrFuncEnd(); MCSection *XData = Asm->OutStreamer->getAssociatedXDataSection( Asm->OutStreamer->getCurrentSectionOnly()); Asm->OutStreamer->SwitchSection(XData); @@ -545,15 +557,17 @@ void WinException::emitCSpecificHandlerTable(const MachineFunction *MF) { OS.AddComment(Comment); }; - // Emit a label assignment with the SEH frame offset so we can use it for - // llvm.eh.recoverfp. - StringRef FLinkageName = - GlobalValue::dropLLVMManglingEscape(MF->getFunction().getName()); - MCSymbol *ParentFrameOffset = - Ctx.getOrCreateParentFrameOffsetSymbol(FLinkageName); - const MCExpr *MCOffset = - MCConstantExpr::create(FuncInfo.SEHSetFrameOffset, Ctx); - Asm->OutStreamer->EmitAssignment(ParentFrameOffset, MCOffset); + if (!isAArch64) { + // Emit a label assignment with the SEH frame offset so we can use it for + // llvm.eh.recoverfp. + StringRef FLinkageName = + GlobalValue::dropLLVMManglingEscape(MF->getFunction().getName()); + MCSymbol *ParentFrameOffset = + Ctx.getOrCreateParentFrameOffsetSymbol(FLinkageName); + const MCExpr *MCOffset = + MCConstantExpr::create(FuncInfo.SEHSetFrameOffset, Ctx); + Asm->OutStreamer->EmitAssignment(ParentFrameOffset, MCOffset); + } // Use the assembler to compute the number of table entries through label // difference and division. @@ -936,8 +950,7 @@ void WinException::emitEHRegistrationOffsetLabel(const WinEHFuncInfo &FuncInfo, int FI = FuncInfo.EHRegNodeFrameIndex; if (FI != INT_MAX) { const TargetFrameLowering *TFI = Asm->MF->getSubtarget().getFrameLowering(); - unsigned UnusedReg; - Offset = TFI->getFrameIndexReference(*Asm->MF, FI, UnusedReg); + Offset = TFI->getNonLocalFrameIndexReference(*Asm->MF, FI); } MCContext &Ctx = Asm->OutContext; diff --git a/lib/CodeGen/AsmPrinter/WinException.h b/lib/CodeGen/AsmPrinter/WinException.h index 37c796f89765..dc5036302131 100644 --- a/lib/CodeGen/AsmPrinter/WinException.h +++ b/lib/CodeGen/AsmPrinter/WinException.h @@ -1,9 +1,8 @@ //===-- WinException.h - Windows Exception Handling ----------*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -86,6 +85,7 @@ class LLVM_LIBRARY_VISIBILITY WinException : public EHStreamer { /// only), it is relative to the frame pointer. int getFrameIndexOffset(int FrameIndex, const WinEHFuncInfo &FuncInfo); + void endFuncletImpl(); public: //===--------------------------------------------------------------------===// // Main entry points. @@ -100,6 +100,8 @@ public: /// immediately after the function entry point. void beginFunction(const MachineFunction *MF) override; + void markFunctionEnd() override; + /// Gather and emit post-function exception information. void endFunction(const MachineFunction *) override; diff --git a/lib/CodeGen/AtomicExpandPass.cpp b/lib/CodeGen/AtomicExpandPass.cpp index 95581c09dd1c..dc7eaf6a5fe7 100644 --- a/lib/CodeGen/AtomicExpandPass.cpp +++ b/lib/CodeGen/AtomicExpandPass.cpp @@ -1,9 +1,8 @@ //===- AtomicExpandPass.cpp - Expand atomic instructions ------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -361,7 +360,7 @@ bool AtomicExpand::bracketInstWithFences(Instruction *I, AtomicOrdering Order) { /// Get the iX type with the same bitwidth as T. IntegerType *AtomicExpand::getCorrespondingIntegerType(Type *T, const DataLayout &DL) { - EVT VT = TLI->getValueType(DL, T); + EVT VT = TLI->getMemValueType(DL, T); unsigned BitWidth = VT.getStoreSizeInBits(); assert(BitWidth == VT.getSizeInBits() && "must be a power of two"); return IntegerType::get(T->getContext(), BitWidth); @@ -382,7 +381,7 @@ LoadInst *AtomicExpand::convertAtomicLoadToIntegerType(LoadInst *LI) { Addr->getType()->getPointerAddressSpace()); Value *NewAddr = Builder.CreateBitCast(Addr, PT); - auto *NewLI = Builder.CreateLoad(NewAddr); + auto *NewLI = Builder.CreateLoad(NewTy, NewAddr); NewLI->setAlignment(LI->getAlignment()); NewLI->setVolatile(LI->isVolatile()); NewLI->setAtomic(LI->getOrdering(), LI->getSyncScopeID()); @@ -431,6 +430,9 @@ bool AtomicExpand::expandAtomicLoadToLL(LoadInst *LI) { bool AtomicExpand::expandAtomicLoadToCmpXchg(LoadInst *LI) { IRBuilder<> Builder(LI); AtomicOrdering Order = LI->getOrdering(); + if (Order == AtomicOrdering::Unordered) + Order = AtomicOrdering::Monotonic; + Value *Addr = LI->getPointerOperand(); Type *Ty = cast(Addr->getType())->getElementType(); Constant *DummyVal = Constant::getNullValue(Ty); @@ -496,11 +498,26 @@ static void createCmpXchgInstFun(IRBuilder<> &Builder, Value *Addr, Value *Loaded, Value *NewVal, AtomicOrdering MemOpOrder, Value *&Success, Value *&NewLoaded) { + Type *OrigTy = NewVal->getType(); + + // This code can go away when cmpxchg supports FP types. + bool NeedBitcast = OrigTy->isFloatingPointTy(); + if (NeedBitcast) { + IntegerType *IntTy = Builder.getIntNTy(OrigTy->getPrimitiveSizeInBits()); + unsigned AS = Addr->getType()->getPointerAddressSpace(); + Addr = Builder.CreateBitCast(Addr, IntTy->getPointerTo(AS)); + NewVal = Builder.CreateBitCast(NewVal, IntTy); + Loaded = Builder.CreateBitCast(Loaded, IntTy); + } + Value* Pair = Builder.CreateAtomicCmpXchg( Addr, Loaded, NewVal, MemOpOrder, AtomicCmpXchgInst::getStrongestFailureOrdering(MemOpOrder)); Success = Builder.CreateExtractValue(Pair, 1, "success"); NewLoaded = Builder.CreateExtractValue(Pair, 0, "newloaded"); + + if (NeedBitcast) + NewLoaded = Builder.CreateBitCast(NewLoaded, OrigTy); } /// Emit IR to implement the given atomicrmw operation on values in registers, @@ -535,6 +552,10 @@ static Value *performAtomicOp(AtomicRMWInst::BinOp Op, IRBuilder<> &Builder, case AtomicRMWInst::UMin: NewVal = Builder.CreateICmpULE(Loaded, Inc); return Builder.CreateSelect(NewVal, Loaded, Inc, "new"); + case AtomicRMWInst::FAdd: + return Builder.CreateFAdd(Loaded, Inc, "new"); + case AtomicRMWInst::FSub: + return Builder.CreateFSub(Loaded, Inc, "new"); default: llvm_unreachable("Unknown atomic op"); } @@ -564,6 +585,10 @@ bool AtomicExpand::tryExpandAtomicRMW(AtomicRMWInst *AI) { unsigned MinCASSize = TLI->getMinCmpXchgSizeInBits() / 8; unsigned ValueSize = getAtomicOpSize(AI); if (ValueSize < MinCASSize) { + // TODO: Handle atomicrmw fadd/fsub + if (AI->getType()->isFloatingPointTy()) + return false; + expandPartwordAtomicRMW(AI, TargetLoweringBase::AtomicExpansionKind::CmpXChg); } else { @@ -1090,11 +1115,11 @@ bool AtomicExpand::expandAtomicCmpXchg(AtomicCmpXchgInst *CI) { bool HasReleasedLoadBB = !CI->isWeak() && ShouldInsertFencesForAtomic && SuccessOrder != AtomicOrdering::Monotonic && SuccessOrder != AtomicOrdering::Acquire && - !F->optForMinSize(); + !F->hasMinSize(); // There's no overhead for sinking the release barrier in a weak cmpxchg, so // do it even on minsize. - bool UseUnconditionalReleaseBarrier = F->optForMinSize() && !CI->isWeak(); + bool UseUnconditionalReleaseBarrier = F->hasMinSize() && !CI->isWeak(); // Given: cmpxchg some_op iN* %addr, iN %desired, iN %new success_ord fail_ord // @@ -1533,6 +1558,8 @@ static ArrayRef GetRMWLibcall(AtomicRMWInst::BinOp Op) { case AtomicRMWInst::Min: case AtomicRMWInst::UMax: case AtomicRMWInst::UMin: + case AtomicRMWInst::FAdd: + case AtomicRMWInst::FSub: // No atomic libcalls are available for max/min/umax/umin. return {}; } @@ -1671,16 +1698,25 @@ bool AtomicExpand::expandAtomicOpToLibcall( } // 'ptr' argument. - Value *PtrVal = - Builder.CreateBitCast(PointerOperand, Type::getInt8PtrTy(Ctx)); + // note: This assumes all address spaces share a common libfunc + // implementation and that addresses are convertable. For systems without + // that property, we'd need to extend this mechanism to support AS-specific + // families of atomic intrinsics. + auto PtrTypeAS = PointerOperand->getType()->getPointerAddressSpace(); + Value *PtrVal = Builder.CreateBitCast(PointerOperand, + Type::getInt8PtrTy(Ctx, PtrTypeAS)); + PtrVal = Builder.CreateAddrSpaceCast(PtrVal, Type::getInt8PtrTy(Ctx)); Args.push_back(PtrVal); // 'expected' argument, if present. if (CASExpected) { AllocaCASExpected = AllocaBuilder.CreateAlloca(CASExpected->getType()); AllocaCASExpected->setAlignment(AllocaAlignment); + unsigned AllocaAS = AllocaCASExpected->getType()->getPointerAddressSpace(); + AllocaCASExpected_i8 = - Builder.CreateBitCast(AllocaCASExpected, Type::getInt8PtrTy(Ctx)); + Builder.CreateBitCast(AllocaCASExpected, + Type::getInt8PtrTy(Ctx, AllocaAS)); Builder.CreateLifetimeStart(AllocaCASExpected_i8, SizeVal64); Builder.CreateAlignedStore(CASExpected, AllocaCASExpected, AllocaAlignment); Args.push_back(AllocaCASExpected_i8); @@ -1707,8 +1743,9 @@ bool AtomicExpand::expandAtomicOpToLibcall( if (!CASExpected && HasResult && !UseSizedLibcall) { AllocaResult = AllocaBuilder.CreateAlloca(I->getType()); AllocaResult->setAlignment(AllocaAlignment); + unsigned AllocaAS = AllocaResult->getType()->getPointerAddressSpace(); AllocaResult_i8 = - Builder.CreateBitCast(AllocaResult, Type::getInt8PtrTy(Ctx)); + Builder.CreateBitCast(AllocaResult, Type::getInt8PtrTy(Ctx, AllocaAS)); Builder.CreateLifetimeStart(AllocaResult_i8, SizeVal64); Args.push_back(AllocaResult_i8); } @@ -1734,7 +1771,7 @@ bool AtomicExpand::expandAtomicOpToLibcall( for (Value *Arg : Args) ArgTys.push_back(Arg->getType()); FunctionType *FnType = FunctionType::get(ResultTy, ArgTys, false); - Constant *LibcallFn = + FunctionCallee LibcallFn = M->getOrInsertFunction(TLI->getLibcallName(RTLibType), FnType, Attr); CallInst *Call = Builder.CreateCall(LibcallFn, Args); Call->setAttributes(Attr); @@ -1749,8 +1786,8 @@ bool AtomicExpand::expandAtomicOpToLibcall( // from call} Type *FinalResultTy = I->getType(); Value *V = UndefValue::get(FinalResultTy); - Value *ExpectedOut = - Builder.CreateAlignedLoad(AllocaCASExpected, AllocaAlignment); + Value *ExpectedOut = Builder.CreateAlignedLoad( + CASExpected->getType(), AllocaCASExpected, AllocaAlignment); Builder.CreateLifetimeEnd(AllocaCASExpected_i8, SizeVal64); V = Builder.CreateInsertValue(V, ExpectedOut, 0); V = Builder.CreateInsertValue(V, Result, 1); @@ -1760,7 +1797,8 @@ bool AtomicExpand::expandAtomicOpToLibcall( if (UseSizedLibcall) V = Builder.CreateBitOrPointerCast(Result, I->getType()); else { - V = Builder.CreateAlignedLoad(AllocaResult, AllocaAlignment); + V = Builder.CreateAlignedLoad(I->getType(), AllocaResult, + AllocaAlignment); Builder.CreateLifetimeEnd(AllocaResult_i8, SizeVal64); } I->replaceAllUsesWith(V); diff --git a/lib/CodeGen/BasicTargetTransformInfo.cpp b/lib/CodeGen/BasicTargetTransformInfo.cpp index d11f375b176e..57cefae2066a 100644 --- a/lib/CodeGen/BasicTargetTransformInfo.cpp +++ b/lib/CodeGen/BasicTargetTransformInfo.cpp @@ -1,9 +1,8 @@ //===- BasicTargetTransformInfo.cpp - Basic target-independent TTI impl ---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/lib/CodeGen/BranchFolding.cpp b/lib/CodeGen/BranchFolding.cpp index efbfd5f4ab2c..fb54b5d6c8d8 100644 --- a/lib/CodeGen/BranchFolding.cpp +++ b/lib/CodeGen/BranchFolding.cpp @@ -1,9 +1,8 @@ //===- BranchFolding.cpp - Fold machine code branch instructions ----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -722,7 +721,7 @@ ProfitableToMerge(MachineBasicBlock *MBB1, MachineBasicBlock *MBB2, // branch instruction, which is likely to be smaller than the 2 // instructions that would be deleted in the merge. MachineFunction *MF = MBB1->getParent(); - return EffectiveTailLen >= 2 && MF->getFunction().optForSize() && + return EffectiveTailLen >= 2 && MF->getFunction().hasOptSize() && (I1 == MBB1->begin() || I2 == MBB2->begin()); } @@ -1071,31 +1070,29 @@ bool BranchFolder::TryTailMergeBlocks(MachineBasicBlock *SuccBB, bool BranchFolder::TailMergeBlocks(MachineFunction &MF) { bool MadeChange = false; - if (!EnableTailMerge) return MadeChange; + if (!EnableTailMerge) + return MadeChange; // First find blocks with no successors. - // Block placement does not create new tail merging opportunities for these - // blocks. - if (!AfterBlockPlacement) { - MergePotentials.clear(); - for (MachineBasicBlock &MBB : MF) { - if (MergePotentials.size() == TailMergeThreshold) - break; - if (!TriedMerging.count(&MBB) && MBB.succ_empty()) - MergePotentials.push_back(MergePotentialsElt(HashEndOfMBB(MBB), &MBB)); - } - - // If this is a large problem, avoid visiting the same basic blocks - // multiple times. + // Block placement may create new tail merging opportunities for these blocks. + MergePotentials.clear(); + for (MachineBasicBlock &MBB : MF) { if (MergePotentials.size() == TailMergeThreshold) - for (unsigned i = 0, e = MergePotentials.size(); i != e; ++i) - TriedMerging.insert(MergePotentials[i].getBlock()); - - // See if we can do any tail merging on those. - if (MergePotentials.size() >= 2) - MadeChange |= TryTailMergeBlocks(nullptr, nullptr, MinCommonTailLength); + break; + if (!TriedMerging.count(&MBB) && MBB.succ_empty()) + MergePotentials.push_back(MergePotentialsElt(HashEndOfMBB(MBB), &MBB)); } + // If this is a large problem, avoid visiting the same basic blocks + // multiple times. + if (MergePotentials.size() == TailMergeThreshold) + for (unsigned i = 0, e = MergePotentials.size(); i != e; ++i) + TriedMerging.insert(MergePotentials[i].getBlock()); + + // See if we can do any tail merging on those. + if (MergePotentials.size() >= 2) + MadeChange |= TryTailMergeBlocks(nullptr, nullptr, MinCommonTailLength); + // Look at blocks (IBB) with multiple predecessors (PBB). // We change each predecessor to a canonical form, by // (1) temporarily removing any unconditional branch from the predecessor @@ -1183,29 +1180,6 @@ bool BranchFolder::TailMergeBlocks(MachineFunction &MF) { } } - // Failing case: the only way IBB can be reached from PBB is via - // exception handling. Happens for landing pads. Would be nice to have - // a bit in the edge so we didn't have to do all this. - if (IBB->isEHPad()) { - MachineFunction::iterator IP = ++PBB->getIterator(); - MachineBasicBlock *PredNextBB = nullptr; - if (IP != MF.end()) - PredNextBB = &*IP; - if (!TBB) { - if (IBB != PredNextBB) // fallthrough - continue; - } else if (FBB) { - if (TBB != IBB && FBB != IBB) // cbr then ubr - continue; - } else if (Cond.empty()) { - if (TBB != IBB) // ubr - continue; - } else { - if (TBB != IBB && IBB != PredNextBB) // cbr - continue; - } - } - // Remove the unconditional branch at the end, if any. if (TBB && (Cond.empty() || FBB)) { DebugLoc dl = PBB->findBranchDebugLoc(); @@ -1598,7 +1572,7 @@ ReoptimizeBlock: } if (!IsEmptyBlock(MBB) && MBB->pred_size() == 1 && - MF.getFunction().optForSize()) { + MF.getFunction().hasOptSize()) { // Changing "Jcc foo; foo: jmp bar;" into "Jcc bar;" might change the branch // direction, thereby defeating careful block placement and regressing // performance. Therefore, only consider this for optsize functions. diff --git a/lib/CodeGen/BranchFolding.h b/lib/CodeGen/BranchFolding.h index accd0ab7317b..761ff9c7d54e 100644 --- a/lib/CodeGen/BranchFolding.h +++ b/lib/CodeGen/BranchFolding.h @@ -1,9 +1,8 @@ //===- BranchFolding.h - Fold machine code branch instructions --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/BranchRelaxation.cpp b/lib/CodeGen/BranchRelaxation.cpp index c092da2b6602..3ad6266d4f35 100644 --- a/lib/CodeGen/BranchRelaxation.cpp +++ b/lib/CodeGen/BranchRelaxation.cpp @@ -1,9 +1,8 @@ //===- BranchRelaxation.cpp -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/BreakFalseDeps.cpp b/lib/CodeGen/BreakFalseDeps.cpp index 210699cbf239..cc4b2caa9bed 100644 --- a/lib/CodeGen/BreakFalseDeps.cpp +++ b/lib/CodeGen/BreakFalseDeps.cpp @@ -1,9 +1,8 @@ //==- llvm/CodeGen/BreakFalseDeps.cpp - Break False Dependency Fix -*- C++ -*==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/BuiltinGCs.cpp b/lib/CodeGen/BuiltinGCs.cpp index 93939e573b7b..bfc10cb3fef2 100644 --- a/lib/CodeGen/BuiltinGCs.cpp +++ b/lib/CodeGen/BuiltinGCs.cpp @@ -1,9 +1,8 @@ //===- BuiltinGCs.cpp - Boilerplate for our built in GC types -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/CFIInstrInserter.cpp b/lib/CodeGen/CFIInstrInserter.cpp index c4799855a2b3..1a4d54231cfd 100644 --- a/lib/CodeGen/CFIInstrInserter.cpp +++ b/lib/CodeGen/CFIInstrInserter.cpp @@ -1,9 +1,8 @@ //===------ CFIInstrInserter.cpp - Insert additional CFI instructions -----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/CalcSpillWeights.cpp b/lib/CodeGen/CalcSpillWeights.cpp index 02347b9f0b5c..7164fdfb7886 100644 --- a/lib/CodeGen/CalcSpillWeights.cpp +++ b/lib/CodeGen/CalcSpillWeights.cpp @@ -1,9 +1,8 @@ //===- CalcSpillWeights.cpp -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/CallingConvLower.cpp b/lib/CodeGen/CallingConvLower.cpp index 3593089b206d..497fcb147849 100644 --- a/lib/CodeGen/CallingConvLower.cpp +++ b/lib/CodeGen/CallingConvLower.cpp @@ -1,9 +1,8 @@ //===-- CallingConvLower.cpp - Calling Conventions ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/CodeGen.cpp b/lib/CodeGen/CodeGen.cpp index 66166482c78b..c37ed57781d4 100644 --- a/lib/CodeGen/CodeGen.cpp +++ b/lib/CodeGen/CodeGen.cpp @@ -1,9 +1,8 @@ //===-- CodeGen.cpp -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -31,14 +30,15 @@ void llvm::initializeCodeGen(PassRegistry &Registry) { initializeEarlyIfConverterPass(Registry); initializeEarlyMachineLICMPass(Registry); initializeEarlyTailDuplicatePass(Registry); - initializeExpandISelPseudosPass(Registry); initializeExpandMemCmpPassPass(Registry); initializeExpandPostRAPass(Registry); initializeFEntryInserterPass(Registry); + initializeFinalizeISelPass(Registry); initializeFinalizeMachineBundlesPass(Registry); initializeFuncletLayoutPass(Registry); initializeGCMachineCodeAnalysisPass(Registry); initializeGCModuleInfoPass(Registry); + initializeHardwareLoopsPass(Registry); initializeIfConverterPass(Registry); initializeImplicitNullChecksPass(Registry); initializeIndirectBrExpandPassPass(Registry); diff --git a/lib/CodeGen/CodeGenPrepare.cpp b/lib/CodeGen/CodeGenPrepare.cpp index c35f8666fa3c..52b4bbea012b 100644 --- a/lib/CodeGen/CodeGenPrepare.cpp +++ b/lib/CodeGen/CodeGenPrepare.cpp @@ -1,9 +1,8 @@ //===- CodeGenPrepare.cpp - Prepare a function for code generation --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -16,6 +15,7 @@ #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallPtrSet.h" @@ -32,6 +32,7 @@ #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Transforms/Utils/Local.h" #include "llvm/Analysis/ValueTracking.h" +#include "llvm/Analysis/VectorUtils.h" #include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/SelectionDAGNodes.h" @@ -292,15 +293,16 @@ class TypePromotionTransaction; /// Keep track of SExt promoted. ValueToSExts ValToSExtendedUses; - /// True if CFG is modified in any way. - bool ModifiedDT; - /// True if optimizing for size. bool OptSize; /// DataLayout for the Function being processed. const DataLayout *DL = nullptr; + /// Building the dominator tree can be expensive, so we only build it + /// lazily and update it when required. + std::unique_ptr DT; + public: static char ID; // Pass identification, replacement for typeid @@ -339,6 +341,13 @@ class TypePromotionTransaction; } } + // Get the DominatorTree, building if necessary. + DominatorTree &getDT(Function &F) { + if (!DT) + DT = llvm::make_unique(F); + return *DT; + } + bool eliminateFallThrough(Function &F); bool eliminateMostlyEmptyBlocks(Function &F); BasicBlock *findDestBlockOfMergeableEmptyBlock(BasicBlock *BB); @@ -355,11 +364,12 @@ class TypePromotionTransaction; bool optimizeExt(Instruction *&I); bool optimizeExtUses(Instruction *I); bool optimizeLoadExt(LoadInst *Load); + bool optimizeShiftInst(BinaryOperator *BO); bool optimizeSelectInst(SelectInst *SI); bool optimizeShuffleVectorInst(ShuffleVectorInst *SVI); bool optimizeSwitchInst(SwitchInst *SI); bool optimizeExtractElementInst(Instruction *Inst); - bool dupRetToEnableTailCallOpts(BasicBlock *BB); + bool dupRetToEnableTailCallOpts(BasicBlock *BB, bool &ModifiedDT); bool placeDbgValues(Function &F); bool canFormExtLd(const SmallVectorImpl &MovedExts, LoadInst *&LI, Instruction *&Inst, bool HasPromoted); @@ -374,8 +384,15 @@ class TypePromotionTransaction; bool AllowPromotionWithoutCommonHeader, bool HasPromoted, TypePromotionTransaction &TPT, SmallVectorImpl &SpeculativelyMovedExts); - bool splitBranchCondition(Function &F); + bool splitBranchCondition(Function &F, bool &ModifiedDT); bool simplifyOffsetableRelocate(Instruction &I); + + bool tryToSinkFreeOperands(Instruction *I); + bool replaceMathCmpWithIntrinsic(BinaryOperator *BO, CmpInst *Cmp, + Intrinsic::ID IID); + bool optimizeCmp(CmpInst *Cmp, bool &ModifiedDT); + bool combineToUSubWithOverflow(CmpInst *Cmp, bool &ModifiedDT); + bool combineToUAddWithOverflow(CmpInst *Cmp, bool &ModifiedDT); }; } // end anonymous namespace @@ -401,7 +418,6 @@ bool CodeGenPrepare::runOnFunction(Function &F) { InsertedInsts.clear(); PromotedInsts.clear(); - ModifiedDT = false; if (auto *TPC = getAnalysisIfAvailable()) { TM = &TPC->getTM(); SubtargetInfo = TM->getSubtargetImpl(F); @@ -413,7 +429,7 @@ bool CodeGenPrepare::runOnFunction(Function &F) { LI = &getAnalysis().getLoopInfo(); BPI.reset(new BranchProbabilityInfo(F, *LI)); BFI.reset(new BlockFrequencyInfo(F, *BPI, *LI)); - OptSize = F.optForSize(); + OptSize = F.hasOptSize(); ProfileSummaryInfo *PSI = &getAnalysis().getPSI(); @@ -444,8 +460,9 @@ bool CodeGenPrepare::runOnFunction(Function &F) { // unconditional branch. EverMadeChange |= eliminateMostlyEmptyBlocks(F); + bool ModifiedDT = false; if (!DisableBranchOpts) - EverMadeChange |= splitBranchCondition(F); + EverMadeChange |= splitBranchCondition(F, ModifiedDT); // Split some critical edges where one of the sources is an indirect branch, // to help generate sane code for PHIs involving such edges. @@ -454,6 +471,7 @@ bool CodeGenPrepare::runOnFunction(Function &F) { bool MadeChange = true; while (MadeChange) { MadeChange = false; + DT.reset(); for (Function::iterator I = F.begin(); I != F.end(); ) { BasicBlock *BB = &*I++; bool ModifiedDTOnIteration = false; @@ -654,6 +672,16 @@ bool CodeGenPrepare::isMergingEmptyBlockProfitable(BasicBlock *BB, BB->getSinglePredecessor()->getSingleSuccessor())) return false; + // Skip merging if the block's successor is also a successor to any callbr + // that leads to this block. + // FIXME: Is this really needed? Is this a correctness issue? + for (pred_iterator PI = pred_begin(BB), E = pred_end(BB); PI != E; ++PI) { + if (auto *CBI = dyn_cast((*PI)->getTerminator())) + for (unsigned i = 0, e = CBI->getNumSuccessors(); i != e; ++i) + if (DestBB == CBI->getSuccessor(i)) + return false; + } + // Try to skip merging if the unique predecessor of BB is terminated by a // switch or indirect branch instruction, and BB is used as an incoming block // of PHIs in DestBB. In such case, merging BB and DestBB would cause ISel to @@ -1040,7 +1068,7 @@ bool CodeGenPrepare::simplifyOffsetableRelocate(Instruction &I) { return MadeChange; } -/// SinkCast - Sink the specified cast instruction into its user blocks +/// Sink the specified cast instruction into its user blocks. static bool SinkCast(CastInst *CI) { BasicBlock *DefBB = CI->getParent(); @@ -1114,8 +1142,8 @@ static bool OptimizeNoopCopyExpression(CastInst *CI, const TargetLowering &TLI, // Sink only "cheap" (or nop) address-space casts. This is a weaker condition // than sinking only nop casts, but is helpful on some platforms. if (auto *ASC = dyn_cast(CI)) { - if (!TLI.isCheapAddrSpaceCast(ASC->getSrcAddressSpace(), - ASC->getDestAddressSpace())) + if (!TLI.isFreeAddrSpaceCast(ASC->getSrcAddressSpace(), + ASC->getDestAddressSpace())) return false; } @@ -1148,54 +1176,169 @@ static bool OptimizeNoopCopyExpression(CastInst *CI, const TargetLowering &TLI, return SinkCast(CI); } -/// Try to combine CI into a call to the llvm.uadd.with.overflow intrinsic if -/// possible. -/// -/// Return true if any changes were made. -static bool CombineUAddWithOverflow(CmpInst *CI) { - Value *A, *B; - Instruction *AddI; - if (!match(CI, - m_UAddWithOverflow(m_Value(A), m_Value(B), m_Instruction(AddI)))) +bool CodeGenPrepare::replaceMathCmpWithIntrinsic(BinaryOperator *BO, + CmpInst *Cmp, + Intrinsic::ID IID) { + if (BO->getParent() != Cmp->getParent()) { + // We used to use a dominator tree here to allow multi-block optimization. + // But that was problematic because: + // 1. It could cause a perf regression by hoisting the math op into the + // critical path. + // 2. It could cause a perf regression by creating a value that was live + // across multiple blocks and increasing register pressure. + // 3. Use of a dominator tree could cause large compile-time regression. + // This is because we recompute the DT on every change in the main CGP + // run-loop. The recomputing is probably unnecessary in many cases, so if + // that was fixed, using a DT here would be ok. + return false; + } + + // We allow matching the canonical IR (add X, C) back to (usubo X, -C). + Value *Arg0 = BO->getOperand(0); + Value *Arg1 = BO->getOperand(1); + if (BO->getOpcode() == Instruction::Add && + IID == Intrinsic::usub_with_overflow) { + assert(isa(Arg1) && "Unexpected input for usubo"); + Arg1 = ConstantExpr::getNeg(cast(Arg1)); + } + + // Insert at the first instruction of the pair. + Instruction *InsertPt = nullptr; + for (Instruction &Iter : *Cmp->getParent()) { + if (&Iter == BO || &Iter == Cmp) { + InsertPt = &Iter; + break; + } + } + assert(InsertPt != nullptr && "Parent block did not contain cmp or binop"); + + IRBuilder<> Builder(InsertPt); + Value *MathOV = Builder.CreateBinaryIntrinsic(IID, Arg0, Arg1); + Value *Math = Builder.CreateExtractValue(MathOV, 0, "math"); + Value *OV = Builder.CreateExtractValue(MathOV, 1, "ov"); + BO->replaceAllUsesWith(Math); + Cmp->replaceAllUsesWith(OV); + BO->eraseFromParent(); + Cmp->eraseFromParent(); + return true; +} + +/// Match special-case patterns that check for unsigned add overflow. +static bool matchUAddWithOverflowConstantEdgeCases(CmpInst *Cmp, + BinaryOperator *&Add) { + // Add = add A, 1; Cmp = icmp eq A,-1 (overflow if A is max val) + // Add = add A,-1; Cmp = icmp ne A, 0 (overflow if A is non-zero) + Value *A = Cmp->getOperand(0), *B = Cmp->getOperand(1); + + // We are not expecting non-canonical/degenerate code. Just bail out. + if (isa(A)) + return false; + + ICmpInst::Predicate Pred = Cmp->getPredicate(); + if (Pred == ICmpInst::ICMP_EQ && match(B, m_AllOnes())) + B = ConstantInt::get(B->getType(), 1); + else if (Pred == ICmpInst::ICMP_NE && match(B, m_ZeroInt())) + B = ConstantInt::get(B->getType(), -1); + else return false; - Type *Ty = AddI->getType(); - if (!isa(Ty)) + // Check the users of the variable operand of the compare looking for an add + // with the adjusted constant. + for (User *U : A->users()) { + if (match(U, m_Add(m_Specific(A), m_Specific(B)))) { + Add = cast(U); + return true; + } + } + return false; +} + +/// Try to combine the compare into a call to the llvm.uadd.with.overflow +/// intrinsic. Return true if any changes were made. +bool CodeGenPrepare::combineToUAddWithOverflow(CmpInst *Cmp, + bool &ModifiedDT) { + Value *A, *B; + BinaryOperator *Add; + if (!match(Cmp, m_UAddWithOverflow(m_Value(A), m_Value(B), m_BinOp(Add)))) + if (!matchUAddWithOverflowConstantEdgeCases(Cmp, Add)) + return false; + + if (!TLI->shouldFormOverflowOp(ISD::UADDO, + TLI->getValueType(*DL, Add->getType()))) return false; - // We don't want to move around uses of condition values this late, so we we + // We don't want to move around uses of condition values this late, so we // check if it is legal to create the call to the intrinsic in the basic - // block containing the icmp: + // block containing the icmp. + if (Add->getParent() != Cmp->getParent() && !Add->hasOneUse()) + return false; - if (AddI->getParent() != CI->getParent() && !AddI->hasOneUse()) + if (!replaceMathCmpWithIntrinsic(Add, Cmp, Intrinsic::uadd_with_overflow)) return false; -#ifndef NDEBUG - // Someday m_UAddWithOverflow may get smarter, but this is a safe assumption - // for now: - if (AddI->hasOneUse()) - assert(*AddI->user_begin() == CI && "expected!"); -#endif + // Reset callers - do not crash by iterating over a dead instruction. + ModifiedDT = true; + return true; +} + +bool CodeGenPrepare::combineToUSubWithOverflow(CmpInst *Cmp, + bool &ModifiedDT) { + // We are not expecting non-canonical/degenerate code. Just bail out. + Value *A = Cmp->getOperand(0), *B = Cmp->getOperand(1); + if (isa(A) && isa(B)) + return false; + + // Convert (A u> B) to (A u< B) to simplify pattern matching. + ICmpInst::Predicate Pred = Cmp->getPredicate(); + if (Pred == ICmpInst::ICMP_UGT) { + std::swap(A, B); + Pred = ICmpInst::ICMP_ULT; + } + // Convert special-case: (A == 0) is the same as (A u< 1). + if (Pred == ICmpInst::ICMP_EQ && match(B, m_ZeroInt())) { + B = ConstantInt::get(B->getType(), 1); + Pred = ICmpInst::ICMP_ULT; + } + // Convert special-case: (A != 0) is the same as (0 u< A). + if (Pred == ICmpInst::ICMP_NE && match(B, m_ZeroInt())) { + std::swap(A, B); + Pred = ICmpInst::ICMP_ULT; + } + if (Pred != ICmpInst::ICMP_ULT) + return false; + + // Walk the users of a variable operand of a compare looking for a subtract or + // add with that same operand. Also match the 2nd operand of the compare to + // the add/sub, but that may be a negated constant operand of an add. + Value *CmpVariableOperand = isa(A) ? B : A; + BinaryOperator *Sub = nullptr; + for (User *U : CmpVariableOperand->users()) { + // A - B, A u< B --> usubo(A, B) + if (match(U, m_Sub(m_Specific(A), m_Specific(B)))) { + Sub = cast(U); + break; + } + + // A + (-C), A u< C (canonicalized form of (sub A, C)) + const APInt *CmpC, *AddC; + if (match(U, m_Add(m_Specific(A), m_APInt(AddC))) && + match(B, m_APInt(CmpC)) && *AddC == -(*CmpC)) { + Sub = cast(U); + break; + } + } + if (!Sub) + return false; + + if (!TLI->shouldFormOverflowOp(ISD::USUBO, + TLI->getValueType(*DL, Sub->getType()))) + return false; + + if (!replaceMathCmpWithIntrinsic(Sub, Cmp, Intrinsic::usub_with_overflow)) + return false; - Module *M = CI->getModule(); - Value *F = Intrinsic::getDeclaration(M, Intrinsic::uadd_with_overflow, Ty); - - auto *InsertPt = AddI->hasOneUse() ? CI : AddI; - - DebugLoc Loc = CI->getDebugLoc(); - auto *UAddWithOverflow = - CallInst::Create(F, {A, B}, "uadd.overflow", InsertPt); - UAddWithOverflow->setDebugLoc(Loc); - auto *UAdd = ExtractValueInst::Create(UAddWithOverflow, 0, "uadd", InsertPt); - UAdd->setDebugLoc(Loc); - auto *Overflow = - ExtractValueInst::Create(UAddWithOverflow, 1, "overflow", InsertPt); - Overflow->setDebugLoc(Loc); - - CI->replaceAllUsesWith(Overflow); - AddI->replaceAllUsesWith(UAdd); - CI->eraseFromParent(); - AddI->eraseFromParent(); + // Reset callers - do not crash by iterating over a dead instruction. + ModifiedDT = true; return true; } @@ -1205,18 +1348,19 @@ static bool CombineUAddWithOverflow(CmpInst *CI) { /// lose; some adjustment may be wanted there. /// /// Return true if any changes are made. -static bool SinkCmpExpression(CmpInst *CI, const TargetLowering *TLI) { - BasicBlock *DefBB = CI->getParent(); +static bool sinkCmpExpression(CmpInst *Cmp, const TargetLowering &TLI) { + if (TLI.hasMultipleConditionRegisters()) + return false; // Avoid sinking soft-FP comparisons, since this can move them into a loop. - if (TLI && TLI->useSoftFloat() && isa(CI)) + if (TLI.useSoftFloat() && isa(Cmp)) return false; // Only insert a cmp in each block once. DenseMap InsertedCmps; bool MadeChange = false; - for (Value::user_iterator UI = CI->user_begin(), E = CI->user_end(); + for (Value::user_iterator UI = Cmp->user_begin(), E = Cmp->user_end(); UI != E; ) { Use &TheUse = UI.getUse(); Instruction *User = cast(*UI); @@ -1230,6 +1374,7 @@ static bool SinkCmpExpression(CmpInst *CI, const TargetLowering *TLI) { // Figure out which BB this cmp is used in. BasicBlock *UserBB = User->getParent(); + BasicBlock *DefBB = Cmp->getParent(); // If this user is in the same block as the cmp, don't change the cmp. if (UserBB == DefBB) continue; @@ -1241,10 +1386,11 @@ static bool SinkCmpExpression(CmpInst *CI, const TargetLowering *TLI) { BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt(); assert(InsertPt != UserBB->end()); InsertedCmp = - CmpInst::Create(CI->getOpcode(), CI->getPredicate(), - CI->getOperand(0), CI->getOperand(1), "", &*InsertPt); + CmpInst::Create(Cmp->getOpcode(), Cmp->getPredicate(), + Cmp->getOperand(0), Cmp->getOperand(1), "", + &*InsertPt); // Propagate the debug info. - InsertedCmp->setDebugLoc(CI->getDebugLoc()); + InsertedCmp->setDebugLoc(Cmp->getDebugLoc()); } // Replace a use of the cmp with a use of the new cmp. @@ -1254,19 +1400,22 @@ static bool SinkCmpExpression(CmpInst *CI, const TargetLowering *TLI) { } // If we removed all uses, nuke the cmp. - if (CI->use_empty()) { - CI->eraseFromParent(); + if (Cmp->use_empty()) { + Cmp->eraseFromParent(); MadeChange = true; } return MadeChange; } -static bool OptimizeCmpExpression(CmpInst *CI, const TargetLowering *TLI) { - if (SinkCmpExpression(CI, TLI)) +bool CodeGenPrepare::optimizeCmp(CmpInst *Cmp, bool &ModifiedDT) { + if (sinkCmpExpression(Cmp, *TLI)) return true; - if (CombineUAddWithOverflow(CI)) + if (combineToUAddWithOverflow(Cmp, ModifiedDT)) + return true; + + if (combineToUSubWithOverflow(Cmp, ModifiedDT)) return true; return false; @@ -1301,7 +1450,7 @@ static bool sinkAndCmp0Expression(Instruction *AndI, for (auto *U : AndI->users()) { Instruction *User = cast(U); - // Only sink for and mask feeding icmp with 0. + // Only sink 'and' feeding icmp with 0. if (!isa(User)) return false; @@ -1704,9 +1853,23 @@ bool CodeGenPrepare::optimizeCallInst(CallInst *CI, bool &ModifiedDT) { if (II) { switch (II->getIntrinsicID()) { default: break; + case Intrinsic::experimental_widenable_condition: { + // Give up on future widening oppurtunties so that we can fold away dead + // paths and merge blocks before going into block-local instruction + // selection. + if (II->use_empty()) { + II->eraseFromParent(); + return true; + } + Constant *RetVal = ConstantInt::getTrue(II->getContext()); + resetIteratorIfInvalidatedWhileCalling(BB, [&]() { + replaceAndRecursivelySimplify(CI, RetVal, TLInfo, nullptr); + }); + return true; + } case Intrinsic::objectsize: { // Lower all uses of llvm.objectsize.* - ConstantInt *RetVal = + Value *RetVal = lowerObjectSizeCall(II, *DL, TLInfo, /*MustSucceed=*/true); resetIteratorIfInvalidatedWhileCalling(BB, [&]() { @@ -1735,6 +1898,7 @@ bool CodeGenPrepare::optimizeCallInst(CallInst *CI, bool &ModifiedDT) { InsertedInsts.insert(ExtVal); return true; } + case Intrinsic::launder_invariant_group: case Intrinsic::strip_invariant_group: { Value *ArgVal = II->getArgOperand(0); @@ -1818,7 +1982,7 @@ bool CodeGenPrepare::optimizeCallInst(CallInst *CI, bool &ModifiedDT) { /// %tmp2 = tail call i32 @f2() /// ret i32 %tmp2 /// @endcode -bool CodeGenPrepare::dupRetToEnableTailCallOpts(BasicBlock *BB) { +bool CodeGenPrepare::dupRetToEnableTailCallOpts(BasicBlock *BB, bool &ModifiedDT) { if (!TLI) return false; @@ -1846,10 +2010,8 @@ bool CodeGenPrepare::dupRetToEnableTailCallOpts(BasicBlock *BB) { // return is the first instruction in the block. if (PN) { BasicBlock::iterator BI = BB->begin(); - do { ++BI; } while (isa(BI)); - if (&*BI == BCI) - // Also skip over the bitcast. - ++BI; + // Skip over debug and the bitcast. + do { ++BI; } while (isa(BI) || &*BI == BCI); if (&*BI != RetI) return false; } else { @@ -1865,7 +2027,9 @@ bool CodeGenPrepare::dupRetToEnableTailCallOpts(BasicBlock *BB) { SmallVector TailCalls; if (PN) { for (unsigned I = 0, E = PN->getNumIncomingValues(); I != E; ++I) { - CallInst *CI = dyn_cast(PN->getIncomingValue(I)); + // Look through bitcasts. + Value *IncomingVal = PN->getIncomingValue(I)->stripPointerCasts(); + CallInst *CI = dyn_cast(IncomingVal); // Make sure the phi value is indeed produced by the tail call. if (CI && CI->hasOneUse() && CI->getParent() == PN->getIncomingBlock(I) && TLI->mayBeEmittedAsTailCall(CI) && @@ -1929,6 +2093,7 @@ struct ExtAddrMode : public TargetLowering::AddrMode { Value *BaseReg = nullptr; Value *ScaledReg = nullptr; Value *OriginalValue = nullptr; + bool InBounds = true; enum FieldName { NoField = 0x00, @@ -1940,6 +2105,7 @@ struct ExtAddrMode : public TargetLowering::AddrMode { MultipleFields = 0xff }; + ExtAddrMode() = default; void print(raw_ostream &OS) const; @@ -1958,6 +2124,10 @@ struct ExtAddrMode : public TargetLowering::AddrMode { ScaledReg->getType() != other.ScaledReg->getType()) return MultipleFields; + // Conservatively reject 'inbounds' mismatches. + if (InBounds != other.InBounds) + return MultipleFields; + // Check each field to see if it differs. unsigned Result = NoField; if (BaseReg != other.BaseReg) @@ -2056,6 +2226,8 @@ static inline raw_ostream &operator<<(raw_ostream &OS, const ExtAddrMode &AM) { void ExtAddrMode::print(raw_ostream &OS) const { bool NeedPlus = false; OS << "["; + if (InBounds) + OS << "inbounds "; if (BaseGV) { OS << (NeedPlus ? " + " : "") << "GV:"; @@ -3126,6 +3298,8 @@ private: PhiNodeSet &PhiNodesToMatch) { SmallVector WorkList; Matcher.insert({ PHI, Candidate }); + SmallSet MatchedPHIs; + MatchedPHIs.insert(PHI); WorkList.push_back({ PHI, Candidate }); SmallSet Visited; while (!WorkList.empty()) { @@ -3158,8 +3332,10 @@ private: if (Matcher.count({ FirstPhi, SecondPhi })) continue; // So the values are different and does not match. So we need them to - // match. - Matcher.insert({ FirstPhi, SecondPhi }); + // match. (But we register no more than one match per PHI node, so that + // we won't later try to replace them twice.) + if (!MatchedPHIs.insert(FirstPhi).second) + Matcher.insert({ FirstPhi, SecondPhi }); // But me must check it. WorkList.push_back({ FirstPhi, SecondPhi }); } @@ -3354,6 +3530,7 @@ bool AddressingModeMatcher::matchScaledValue(Value *ScaleReg, int64_t Scale, ConstantInt *CI = nullptr; Value *AddLHS = nullptr; if (isa(ScaleReg) && // not a constant expr. match(ScaleReg, m_Add(m_Value(AddLHS), m_ConstantInt(CI)))) { + TestAddrMode.InBounds = false; TestAddrMode.ScaledReg = AddLHS; TestAddrMode.BaseOffs += CI->getSExtValue()*TestAddrMode.Scale; @@ -3928,6 +4105,7 @@ bool AddressingModeMatcher::matchOperationAddr(User *AddrInst, unsigned Opcode, TypePromotionTransaction::ConstRestorationPt LastKnownGood = TPT.getRestorationPoint(); + AddrMode.InBounds = false; if (matchAddr(AddrInst->getOperand(1), Depth+1) && matchAddr(AddrInst->getOperand(0), Depth+1)) return true; @@ -3954,6 +4132,7 @@ bool AddressingModeMatcher::matchOperationAddr(User *AddrInst, unsigned Opcode, case Instruction::Mul: case Instruction::Shl: { // Can only handle X*C and X << C. + AddrMode.InBounds = false; ConstantInt *RHS = dyn_cast(AddrInst->getOperand(1)); if (!RHS || RHS->getBitWidth() > 64) return false; @@ -4005,8 +4184,11 @@ bool AddressingModeMatcher::matchOperationAddr(User *AddrInst, unsigned Opcode, if (ConstantOffset == 0 || TLI.isLegalAddressingMode(DL, AddrMode, AccessTy, AddrSpace)) { // Check to see if we can fold the base pointer in too. - if (matchAddr(AddrInst->getOperand(0), Depth+1)) + if (matchAddr(AddrInst->getOperand(0), Depth+1)) { + if (!cast(AddrInst)->isInBounds()) + AddrMode.InBounds = false; return true; + } } else if (EnableGEPOffsetSplit && isa(AddrInst) && TLI.shouldConsiderGEPOffsetSplit() && Depth == 0 && ConstantOffset > 0) { @@ -4020,15 +4202,11 @@ bool AddressingModeMatcher::matchOperationAddr(User *AddrInst, unsigned Opcode, if (isa(Base) || isa(Base) || (BaseI && !isa(BaseI) && !isa(BaseI))) { - // If the base is an instruction, make sure the GEP is not in the same - // basic block as the base. If the base is an argument or global - // value, make sure the GEP is not in the entry block. Otherwise, - // instruction selection can undo the split. Also make sure the - // parent block allows inserting non-PHI instructions before the - // terminator. + // Make sure the parent block allows inserting non-PHI instructions + // before the terminator. BasicBlock *Parent = BaseI ? BaseI->getParent() : &GEP->getFunction()->getEntryBlock(); - if (GEP->getParent() != Parent && !Parent->getTerminator()->isEHPad()) + if (!Parent->getTerminator()->isEHPad()) LargeOffsetGEP = std::make_pair(GEP, ConstantOffset); } } @@ -4042,6 +4220,8 @@ bool AddressingModeMatcher::matchOperationAddr(User *AddrInst, unsigned Opcode, // See if the scale and offset amount is valid for this target. AddrMode.BaseOffs += ConstantOffset; + if (!cast(AddrInst)->isInBounds()) + AddrMode.InBounds = false; // Match the base operand of the GEP. if (!matchAddr(AddrInst->getOperand(0), Depth+1)) { @@ -4268,7 +4448,7 @@ static bool FindAllMemoryUses( if (!MightBeFoldableInst(I)) return true; - const bool OptSize = I->getFunction()->optForSize(); + const bool OptSize = I->getFunction()->hasOptSize(); // Loop over all the uses, recursively processing them. for (Use &U : I->uses()) { @@ -4556,8 +4736,7 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr, InsertedInsts, PromotedInsts, TPT, LargeOffsetGEP); GetElementPtrInst *GEP = LargeOffsetGEP.first; - if (GEP && GEP->getParent() != MemoryInst->getParent() && - !NewGEPBases.count(GEP)) { + if (GEP && !NewGEPBases.count(GEP)) { // If splitting the underlying data structure can reduce the offset of a // GEP, collect the GEP. Skip the GEPs that are the new bases of // previously split data structures. @@ -4727,7 +4906,11 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr, // SDAG consecutive load/store merging. if (ResultPtr->getType() != I8PtrTy) ResultPtr = Builder.CreatePointerCast(ResultPtr, I8PtrTy); - ResultPtr = Builder.CreateGEP(I8Ty, ResultPtr, ResultIndex, "sunkaddr"); + ResultPtr = + AddrMode.InBounds + ? Builder.CreateInBoundsGEP(I8Ty, ResultPtr, ResultIndex, + "sunkaddr") + : Builder.CreateGEP(I8Ty, ResultPtr, ResultIndex, "sunkaddr"); } ResultIndex = V; @@ -4738,7 +4921,11 @@ bool CodeGenPrepare::optimizeMemoryInst(Instruction *MemoryInst, Value *Addr, } else { if (ResultPtr->getType() != I8PtrTy) ResultPtr = Builder.CreatePointerCast(ResultPtr, I8PtrTy); - SunkAddr = Builder.CreateGEP(I8Ty, ResultPtr, ResultIndex, "sunkaddr"); + SunkAddr = + AddrMode.InBounds + ? Builder.CreateInBoundsGEP(I8Ty, ResultPtr, ResultIndex, + "sunkaddr") + : Builder.CreateGEP(I8Ty, ResultPtr, ResultIndex, "sunkaddr"); } if (SunkAddr->getType() != Addr->getType()) @@ -5037,7 +5224,6 @@ bool CodeGenPrepare::tryToPromoteExts( /// Merging redundant sexts when one is dominating the other. bool CodeGenPrepare::mergeSExts(Function &F) { - DominatorTree DT(F); bool Changed = false; for (auto &Entry : ValToSExtendedUses) { SExts &Insts = Entry.second; @@ -5048,7 +5234,7 @@ bool CodeGenPrepare::mergeSExts(Function &F) { continue; bool inserted = false; for (auto &Pt : CurPts) { - if (DT.dominates(Inst, Pt)) { + if (getDT(F).dominates(Inst, Pt)) { Pt->replaceAllUsesWith(Inst); RemovedInsts.insert(Pt); Pt->removeFromParent(); @@ -5057,7 +5243,7 @@ bool CodeGenPrepare::mergeSExts(Function &F) { Changed = true; break; } - if (!DT.dominates(Pt, Inst)) + if (!getDT(F).dominates(Pt, Inst)) // Give up if we need to merge in a common dominator as the // experiments show it is not profitable. continue; @@ -5715,7 +5901,7 @@ static bool isFormingBranchFromSelectProfitable(const TargetTransformInfo *TTI, static Value *getTrueOrFalseValue( SelectInst *SI, bool isTrue, const SmallPtrSet &Selects) { - Value *V; + Value *V = nullptr; for (SelectInst *DefSI = SI; DefSI != nullptr && Selects.count(DefSI); DefSI = dyn_cast(V)) { @@ -5723,9 +5909,44 @@ static Value *getTrueOrFalseValue( "The condition of DefSI does not match with SI"); V = (isTrue ? DefSI->getTrueValue() : DefSI->getFalseValue()); } + + assert(V && "Failed to get select true/false value"); return V; } +bool CodeGenPrepare::optimizeShiftInst(BinaryOperator *Shift) { + assert(Shift->isShift() && "Expected a shift"); + + // If this is (1) a vector shift, (2) shifts by scalars are cheaper than + // general vector shifts, and (3) the shift amount is a select-of-splatted + // values, hoist the shifts before the select: + // shift Op0, (select Cond, TVal, FVal) --> + // select Cond, (shift Op0, TVal), (shift Op0, FVal) + // + // This is inverting a generic IR transform when we know that the cost of a + // general vector shift is more than the cost of 2 shift-by-scalars. + // We can't do this effectively in SDAG because we may not be able to + // determine if the select operands are splats from within a basic block. + Type *Ty = Shift->getType(); + if (!Ty->isVectorTy() || !TLI->isVectorShiftByScalarCheap(Ty)) + return false; + Value *Cond, *TVal, *FVal; + if (!match(Shift->getOperand(1), + m_OneUse(m_Select(m_Value(Cond), m_Value(TVal), m_Value(FVal))))) + return false; + if (!isSplatValue(TVal) || !isSplatValue(FVal)) + return false; + + IRBuilder<> Builder(Shift); + BinaryOperator::BinaryOps Opcode = Shift->getOpcode(); + Value *NewTVal = Builder.CreateBinOp(Opcode, Shift->getOperand(0), TVal); + Value *NewFVal = Builder.CreateBinOp(Opcode, Shift->getOperand(0), FVal); + Value *NewSel = Builder.CreateSelect(Cond, NewTVal, NewFVal); + Shift->replaceAllUsesWith(NewSel); + Shift->eraseFromParent(); + return true; +} + /// If we have a SelectInst that will likely profit from branch prediction, /// turn it into a branch. bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) { @@ -5769,7 +5990,11 @@ bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) { !isFormingBranchFromSelectProfitable(TTI, TLI, SI)) return false; - ModifiedDT = true; + // The DominatorTree needs to be rebuilt by any consumers after this + // transformation. We simply reset here rather than setting the ModifiedDT + // flag to avoid restarting the function walk in runOnFunction for each + // select optimized. + DT.reset(); // Transform a sequence like this: // start: @@ -5943,6 +6168,7 @@ bool CodeGenPrepare::optimizeShuffleVectorInst(ShuffleVectorInst *SVI) { InsertedShuffle = new ShuffleVectorInst(SVI->getOperand(0), SVI->getOperand(1), SVI->getOperand(2), "", &*InsertPt); + InsertedShuffle->setDebugLoc(SVI->getDebugLoc()); } UI->replaceUsesOfWith(SVI, InsertedShuffle); @@ -5958,6 +6184,48 @@ bool CodeGenPrepare::optimizeShuffleVectorInst(ShuffleVectorInst *SVI) { return MadeChange; } +bool CodeGenPrepare::tryToSinkFreeOperands(Instruction *I) { + // If the operands of I can be folded into a target instruction together with + // I, duplicate and sink them. + SmallVector OpsToSink; + if (!TLI || !TLI->shouldSinkOperands(I, OpsToSink)) + return false; + + // OpsToSink can contain multiple uses in a use chain (e.g. + // (%u1 with %u1 = shufflevector), (%u2 with %u2 = zext %u1)). The dominating + // uses must come first, which means they are sunk first, temporarily creating + // invalid IR. This will be fixed once their dominated users are sunk and + // updated. + BasicBlock *TargetBB = I->getParent(); + bool Changed = false; + SmallVector ToReplace; + for (Use *U : OpsToSink) { + auto *UI = cast(U->get()); + if (UI->getParent() == TargetBB || isa(UI)) + continue; + ToReplace.push_back(U); + } + + SmallPtrSet MaybeDead; + for (Use *U : ToReplace) { + auto *UI = cast(U->get()); + Instruction *NI = UI->clone(); + MaybeDead.insert(UI); + LLVM_DEBUG(dbgs() << "Sinking " << *UI << " to user " << *I << "\n"); + NI->insertBefore(I); + InsertedInsts.insert(NI); + U->set(NI); + Changed = true; + } + + // Remove instructions that are dead after sinking. + for (auto *I : MaybeDead) + if (!I->hasNUsesOrMore(1)) + I->eraseFromParent(); + + return Changed; +} + bool CodeGenPrepare::optimizeSwitchInst(SwitchInst *SI) { if (!TLI || !DL) return false; @@ -6412,14 +6680,17 @@ static bool splitMergedValStore(StoreInst &SI, const DataLayout &DL, const TargetLowering &TLI) { // Handle simple but common cases only. Type *StoreType = SI.getValueOperand()->getType(); - if (DL.getTypeStoreSizeInBits(StoreType) != DL.getTypeSizeInBits(StoreType) || + if (!DL.typeSizeEqualsStoreSize(StoreType) || DL.getTypeSizeInBits(StoreType) == 0) return false; unsigned HalfValBitSize = DL.getTypeSizeInBits(StoreType) / 2; Type *SplitStoreType = Type::getIntNTy(SI.getContext(), HalfValBitSize); - if (DL.getTypeStoreSizeInBits(SplitStoreType) != - DL.getTypeSizeInBits(SplitStoreType)) + if (!DL.typeSizeEqualsStoreSize(SplitStoreType)) + return false; + + // Don't split the store if it is volatile. + if (SI.isVolatile()) return false; // Match the following patterns: @@ -6658,11 +6929,13 @@ bool CodeGenPrepare::optimizeInst(Instruction *I, bool &ModifiedDT) { if (InsertedInsts.count(I)) return false; + // TODO: Move into the switch on opcode below here. if (PHINode *P = dyn_cast(I)) { // It is possible for very late stage optimizations (such as SimplifyCFG) // to introduce PHI nodes too late to be cleaned up. If we detect such a // trivial PHI, go ahead and zap it here. if (Value *V = SimplifyInstruction(P, {*DL, TLInfo})) { + LargeOffsetGEPMap.erase(P); P->replaceAllUsesWith(V); P->eraseFromParent(); ++NumPHIsElim; @@ -6700,9 +6973,9 @@ bool CodeGenPrepare::optimizeInst(Instruction *I, bool &ModifiedDT) { return false; } - if (CmpInst *CI = dyn_cast(I)) - if (!TLI || !TLI->hasMultipleConditionRegisters()) - return OptimizeCmpExpression(CI, TLI); + if (auto *Cmp = dyn_cast(I)) + if (TLI && optimizeCmp(Cmp, ModifiedDT)) + return true; if (LoadInst *LI = dyn_cast(I)) { LI->setMetadata(LLVMContext::MD_invariant_group, nullptr); @@ -6745,13 +7018,13 @@ bool CodeGenPrepare::optimizeInst(Instruction *I, bool &ModifiedDT) { EnableAndCmpSinking && TLI) return sinkAndCmp0Expression(BinOp, *TLI, InsertedInsts); + // TODO: Move this into the switch on opcode - it handles shifts already. if (BinOp && (BinOp->getOpcode() == Instruction::AShr || BinOp->getOpcode() == Instruction::LShr)) { ConstantInt *CI = dyn_cast(BinOp->getOperand(1)); if (TLI && CI && TLI->hasExtractBitsInsn()) - return OptimizeExtractBits(BinOp, CI, *TLI, *DL); - - return false; + if (OptimizeExtractBits(BinOp, CI, *TLI, *DL)) + return true; } if (GetElementPtrInst *GEPI = dyn_cast(I)) { @@ -6772,20 +7045,25 @@ bool CodeGenPrepare::optimizeInst(Instruction *I, bool &ModifiedDT) { return false; } - if (CallInst *CI = dyn_cast(I)) - return optimizeCallInst(CI, ModifiedDT); - - if (SelectInst *SI = dyn_cast(I)) - return optimizeSelectInst(SI); - - if (ShuffleVectorInst *SVI = dyn_cast(I)) - return optimizeShuffleVectorInst(SVI); - - if (auto *Switch = dyn_cast(I)) - return optimizeSwitchInst(Switch); + if (tryToSinkFreeOperands(I)) + return true; - if (isa(I)) - return optimizeExtractElementInst(I); + switch (I->getOpcode()) { + case Instruction::Shl: + case Instruction::LShr: + case Instruction::AShr: + return optimizeShiftInst(cast(I)); + case Instruction::Call: + return optimizeCallInst(cast(I), ModifiedDT); + case Instruction::Select: + return optimizeSelectInst(cast(I)); + case Instruction::ShuffleVector: + return optimizeShuffleVectorInst(cast(I)); + case Instruction::Switch: + return optimizeSwitchInst(cast(I)); + case Instruction::ExtractElement: + return optimizeExtractElementInst(cast(I)); + } return false; } @@ -6833,7 +7111,7 @@ bool CodeGenPrepare::optimizeBlock(BasicBlock &BB, bool &ModifiedDT) { } } } - MadeChange |= dupRetToEnableTailCallOpts(&BB); + MadeChange |= dupRetToEnableTailCallOpts(&BB, ModifiedDT); return MadeChange; } @@ -6909,7 +7187,7 @@ static void scaleWeights(uint64_t &NewTrue, uint64_t &NewFalse) { /// /// FIXME: Remove the (equivalent?) implementation in SelectionDAG. /// -bool CodeGenPrepare::splitBranchCondition(Function &F) { +bool CodeGenPrepare::splitBranchCondition(Function &F, bool &ModifiedDT) { if (!TM || !TM->Options.EnableFastISel || !TLI || TLI->isJumpExpensive()) return false; @@ -6983,11 +7261,7 @@ bool CodeGenPrepare::splitBranchCondition(Function &F) { std::swap(TBB, FBB); // Replace the old BB with the new BB. - for (PHINode &PN : TBB->phis()) { - int i; - while ((i = PN.getBasicBlockIndex(&BB)) >= 0) - PN.setIncomingBlock(i, TmpBB); - } + TBB->replacePhiUsesWith(&BB, TmpBB); // Add another incoming edge form the new BB. for (PHINode &PN : FBB->phis()) { @@ -7066,10 +7340,7 @@ bool CodeGenPrepare::splitBranchCondition(Function &F) { } } - // Note: No point in getting fancy here, since the DT info is never - // available to CodeGenPrepare. ModifiedDT = true; - MadeChange = true; LLVM_DEBUG(dbgs() << "After branch condition splitting\n"; BB.dump(); diff --git a/lib/CodeGen/CriticalAntiDepBreaker.cpp b/lib/CodeGen/CriticalAntiDepBreaker.cpp index 5a5960b16130..4144c243a341 100644 --- a/lib/CodeGen/CriticalAntiDepBreaker.cpp +++ b/lib/CodeGen/CriticalAntiDepBreaker.cpp @@ -1,9 +1,8 @@ //===- CriticalAntiDepBreaker.cpp - Anti-dep breaker ----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/CriticalAntiDepBreaker.h b/lib/CodeGen/CriticalAntiDepBreaker.h index 09c4423a2f05..4e127ce525c8 100644 --- a/lib/CodeGen/CriticalAntiDepBreaker.h +++ b/lib/CodeGen/CriticalAntiDepBreaker.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/CriticalAntiDepBreaker.h - Anti-Dep Support -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/DFAPacketizer.cpp b/lib/CodeGen/DFAPacketizer.cpp index 68034afe98d5..b99be5d7a87c 100644 --- a/lib/CodeGen/DFAPacketizer.cpp +++ b/lib/CodeGen/DFAPacketizer.cpp @@ -1,9 +1,8 @@ //=- llvm/CodeGen/DFAPacketizer.cpp - DFA Packetizer for VLIW -*- C++ -*-=====// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // This class implements a deterministic finite automaton (DFA) based diff --git a/lib/CodeGen/DeadMachineInstructionElim.cpp b/lib/CodeGen/DeadMachineInstructionElim.cpp index ff44c5660bad..049ce7063307 100644 --- a/lib/CodeGen/DeadMachineInstructionElim.cpp +++ b/lib/CodeGen/DeadMachineInstructionElim.cpp @@ -1,9 +1,8 @@ //===- DeadMachineInstructionElim.cpp - Remove dead machine instructions --===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -82,9 +81,11 @@ bool DeadMachineInstructionElim::isDead(const MachineInstr *MI) const { if (LivePhysRegs.test(Reg) || MRI->isReserved(Reg)) return false; } else { - if (!MRI->use_nodbg_empty(Reg)) - // This def has a non-debug use. Don't delete the instruction! - return false; + for (const MachineInstr &Use : MRI->use_nodbg_instructions(Reg)) { + if (&Use != MI) + // This def has a non-debug use. Don't delete the instruction! + return false; + } } } } diff --git a/lib/CodeGen/DetectDeadLanes.cpp b/lib/CodeGen/DetectDeadLanes.cpp index c83db476a4de..fe78acf4d80a 100644 --- a/lib/CodeGen/DetectDeadLanes.cpp +++ b/lib/CodeGen/DetectDeadLanes.cpp @@ -1,9 +1,8 @@ //===- DetectDeadLanes.cpp - SubRegister Lane Usage Analysis --*- C++ -*---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/DwarfEHPrepare.cpp b/lib/CodeGen/DwarfEHPrepare.cpp index 4586649d17f0..ddd6cec5a178 100644 --- a/lib/CodeGen/DwarfEHPrepare.cpp +++ b/lib/CodeGen/DwarfEHPrepare.cpp @@ -1,9 +1,8 @@ //===- DwarfEHPrepare - Prepare exception handling for code generation ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -46,7 +45,7 @@ namespace { class DwarfEHPrepare : public FunctionPass { // RewindFunction - _Unwind_Resume or the target equivalent. - Constant *RewindFunction = nullptr; + FunctionCallee RewindFunction = nullptr; DominatorTree *DT = nullptr; const TargetLowering *TLI = nullptr; @@ -146,7 +145,7 @@ size_t DwarfEHPrepare::pruneUnreachableResumes( size_t ResumeIndex = 0; for (auto *RI : Resumes) { for (auto *LP : CleanupLPads) { - if (isPotentiallyReachable(LP, RI, DT)) { + if (isPotentiallyReachable(LP, RI, nullptr, DT)) { ResumeReachable.set(ResumeIndex); break; } diff --git a/lib/CodeGen/EarlyIfConversion.cpp b/lib/CodeGen/EarlyIfConversion.cpp index 364e1f030942..0a83760befaa 100644 --- a/lib/CodeGen/EarlyIfConversion.cpp +++ b/lib/CodeGen/EarlyIfConversion.cpp @@ -1,9 +1,8 @@ //===-- EarlyIfConversion.cpp - If-conversion on SSA form machine code ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/EdgeBundles.cpp b/lib/CodeGen/EdgeBundles.cpp index 54c53eb16312..486720cadd27 100644 --- a/lib/CodeGen/EdgeBundles.cpp +++ b/lib/CodeGen/EdgeBundles.cpp @@ -1,9 +1,8 @@ //===-------- EdgeBundles.cpp - Bundles of CFG edges ----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -28,7 +27,7 @@ ViewEdgeBundles("view-edge-bundles", cl::Hidden, char EdgeBundles::ID = 0; INITIALIZE_PASS(EdgeBundles, "edge-bundles", "Bundle Machine CFG Edges", - /* cfg = */true, /* analysis = */ true) + /* cfg = */true, /* is_analysis = */ true) char &llvm::EdgeBundlesID = EdgeBundles::ID; diff --git a/lib/CodeGen/ExecutionDomainFix.cpp b/lib/CodeGen/ExecutionDomainFix.cpp index 458dcf2b0e26..a2dd5eee33b7 100644 --- a/lib/CodeGen/ExecutionDomainFix.cpp +++ b/lib/CodeGen/ExecutionDomainFix.cpp @@ -1,9 +1,8 @@ //===- ExecutionDomainFix.cpp - Fix execution domain issues ----*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -337,11 +336,10 @@ void ExecutionDomainFix::visitSoftInstr(MachineInstr *mi, unsigned mask) { } // Sorted insertion. // Enables giving priority to the latest domains during merging. - auto I = std::upper_bound( - Regs.begin(), Regs.end(), rx, [&](int LHS, const int RHS) { - return RDA->getReachingDef(mi, RC->getRegister(LHS)) < - RDA->getReachingDef(mi, RC->getRegister(RHS)); - }); + const int Def = RDA->getReachingDef(mi, RC->getRegister(rx)); + auto I = partition_point(Regs, [&](int I) { + return RDA->getReachingDef(mi, RC->getRegister(I)) <= Def; + }); Regs.insert(I, rx); } diff --git a/lib/CodeGen/ExpandISelPseudos.cpp b/lib/CodeGen/ExpandISelPseudos.cpp deleted file mode 100644 index ec586a2caea3..000000000000 --- a/lib/CodeGen/ExpandISelPseudos.cpp +++ /dev/null @@ -1,74 +0,0 @@ -//===-- llvm/CodeGen/ExpandISelPseudos.cpp ----------------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// Expand Pseudo-instructions produced by ISel. These are usually to allow -// the expansion to contain control flow, such as a conditional move -// implemented with a conditional branch and a phi, or an atomic operation -// implemented with a loop. -// -//===----------------------------------------------------------------------===// - -#include "llvm/CodeGen/MachineFunction.h" -#include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/Passes.h" -#include "llvm/CodeGen/TargetLowering.h" -#include "llvm/CodeGen/TargetSubtargetInfo.h" -#include "llvm/Support/Debug.h" -using namespace llvm; - -#define DEBUG_TYPE "expand-isel-pseudos" - -namespace { - class ExpandISelPseudos : public MachineFunctionPass { - public: - static char ID; // Pass identification, replacement for typeid - ExpandISelPseudos() : MachineFunctionPass(ID) {} - - private: - bool runOnMachineFunction(MachineFunction &MF) override; - - void getAnalysisUsage(AnalysisUsage &AU) const override { - MachineFunctionPass::getAnalysisUsage(AU); - } - }; -} // end anonymous namespace - -char ExpandISelPseudos::ID = 0; -char &llvm::ExpandISelPseudosID = ExpandISelPseudos::ID; -INITIALIZE_PASS(ExpandISelPseudos, DEBUG_TYPE, - "Expand ISel Pseudo-instructions", false, false) - -bool ExpandISelPseudos::runOnMachineFunction(MachineFunction &MF) { - bool Changed = false; - const TargetLowering *TLI = MF.getSubtarget().getTargetLowering(); - - // Iterate through each instruction in the function, looking for pseudos. - for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) { - MachineBasicBlock *MBB = &*I; - for (MachineBasicBlock::iterator MBBI = MBB->begin(), MBBE = MBB->end(); - MBBI != MBBE; ) { - MachineInstr &MI = *MBBI++; - - // If MI is a pseudo, expand it. - if (MI.usesCustomInsertionHook()) { - Changed = true; - MachineBasicBlock *NewMBB = TLI->EmitInstrWithCustomInserter(MI, MBB); - // The expansion may involve new basic blocks. - if (NewMBB != MBB) { - MBB = NewMBB; - I = NewMBB->getIterator(); - MBBI = NewMBB->begin(); - MBBE = NewMBB->end(); - } - } - } - } - - return Changed; -} diff --git a/lib/CodeGen/ExpandMemCmp.cpp b/lib/CodeGen/ExpandMemCmp.cpp index ee7683adbcdd..b425482e6adf 100644 --- a/lib/CodeGen/ExpandMemCmp.cpp +++ b/lib/CodeGen/ExpandMemCmp.cpp @@ -1,9 +1,8 @@ //===--- ExpandMemCmp.cpp - Expand memcmp() to load/stores ----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -37,6 +36,14 @@ static cl::opt MemCmpEqZeroNumLoadsPerBlock( cl::desc("The number of loads per basic block for inline expansion of " "memcmp that is only being compared against zero.")); +static cl::opt MaxLoadsPerMemcmp( + "max-loads-per-memcmp", cl::Hidden, + cl::desc("Set maximum number of loads used in expanded memcmp")); + +static cl::opt MaxLoadsPerMemcmpOptSize( + "max-loads-per-memcmp-opt-size", cl::Hidden, + cl::desc("Set maximum number of loads used in expanded memcmp for -Os/Oz")); + namespace { @@ -106,8 +113,7 @@ class MemCmpExpansion { public: MemCmpExpansion(CallInst *CI, uint64_t Size, const TargetTransformInfo::MemCmpExpansionOptions &Options, - unsigned MaxNumLoads, const bool IsUsedForZeroCmp, - unsigned MaxLoadsPerBlockForZeroCmp, const DataLayout &TheDataLayout); + const bool IsUsedForZeroCmp, const DataLayout &TheDataLayout); unsigned getNumBlocks(); uint64_t getNumLoads() const { return LoadSequence.size(); } @@ -196,16 +202,10 @@ MemCmpExpansion::computeOverlappingLoadSequence(uint64_t Size, MemCmpExpansion::MemCmpExpansion( CallInst *const CI, uint64_t Size, const TargetTransformInfo::MemCmpExpansionOptions &Options, - const unsigned MaxNumLoads, const bool IsUsedForZeroCmp, - const unsigned MaxLoadsPerBlockForZeroCmp, const DataLayout &TheDataLayout) - : CI(CI), - Size(Size), - MaxLoadSize(0), - NumLoadsNonOneByte(0), - NumLoadsPerBlockForZeroCmp(MaxLoadsPerBlockForZeroCmp), - IsUsedForZeroCmp(IsUsedForZeroCmp), - DL(TheDataLayout), - Builder(CI) { + const bool IsUsedForZeroCmp, const DataLayout &TheDataLayout) + : CI(CI), Size(Size), MaxLoadSize(0), NumLoadsNonOneByte(0), + NumLoadsPerBlockForZeroCmp(Options.NumLoadsPerBlock), + IsUsedForZeroCmp(IsUsedForZeroCmp), DL(TheDataLayout), Builder(CI) { assert(Size > 0 && "zero blocks"); // Scale the max size down if the target can load more bytes than we need. llvm::ArrayRef LoadSizes(Options.LoadSizes); @@ -216,17 +216,17 @@ MemCmpExpansion::MemCmpExpansion( MaxLoadSize = LoadSizes.front(); // Compute the decomposition. unsigned GreedyNumLoadsNonOneByte = 0; - LoadSequence = computeGreedyLoadSequence(Size, LoadSizes, MaxNumLoads, + LoadSequence = computeGreedyLoadSequence(Size, LoadSizes, Options.MaxNumLoads, GreedyNumLoadsNonOneByte); NumLoadsNonOneByte = GreedyNumLoadsNonOneByte; - assert(LoadSequence.size() <= MaxNumLoads && "broken invariant"); + assert(LoadSequence.size() <= Options.MaxNumLoads && "broken invariant"); // If we allow overlapping loads and the load sequence is not already optimal, // use overlapping loads. if (Options.AllowOverlappingLoads && (LoadSequence.empty() || LoadSequence.size() > 2)) { unsigned OverlappingNumLoadsNonOneByte = 0; auto OverlappingLoads = computeOverlappingLoadSequence( - Size, MaxLoadSize, MaxNumLoads, OverlappingNumLoadsNonOneByte); + Size, MaxLoadSize, Options.MaxNumLoads, OverlappingNumLoadsNonOneByte); if (!OverlappingLoads.empty() && (LoadSequence.empty() || OverlappingLoads.size() < LoadSequence.size())) { @@ -234,7 +234,7 @@ MemCmpExpansion::MemCmpExpansion( NumLoadsNonOneByte = OverlappingNumLoadsNonOneByte; } } - assert(LoadSequence.size() <= MaxNumLoads && "broken invariant"); + assert(LoadSequence.size() <= Options.MaxNumLoads && "broken invariant"); } unsigned MemCmpExpansion::getNumBlocks() { @@ -316,7 +316,7 @@ Value *MemCmpExpansion::getCompareLoadPairs(unsigned BlockIndex, assert(LoadIndex < getNumLoads() && "getCompareLoadPairs() called with no remaining loads"); std::vector XorList, OrList; - Value *Diff; + Value *Diff = nullptr; const unsigned NumLoads = std::min(getNumLoads() - LoadIndex, NumLoadsPerBlockForZeroCmp); @@ -393,6 +393,8 @@ Value *MemCmpExpansion::getCompareLoadPairs(unsigned BlockIndex, while (OrList.size() != 1) { OrList = pairWiseOr(OrList); } + + assert(Diff && "Failed to find comparison diff"); Cmp = Builder.CreateICmpNE(OrList[0], ConstantInt::get(Diff->getType(), 0)); } @@ -722,7 +724,7 @@ static bool expandMemCmp(CallInst *CI, const TargetTransformInfo *TTI, NumMemCmpCalls++; // Early exit from expansion if -Oz. - if (CI->getFunction()->optForMinSize()) + if (CI->getFunction()->hasMinSize()) return false; // Early exit from expansion if size is not a constant. @@ -739,18 +741,21 @@ static bool expandMemCmp(CallInst *CI, const TargetTransformInfo *TTI, // TTI call to check if target would like to expand memcmp. Also, get the // available load sizes. const bool IsUsedForZeroCmp = isOnlyUsedInZeroEqualityComparison(CI); - const auto *const Options = TTI->enableMemCmpExpansion(IsUsedForZeroCmp); + auto Options = TTI->enableMemCmpExpansion(CI->getFunction()->hasOptSize(), + IsUsedForZeroCmp); if (!Options) return false; - const unsigned MaxNumLoads = - TLI->getMaxExpandSizeMemcmp(CI->getFunction()->optForSize()); + if (MemCmpEqZeroNumLoadsPerBlock.getNumOccurrences()) + Options.NumLoadsPerBlock = MemCmpEqZeroNumLoadsPerBlock; + + if (CI->getFunction()->hasOptSize() && + MaxLoadsPerMemcmpOptSize.getNumOccurrences()) + Options.MaxNumLoads = MaxLoadsPerMemcmpOptSize; - unsigned NumLoadsPerBlock = MemCmpEqZeroNumLoadsPerBlock.getNumOccurrences() - ? MemCmpEqZeroNumLoadsPerBlock - : TLI->getMemcmpEqZeroLoadsPerBlock(); + if (!CI->getFunction()->hasOptSize() && MaxLoadsPerMemcmp.getNumOccurrences()) + Options.MaxNumLoads = MaxLoadsPerMemcmp; - MemCmpExpansion Expansion(CI, SizeVal, *Options, MaxNumLoads, - IsUsedForZeroCmp, NumLoadsPerBlock, *DL); + MemCmpExpansion Expansion(CI, SizeVal, Options, IsUsedForZeroCmp, *DL); // Don't expand if this will require more loads than desired by the target. if (Expansion.getNumLoads() == 0) { @@ -824,7 +829,8 @@ bool ExpandMemCmpPass::runOnBlock( } LibFunc Func; if (TLI->getLibFunc(ImmutableCallSite(CI), Func) && - Func == LibFunc_memcmp && expandMemCmp(CI, TTI, TL, &DL)) { + (Func == LibFunc_memcmp || Func == LibFunc_bcmp) && + expandMemCmp(CI, TTI, TL, &DL)) { return true; } } diff --git a/lib/CodeGen/ExpandPostRAPseudos.cpp b/lib/CodeGen/ExpandPostRAPseudos.cpp index f2a2bcbb94b1..0ab70aff7dc4 100644 --- a/lib/CodeGen/ExpandPostRAPseudos.cpp +++ b/lib/CodeGen/ExpandPostRAPseudos.cpp @@ -1,9 +1,8 @@ //===-- ExpandPostRAPseudos.cpp - Pseudo instruction expansion pass -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/ExpandReductions.cpp b/lib/CodeGen/ExpandReductions.cpp index 7552ba8cd85d..1069a2423b8b 100644 --- a/lib/CodeGen/ExpandReductions.cpp +++ b/lib/CodeGen/ExpandReductions.cpp @@ -1,9 +1,8 @@ //===--- ExpandReductions.cpp - Expand experimental reduction intrinsics --===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -30,9 +29,9 @@ namespace { unsigned getOpcode(Intrinsic::ID ID) { switch (ID) { - case Intrinsic::experimental_vector_reduce_fadd: + case Intrinsic::experimental_vector_reduce_v2_fadd: return Instruction::FAdd; - case Intrinsic::experimental_vector_reduce_fmul: + case Intrinsic::experimental_vector_reduce_v2_fmul: return Instruction::FMul; case Intrinsic::experimental_vector_reduce_add: return Instruction::Add; @@ -84,22 +83,33 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI) { Worklist.push_back(II); for (auto *II : Worklist) { + if (!TTI->shouldExpandReduction(II)) + continue; + + FastMathFlags FMF = + isa(II) ? II->getFastMathFlags() : FastMathFlags{}; + Intrinsic::ID ID = II->getIntrinsicID(); + RecurrenceDescriptor::MinMaxRecurrenceKind MRK = getMRK(ID); + + Value *Rdx = nullptr; IRBuilder<> Builder(II); - bool IsOrdered = false; - Value *Acc = nullptr; - Value *Vec = nullptr; - auto ID = II->getIntrinsicID(); - auto MRK = RecurrenceDescriptor::MRK_Invalid; + IRBuilder<>::FastMathFlagGuard FMFGuard(Builder); + Builder.setFastMathFlags(FMF); switch (ID) { - case Intrinsic::experimental_vector_reduce_fadd: - case Intrinsic::experimental_vector_reduce_fmul: + case Intrinsic::experimental_vector_reduce_v2_fadd: + case Intrinsic::experimental_vector_reduce_v2_fmul: { // FMFs must be attached to the call, otherwise it's an ordered reduction // and it can't be handled by generating a shuffle sequence. - if (!II->getFastMathFlags().isFast()) - IsOrdered = true; - Acc = II->getArgOperand(0); - Vec = II->getArgOperand(1); - break; + Value *Acc = II->getArgOperand(0); + Value *Vec = II->getArgOperand(1); + if (!FMF.allowReassoc()) + Rdx = getOrderedReduction(Builder, Acc, Vec, getOpcode(ID), MRK); + else { + Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), MRK); + Rdx = Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(ID), + Acc, Rdx, "bin.rdx"); + } + } break; case Intrinsic::experimental_vector_reduce_add: case Intrinsic::experimental_vector_reduce_mul: case Intrinsic::experimental_vector_reduce_and: @@ -110,18 +120,13 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI) { case Intrinsic::experimental_vector_reduce_umax: case Intrinsic::experimental_vector_reduce_umin: case Intrinsic::experimental_vector_reduce_fmax: - case Intrinsic::experimental_vector_reduce_fmin: - Vec = II->getArgOperand(0); - MRK = getMRK(ID); - break; + case Intrinsic::experimental_vector_reduce_fmin: { + Value *Vec = II->getArgOperand(0); + Rdx = getShuffleReduction(Builder, Vec, getOpcode(ID), MRK); + } break; default: continue; } - if (!TTI->shouldExpandReduction(II)) - continue; - Value *Rdx = - IsOrdered ? getOrderedReduction(Builder, Acc, Vec, getOpcode(ID), MRK) - : getShuffleReduction(Builder, Vec, getOpcode(ID), MRK); II->replaceAllUsesWith(Rdx); II->eraseFromParent(); Changed = true; diff --git a/lib/CodeGen/FEntryInserter.cpp b/lib/CodeGen/FEntryInserter.cpp index 4ddf9f92836c..a122f490884e 100644 --- a/lib/CodeGen/FEntryInserter.cpp +++ b/lib/CodeGen/FEntryInserter.cpp @@ -1,9 +1,8 @@ //===-- FEntryInsertion.cpp - Patchable prologues for LLVM -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/FaultMaps.cpp b/lib/CodeGen/FaultMaps.cpp index 361558a0e562..600f72d320eb 100644 --- a/lib/CodeGen/FaultMaps.cpp +++ b/lib/CodeGen/FaultMaps.cpp @@ -1,9 +1,8 @@ //===- FaultMaps.cpp ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/FinalizeISel.cpp b/lib/CodeGen/FinalizeISel.cpp new file mode 100644 index 000000000000..772d7f71bb37 --- /dev/null +++ b/lib/CodeGen/FinalizeISel.cpp @@ -0,0 +1,76 @@ +//===-- llvm/CodeGen/FinalizeISel.cpp ---------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// This pass expands Pseudo-instructions produced by ISel, fixes register +/// reservations and may do machine frame information adjustments. +/// The pseudo instructions are used to allow the expansion to contain control +/// flow, such as a conditional move implemented with a conditional branch and a +/// phi, or an atomic operation implemented with a loop. +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/CodeGen/TargetLowering.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" +#include "llvm/Support/Debug.h" +using namespace llvm; + +#define DEBUG_TYPE "finalize-isel" + +namespace { + class FinalizeISel : public MachineFunctionPass { + public: + static char ID; // Pass identification, replacement for typeid + FinalizeISel() : MachineFunctionPass(ID) {} + + private: + bool runOnMachineFunction(MachineFunction &MF) override; + + void getAnalysisUsage(AnalysisUsage &AU) const override { + MachineFunctionPass::getAnalysisUsage(AU); + } + }; +} // end anonymous namespace + +char FinalizeISel::ID = 0; +char &llvm::FinalizeISelID = FinalizeISel::ID; +INITIALIZE_PASS(FinalizeISel, DEBUG_TYPE, + "Finalize ISel and expand pseudo-instructions", false, false) + +bool FinalizeISel::runOnMachineFunction(MachineFunction &MF) { + bool Changed = false; + const TargetLowering *TLI = MF.getSubtarget().getTargetLowering(); + + // Iterate through each instruction in the function, looking for pseudos. + for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) { + MachineBasicBlock *MBB = &*I; + for (MachineBasicBlock::iterator MBBI = MBB->begin(), MBBE = MBB->end(); + MBBI != MBBE; ) { + MachineInstr &MI = *MBBI++; + + // If MI is a pseudo, expand it. + if (MI.usesCustomInsertionHook()) { + Changed = true; + MachineBasicBlock *NewMBB = TLI->EmitInstrWithCustomInserter(MI, MBB); + // The expansion may involve new basic blocks. + if (NewMBB != MBB) { + MBB = NewMBB; + I = NewMBB->getIterator(); + MBBI = NewMBB->begin(); + MBBE = NewMBB->end(); + } + } + } + } + + TLI->finalizeLowering(MF); + + return Changed; +} diff --git a/lib/CodeGen/FuncletLayout.cpp b/lib/CodeGen/FuncletLayout.cpp index 581cd423f2d4..75f6d0b8f0bf 100644 --- a/lib/CodeGen/FuncletLayout.cpp +++ b/lib/CodeGen/FuncletLayout.cpp @@ -1,9 +1,8 @@ //===-- FuncletLayout.cpp - Contiguously lay out funclets -----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/GCMetadata.cpp b/lib/CodeGen/GCMetadata.cpp index 1c80556dfef5..9c53550eaa9d 100644 --- a/lib/CodeGen/GCMetadata.cpp +++ b/lib/CodeGen/GCMetadata.cpp @@ -1,9 +1,8 @@ //===-- GCMetadata.cpp - Garbage collector metadata -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/GCMetadataPrinter.cpp b/lib/CodeGen/GCMetadataPrinter.cpp index bc7beb6f6c2d..500dba9aea37 100644 --- a/lib/CodeGen/GCMetadataPrinter.cpp +++ b/lib/CodeGen/GCMetadataPrinter.cpp @@ -1,9 +1,8 @@ //===- GCMetadataPrinter.cpp - Garbage collection infrastructure ----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/GCRootLowering.cpp b/lib/CodeGen/GCRootLowering.cpp index e8ccd84b0b93..90571d090bfb 100644 --- a/lib/CodeGen/GCRootLowering.cpp +++ b/lib/CodeGen/GCRootLowering.cpp @@ -1,9 +1,8 @@ //===-- GCRootLowering.cpp - Garbage collection infrastructure ------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -214,7 +213,7 @@ bool LowerIntrinsics::DoLowering(Function &F, GCStrategy &S) { } case Intrinsic::gcread: { // Replace a read barrier with a simple load. - Value *Ld = new LoadInst(CI->getArgOperand(1), "", CI); + Value *Ld = new LoadInst(CI->getType(), CI->getArgOperand(1), "", CI); Ld->takeName(CI); CI->replaceAllUsesWith(Ld); CI->eraseFromParent(); diff --git a/lib/CodeGen/GCStrategy.cpp b/lib/CodeGen/GCStrategy.cpp index 6be4c16c6301..43d06b0f82e9 100644 --- a/lib/CodeGen/GCStrategy.cpp +++ b/lib/CodeGen/GCStrategy.cpp @@ -1,9 +1,8 @@ //===- GCStrategy.cpp - Garbage Collector Description ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/GlobalISel/CSEInfo.cpp b/lib/CodeGen/GlobalISel/CSEInfo.cpp index 89c525c5ba15..4518dbee1a9f 100644 --- a/lib/CodeGen/GlobalISel/CSEInfo.cpp +++ b/lib/CodeGen/GlobalISel/CSEInfo.cpp @@ -1,9 +1,8 @@ //===- CSEInfo.cpp ------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -28,8 +27,8 @@ void UniqueMachineInstr::Profile(FoldingSetNodeID &ID) { } /// ----------------------------------------- -/// --------- CSEConfig ---------- /// -bool CSEConfig::shouldCSEOpc(unsigned Opc) { +/// --------- CSEConfigFull ---------- /// +bool CSEConfigFull::shouldCSEOpc(unsigned Opc) { switch (Opc) { default: break; @@ -61,6 +60,17 @@ bool CSEConfig::shouldCSEOpc(unsigned Opc) { bool CSEConfigConstantOnly::shouldCSEOpc(unsigned Opc) { return Opc == TargetOpcode::G_CONSTANT; } + +std::unique_ptr +llvm::getStandardCSEConfigForOpt(CodeGenOpt::Level Level) { + std::unique_ptr Config; + if (Level == CodeGenOpt::None) + Config = make_unique(); + else + Config = make_unique(); + return Config; +} + /// ----------------------------------------- /// -------- GISelCSEInfo -------------// @@ -139,7 +149,7 @@ MachineInstr *GISelCSEInfo::getMachineInstrIfExists(FoldingSetNodeID &ID, void *&InsertPos) { handleRecordedInsts(); if (auto *Inst = getNodeIfExists(ID, MBB, InsertPos)) { - LLVM_DEBUG(dbgs() << "CSEInfo: Found Instr " << *Inst->MI << "\n";); + LLVM_DEBUG(dbgs() << "CSEInfo::Found Instr " << *Inst->MI;); return const_cast(Inst->MI); } return nullptr; @@ -158,14 +168,14 @@ void GISelCSEInfo::countOpcodeHit(unsigned Opc) { void GISelCSEInfo::recordNewInstruction(MachineInstr *MI) { if (shouldCSE(MI->getOpcode())) { TemporaryInsts.insert(MI); - LLVM_DEBUG(dbgs() << "CSEInfo: Recording new MI" << *MI << "\n";); + LLVM_DEBUG(dbgs() << "CSEInfo::Recording new MI " << *MI); } } void GISelCSEInfo::handleRecordedInst(MachineInstr *MI) { assert(shouldCSE(MI->getOpcode()) && "Invalid instruction for CSE"); auto *UMI = InstrMapping.lookup(MI); - LLVM_DEBUG(dbgs() << "CSEInfo: Handling recorded MI" << *MI << "\n";); + LLVM_DEBUG(dbgs() << "CSEInfo::Handling recorded MI " << *MI); if (UMI) { // Invalidate this MI. invalidateUniqueMachineInstr(UMI); @@ -224,14 +234,14 @@ void GISelCSEInfo::analyze(MachineFunction &MF) { for (MachineInstr &MI : MBB) { if (!shouldCSE(MI.getOpcode())) continue; - LLVM_DEBUG(dbgs() << "CSEInfo::Add MI: " << MI << "\n";); + LLVM_DEBUG(dbgs() << "CSEInfo::Add MI: " << MI); insertInstr(&MI); } } } void GISelCSEInfo::releaseMemory() { - // print(); + print(); CSEMap.clear(); InstrMapping.clear(); UniqueInstrAllocator.Reset(); @@ -245,11 +255,11 @@ void GISelCSEInfo::releaseMemory() { } void GISelCSEInfo::print() { -#ifndef NDEBUG - for (auto &It : OpcodeHitTable) { - dbgs() << "CSE Count for Opc " << It.first << " : " << It.second << "\n"; - }; -#endif + LLVM_DEBUG(for (auto &It + : OpcodeHitTable) { + dbgs() << "CSEInfo::CSE Hit for Opc " << It.first << " : " << It.second + << "\n"; + };); } /// ----------------------------------------- // ---- Profiling methods for FoldingSetNode --- // @@ -349,8 +359,9 @@ const GISelInstProfileBuilder &GISelInstProfileBuilder::addNodeIDMachineOperand( return *this; } -GISelCSEInfo &GISelCSEAnalysisWrapper::get(std::unique_ptr CSEOpt, - bool Recompute) { +GISelCSEInfo & +GISelCSEAnalysisWrapper::get(std::unique_ptr CSEOpt, + bool Recompute) { if (!AlreadyComputed || Recompute) { Info.setCSEConfig(std::move(CSEOpt)); Info.analyze(*MF); diff --git a/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp b/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp index 863efe0c3e34..461bc6038c2c 100644 --- a/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp +++ b/lib/CodeGen/GlobalISel/CSEMIRBuilder.cpp @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/GlobalISel/CSEMIRBuilder.cpp - MIBuilder--*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -40,6 +39,7 @@ CSEMIRBuilder::getDominatingInstrForID(FoldingSetNodeID &ID, MachineInstr *MI = CSEInfo->getMachineInstrIfExists(ID, CurMBB, NodeInsertPos); if (MI) { + CSEInfo->countOpcodeHit(MI->getOpcode()); auto CurrPos = getInsertPt(); if (!dominates(MI, CurrPos)) CurMBB->splice(CurrPos, CurMBB, MI); @@ -195,6 +195,12 @@ MachineInstrBuilder CSEMIRBuilder::buildConstant(const DstOp &Res, constexpr unsigned Opc = TargetOpcode::G_CONSTANT; if (!canPerformCSEForOpc(Opc)) return MachineIRBuilder::buildConstant(Res, Val); + + // For vectors, CSE the element only for now. + LLT Ty = Res.getLLTTy(*getMRI()); + if (Ty.isVector()) + return buildSplatVector(Res, buildConstant(Ty.getElementType(), Val)); + FoldingSetNodeID ID; GISelInstProfileBuilder ProfBuilder(ID, *getMRI()); void *InsertPos = nullptr; @@ -206,6 +212,7 @@ MachineInstrBuilder CSEMIRBuilder::buildConstant(const DstOp &Res, // Handle generating copies here. return generateCopiesIfRequired({Res}, MIB); } + MachineInstrBuilder NewMIB = MachineIRBuilder::buildConstant(Res, Val); return memoizeMI(NewMIB, InsertPos); } @@ -215,6 +222,12 @@ MachineInstrBuilder CSEMIRBuilder::buildFConstant(const DstOp &Res, constexpr unsigned Opc = TargetOpcode::G_FCONSTANT; if (!canPerformCSEForOpc(Opc)) return MachineIRBuilder::buildFConstant(Res, Val); + + // For vectors, CSE the element only for now. + LLT Ty = Res.getLLTTy(*getMRI()); + if (Ty.isVector()) + return buildSplatVector(Res, buildFConstant(Ty.getElementType(), Val)); + FoldingSetNodeID ID; GISelInstProfileBuilder ProfBuilder(ID, *getMRI()); void *InsertPos = nullptr; diff --git a/lib/CodeGen/GlobalISel/CallLowering.cpp b/lib/CodeGen/GlobalISel/CallLowering.cpp index 724ecedf3b3f..a5d8205a34a8 100644 --- a/lib/CodeGen/GlobalISel/CallLowering.cpp +++ b/lib/CodeGen/GlobalISel/CallLowering.cpp @@ -1,9 +1,8 @@ //===-- lib/CodeGen/GlobalISel/CallLowering.cpp - Call lowering -----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -13,6 +12,7 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/GlobalISel/CallLowering.h" +#include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" @@ -21,13 +21,17 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/Module.h" +#define DEBUG_TYPE "call-lowering" + using namespace llvm; void CallLowering::anchor() {} -bool CallLowering::lowerCall( - MachineIRBuilder &MIRBuilder, ImmutableCallSite CS, unsigned ResReg, - ArrayRef ArgRegs, std::function GetCalleeReg) const { +bool CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, ImmutableCallSite CS, + ArrayRef ResRegs, + ArrayRef> ArgRegs, + Register SwiftErrorVReg, + std::function GetCalleeReg) const { auto &DL = CS.getParent()->getParent()->getParent()->getDataLayout(); // First step is to marshall all the function's parameters into the correct @@ -40,8 +44,8 @@ bool CallLowering::lowerCall( ArgInfo OrigArg{ArgRegs[i], Arg->getType(), ISD::ArgFlagsTy{}, i < NumFixedArgs}; setArgFlags(OrigArg, i + AttributeList::FirstArgIndex, DL, CS); - // We don't currently support swifterror or swiftself args. - if (OrigArg.Flags.isSwiftError() || OrigArg.Flags.isSwiftSelf()) + // We don't currently support swiftself args. + if (OrigArg.Flags.isSwiftSelf()) return false; OrigArgs.push_back(OrigArg); ++i; @@ -53,11 +57,12 @@ bool CallLowering::lowerCall( else Callee = MachineOperand::CreateReg(GetCalleeReg(), false); - ArgInfo OrigRet{ResReg, CS.getType(), ISD::ArgFlagsTy{}}; + ArgInfo OrigRet{ResRegs, CS.getType(), ISD::ArgFlagsTy{}}; if (!OrigRet.Ty->isVoidTy()) setArgFlags(OrigRet, AttributeList::ReturnIndex, DL, CS); - return lowerCall(MIRBuilder, CS.getCallingConv(), Callee, OrigRet, OrigArgs); + return lowerCall(MIRBuilder, CS.getCallingConv(), Callee, OrigRet, OrigArgs, + SwiftErrorVReg); } template @@ -84,7 +89,10 @@ void CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx, if (Arg.Flags.isByVal() || Arg.Flags.isInAlloca()) { Type *ElementTy = cast(Arg.Ty)->getElementType(); - Arg.Flags.setByValSize(DL.getTypeAllocSize(ElementTy)); + + auto Ty = Attrs.getAttribute(OpIdx, Attribute::ByVal).getValueAsType(); + Arg.Flags.setByValSize(DL.getTypeAllocSize(Ty ? Ty : ElementTy)); + // For ByVal, alignment should be passed from FE. BE will guess if // this info is not there but there are cases it cannot get right. unsigned FrameAlign; @@ -109,21 +117,78 @@ CallLowering::setArgFlags(CallLowering::ArgInfo &Arg, unsigned OpIdx, const DataLayout &DL, const CallInst &FuncInfo) const; +Register CallLowering::packRegs(ArrayRef SrcRegs, Type *PackedTy, + MachineIRBuilder &MIRBuilder) const { + assert(SrcRegs.size() > 1 && "Nothing to pack"); + + const DataLayout &DL = MIRBuilder.getMF().getDataLayout(); + MachineRegisterInfo *MRI = MIRBuilder.getMRI(); + + LLT PackedLLT = getLLTForType(*PackedTy, DL); + + SmallVector LLTs; + SmallVector Offsets; + computeValueLLTs(DL, *PackedTy, LLTs, &Offsets); + assert(LLTs.size() == SrcRegs.size() && "Regs / types mismatch"); + + Register Dst = MRI->createGenericVirtualRegister(PackedLLT); + MIRBuilder.buildUndef(Dst); + for (unsigned i = 0; i < SrcRegs.size(); ++i) { + Register NewDst = MRI->createGenericVirtualRegister(PackedLLT); + MIRBuilder.buildInsert(NewDst, Dst, SrcRegs[i], Offsets[i]); + Dst = NewDst; + } + + return Dst; +} + +void CallLowering::unpackRegs(ArrayRef DstRegs, Register SrcReg, + Type *PackedTy, + MachineIRBuilder &MIRBuilder) const { + assert(DstRegs.size() > 1 && "Nothing to unpack"); + + const DataLayout &DL = MIRBuilder.getMF().getDataLayout(); + + SmallVector LLTs; + SmallVector Offsets; + computeValueLLTs(DL, *PackedTy, LLTs, &Offsets); + assert(LLTs.size() == DstRegs.size() && "Regs / types mismatch"); + + for (unsigned i = 0; i < DstRegs.size(); ++i) + MIRBuilder.buildExtract(DstRegs[i], SrcReg, Offsets[i]); +} + bool CallLowering::handleAssignments(MachineIRBuilder &MIRBuilder, ArrayRef Args, ValueHandler &Handler) const { MachineFunction &MF = MIRBuilder.getMF(); const Function &F = MF.getFunction(); - const DataLayout &DL = F.getParent()->getDataLayout(); - SmallVector ArgLocs; CCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs, F.getContext()); + return handleAssignments(CCInfo, ArgLocs, MIRBuilder, Args, Handler); +} + +bool CallLowering::handleAssignments(CCState &CCInfo, + SmallVectorImpl &ArgLocs, + MachineIRBuilder &MIRBuilder, + ArrayRef Args, + ValueHandler &Handler) const { + MachineFunction &MF = MIRBuilder.getMF(); + const Function &F = MF.getFunction(); + const DataLayout &DL = F.getParent()->getDataLayout(); unsigned NumArgs = Args.size(); for (unsigned i = 0; i != NumArgs; ++i) { MVT CurVT = MVT::getVT(Args[i].Ty); - if (Handler.assignArg(i, CurVT, CurVT, CCValAssign::Full, Args[i], CCInfo)) - return false; + if (Handler.assignArg(i, CurVT, CurVT, CCValAssign::Full, Args[i], CCInfo)) { + // Try to use the register type if we couldn't assign the VT. + if (!Handler.isArgumentHandler() || !CurVT.isValid()) + return false; + CurVT = TLI->getRegisterTypeForCallingConv( + F.getContext(), F.getCallingConv(), EVT(CurVT)); + if (Handler.assignArg(i, CurVT, CurVT, CCValAssign::Full, Args[i], CCInfo)) + return false; + } } for (unsigned i = 0, e = Args.size(), j = 0; i != e; ++i, ++j) { @@ -137,16 +202,49 @@ bool CallLowering::handleAssignments(MachineIRBuilder &MIRBuilder, continue; } - if (VA.isRegLoc()) - Handler.assignValueToReg(Args[i].Reg, VA.getLocReg(), VA); - else if (VA.isMemLoc()) { - unsigned Size = VA.getValVT() == MVT::iPTR - ? DL.getPointerSize() - : alignTo(VA.getValVT().getSizeInBits(), 8) / 8; + assert(Args[i].Regs.size() == 1 && + "Can't handle multiple virtual regs yet"); + + // FIXME: Pack registers if we have more than one. + Register ArgReg = Args[i].Regs[0]; + + if (VA.isRegLoc()) { + MVT OrigVT = MVT::getVT(Args[i].Ty); + MVT VAVT = VA.getValVT(); + if (Handler.isArgumentHandler() && VAVT != OrigVT) { + if (VAVT.getSizeInBits() < OrigVT.getSizeInBits()) + return false; // Can't handle this type of arg yet. + const LLT VATy(VAVT); + Register NewReg = + MIRBuilder.getMRI()->createGenericVirtualRegister(VATy); + Handler.assignValueToReg(NewReg, VA.getLocReg(), VA); + // If it's a vector type, we either need to truncate the elements + // or do an unmerge to get the lower block of elements. + if (VATy.isVector() && + VATy.getNumElements() > OrigVT.getVectorNumElements()) { + const LLT OrigTy(OrigVT); + // Just handle the case where the VA type is 2 * original type. + if (VATy.getNumElements() != OrigVT.getVectorNumElements() * 2) { + LLVM_DEBUG(dbgs() + << "Incoming promoted vector arg has too many elts"); + return false; + } + auto Unmerge = MIRBuilder.buildUnmerge({OrigTy, OrigTy}, {NewReg}); + MIRBuilder.buildCopy(ArgReg, Unmerge.getReg(0)); + } else { + MIRBuilder.buildTrunc(ArgReg, {NewReg}).getReg(0); + } + } else { + Handler.assignValueToReg(ArgReg, VA.getLocReg(), VA); + } + } else if (VA.isMemLoc()) { + MVT VT = MVT::getVT(Args[i].Ty); + unsigned Size = VT == MVT::iPTR ? DL.getPointerSize() + : alignTo(VT.getSizeInBits(), 8) / 8; unsigned Offset = VA.getLocMemOffset(); MachinePointerInfo MPO; - unsigned StackAddr = Handler.getStackAddress(Size, Offset, MPO); - Handler.assignValueToAddress(Args[i].Reg, StackAddr, Size, MPO, VA); + Register StackAddr = Handler.getStackAddress(Size, Offset, MPO); + Handler.assignValueToAddress(ArgReg, StackAddr, Size, MPO, VA); } else { // FIXME: Support byvals and other weirdness return false; @@ -155,9 +253,11 @@ bool CallLowering::handleAssignments(MachineIRBuilder &MIRBuilder, return true; } -unsigned CallLowering::ValueHandler::extendRegister(unsigned ValReg, +Register CallLowering::ValueHandler::extendRegister(Register ValReg, CCValAssign &VA) { LLT LocTy{VA.getLocVT()}; + if (LocTy.getSizeInBits() == MRI.getType(ValReg).getSizeInBits()) + return ValReg; switch (VA.getLocInfo()) { default: break; case CCValAssign::Full: @@ -170,12 +270,12 @@ unsigned CallLowering::ValueHandler::extendRegister(unsigned ValReg, return MIB->getOperand(0).getReg(); } case CCValAssign::SExt: { - unsigned NewReg = MRI.createGenericVirtualRegister(LocTy); + Register NewReg = MRI.createGenericVirtualRegister(LocTy); MIRBuilder.buildSExt(NewReg, ValReg); return NewReg; } case CCValAssign::ZExt: { - unsigned NewReg = MRI.createGenericVirtualRegister(LocTy); + Register NewReg = MRI.createGenericVirtualRegister(LocTy); MIRBuilder.buildZExt(NewReg, ValReg); return NewReg; } diff --git a/lib/CodeGen/GlobalISel/Combiner.cpp b/lib/CodeGen/GlobalISel/Combiner.cpp index 45b0e36fd7d9..31cb1dbbc9b5 100644 --- a/lib/CodeGen/GlobalISel/Combiner.cpp +++ b/lib/CodeGen/GlobalISel/Combiner.cpp @@ -1,9 +1,8 @@ //===-- lib/CodeGen/GlobalISel/Combiner.cpp -------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -51,7 +50,7 @@ public: } void erasingInstr(MachineInstr &MI) override { - LLVM_DEBUG(dbgs() << "Erased: " << MI << "\n"); + LLVM_DEBUG(dbgs() << "Erasing: " << MI << "\n"); WorkList.remove(&MI); } void createdInstr(MachineInstr &MI) override { @@ -130,9 +129,10 @@ bool Combiner::combineMachineInstrs(MachineFunction &MF, CurMI->eraseFromParentAndMarkDBGValuesForRemoval(); continue; } - WorkList.insert(CurMI); + WorkList.deferred_insert(CurMI); } } + WorkList.finalize(); // Main Loop. Process the instructions here. while (!WorkList.empty()) { MachineInstr *CurrInst = WorkList.pop_back_val(); diff --git a/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/lib/CodeGen/GlobalISel/CombinerHelper.cpp index b1c5670a6dec..9cbf3dd83ff1 100644 --- a/lib/CodeGen/GlobalISel/CombinerHelper.cpp +++ b/lib/CodeGen/GlobalISel/CombinerHelper.cpp @@ -1,9 +1,8 @@ //===-- lib/CodeGen/GlobalISel/GICombinerHelper.cpp -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "llvm/CodeGen/GlobalISel/CombinerHelper.h" @@ -23,8 +22,8 @@ CombinerHelper::CombinerHelper(GISelChangeObserver &Observer, MachineIRBuilder &B) : Builder(B), MRI(Builder.getMF().getRegInfo()), Observer(Observer) {} -void CombinerHelper::replaceRegWith(MachineRegisterInfo &MRI, unsigned FromReg, - unsigned ToReg) const { +void CombinerHelper::replaceRegWith(MachineRegisterInfo &MRI, Register FromReg, + Register ToReg) const { Observer.changingAllUsesOfReg(MRI, FromReg); if (MRI.constrainRegAttrs(ToReg, FromReg)) @@ -37,7 +36,7 @@ void CombinerHelper::replaceRegWith(MachineRegisterInfo &MRI, unsigned FromReg, void CombinerHelper::replaceRegOpWith(MachineRegisterInfo &MRI, MachineOperand &FromRegOp, - unsigned ToReg) const { + Register ToReg) const { assert(FromRegOp.getParent() && "Expected an operand in an MI"); Observer.changingInstr(*FromRegOp.getParent()); @@ -47,6 +46,13 @@ void CombinerHelper::replaceRegOpWith(MachineRegisterInfo &MRI, } bool CombinerHelper::tryCombineCopy(MachineInstr &MI) { + if (matchCombineCopy(MI)) { + applyCombineCopy(MI); + return true; + } + return false; +} +bool CombinerHelper::matchCombineCopy(MachineInstr &MI) { if (MI.getOpcode() != TargetOpcode::COPY) return false; unsigned DstReg = MI.getOperand(0).getReg(); @@ -55,20 +61,18 @@ bool CombinerHelper::tryCombineCopy(MachineInstr &MI) { LLT SrcTy = MRI.getType(SrcReg); // Simple Copy Propagation. // a(sx) = COPY b(sx) -> Replace all uses of a with b. - if (DstTy.isValid() && SrcTy.isValid() && DstTy == SrcTy) { - MI.eraseFromParent(); - replaceRegWith(MRI, DstReg, SrcReg); + if (DstTy.isValid() && SrcTy.isValid() && DstTy == SrcTy) return true; - } return false; } +void CombinerHelper::applyCombineCopy(MachineInstr &MI) { + unsigned DstReg = MI.getOperand(0).getReg(); + unsigned SrcReg = MI.getOperand(1).getReg(); + MI.eraseFromParent(); + replaceRegWith(MRI, DstReg, SrcReg); +} namespace { -struct PreferredTuple { - LLT Ty; // The result type of the extend. - unsigned ExtendOpcode; // G_ANYEXT/G_SEXT/G_ZEXT - MachineInstr *MI; -}; /// Select a preference between two uses. CurrentUse is the current preference /// while *ForCandidate is attributes of the candidate under consideration. @@ -127,7 +131,8 @@ PreferredTuple ChoosePreferredUse(PreferredTuple &CurrentUse, /// want to try harder to find a dominating block. static void InsertInsnsWithoutSideEffectsBeforeUse( MachineIRBuilder &Builder, MachineInstr &DefMI, MachineOperand &UseMO, - std::function + std::function Inserter) { MachineInstr &UseMI = *UseMO.getParent(); @@ -143,26 +148,26 @@ static void InsertInsnsWithoutSideEffectsBeforeUse( // the def instead of at the start of the block. if (InsertBB == DefMI.getParent()) { MachineBasicBlock::iterator InsertPt = &DefMI; - Inserter(InsertBB, std::next(InsertPt)); + Inserter(InsertBB, std::next(InsertPt), UseMO); return; } // Otherwise we want the start of the BB - Inserter(InsertBB, InsertBB->getFirstNonPHI()); + Inserter(InsertBB, InsertBB->getFirstNonPHI(), UseMO); } } // end anonymous namespace bool CombinerHelper::tryCombineExtendingLoads(MachineInstr &MI) { - struct InsertionPoint { - MachineOperand *UseMO; - MachineBasicBlock *InsertIntoBB; - MachineBasicBlock::iterator InsertBefore; - InsertionPoint(MachineOperand *UseMO, MachineBasicBlock *InsertIntoBB, - MachineBasicBlock::iterator InsertBefore) - : UseMO(UseMO), InsertIntoBB(InsertIntoBB), InsertBefore(InsertBefore) { - } - }; + PreferredTuple Preferred; + if (matchCombineExtendingLoads(MI, Preferred)) { + applyCombineExtendingLoads(MI, Preferred); + return true; + } + return false; +} +bool CombinerHelper::matchCombineExtendingLoads(MachineInstr &MI, + PreferredTuple &Preferred) { // We match the loads and follow the uses to the extend instead of matching // the extends and following the def to the load. This is because the load // must remain in the same position for correctness (unless we also add code @@ -182,6 +187,19 @@ bool CombinerHelper::tryCombineExtendingLoads(MachineInstr &MI) { if (!LoadValueTy.isScalar()) return false; + // Most architectures are going to legalize getOperand(0).getReg(); + Register ChosenDstReg = Preferred.MI->getOperand(0).getReg(); + + // Inserter to insert a truncate back to the original type at a given point + // with some basic CSE to limit truncate duplication to one per BB. + DenseMap EmittedInsns; + auto InsertTruncAt = [&](MachineBasicBlock *InsertIntoBB, + MachineBasicBlock::iterator InsertBefore, + MachineOperand &UseMO) { + MachineInstr *PreviouslyEmitted = EmittedInsns.lookup(InsertIntoBB); + if (PreviouslyEmitted) { + Observer.changingInstr(*UseMO.getParent()); + UseMO.setReg(PreviouslyEmitted->getOperand(0).getReg()); + Observer.changedInstr(*UseMO.getParent()); + return; + } + + Builder.setInsertPt(*InsertIntoBB, InsertBefore); + Register NewDstReg = MRI.cloneVirtualRegister(MI.getOperand(0).getReg()); + MachineInstr *NewMI = Builder.buildTrunc(NewDstReg, ChosenDstReg); + EmittedInsns[InsertIntoBB] = NewMI; + replaceRegOpWith(MRI, UseMO, NewDstReg); + }; + Observer.changingInstr(MI); MI.setDesc( Builder.getTII().get(Preferred.ExtendOpcode == TargetOpcode::G_SEXT @@ -223,10 +267,13 @@ bool CombinerHelper::tryCombineExtendingLoads(MachineInstr &MI) { : TargetOpcode::G_LOAD)); // Rewrite all the uses to fix up the types. - SmallVector ScheduleForErase; - SmallVector ScheduleForInsert; - for (auto &UseMO : MRI.use_operands(LoadValue.getReg())) { - MachineInstr *UseMI = UseMO.getParent(); + auto &LoadValue = MI.getOperand(0); + SmallVector Uses; + for (auto &UseMO : MRI.use_operands(LoadValue.getReg())) + Uses.push_back(&UseMO); + + for (auto *UseMO : Uses) { + MachineInstr *UseMI = UseMO->getParent(); // If the extend is compatible with the preferred extend then we should fix // up the type and extend so that it uses the preferred use. @@ -247,7 +294,8 @@ bool CombinerHelper::tryCombineExtendingLoads(MachineInstr &MI) { // %2:_(s32) = G_SEXTLOAD ... // ... = ... %2(s32) replaceRegWith(MRI, UseDstReg, ChosenDstReg); - ScheduleForErase.push_back(UseMO.getParent()); + Observer.erasingInstr(*UseMO->getParent()); + UseMO->getParent()->eraseFromParent(); } else if (Preferred.Ty.getSizeInBits() < UseDstTy.getSizeInBits()) { // If the preferred size is smaller, then keep the extend but extend // from the result of the extending load. For example: @@ -272,59 +320,87 @@ bool CombinerHelper::tryCombineExtendingLoads(MachineInstr &MI) { // %4:_(s8) = G_TRUNC %2:_(s32) // %3:_(s64) = G_ZEXT %2:_(s8) // ... = ... %3(s64) - InsertInsnsWithoutSideEffectsBeforeUse( - Builder, MI, UseMO, - [&](MachineBasicBlock *InsertIntoBB, - MachineBasicBlock::iterator InsertBefore) { - ScheduleForInsert.emplace_back(&UseMO, InsertIntoBB, InsertBefore); - }); + InsertInsnsWithoutSideEffectsBeforeUse(Builder, MI, *UseMO, + InsertTruncAt); } continue; } // The use is (one of) the uses of the preferred use we chose earlier. // We're going to update the load to def this value later so just erase // the old extend. - ScheduleForErase.push_back(UseMO.getParent()); + Observer.erasingInstr(*UseMO->getParent()); + UseMO->getParent()->eraseFromParent(); continue; } // The use isn't an extend. Truncate back to the type we originally loaded. // This is free on many targets. - InsertInsnsWithoutSideEffectsBeforeUse( - Builder, MI, UseMO, - [&](MachineBasicBlock *InsertIntoBB, - MachineBasicBlock::iterator InsertBefore) { - ScheduleForInsert.emplace_back(&UseMO, InsertIntoBB, InsertBefore); - }); + InsertInsnsWithoutSideEffectsBeforeUse(Builder, MI, *UseMO, InsertTruncAt); } - DenseMap EmittedInsns; - for (auto &InsertionInfo : ScheduleForInsert) { - MachineOperand *UseMO = InsertionInfo.UseMO; - MachineBasicBlock *InsertIntoBB = InsertionInfo.InsertIntoBB; - MachineBasicBlock::iterator InsertBefore = InsertionInfo.InsertBefore; - - MachineInstr *PreviouslyEmitted = EmittedInsns.lookup(InsertIntoBB); - if (PreviouslyEmitted) { - Observer.changingInstr(*UseMO->getParent()); - UseMO->setReg(PreviouslyEmitted->getOperand(0).getReg()); - Observer.changedInstr(*UseMO->getParent()); - continue; - } - - Builder.setInsertPt(*InsertIntoBB, InsertBefore); - unsigned NewDstReg = MRI.cloneVirtualRegister(MI.getOperand(0).getReg()); - MachineInstr *NewMI = Builder.buildTrunc(NewDstReg, ChosenDstReg); - EmittedInsns[InsertIntoBB] = NewMI; - replaceRegOpWith(MRI, *UseMO, NewDstReg); - } - for (auto &EraseMI : ScheduleForErase) { - Observer.erasingInstr(*EraseMI); - EraseMI->eraseFromParent(); - } MI.getOperand(0).setReg(ChosenDstReg); Observer.changedInstr(MI); +} + +bool CombinerHelper::matchCombineBr(MachineInstr &MI) { + assert(MI.getOpcode() == TargetOpcode::G_BR && "Expected a G_BR"); + // Try to match the following: + // bb1: + // %c(s32) = G_ICMP pred, %a, %b + // %c1(s1) = G_TRUNC %c(s32) + // G_BRCOND %c1, %bb2 + // G_BR %bb3 + // bb2: + // ... + // bb3: + + // The above pattern does not have a fall through to the successor bb2, always + // resulting in a branch no matter which path is taken. Here we try to find + // and replace that pattern with conditional branch to bb3 and otherwise + // fallthrough to bb2. + + MachineBasicBlock *MBB = MI.getParent(); + MachineBasicBlock::iterator BrIt(MI); + if (BrIt == MBB->begin()) + return false; + assert(std::next(BrIt) == MBB->end() && "expected G_BR to be a terminator"); + + MachineInstr *BrCond = &*std::prev(BrIt); + if (BrCond->getOpcode() != TargetOpcode::G_BRCOND) + return false; + // Check that the next block is the conditional branch target. + if (!MBB->isLayoutSuccessor(BrCond->getOperand(1).getMBB())) + return false; + + MachineInstr *CmpMI = MRI.getVRegDef(BrCond->getOperand(0).getReg()); + if (!CmpMI || CmpMI->getOpcode() != TargetOpcode::G_ICMP || + !MRI.hasOneUse(CmpMI->getOperand(0).getReg())) + return false; + return true; +} + +bool CombinerHelper::tryCombineBr(MachineInstr &MI) { + if (!matchCombineBr(MI)) + return false; + MachineBasicBlock *BrTarget = MI.getOperand(0).getMBB(); + MachineBasicBlock::iterator BrIt(MI); + MachineInstr *BrCond = &*std::prev(BrIt); + MachineInstr *CmpMI = MRI.getVRegDef(BrCond->getOperand(0).getReg()); + + CmpInst::Predicate InversePred = CmpInst::getInversePredicate( + (CmpInst::Predicate)CmpMI->getOperand(1).getPredicate()); + + // Invert the G_ICMP condition. + Observer.changingInstr(*CmpMI); + CmpMI->getOperand(1).setPredicate(InversePred); + Observer.changedInstr(*CmpMI); + + // Change the conditional branch target. + Observer.changingInstr(*BrCond); + BrCond->getOperand(1).setMBB(BrTarget); + Observer.changedInstr(*BrCond); + MI.eraseFromParent(); return true; } diff --git a/lib/CodeGen/GlobalISel/GISelChangeObserver.cpp b/lib/CodeGen/GlobalISel/GISelChangeObserver.cpp index c693acbbf10b..62b903c30b89 100644 --- a/lib/CodeGen/GlobalISel/GISelChangeObserver.cpp +++ b/lib/CodeGen/GlobalISel/GISelChangeObserver.cpp @@ -1,9 +1,8 @@ //===-- lib/CodeGen/GlobalISel/GISelChangeObserver.cpp --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -27,6 +26,7 @@ void GISelChangeObserver::changingAllUsesOfReg( void GISelChangeObserver::finishedChangingAllUsesOfReg() { for (auto *ChangedMI : ChangingAllUsesOfReg) changedInstr(*ChangedMI); + ChangingAllUsesOfReg.clear(); } RAIIDelegateInstaller::RAIIDelegateInstaller(MachineFunction &MF, diff --git a/lib/CodeGen/GlobalISel/GlobalISel.cpp b/lib/CodeGen/GlobalISel/GlobalISel.cpp index 00c6a9d63158..e0391e6f6467 100644 --- a/lib/CodeGen/GlobalISel/GlobalISel.cpp +++ b/lib/CodeGen/GlobalISel/GlobalISel.cpp @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/GlobalISel/GlobalIsel.cpp --- GlobalISel ----*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/lib/CodeGen/GlobalISel/IRTranslator.cpp b/lib/CodeGen/GlobalISel/IRTranslator.cpp index 95f6274aa068..6e99bdbd8264 100644 --- a/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/IRTranslator.cpp - IRTranslator ---*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -16,8 +15,11 @@ #include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/Analysis/BranchProbabilityInfo.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" +#include "llvm/Analysis/ValueTracking.h" #include "llvm/CodeGen/Analysis.h" +#include "llvm/CodeGen/FunctionLoweringInfo.h" #include "llvm/CodeGen/GlobalISel/CallLowering.h" #include "llvm/CodeGen/GlobalISel/GISelChangeObserver.h" #include "llvm/CodeGen/LowLevelType.h" @@ -106,9 +108,7 @@ static void reportTranslationError(MachineFunction &MF, ORE.emit(R); } -IRTranslator::IRTranslator() : MachineFunctionPass(ID) { - initializeIRTranslatorPass(*PassRegistry::getPassRegistry()); -} +IRTranslator::IRTranslator() : MachineFunctionPass(ID) { } #ifndef NDEBUG namespace { @@ -136,7 +136,11 @@ public: LLVM_DEBUG(dbgs() << "Checking DILocation from " << *CurrInst << " was copied to " << MI); #endif - assert(CurrInst->getDebugLoc() == MI.getDebugLoc() && + // We allow insts in the entry block to have a debug loc line of 0 because + // they could have originated from constants, and we don't want a jumpy + // debug experience. + assert((CurrInst->getDebugLoc() == MI.getDebugLoc() || + MI.getDebugLoc().getLine() == 0) && "Line info was not transferred to all instructions"); } }; @@ -152,36 +156,6 @@ void IRTranslator::getAnalysisUsage(AnalysisUsage &AU) const { MachineFunctionPass::getAnalysisUsage(AU); } -static void computeValueLLTs(const DataLayout &DL, Type &Ty, - SmallVectorImpl &ValueTys, - SmallVectorImpl *Offsets = nullptr, - uint64_t StartingOffset = 0) { - // Given a struct type, recursively traverse the elements. - if (StructType *STy = dyn_cast(&Ty)) { - const StructLayout *SL = DL.getStructLayout(STy); - for (unsigned I = 0, E = STy->getNumElements(); I != E; ++I) - computeValueLLTs(DL, *STy->getElementType(I), ValueTys, Offsets, - StartingOffset + SL->getElementOffset(I)); - return; - } - // Given an array type, recursively traverse the elements. - if (ArrayType *ATy = dyn_cast(&Ty)) { - Type *EltTy = ATy->getElementType(); - uint64_t EltSize = DL.getTypeAllocSize(EltTy); - for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) - computeValueLLTs(DL, *EltTy, ValueTys, Offsets, - StartingOffset + i * EltSize); - return; - } - // Interpret void as zero return values. - if (Ty.isVoidTy()) - return; - // Base case: we can get an LLT for this LLVM IR type. - ValueTys.push_back(getLLTForType(Ty, DL)); - if (Offsets != nullptr) - Offsets->push_back(StartingOffset * 8); -} - IRTranslator::ValueToVRegInfo::VRegListT & IRTranslator::allocateVRegs(const Value &Val) { assert(!VMap.contains(Val) && "Value already allocated in VMap"); @@ -195,7 +169,7 @@ IRTranslator::allocateVRegs(const Value &Val) { return *Regs; } -ArrayRef IRTranslator::getOrCreateVRegs(const Value &Val) { +ArrayRef IRTranslator::getOrCreateVRegs(const Value &Val) { auto VRegsIt = VMap.findVRegs(Val); if (VRegsIt != VMap.vregs_end()) return *VRegsIt->second; @@ -249,7 +223,7 @@ int IRTranslator::getOrCreateFrameIndex(const AllocaInst &AI) { if (FrameIndices.find(&AI) != FrameIndices.end()) return FrameIndices[&AI]; - unsigned ElementSize = DL->getTypeStoreSize(AI.getAllocatedType()); + unsigned ElementSize = DL->getTypeAllocSize(AI.getAllocatedType()); unsigned Size = ElementSize * cast(AI.getArraySize())->getZExtValue(); @@ -311,21 +285,20 @@ void IRTranslator::addMachineCFGPred(CFGEdge Edge, MachineBasicBlock *NewPred) { bool IRTranslator::translateBinaryOp(unsigned Opcode, const User &U, MachineIRBuilder &MIRBuilder) { - // FIXME: handle signed/unsigned wrapping flags. - // Get or create a virtual register for each value. // Unless the value is a Constant => loadimm cst? // or inline constant each time? // Creation of a virtual register needs to have a size. - unsigned Op0 = getOrCreateVReg(*U.getOperand(0)); - unsigned Op1 = getOrCreateVReg(*U.getOperand(1)); - unsigned Res = getOrCreateVReg(U); - auto FBinOp = MIRBuilder.buildInstr(Opcode).addDef(Res).addUse(Op0).addUse(Op1); + Register Op0 = getOrCreateVReg(*U.getOperand(0)); + Register Op1 = getOrCreateVReg(*U.getOperand(1)); + Register Res = getOrCreateVReg(U); + uint16_t Flags = 0; if (isa(U)) { - MachineInstr *FBinOpMI = FBinOp.getInstr(); const Instruction &I = cast(U); - FBinOpMI->copyIRFlags(I); + Flags = MachineInstr::copyFlagsFromInstruction(I); } + + MIRBuilder.buildInstr(Opcode, {Res}, {Op0, Op1}, Flags); return true; } @@ -333,27 +306,38 @@ bool IRTranslator::translateFSub(const User &U, MachineIRBuilder &MIRBuilder) { // -0.0 - X --> G_FNEG if (isa(U.getOperand(0)) && U.getOperand(0) == ConstantFP::getZeroValueForNegation(U.getType())) { - MIRBuilder.buildInstr(TargetOpcode::G_FNEG) - .addDef(getOrCreateVReg(U)) - .addUse(getOrCreateVReg(*U.getOperand(1))); + Register Op1 = getOrCreateVReg(*U.getOperand(1)); + Register Res = getOrCreateVReg(U); + uint16_t Flags = 0; + if (isa(U)) { + const Instruction &I = cast(U); + Flags = MachineInstr::copyFlagsFromInstruction(I); + } + // Negate the last operand of the FSUB + MIRBuilder.buildInstr(TargetOpcode::G_FNEG, {Res}, {Op1}, Flags); return true; } return translateBinaryOp(TargetOpcode::G_FSUB, U, MIRBuilder); } bool IRTranslator::translateFNeg(const User &U, MachineIRBuilder &MIRBuilder) { - MIRBuilder.buildInstr(TargetOpcode::G_FNEG) - .addDef(getOrCreateVReg(U)) - .addUse(getOrCreateVReg(*U.getOperand(1))); + Register Op0 = getOrCreateVReg(*U.getOperand(0)); + Register Res = getOrCreateVReg(U); + uint16_t Flags = 0; + if (isa(U)) { + const Instruction &I = cast(U); + Flags = MachineInstr::copyFlagsFromInstruction(I); + } + MIRBuilder.buildInstr(TargetOpcode::G_FNEG, {Res}, {Op0}, Flags); return true; } bool IRTranslator::translateCompare(const User &U, MachineIRBuilder &MIRBuilder) { const CmpInst *CI = dyn_cast(&U); - unsigned Op0 = getOrCreateVReg(*U.getOperand(0)); - unsigned Op1 = getOrCreateVReg(*U.getOperand(1)); - unsigned Res = getOrCreateVReg(U); + Register Op0 = getOrCreateVReg(*U.getOperand(0)); + Register Op1 = getOrCreateVReg(*U.getOperand(1)); + Register Res = getOrCreateVReg(U); CmpInst::Predicate Pred = CI ? CI->getPredicate() : static_cast( cast(U).getPredicate()); @@ -366,8 +350,8 @@ bool IRTranslator::translateCompare(const User &U, MIRBuilder.buildCopy( Res, getOrCreateVReg(*Constant::getAllOnesValue(CI->getType()))); else { - auto FCmp = MIRBuilder.buildFCmp(Pred, Res, Op0, Op1); - FCmp->copyIRFlags(*CI); + MIRBuilder.buildInstr(TargetOpcode::G_FCMP, {Res}, {Pred, Op0, Op1}, + MachineInstr::copyFlagsFromInstruction(*CI)); } return true; @@ -379,15 +363,20 @@ bool IRTranslator::translateRet(const User &U, MachineIRBuilder &MIRBuilder) { if (Ret && DL->getTypeStoreSize(Ret->getType()) == 0) Ret = nullptr; - ArrayRef VRegs; + ArrayRef VRegs; if (Ret) VRegs = getOrCreateVRegs(*Ret); + Register SwiftErrorVReg = 0; + if (CLI->supportSwiftError() && SwiftError.getFunctionArg()) { + SwiftErrorVReg = SwiftError.getOrCreateVRegUseAt( + &RI, &MIRBuilder.getMBB(), SwiftError.getFunctionArg()); + } + // The target may mess up with the insertion point, but // this is not important as a return is the last instruction // of the block anyway. - - return CLI->lowerReturn(MIRBuilder, Ret, VRegs); + return CLI->lowerReturn(MIRBuilder, Ret, VRegs, SwiftErrorVReg); } bool IRTranslator::translateBr(const User &U, MachineIRBuilder &MIRBuilder) { @@ -395,7 +384,7 @@ bool IRTranslator::translateBr(const User &U, MachineIRBuilder &MIRBuilder) { unsigned Succ = 0; if (!BrInst.isUnconditional()) { // We want a G_BRCOND to the true BB followed by an unconditional branch. - unsigned Tst = getOrCreateVReg(*BrInst.getCondition()); + Register Tst = getOrCreateVReg(*BrInst.getCondition()); const BasicBlock &TrueTgt = *cast(BrInst.getSuccessor(Succ++)); MachineBasicBlock &TrueBB = getMBB(TrueTgt); MIRBuilder.buildBrCond(Tst, TrueBB); @@ -415,48 +404,429 @@ bool IRTranslator::translateBr(const User &U, MachineIRBuilder &MIRBuilder) { return true; } -bool IRTranslator::translateSwitch(const User &U, - MachineIRBuilder &MIRBuilder) { - // For now, just translate as a chain of conditional branches. - // FIXME: could we share most of the logic/code in - // SelectionDAGBuilder::visitSwitch between SelectionDAG and GlobalISel? - // At first sight, it seems most of the logic in there is independent of - // SelectionDAG-specifics and a lot of work went in to optimize switch - // lowering in there. - - const SwitchInst &SwInst = cast(U); - const unsigned SwCondValue = getOrCreateVReg(*SwInst.getCondition()); - const BasicBlock *OrigBB = SwInst.getParent(); - - LLT LLTi1 = getLLTForType(*Type::getInt1Ty(U.getContext()), *DL); - for (auto &CaseIt : SwInst.cases()) { - const unsigned CaseValueReg = getOrCreateVReg(*CaseIt.getCaseValue()); - const unsigned Tst = MRI->createGenericVirtualRegister(LLTi1); - MIRBuilder.buildICmp(CmpInst::ICMP_EQ, Tst, CaseValueReg, SwCondValue); - MachineBasicBlock &CurMBB = MIRBuilder.getMBB(); - const BasicBlock *TrueBB = CaseIt.getCaseSuccessor(); - MachineBasicBlock &TrueMBB = getMBB(*TrueBB); - - MIRBuilder.buildBrCond(Tst, TrueMBB); - CurMBB.addSuccessor(&TrueMBB); - addMachineCFGPred({OrigBB, TrueBB}, &CurMBB); - - MachineBasicBlock *FalseMBB = - MF->CreateMachineBasicBlock(SwInst.getParent()); - // Insert the comparison blocks one after the other. - MF->insert(std::next(CurMBB.getIterator()), FalseMBB); - MIRBuilder.buildBr(*FalseMBB); - CurMBB.addSuccessor(FalseMBB); - - MIRBuilder.setMBB(*FalseMBB); - } - // handle default case - const BasicBlock *DefaultBB = SwInst.getDefaultDest(); - MachineBasicBlock &DefaultMBB = getMBB(*DefaultBB); - MIRBuilder.buildBr(DefaultMBB); - MachineBasicBlock &CurMBB = MIRBuilder.getMBB(); - CurMBB.addSuccessor(&DefaultMBB); - addMachineCFGPred({OrigBB, DefaultBB}, &CurMBB); +void IRTranslator::addSuccessorWithProb(MachineBasicBlock *Src, + MachineBasicBlock *Dst, + BranchProbability Prob) { + if (!FuncInfo.BPI) { + Src->addSuccessorWithoutProb(Dst); + return; + } + if (Prob.isUnknown()) + Prob = getEdgeProbability(Src, Dst); + Src->addSuccessor(Dst, Prob); +} + +BranchProbability +IRTranslator::getEdgeProbability(const MachineBasicBlock *Src, + const MachineBasicBlock *Dst) const { + const BasicBlock *SrcBB = Src->getBasicBlock(); + const BasicBlock *DstBB = Dst->getBasicBlock(); + if (!FuncInfo.BPI) { + // If BPI is not available, set the default probability as 1 / N, where N is + // the number of successors. + auto SuccSize = std::max(succ_size(SrcBB), 1); + return BranchProbability(1, SuccSize); + } + return FuncInfo.BPI->getEdgeProbability(SrcBB, DstBB); +} + +bool IRTranslator::translateSwitch(const User &U, MachineIRBuilder &MIB) { + using namespace SwitchCG; + // Extract cases from the switch. + const SwitchInst &SI = cast(U); + BranchProbabilityInfo *BPI = FuncInfo.BPI; + CaseClusterVector Clusters; + Clusters.reserve(SI.getNumCases()); + for (auto &I : SI.cases()) { + MachineBasicBlock *Succ = &getMBB(*I.getCaseSuccessor()); + assert(Succ && "Could not find successor mbb in mapping"); + const ConstantInt *CaseVal = I.getCaseValue(); + BranchProbability Prob = + BPI ? BPI->getEdgeProbability(SI.getParent(), I.getSuccessorIndex()) + : BranchProbability(1, SI.getNumCases() + 1); + Clusters.push_back(CaseCluster::range(CaseVal, CaseVal, Succ, Prob)); + } + + MachineBasicBlock *DefaultMBB = &getMBB(*SI.getDefaultDest()); + + // Cluster adjacent cases with the same destination. We do this at all + // optimization levels because it's cheap to do and will make codegen faster + // if there are many clusters. + sortAndRangeify(Clusters); + + MachineBasicBlock *SwitchMBB = &getMBB(*SI.getParent()); + + // If there is only the default destination, jump there directly. + if (Clusters.empty()) { + SwitchMBB->addSuccessor(DefaultMBB); + if (DefaultMBB != SwitchMBB->getNextNode()) + MIB.buildBr(*DefaultMBB); + return true; + } + + SL->findJumpTables(Clusters, &SI, DefaultMBB); + + LLVM_DEBUG({ + dbgs() << "Case clusters: "; + for (const CaseCluster &C : Clusters) { + if (C.Kind == CC_JumpTable) + dbgs() << "JT:"; + if (C.Kind == CC_BitTests) + dbgs() << "BT:"; + + C.Low->getValue().print(dbgs(), true); + if (C.Low != C.High) { + dbgs() << '-'; + C.High->getValue().print(dbgs(), true); + } + dbgs() << ' '; + } + dbgs() << '\n'; + }); + + assert(!Clusters.empty()); + SwitchWorkList WorkList; + CaseClusterIt First = Clusters.begin(); + CaseClusterIt Last = Clusters.end() - 1; + auto DefaultProb = getEdgeProbability(SwitchMBB, DefaultMBB); + WorkList.push_back({SwitchMBB, First, Last, nullptr, nullptr, DefaultProb}); + + // FIXME: At the moment we don't do any splitting optimizations here like + // SelectionDAG does, so this worklist only has one entry. + while (!WorkList.empty()) { + SwitchWorkListItem W = WorkList.back(); + WorkList.pop_back(); + if (!lowerSwitchWorkItem(W, SI.getCondition(), SwitchMBB, DefaultMBB, MIB)) + return false; + } + return true; +} + +void IRTranslator::emitJumpTable(SwitchCG::JumpTable &JT, + MachineBasicBlock *MBB) { + // Emit the code for the jump table + assert(JT.Reg != -1U && "Should lower JT Header first!"); + MachineIRBuilder MIB(*MBB->getParent()); + MIB.setMBB(*MBB); + MIB.setDebugLoc(CurBuilder->getDebugLoc()); + + Type *PtrIRTy = Type::getInt8PtrTy(MF->getFunction().getContext()); + const LLT PtrTy = getLLTForType(*PtrIRTy, *DL); + + auto Table = MIB.buildJumpTable(PtrTy, JT.JTI); + MIB.buildBrJT(Table.getReg(0), JT.JTI, JT.Reg); +} + +bool IRTranslator::emitJumpTableHeader(SwitchCG::JumpTable &JT, + SwitchCG::JumpTableHeader &JTH, + MachineBasicBlock *HeaderBB) { + MachineIRBuilder MIB(*HeaderBB->getParent()); + MIB.setMBB(*HeaderBB); + MIB.setDebugLoc(CurBuilder->getDebugLoc()); + + const Value &SValue = *JTH.SValue; + // Subtract the lowest switch case value from the value being switched on. + const LLT SwitchTy = getLLTForType(*SValue.getType(), *DL); + Register SwitchOpReg = getOrCreateVReg(SValue); + auto FirstCst = MIB.buildConstant(SwitchTy, JTH.First); + auto Sub = MIB.buildSub({SwitchTy}, SwitchOpReg, FirstCst); + + // This value may be smaller or larger than the target's pointer type, and + // therefore require extension or truncating. + Type *PtrIRTy = SValue.getType()->getPointerTo(); + const LLT PtrScalarTy = LLT::scalar(DL->getTypeSizeInBits(PtrIRTy)); + Sub = MIB.buildZExtOrTrunc(PtrScalarTy, Sub); + + JT.Reg = Sub.getReg(0); + + if (JTH.OmitRangeCheck) { + if (JT.MBB != HeaderBB->getNextNode()) + MIB.buildBr(*JT.MBB); + return true; + } + + // Emit the range check for the jump table, and branch to the default block + // for the switch statement if the value being switched on exceeds the + // largest case in the switch. + auto Cst = getOrCreateVReg( + *ConstantInt::get(SValue.getType(), JTH.Last - JTH.First)); + Cst = MIB.buildZExtOrTrunc(PtrScalarTy, Cst).getReg(0); + auto Cmp = MIB.buildICmp(CmpInst::ICMP_UGT, LLT::scalar(1), Sub, Cst); + + auto BrCond = MIB.buildBrCond(Cmp.getReg(0), *JT.Default); + + // Avoid emitting unnecessary branches to the next block. + if (JT.MBB != HeaderBB->getNextNode()) + BrCond = MIB.buildBr(*JT.MBB); + return true; +} + +void IRTranslator::emitSwitchCase(SwitchCG::CaseBlock &CB, + MachineBasicBlock *SwitchBB, + MachineIRBuilder &MIB) { + Register CondLHS = getOrCreateVReg(*CB.CmpLHS); + Register Cond; + DebugLoc OldDbgLoc = MIB.getDebugLoc(); + MIB.setDebugLoc(CB.DbgLoc); + MIB.setMBB(*CB.ThisBB); + + if (CB.PredInfo.NoCmp) { + // Branch or fall through to TrueBB. + addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb); + addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()}, + CB.ThisBB); + CB.ThisBB->normalizeSuccProbs(); + if (CB.TrueBB != CB.ThisBB->getNextNode()) + MIB.buildBr(*CB.TrueBB); + MIB.setDebugLoc(OldDbgLoc); + return; + } + + const LLT i1Ty = LLT::scalar(1); + // Build the compare. + if (!CB.CmpMHS) { + Register CondRHS = getOrCreateVReg(*CB.CmpRHS); + Cond = MIB.buildICmp(CB.PredInfo.Pred, i1Ty, CondLHS, CondRHS).getReg(0); + } else { + assert(CB.PredInfo.Pred == CmpInst::ICMP_ULE && + "Can only handle ULE ranges"); + + const APInt& Low = cast(CB.CmpLHS)->getValue(); + const APInt& High = cast(CB.CmpRHS)->getValue(); + + Register CmpOpReg = getOrCreateVReg(*CB.CmpMHS); + if (cast(CB.CmpLHS)->isMinValue(true)) { + Register CondRHS = getOrCreateVReg(*CB.CmpRHS); + Cond = + MIB.buildICmp(CmpInst::ICMP_ULE, i1Ty, CmpOpReg, CondRHS).getReg(0); + } else { + const LLT &CmpTy = MRI->getType(CmpOpReg); + auto Sub = MIB.buildSub({CmpTy}, CmpOpReg, CondLHS); + auto Diff = MIB.buildConstant(CmpTy, High - Low); + Cond = MIB.buildICmp(CmpInst::ICMP_ULE, i1Ty, Sub, Diff).getReg(0); + } + } + + // Update successor info + addSuccessorWithProb(CB.ThisBB, CB.TrueBB, CB.TrueProb); + + addMachineCFGPred({SwitchBB->getBasicBlock(), CB.TrueBB->getBasicBlock()}, + CB.ThisBB); + + // TrueBB and FalseBB are always different unless the incoming IR is + // degenerate. This only happens when running llc on weird IR. + if (CB.TrueBB != CB.FalseBB) + addSuccessorWithProb(CB.ThisBB, CB.FalseBB, CB.FalseProb); + CB.ThisBB->normalizeSuccProbs(); + + // if (SwitchBB->getBasicBlock() != CB.FalseBB->getBasicBlock()) + addMachineCFGPred({SwitchBB->getBasicBlock(), CB.FalseBB->getBasicBlock()}, + CB.ThisBB); + + // If the lhs block is the next block, invert the condition so that we can + // fall through to the lhs instead of the rhs block. + if (CB.TrueBB == CB.ThisBB->getNextNode()) { + std::swap(CB.TrueBB, CB.FalseBB); + auto True = MIB.buildConstant(i1Ty, 1); + Cond = MIB.buildInstr(TargetOpcode::G_XOR, {i1Ty}, {Cond, True}, None) + .getReg(0); + } + + MIB.buildBrCond(Cond, *CB.TrueBB); + MIB.buildBr(*CB.FalseBB); + MIB.setDebugLoc(OldDbgLoc); +} + +bool IRTranslator::lowerJumpTableWorkItem(SwitchCG::SwitchWorkListItem W, + MachineBasicBlock *SwitchMBB, + MachineBasicBlock *CurMBB, + MachineBasicBlock *DefaultMBB, + MachineIRBuilder &MIB, + MachineFunction::iterator BBI, + BranchProbability UnhandledProbs, + SwitchCG::CaseClusterIt I, + MachineBasicBlock *Fallthrough, + bool FallthroughUnreachable) { + using namespace SwitchCG; + MachineFunction *CurMF = SwitchMBB->getParent(); + // FIXME: Optimize away range check based on pivot comparisons. + JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first; + SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second; + BranchProbability DefaultProb = W.DefaultProb; + + // The jump block hasn't been inserted yet; insert it here. + MachineBasicBlock *JumpMBB = JT->MBB; + CurMF->insert(BBI, JumpMBB); + + // Since the jump table block is separate from the switch block, we need + // to keep track of it as a machine predecessor to the default block, + // otherwise we lose the phi edges. + addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()}, + CurMBB); + addMachineCFGPred({SwitchMBB->getBasicBlock(), DefaultMBB->getBasicBlock()}, + JumpMBB); + + auto JumpProb = I->Prob; + auto FallthroughProb = UnhandledProbs; + + // If the default statement is a target of the jump table, we evenly + // distribute the default probability to successors of CurMBB. Also + // update the probability on the edge from JumpMBB to Fallthrough. + for (MachineBasicBlock::succ_iterator SI = JumpMBB->succ_begin(), + SE = JumpMBB->succ_end(); + SI != SE; ++SI) { + if (*SI == DefaultMBB) { + JumpProb += DefaultProb / 2; + FallthroughProb -= DefaultProb / 2; + JumpMBB->setSuccProbability(SI, DefaultProb / 2); + JumpMBB->normalizeSuccProbs(); + } else { + // Also record edges from the jump table block to it's successors. + addMachineCFGPred({SwitchMBB->getBasicBlock(), (*SI)->getBasicBlock()}, + JumpMBB); + } + } + + // Skip the range check if the fallthrough block is unreachable. + if (FallthroughUnreachable) + JTH->OmitRangeCheck = true; + + if (!JTH->OmitRangeCheck) + addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb); + addSuccessorWithProb(CurMBB, JumpMBB, JumpProb); + CurMBB->normalizeSuccProbs(); + + // The jump table header will be inserted in our current block, do the + // range check, and fall through to our fallthrough block. + JTH->HeaderBB = CurMBB; + JT->Default = Fallthrough; // FIXME: Move Default to JumpTableHeader. + + // If we're in the right place, emit the jump table header right now. + if (CurMBB == SwitchMBB) { + if (!emitJumpTableHeader(*JT, *JTH, CurMBB)) + return false; + JTH->Emitted = true; + } + return true; +} +bool IRTranslator::lowerSwitchRangeWorkItem(SwitchCG::CaseClusterIt I, + Value *Cond, + MachineBasicBlock *Fallthrough, + bool FallthroughUnreachable, + BranchProbability UnhandledProbs, + MachineBasicBlock *CurMBB, + MachineIRBuilder &MIB, + MachineBasicBlock *SwitchMBB) { + using namespace SwitchCG; + const Value *RHS, *LHS, *MHS; + CmpInst::Predicate Pred; + if (I->Low == I->High) { + // Check Cond == I->Low. + Pred = CmpInst::ICMP_EQ; + LHS = Cond; + RHS = I->Low; + MHS = nullptr; + } else { + // Check I->Low <= Cond <= I->High. + Pred = CmpInst::ICMP_ULE; + LHS = I->Low; + MHS = Cond; + RHS = I->High; + } + + // If Fallthrough is unreachable, fold away the comparison. + // The false probability is the sum of all unhandled cases. + CaseBlock CB(Pred, FallthroughUnreachable, LHS, RHS, MHS, I->MBB, Fallthrough, + CurMBB, MIB.getDebugLoc(), I->Prob, UnhandledProbs); + + emitSwitchCase(CB, SwitchMBB, MIB); + return true; +} + +bool IRTranslator::lowerSwitchWorkItem(SwitchCG::SwitchWorkListItem W, + Value *Cond, + MachineBasicBlock *SwitchMBB, + MachineBasicBlock *DefaultMBB, + MachineIRBuilder &MIB) { + using namespace SwitchCG; + MachineFunction *CurMF = FuncInfo.MF; + MachineBasicBlock *NextMBB = nullptr; + MachineFunction::iterator BBI(W.MBB); + if (++BBI != FuncInfo.MF->end()) + NextMBB = &*BBI; + + if (EnableOpts) { + // Here, we order cases by probability so the most likely case will be + // checked first. However, two clusters can have the same probability in + // which case their relative ordering is non-deterministic. So we use Low + // as a tie-breaker as clusters are guaranteed to never overlap. + llvm::sort(W.FirstCluster, W.LastCluster + 1, + [](const CaseCluster &a, const CaseCluster &b) { + return a.Prob != b.Prob + ? a.Prob > b.Prob + : a.Low->getValue().slt(b.Low->getValue()); + }); + + // Rearrange the case blocks so that the last one falls through if possible + // without changing the order of probabilities. + for (CaseClusterIt I = W.LastCluster; I > W.FirstCluster;) { + --I; + if (I->Prob > W.LastCluster->Prob) + break; + if (I->Kind == CC_Range && I->MBB == NextMBB) { + std::swap(*I, *W.LastCluster); + break; + } + } + } + + // Compute total probability. + BranchProbability DefaultProb = W.DefaultProb; + BranchProbability UnhandledProbs = DefaultProb; + for (CaseClusterIt I = W.FirstCluster; I <= W.LastCluster; ++I) + UnhandledProbs += I->Prob; + + MachineBasicBlock *CurMBB = W.MBB; + for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) { + bool FallthroughUnreachable = false; + MachineBasicBlock *Fallthrough; + if (I == W.LastCluster) { + // For the last cluster, fall through to the default destination. + Fallthrough = DefaultMBB; + FallthroughUnreachable = isa( + DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg()); + } else { + Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock()); + CurMF->insert(BBI, Fallthrough); + } + UnhandledProbs -= I->Prob; + + switch (I->Kind) { + case CC_BitTests: { + LLVM_DEBUG(dbgs() << "Switch to bit test optimization unimplemented"); + return false; // Bit tests currently unimplemented. + } + case CC_JumpTable: { + if (!lowerJumpTableWorkItem(W, SwitchMBB, CurMBB, DefaultMBB, MIB, BBI, + UnhandledProbs, I, Fallthrough, + FallthroughUnreachable)) { + LLVM_DEBUG(dbgs() << "Failed to lower jump table"); + return false; + } + break; + } + case CC_Range: { + if (!lowerSwitchRangeWorkItem(I, Cond, Fallthrough, + FallthroughUnreachable, UnhandledProbs, + CurMBB, MIB, SwitchMBB)) { + LLVM_DEBUG(dbgs() << "Failed to lower switch range"); + return false; + } + break; + } + } + CurMBB = Fallthrough; + } return true; } @@ -465,7 +835,7 @@ bool IRTranslator::translateIndirectBr(const User &U, MachineIRBuilder &MIRBuilder) { const IndirectBrInst &BrInst = cast(U); - const unsigned Tgt = getOrCreateVReg(*BrInst.getAddress()); + const Register Tgt = getOrCreateVReg(*BrInst.getAddress()); MIRBuilder.buildBrIndirect(Tgt); // Link successors. @@ -476,6 +846,14 @@ bool IRTranslator::translateIndirectBr(const User &U, return true; } +static bool isSwiftError(const Value *V) { + if (auto Arg = dyn_cast(V)) + return Arg->hasSwiftErrorAttr(); + if (auto AI = dyn_cast(V)) + return AI->isSwiftError(); + return false; +} + bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) { const LoadInst &LI = cast(U); @@ -486,13 +864,25 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) { if (DL->getTypeStoreSize(LI.getType()) == 0) return true; - ArrayRef Regs = getOrCreateVRegs(LI); + ArrayRef Regs = getOrCreateVRegs(LI); ArrayRef Offsets = *VMap.getOffsets(LI); - unsigned Base = getOrCreateVReg(*LI.getPointerOperand()); + Register Base = getOrCreateVReg(*LI.getPointerOperand()); + + Type *OffsetIRTy = DL->getIntPtrType(LI.getPointerOperandType()); + LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL); + + if (CLI->supportSwiftError() && isSwiftError(LI.getPointerOperand())) { + assert(Regs.size() == 1 && "swifterror should be single pointer"); + Register VReg = SwiftError.getOrCreateVRegUseAt(&LI, &MIRBuilder.getMBB(), + LI.getPointerOperand()); + MIRBuilder.buildCopy(Regs[0], VReg); + return true; + } + for (unsigned i = 0; i < Regs.size(); ++i) { - unsigned Addr = 0; - MIRBuilder.materializeGEP(Addr, Base, LLT::scalar(64), Offsets[i] / 8); + Register Addr; + MIRBuilder.materializeGEP(Addr, Base, OffsetTy, Offsets[i] / 8); MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i] / 8); unsigned BaseAlign = getMemOpAlignment(LI); @@ -515,13 +905,25 @@ bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) { if (DL->getTypeStoreSize(SI.getValueOperand()->getType()) == 0) return true; - ArrayRef Vals = getOrCreateVRegs(*SI.getValueOperand()); + ArrayRef Vals = getOrCreateVRegs(*SI.getValueOperand()); ArrayRef Offsets = *VMap.getOffsets(*SI.getValueOperand()); - unsigned Base = getOrCreateVReg(*SI.getPointerOperand()); + Register Base = getOrCreateVReg(*SI.getPointerOperand()); + + Type *OffsetIRTy = DL->getIntPtrType(SI.getPointerOperandType()); + LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL); + + if (CLI->supportSwiftError() && isSwiftError(SI.getPointerOperand())) { + assert(Vals.size() == 1 && "swifterror should be single pointer"); + + Register VReg = SwiftError.getOrCreateVRegDefAt(&SI, &MIRBuilder.getMBB(), + SI.getPointerOperand()); + MIRBuilder.buildCopy(VReg, Vals[0]); + return true; + } for (unsigned i = 0; i < Vals.size(); ++i) { - unsigned Addr = 0; - MIRBuilder.materializeGEP(Addr, Base, LLT::scalar(64), Offsets[i] / 8); + Register Addr; + MIRBuilder.materializeGEP(Addr, Base, OffsetTy, Offsets[i] / 8); MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i] / 8); unsigned BaseAlign = getMemOpAlignment(SI); @@ -562,10 +964,9 @@ bool IRTranslator::translateExtractValue(const User &U, MachineIRBuilder &MIRBuilder) { const Value *Src = U.getOperand(0); uint64_t Offset = getOffsetFromIndices(U, *DL); - ArrayRef SrcRegs = getOrCreateVRegs(*Src); + ArrayRef SrcRegs = getOrCreateVRegs(*Src); ArrayRef Offsets = *VMap.getOffsets(*Src); - unsigned Idx = std::lower_bound(Offsets.begin(), Offsets.end(), Offset) - - Offsets.begin(); + unsigned Idx = llvm::lower_bound(Offsets, Offset) - Offsets.begin(); auto &DstRegs = allocateVRegs(U); for (unsigned i = 0; i < DstRegs.size(); ++i) @@ -580,8 +981,8 @@ bool IRTranslator::translateInsertValue(const User &U, uint64_t Offset = getOffsetFromIndices(U, *DL); auto &DstRegs = allocateVRegs(U); ArrayRef DstOffsets = *VMap.getOffsets(U); - ArrayRef SrcRegs = getOrCreateVRegs(*Src); - ArrayRef InsertedRegs = getOrCreateVRegs(*U.getOperand(1)); + ArrayRef SrcRegs = getOrCreateVRegs(*Src); + ArrayRef InsertedRegs = getOrCreateVRegs(*U.getOperand(1)); auto InsertedIt = InsertedRegs.begin(); for (unsigned i = 0; i < DstRegs.size(); ++i) { @@ -596,19 +997,19 @@ bool IRTranslator::translateInsertValue(const User &U, bool IRTranslator::translateSelect(const User &U, MachineIRBuilder &MIRBuilder) { - unsigned Tst = getOrCreateVReg(*U.getOperand(0)); - ArrayRef ResRegs = getOrCreateVRegs(U); - ArrayRef Op0Regs = getOrCreateVRegs(*U.getOperand(1)); - ArrayRef Op1Regs = getOrCreateVRegs(*U.getOperand(2)); + Register Tst = getOrCreateVReg(*U.getOperand(0)); + ArrayRef ResRegs = getOrCreateVRegs(U); + ArrayRef Op0Regs = getOrCreateVRegs(*U.getOperand(1)); + ArrayRef Op1Regs = getOrCreateVRegs(*U.getOperand(2)); const SelectInst &SI = cast(U); - const CmpInst *Cmp = dyn_cast(SI.getCondition()); + uint16_t Flags = 0; + if (const CmpInst *Cmp = dyn_cast(SI.getCondition())) + Flags = MachineInstr::copyFlagsFromInstruction(*Cmp); + for (unsigned i = 0; i < ResRegs.size(); ++i) { - auto Select = - MIRBuilder.buildSelect(ResRegs[i], Tst, Op0Regs[i], Op1Regs[i]); - if (Cmp && isa(Cmp)) { - Select->copyIRFlags(*Cmp); - } + MIRBuilder.buildInstr(TargetOpcode::G_SELECT, {ResRegs[i]}, + {Tst, Op0Regs[i], Op1Regs[i]}, Flags); } return true; @@ -619,7 +1020,7 @@ bool IRTranslator::translateBitCast(const User &U, // If we're bitcasting to the source type, we can reuse the source vreg. if (getLLTForType(*U.getOperand(0)->getType(), *DL) == getLLTForType(*U.getType(), *DL)) { - unsigned SrcReg = getOrCreateVReg(*U.getOperand(0)); + Register SrcReg = getOrCreateVReg(*U.getOperand(0)); auto &Regs = *VMap.getVRegs(U); // If we already assigned a vreg for this bitcast, we can't change that. // Emit a copy to satisfy the users we already emitted. @@ -636,9 +1037,9 @@ bool IRTranslator::translateBitCast(const User &U, bool IRTranslator::translateCast(unsigned Opcode, const User &U, MachineIRBuilder &MIRBuilder) { - unsigned Op = getOrCreateVReg(*U.getOperand(0)); - unsigned Res = getOrCreateVReg(U); - MIRBuilder.buildInstr(Opcode).addDef(Res).addUse(Op); + Register Op = getOrCreateVReg(*U.getOperand(0)); + Register Res = getOrCreateVReg(U); + MIRBuilder.buildInstr(Opcode, {Res}, {Op}); return true; } @@ -649,7 +1050,7 @@ bool IRTranslator::translateGetElementPtr(const User &U, return false; Value &Op0 = *U.getOperand(0); - unsigned BaseReg = getOrCreateVReg(Op0); + Register BaseReg = getOrCreateVReg(Op0); Type *PtrIRTy = Op0.getType(); LLT PtrTy = getLLTForType(*PtrIRTy, *DL); Type *OffsetIRTy = DL->getIntPtrType(PtrIRTy); @@ -674,43 +1075,43 @@ bool IRTranslator::translateGetElementPtr(const User &U, } if (Offset != 0) { - unsigned NewBaseReg = MRI->createGenericVirtualRegister(PtrTy); - unsigned OffsetReg = - getOrCreateVReg(*ConstantInt::get(OffsetIRTy, Offset)); - MIRBuilder.buildGEP(NewBaseReg, BaseReg, OffsetReg); + Register NewBaseReg = MRI->createGenericVirtualRegister(PtrTy); + LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL); + auto OffsetMIB = MIRBuilder.buildConstant({OffsetTy}, Offset); + MIRBuilder.buildGEP(NewBaseReg, BaseReg, OffsetMIB.getReg(0)); BaseReg = NewBaseReg; Offset = 0; } - unsigned IdxReg = getOrCreateVReg(*Idx); + Register IdxReg = getOrCreateVReg(*Idx); if (MRI->getType(IdxReg) != OffsetTy) { - unsigned NewIdxReg = MRI->createGenericVirtualRegister(OffsetTy); + Register NewIdxReg = MRI->createGenericVirtualRegister(OffsetTy); MIRBuilder.buildSExtOrTrunc(NewIdxReg, IdxReg); IdxReg = NewIdxReg; } // N = N + Idx * ElementSize; // Avoid doing it for ElementSize of 1. - unsigned GepOffsetReg; + Register GepOffsetReg; if (ElementSize != 1) { - unsigned ElementSizeReg = - getOrCreateVReg(*ConstantInt::get(OffsetIRTy, ElementSize)); - GepOffsetReg = MRI->createGenericVirtualRegister(OffsetTy); - MIRBuilder.buildMul(GepOffsetReg, ElementSizeReg, IdxReg); + auto ElementSizeMIB = MIRBuilder.buildConstant( + getLLTForType(*OffsetIRTy, *DL), ElementSize); + MIRBuilder.buildMul(GepOffsetReg, ElementSizeMIB.getReg(0), IdxReg); } else GepOffsetReg = IdxReg; - unsigned NewBaseReg = MRI->createGenericVirtualRegister(PtrTy); + Register NewBaseReg = MRI->createGenericVirtualRegister(PtrTy); MIRBuilder.buildGEP(NewBaseReg, BaseReg, GepOffsetReg); BaseReg = NewBaseReg; } } if (Offset != 0) { - unsigned OffsetReg = getOrCreateVReg(*ConstantInt::get(OffsetIRTy, Offset)); - MIRBuilder.buildGEP(getOrCreateVReg(U), BaseReg, OffsetReg); + auto OffsetMIB = + MIRBuilder.buildConstant(getLLTForType(*OffsetIRTy, *DL), Offset); + MIRBuilder.buildGEP(getOrCreateVReg(U), BaseReg, OffsetMIB.getReg(0)); return true; } @@ -721,6 +1122,19 @@ bool IRTranslator::translateGetElementPtr(const User &U, bool IRTranslator::translateMemfunc(const CallInst &CI, MachineIRBuilder &MIRBuilder, unsigned ID) { + + // If the source is undef, then just emit a nop. + if (isa(CI.getArgOperand(1))) { + switch (ID) { + case Intrinsic::memmove: + case Intrinsic::memcpy: + case Intrinsic::memset: + return true; + default: + break; + } + } + LLT SizeTy = getLLTForType(*CI.getArgOperand(2)->getType(), *DL); Type *DstTy = CI.getArgOperand(0)->getType(); if (cast(DstTy)->getAddressSpace() != 0 || @@ -752,10 +1166,10 @@ bool IRTranslator::translateMemfunc(const CallInst &CI, return CLI->lowerCall(MIRBuilder, CI.getCallingConv(), MachineOperand::CreateES(Callee), - CallLowering::ArgInfo(0, CI.getType()), Args); + CallLowering::ArgInfo({0}, CI.getType()), Args); } -void IRTranslator::getStackGuard(unsigned DstReg, +void IRTranslator::getStackGuard(Register DstReg, MachineIRBuilder &MIRBuilder) { const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo(); MRI->setRegClass(DstReg, TRI->getPointerRegClass(*MF)); @@ -778,7 +1192,7 @@ void IRTranslator::getStackGuard(unsigned DstReg, bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op, MachineIRBuilder &MIRBuilder) { - ArrayRef ResRegs = getOrCreateVRegs(CI); + ArrayRef ResRegs = getOrCreateVRegs(CI); MIRBuilder.buildInstr(Op) .addDef(ResRegs[0]) .addDef(ResRegs[1]) @@ -788,19 +1202,123 @@ bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op, return true; } +unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) { + switch (ID) { + default: + break; + case Intrinsic::bswap: + return TargetOpcode::G_BSWAP; + case Intrinsic::ceil: + return TargetOpcode::G_FCEIL; + case Intrinsic::cos: + return TargetOpcode::G_FCOS; + case Intrinsic::ctpop: + return TargetOpcode::G_CTPOP; + case Intrinsic::exp: + return TargetOpcode::G_FEXP; + case Intrinsic::exp2: + return TargetOpcode::G_FEXP2; + case Intrinsic::fabs: + return TargetOpcode::G_FABS; + case Intrinsic::copysign: + return TargetOpcode::G_FCOPYSIGN; + case Intrinsic::minnum: + return TargetOpcode::G_FMINNUM; + case Intrinsic::maxnum: + return TargetOpcode::G_FMAXNUM; + case Intrinsic::minimum: + return TargetOpcode::G_FMINIMUM; + case Intrinsic::maximum: + return TargetOpcode::G_FMAXIMUM; + case Intrinsic::canonicalize: + return TargetOpcode::G_FCANONICALIZE; + case Intrinsic::floor: + return TargetOpcode::G_FFLOOR; + case Intrinsic::fma: + return TargetOpcode::G_FMA; + case Intrinsic::log: + return TargetOpcode::G_FLOG; + case Intrinsic::log2: + return TargetOpcode::G_FLOG2; + case Intrinsic::log10: + return TargetOpcode::G_FLOG10; + case Intrinsic::nearbyint: + return TargetOpcode::G_FNEARBYINT; + case Intrinsic::pow: + return TargetOpcode::G_FPOW; + case Intrinsic::rint: + return TargetOpcode::G_FRINT; + case Intrinsic::round: + return TargetOpcode::G_INTRINSIC_ROUND; + case Intrinsic::sin: + return TargetOpcode::G_FSIN; + case Intrinsic::sqrt: + return TargetOpcode::G_FSQRT; + case Intrinsic::trunc: + return TargetOpcode::G_INTRINSIC_TRUNC; + } + return Intrinsic::not_intrinsic; +} + +bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI, + Intrinsic::ID ID, + MachineIRBuilder &MIRBuilder) { + + unsigned Op = getSimpleIntrinsicOpcode(ID); + + // Is this a simple intrinsic? + if (Op == Intrinsic::not_intrinsic) + return false; + + // Yes. Let's translate it. + SmallVector VRegs; + for (auto &Arg : CI.arg_operands()) + VRegs.push_back(getOrCreateVReg(*Arg)); + + MIRBuilder.buildInstr(Op, {getOrCreateVReg(CI)}, VRegs, + MachineInstr::copyFlagsFromInstruction(CI)); + return true; +} + bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) { + + // If this is a simple intrinsic (that is, we just need to add a def of + // a vreg, and uses for each arg operand, then translate it. + if (translateSimpleIntrinsic(CI, ID, MIRBuilder)) + return true; + switch (ID) { default: break; case Intrinsic::lifetime_start: - case Intrinsic::lifetime_end: - // Stack coloring is not enabled in O0 (which we care about now) so we can - // drop these. Make sure someone notices when we start compiling at higher - // opts though. - if (MF->getTarget().getOptLevel() != CodeGenOpt::None) - return false; + case Intrinsic::lifetime_end: { + // No stack colouring in O0, discard region information. + if (MF->getTarget().getOptLevel() == CodeGenOpt::None) + return true; + + unsigned Op = ID == Intrinsic::lifetime_start ? TargetOpcode::LIFETIME_START + : TargetOpcode::LIFETIME_END; + + // Get the underlying objects for the location passed on the lifetime + // marker. + SmallVector Allocas; + GetUnderlyingObjects(CI.getArgOperand(1), Allocas, *DL); + + // Iterate over each underlying object, creating lifetime markers for each + // static alloca. Quit if we find a non-static alloca. + for (const Value *V : Allocas) { + const AllocaInst *AI = dyn_cast(V); + if (!AI) + continue; + + if (!AI->isStaticAlloca()) + return true; + + MIRBuilder.buildInstr(Op).addFrameIndex(getOrCreateFrameIndex(*AI)); + } return true; + } case Intrinsic::dbg_declare: { const DbgDeclareInst &DI = cast(CI); assert(DI.getVariable() && "Missing variable"); @@ -848,10 +1366,11 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, Value *Ptr = CI.getArgOperand(0); unsigned ListSize = TLI.getVaListSizeInBits(*DL) / 8; + // FIXME: Get alignment MIRBuilder.buildInstr(TargetOpcode::G_VASTART) .addUse(getOrCreateVReg(*Ptr)) .addMemOperand(MF->getMachineMemOperand( - MachinePointerInfo(Ptr), MachineMemOperand::MOStore, ListSize, 0)); + MachinePointerInfo(Ptr), MachineMemOperand::MOStore, ListSize, 1)); return true; } case Intrinsic::dbg_value: { @@ -868,7 +1387,7 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, } else if (const auto *CI = dyn_cast(V)) { MIRBuilder.buildConstDbgValue(*CI, DI.getVariable(), DI.getExpression()); } else { - unsigned Reg = getOrCreateVReg(*V); + Register Reg = getOrCreateVReg(*V); // FIXME: This does not handle register-indirect values at offset 0. The // direct/indirect thing shouldn't really be handled by something as // implicit as reg+noreg vs reg+imm in the first palce, but it seems @@ -889,94 +1408,25 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, return translateOverflowIntrinsic(CI, TargetOpcode::G_UMULO, MIRBuilder); case Intrinsic::smul_with_overflow: return translateOverflowIntrinsic(CI, TargetOpcode::G_SMULO, MIRBuilder); - case Intrinsic::pow: { - auto Pow = MIRBuilder.buildInstr(TargetOpcode::G_FPOW) - .addDef(getOrCreateVReg(CI)) - .addUse(getOrCreateVReg(*CI.getArgOperand(0))) - .addUse(getOrCreateVReg(*CI.getArgOperand(1))); - Pow->copyIRFlags(CI); - return true; - } - case Intrinsic::exp: { - auto Exp = MIRBuilder.buildInstr(TargetOpcode::G_FEXP) - .addDef(getOrCreateVReg(CI)) - .addUse(getOrCreateVReg(*CI.getArgOperand(0))); - Exp->copyIRFlags(CI); - return true; - } - case Intrinsic::exp2: { - auto Exp2 = MIRBuilder.buildInstr(TargetOpcode::G_FEXP2) - .addDef(getOrCreateVReg(CI)) - .addUse(getOrCreateVReg(*CI.getArgOperand(0))); - Exp2->copyIRFlags(CI); - return true; - } - case Intrinsic::log: { - auto Log = MIRBuilder.buildInstr(TargetOpcode::G_FLOG) - .addDef(getOrCreateVReg(CI)) - .addUse(getOrCreateVReg(*CI.getArgOperand(0))); - Log->copyIRFlags(CI); - return true; - } - case Intrinsic::log2: { - auto Log2 = MIRBuilder.buildInstr(TargetOpcode::G_FLOG2) - .addDef(getOrCreateVReg(CI)) - .addUse(getOrCreateVReg(*CI.getArgOperand(0))); - Log2->copyIRFlags(CI); - return true; - } - case Intrinsic::log10: { - auto Log10 = MIRBuilder.buildInstr(TargetOpcode::G_FLOG10) - .addDef(getOrCreateVReg(CI)) - .addUse(getOrCreateVReg(*CI.getArgOperand(0))); - Log10->copyIRFlags(CI); - return true; - } - case Intrinsic::fabs: { - auto Fabs = MIRBuilder.buildInstr(TargetOpcode::G_FABS) - .addDef(getOrCreateVReg(CI)) - .addUse(getOrCreateVReg(*CI.getArgOperand(0))); - Fabs->copyIRFlags(CI); - return true; - } - case Intrinsic::trunc: - MIRBuilder.buildInstr(TargetOpcode::G_INTRINSIC_TRUNC) - .addDef(getOrCreateVReg(CI)) - .addUse(getOrCreateVReg(*CI.getArgOperand(0))); - return true; - case Intrinsic::round: - MIRBuilder.buildInstr(TargetOpcode::G_INTRINSIC_ROUND) - .addDef(getOrCreateVReg(CI)) - .addUse(getOrCreateVReg(*CI.getArgOperand(0))); - return true; - case Intrinsic::fma: { - auto FMA = MIRBuilder.buildInstr(TargetOpcode::G_FMA) - .addDef(getOrCreateVReg(CI)) - .addUse(getOrCreateVReg(*CI.getArgOperand(0))) - .addUse(getOrCreateVReg(*CI.getArgOperand(1))) - .addUse(getOrCreateVReg(*CI.getArgOperand(2))); - FMA->copyIRFlags(CI); - return true; - } case Intrinsic::fmuladd: { const TargetMachine &TM = MF->getTarget(); const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering(); - unsigned Dst = getOrCreateVReg(CI); - unsigned Op0 = getOrCreateVReg(*CI.getArgOperand(0)); - unsigned Op1 = getOrCreateVReg(*CI.getArgOperand(1)); - unsigned Op2 = getOrCreateVReg(*CI.getArgOperand(2)); + Register Dst = getOrCreateVReg(CI); + Register Op0 = getOrCreateVReg(*CI.getArgOperand(0)); + Register Op1 = getOrCreateVReg(*CI.getArgOperand(1)); + Register Op2 = getOrCreateVReg(*CI.getArgOperand(2)); if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict && TLI.isFMAFasterThanFMulAndFAdd(TLI.getValueType(*DL, CI.getType()))) { // TODO: Revisit this to see if we should move this part of the // lowering to the combiner. - auto FMA = MIRBuilder.buildInstr(TargetOpcode::G_FMA, {Dst}, {Op0, Op1, Op2}); - FMA->copyIRFlags(CI); + MIRBuilder.buildInstr(TargetOpcode::G_FMA, {Dst}, {Op0, Op1, Op2}, + MachineInstr::copyFlagsFromInstruction(CI)); } else { LLT Ty = getLLTForType(*CI.getType(), *DL); - auto FMul = MIRBuilder.buildInstr(TargetOpcode::G_FMUL, {Ty}, {Op0, Op1}); - FMul->copyIRFlags(CI); - auto FAdd = MIRBuilder.buildInstr(TargetOpcode::G_FADD, {Dst}, {FMul, Op2}); - FAdd->copyIRFlags(CI); + auto FMul = MIRBuilder.buildInstr(TargetOpcode::G_FMUL, {Ty}, {Op0, Op1}, + MachineInstr::copyFlagsFromInstruction(CI)); + MIRBuilder.buildInstr(TargetOpcode::G_FADD, {Dst}, {FMul, Op2}, + MachineInstr::copyFlagsFromInstruction(CI)); } return true; } @@ -986,7 +1436,7 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, return translateMemfunc(CI, MIRBuilder, ID); case Intrinsic::eh_typeid_for: { GlobalValue *GV = ExtractTypeInfo(CI.getArgOperand(0)); - unsigned Reg = getOrCreateVReg(CI); + Register Reg = getOrCreateVReg(CI); unsigned TypeID = MF->getTypeIDFor(GV); MIRBuilder.buildConstant(Reg, TypeID); return true; @@ -1008,7 +1458,7 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, return true; case Intrinsic::stackprotector: { LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL); - unsigned GuardVal = MRI->createGenericVirtualRegister(PtrTy); + Register GuardVal = MRI->createGenericVirtualRegister(PtrTy); getStackGuard(GuardVal, MIRBuilder); AllocaInst *Slot = cast(CI.getArgOperand(1)); @@ -1023,6 +1473,34 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, PtrTy.getSizeInBits() / 8, 8)); return true; } + case Intrinsic::stacksave: { + // Save the stack pointer to the location provided by the intrinsic. + Register Reg = getOrCreateVReg(CI); + Register StackPtr = MF->getSubtarget() + .getTargetLowering() + ->getStackPointerRegisterToSaveRestore(); + + // If the target doesn't specify a stack pointer, then fall back. + if (!StackPtr) + return false; + + MIRBuilder.buildCopy(Reg, StackPtr); + return true; + } + case Intrinsic::stackrestore: { + // Restore the stack pointer from the location provided by the intrinsic. + Register Reg = getOrCreateVReg(*CI.getArgOperand(0)); + Register StackPtr = MF->getSubtarget() + .getTargetLowering() + ->getStackPointerRegisterToSaveRestore(); + + // If the target doesn't specify a stack pointer, then fall back. + if (!StackPtr) + return false; + + MIRBuilder.buildCopy(StackPtr, Reg); + return true; + } case Intrinsic::cttz: case Intrinsic::ctlz: { ConstantInt *Cst = cast(CI.getArgOperand(1)); @@ -1037,24 +1515,18 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, .addUse(getOrCreateVReg(*CI.getArgOperand(0))); return true; } - case Intrinsic::ctpop: { - MIRBuilder.buildInstr(TargetOpcode::G_CTPOP) - .addDef(getOrCreateVReg(CI)) - .addUse(getOrCreateVReg(*CI.getArgOperand(0))); - return true; - } case Intrinsic::invariant_start: { LLT PtrTy = getLLTForType(*CI.getArgOperand(0)->getType(), *DL); - unsigned Undef = MRI->createGenericVirtualRegister(PtrTy); + Register Undef = MRI->createGenericVirtualRegister(PtrTy); MIRBuilder.buildUndef(Undef); return true; } case Intrinsic::invariant_end: return true; - case Intrinsic::ceil: - MIRBuilder.buildInstr(TargetOpcode::G_FCEIL) - .addDef(getOrCreateVReg(CI)) - .addUse(getOrCreateVReg(*CI.getArgOperand(0))); + case Intrinsic::assume: + case Intrinsic::var_annotation: + case Intrinsic::sideeffect: + // Discard annotate attributes, assumptions, and artificial side-effects. return true; } return false; @@ -1079,34 +1551,6 @@ bool IRTranslator::translateInlineAsm(const CallInst &CI, return true; } -unsigned IRTranslator::packRegs(const Value &V, - MachineIRBuilder &MIRBuilder) { - ArrayRef Regs = getOrCreateVRegs(V); - ArrayRef Offsets = *VMap.getOffsets(V); - LLT BigTy = getLLTForType(*V.getType(), *DL); - - if (Regs.size() == 1) - return Regs[0]; - - unsigned Dst = MRI->createGenericVirtualRegister(BigTy); - MIRBuilder.buildUndef(Dst); - for (unsigned i = 0; i < Regs.size(); ++i) { - unsigned NewDst = MRI->createGenericVirtualRegister(BigTy); - MIRBuilder.buildInsert(NewDst, Dst, Regs[i], Offsets[i]); - Dst = NewDst; - } - return Dst; -} - -void IRTranslator::unpackRegs(const Value &V, unsigned Src, - MachineIRBuilder &MIRBuilder) { - ArrayRef Regs = getOrCreateVRegs(V); - ArrayRef Offsets = *VMap.getOffsets(V); - - for (unsigned i = 0; i < Regs.size(); ++i) - MIRBuilder.buildExtract(Regs[i], Src, Offsets[i]); -} - bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) { const CallInst &CI = cast(U); auto TII = MF->getTarget().getIntrinsicInfo(); @@ -1126,23 +1570,32 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) { ID = static_cast(TII->getIntrinsicID(F)); } - bool IsSplitType = valueIsSplit(CI); if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic) { - unsigned Res = IsSplitType ? MRI->createGenericVirtualRegister( - getLLTForType(*CI.getType(), *DL)) - : getOrCreateVReg(CI); - - SmallVector Args; - for (auto &Arg: CI.arg_operands()) - Args.push_back(packRegs(*Arg, MIRBuilder)); + ArrayRef Res = getOrCreateVRegs(CI); + + SmallVector, 8> Args; + Register SwiftInVReg = 0; + Register SwiftErrorVReg = 0; + for (auto &Arg: CI.arg_operands()) { + if (CLI->supportSwiftError() && isSwiftError(Arg)) { + assert(SwiftInVReg == 0 && "Expected only one swift error argument"); + LLT Ty = getLLTForType(*Arg->getType(), *DL); + SwiftInVReg = MRI->createGenericVirtualRegister(Ty); + MIRBuilder.buildCopy(SwiftInVReg, SwiftError.getOrCreateVRegUseAt( + &CI, &MIRBuilder.getMBB(), Arg)); + Args.emplace_back(makeArrayRef(SwiftInVReg)); + SwiftErrorVReg = + SwiftError.getOrCreateVRegDefAt(&CI, &MIRBuilder.getMBB(), Arg); + continue; + } + Args.push_back(getOrCreateVRegs(*Arg)); + } MF->getFrameInfo().setHasCalls(true); - bool Success = CLI->lowerCall(MIRBuilder, &CI, Res, Args, [&]() { - return getOrCreateVReg(*CI.getCalledValue()); - }); + bool Success = + CLI->lowerCall(MIRBuilder, &CI, Res, Args, SwiftErrorVReg, + [&]() { return getOrCreateVReg(*CI.getCalledValue()); }); - if (IsSplitType) - unpackRegs(CI, Res, MIRBuilder); return Success; } @@ -1151,35 +1604,39 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) { if (translateKnownIntrinsic(CI, ID, MIRBuilder)) return true; - unsigned Res = 0; - if (!CI.getType()->isVoidTy()) { - if (IsSplitType) - Res = - MRI->createGenericVirtualRegister(getLLTForType(*CI.getType(), *DL)); - else - Res = getOrCreateVReg(CI); - } + ArrayRef ResultRegs; + if (!CI.getType()->isVoidTy()) + ResultRegs = getOrCreateVRegs(CI); + + // Ignore the callsite attributes. Backend code is most likely not expecting + // an intrinsic to sometimes have side effects and sometimes not. MachineInstrBuilder MIB = - MIRBuilder.buildIntrinsic(ID, Res, !CI.doesNotAccessMemory()); + MIRBuilder.buildIntrinsic(ID, ResultRegs, !F->doesNotAccessMemory()); + if (isa(CI)) + MIB->copyIRFlags(CI); for (auto &Arg : CI.arg_operands()) { // Some intrinsics take metadata parameters. Reject them. if (isa(Arg)) return false; - MIB.addUse(packRegs(*Arg, MIRBuilder)); + ArrayRef VRegs = getOrCreateVRegs(*Arg); + if (VRegs.size() > 1) + return false; + MIB.addUse(VRegs[0]); } - if (IsSplitType) - unpackRegs(CI, Res, MIRBuilder); - // Add a MachineMemOperand if it is a target mem intrinsic. const TargetLowering &TLI = *MF->getSubtarget().getTargetLowering(); TargetLowering::IntrinsicInfo Info; // TODO: Add a GlobalISel version of getTgtMemIntrinsic. if (TLI.getTgtMemIntrinsic(Info, CI, *MF, ID)) { + unsigned Align = Info.align; + if (Align == 0) + Align = DL->getABITypeAlignment(Info.memVT.getTypeForEVT(F->getContext())); + uint64_t Size = Info.memVT.getStoreSize(); MIB.addMemOperand(MF->getMachineMemOperand(MachinePointerInfo(Info.ptrVal), - Info.flags, Size, Info.align)); + Info.flags, Size, Align)); } return true; @@ -1215,18 +1672,32 @@ bool IRTranslator::translateInvoke(const User &U, MCSymbol *BeginSymbol = Context.createTempSymbol(); MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(BeginSymbol); - unsigned Res = - MRI->createGenericVirtualRegister(getLLTForType(*I.getType(), *DL)); - SmallVector Args; - for (auto &Arg: I.arg_operands()) - Args.push_back(packRegs(*Arg, MIRBuilder)); + ArrayRef Res; + if (!I.getType()->isVoidTy()) + Res = getOrCreateVRegs(I); + SmallVector, 8> Args; + Register SwiftErrorVReg = 0; + Register SwiftInVReg = 0; + for (auto &Arg : I.arg_operands()) { + if (CLI->supportSwiftError() && isSwiftError(Arg)) { + assert(SwiftInVReg == 0 && "Expected only one swift error argument"); + LLT Ty = getLLTForType(*Arg->getType(), *DL); + SwiftInVReg = MRI->createGenericVirtualRegister(Ty); + MIRBuilder.buildCopy(SwiftInVReg, SwiftError.getOrCreateVRegUseAt( + &I, &MIRBuilder.getMBB(), Arg)); + Args.push_back(makeArrayRef(SwiftInVReg)); + SwiftErrorVReg = + SwiftError.getOrCreateVRegDefAt(&I, &MIRBuilder.getMBB(), Arg); + continue; + } + + Args.push_back(getOrCreateVRegs(*Arg)); + } - if (!CLI->lowerCall(MIRBuilder, &I, Res, Args, + if (!CLI->lowerCall(MIRBuilder, &I, Res, Args, SwiftErrorVReg, [&]() { return getOrCreateVReg(*I.getCalledValue()); })) return false; - unpackRegs(I, Res, MIRBuilder); - MCSymbol *EndSymbol = Context.createTempSymbol(); MIRBuilder.buildInstr(TargetOpcode::EH_LABEL).addSym(EndSymbol); @@ -1241,6 +1712,12 @@ bool IRTranslator::translateInvoke(const User &U, return true; } +bool IRTranslator::translateCallBr(const User &U, + MachineIRBuilder &MIRBuilder) { + // FIXME: Implement this. + return false; +} + bool IRTranslator::translateLandingPad(const User &U, MachineIRBuilder &MIRBuilder) { const LandingPadInst &LP = cast(U); @@ -1270,7 +1747,7 @@ bool IRTranslator::translateLandingPad(const User &U, .addSym(MF->addLandingPad(&MBB)); LLT Ty = getLLTForType(*LP.getType(), *DL); - unsigned Undef = MRI->createGenericVirtualRegister(Ty); + Register Undef = MRI->createGenericVirtualRegister(Ty); MIRBuilder.buildUndef(Undef); SmallVector Tys; @@ -1279,20 +1756,20 @@ bool IRTranslator::translateLandingPad(const User &U, assert(Tys.size() == 2 && "Only two-valued landingpads are supported"); // Mark exception register as live in. - unsigned ExceptionReg = TLI.getExceptionPointerRegister(PersonalityFn); + Register ExceptionReg = TLI.getExceptionPointerRegister(PersonalityFn); if (!ExceptionReg) return false; MBB.addLiveIn(ExceptionReg); - ArrayRef ResRegs = getOrCreateVRegs(LP); + ArrayRef ResRegs = getOrCreateVRegs(LP); MIRBuilder.buildCopy(ResRegs[0], ExceptionReg); - unsigned SelectorReg = TLI.getExceptionSelectorRegister(PersonalityFn); + Register SelectorReg = TLI.getExceptionSelectorRegister(PersonalityFn); if (!SelectorReg) return false; MBB.addLiveIn(SelectorReg); - unsigned PtrVReg = MRI->createGenericVirtualRegister(Tys[0]); + Register PtrVReg = MRI->createGenericVirtualRegister(Tys[0]); MIRBuilder.buildCopy(PtrVReg, SelectorReg); MIRBuilder.buildCast(ResRegs[1], PtrVReg); @@ -1304,10 +1781,10 @@ bool IRTranslator::translateAlloca(const User &U, auto &AI = cast(U); if (AI.isSwiftError()) - return false; + return true; if (AI.isStaticAlloca()) { - unsigned Res = getOrCreateVReg(AI); + Register Res = getOrCreateVReg(AI); int FI = getOrCreateFrameIndex(AI); MIRBuilder.buildFrameIndex(Res, FI); return true; @@ -1322,29 +1799,29 @@ bool IRTranslator::translateAlloca(const User &U, unsigned Align = std::max((unsigned)DL->getPrefTypeAlignment(Ty), AI.getAlignment()); - unsigned NumElts = getOrCreateVReg(*AI.getArraySize()); + Register NumElts = getOrCreateVReg(*AI.getArraySize()); Type *IntPtrIRTy = DL->getIntPtrType(AI.getType()); LLT IntPtrTy = getLLTForType(*IntPtrIRTy, *DL); if (MRI->getType(NumElts) != IntPtrTy) { - unsigned ExtElts = MRI->createGenericVirtualRegister(IntPtrTy); + Register ExtElts = MRI->createGenericVirtualRegister(IntPtrTy); MIRBuilder.buildZExtOrTrunc(ExtElts, NumElts); NumElts = ExtElts; } - unsigned AllocSize = MRI->createGenericVirtualRegister(IntPtrTy); - unsigned TySize = + Register AllocSize = MRI->createGenericVirtualRegister(IntPtrTy); + Register TySize = getOrCreateVReg(*ConstantInt::get(IntPtrIRTy, -DL->getTypeAllocSize(Ty))); MIRBuilder.buildMul(AllocSize, NumElts, TySize); LLT PtrTy = getLLTForType(*AI.getType(), *DL); auto &TLI = *MF->getSubtarget().getTargetLowering(); - unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore(); + Register SPReg = TLI.getStackPointerRegisterToSaveRestore(); - unsigned SPTmp = MRI->createGenericVirtualRegister(PtrTy); + Register SPTmp = MRI->createGenericVirtualRegister(PtrTy); MIRBuilder.buildCopy(SPTmp, SPReg); - unsigned AllocTmp = MRI->createGenericVirtualRegister(PtrTy); + Register AllocTmp = MRI->createGenericVirtualRegister(PtrTy); MIRBuilder.buildGEP(AllocTmp, SPTmp, AllocSize); // Handle alignment. We have to realign if the allocation granule was smaller @@ -1357,7 +1834,7 @@ bool IRTranslator::translateAlloca(const User &U, // Round the size of the allocation up to the stack alignment size // by add SA-1 to the size. This doesn't overflow because we're computing // an address inside an alloca. - unsigned AlignedAlloc = MRI->createGenericVirtualRegister(PtrTy); + Register AlignedAlloc = MRI->createGenericVirtualRegister(PtrTy); MIRBuilder.buildPtrMask(AlignedAlloc, AllocTmp, Log2_32(Align)); AllocTmp = AlignedAlloc; } @@ -1387,7 +1864,7 @@ bool IRTranslator::translateInsertElement(const User &U, // If it is a <1 x Ty> vector, use the scalar as it is // not a legal vector type in LLT. if (U.getType()->getVectorNumElements() == 1) { - unsigned Elt = getOrCreateVReg(*U.getOperand(1)); + Register Elt = getOrCreateVReg(*U.getOperand(1)); auto &Regs = *VMap.getVRegs(U); if (Regs.empty()) { Regs.push_back(Elt); @@ -1398,10 +1875,10 @@ bool IRTranslator::translateInsertElement(const User &U, return true; } - unsigned Res = getOrCreateVReg(U); - unsigned Val = getOrCreateVReg(*U.getOperand(0)); - unsigned Elt = getOrCreateVReg(*U.getOperand(1)); - unsigned Idx = getOrCreateVReg(*U.getOperand(2)); + Register Res = getOrCreateVReg(U); + Register Val = getOrCreateVReg(*U.getOperand(0)); + Register Elt = getOrCreateVReg(*U.getOperand(1)); + Register Idx = getOrCreateVReg(*U.getOperand(2)); MIRBuilder.buildInsertVectorElement(Res, Val, Elt, Idx); return true; } @@ -1411,7 +1888,7 @@ bool IRTranslator::translateExtractElement(const User &U, // If it is a <1 x Ty> vector, use the scalar as it is // not a legal vector type in LLT. if (U.getOperand(0)->getType()->getVectorNumElements() == 1) { - unsigned Elt = getOrCreateVReg(*U.getOperand(0)); + Register Elt = getOrCreateVReg(*U.getOperand(0)); auto &Regs = *VMap.getVRegs(U); if (Regs.empty()) { Regs.push_back(Elt); @@ -1421,11 +1898,11 @@ bool IRTranslator::translateExtractElement(const User &U, } return true; } - unsigned Res = getOrCreateVReg(U); - unsigned Val = getOrCreateVReg(*U.getOperand(0)); + Register Res = getOrCreateVReg(U); + Register Val = getOrCreateVReg(*U.getOperand(0)); const auto &TLI = *MF->getSubtarget().getTargetLowering(); unsigned PreferredVecIdxWidth = TLI.getVectorIdxTy(*DL).getSizeInBits(); - unsigned Idx = 0; + Register Idx; if (auto *CI = dyn_cast(U.getOperand(1))) { if (CI->getBitWidth() != PreferredVecIdxWidth) { APInt NewIdx = CI->getValue().sextOrTrunc(PreferredVecIdxWidth); @@ -1481,11 +1958,11 @@ bool IRTranslator::translateAtomicCmpXchg(const User &U, Type *ValType = ResType->Type::getStructElementType(0); auto Res = getOrCreateVRegs(I); - unsigned OldValRes = Res[0]; - unsigned SuccessRes = Res[1]; - unsigned Addr = getOrCreateVReg(*I.getPointerOperand()); - unsigned Cmp = getOrCreateVReg(*I.getCompareOperand()); - unsigned NewVal = getOrCreateVReg(*I.getNewValOperand()); + Register OldValRes = Res[0]; + Register SuccessRes = Res[1]; + Register Addr = getOrCreateVReg(*I.getPointerOperand()); + Register Cmp = getOrCreateVReg(*I.getCompareOperand()); + Register NewVal = getOrCreateVReg(*I.getNewValOperand()); MIRBuilder.buildAtomicCmpXchgWithSuccess( OldValRes, SuccessRes, Addr, Cmp, NewVal, @@ -1507,9 +1984,9 @@ bool IRTranslator::translateAtomicRMW(const User &U, Type *ResType = I.getType(); - unsigned Res = getOrCreateVReg(I); - unsigned Addr = getOrCreateVReg(*I.getPointerOperand()); - unsigned Val = getOrCreateVReg(*I.getValOperand()); + Register Res = getOrCreateVReg(I); + Register Addr = getOrCreateVReg(*I.getPointerOperand()); + Register Val = getOrCreateVReg(*I.getValOperand()); unsigned Opcode = 0; switch (I.getOperation()) { @@ -1560,6 +2037,14 @@ bool IRTranslator::translateAtomicRMW(const User &U, return true; } +bool IRTranslator::translateFence(const User &U, + MachineIRBuilder &MIRBuilder) { + const FenceInst &Fence = cast(U); + MIRBuilder.buildFence(static_cast(Fence.getOrdering()), + Fence.getSyncScopeID()); + return true; +} + void IRTranslator::finishPendingPhis() { #ifndef NDEBUG DILocationVerifier Verifier; @@ -1569,27 +2054,20 @@ void IRTranslator::finishPendingPhis() { for (auto &Phi : PendingPHIs) { const PHINode *PI = Phi.first; ArrayRef ComponentPHIs = Phi.second; + MachineBasicBlock *PhiMBB = ComponentPHIs[0]->getParent(); EntryBuilder->setDebugLoc(PI->getDebugLoc()); #ifndef NDEBUG Verifier.setCurrentInst(PI); #endif // ifndef NDEBUG - // All MachineBasicBlocks exist, add them to the PHI. We assume IRTranslator - // won't create extra control flow here, otherwise we need to find the - // dominating predecessor here (or perhaps force the weirder IRTranslators - // to provide a simple boundary). - SmallSet HandledPreds; - + SmallSet SeenPreds; for (unsigned i = 0; i < PI->getNumIncomingValues(); ++i) { auto IRPred = PI->getIncomingBlock(i); - if (HandledPreds.count(IRPred)) - continue; - - HandledPreds.insert(IRPred); - ArrayRef ValRegs = getOrCreateVRegs(*PI->getIncomingValue(i)); + ArrayRef ValRegs = getOrCreateVRegs(*PI->getIncomingValue(i)); for (auto Pred : getMachinePredBBs({IRPred, PI->getParent()})) { - assert(Pred->isSuccessor(ComponentPHIs[0]->getParent()) && - "incorrect CFG at MachineBasicBlock level"); + if (SeenPreds.count(Pred) || !PhiMBB->isPredecessor(Pred)) + continue; + SeenPreds.insert(Pred); for (unsigned j = 0; j < ValRegs.size(); ++j) { MachineInstrBuilder MIB(*MF, ComponentPHIs[j]); MIB.addUse(ValRegs[j]); @@ -1611,8 +2089,15 @@ bool IRTranslator::valueIsSplit(const Value &V, bool IRTranslator::translate(const Instruction &Inst) { CurBuilder->setDebugLoc(Inst.getDebugLoc()); - EntryBuilder->setDebugLoc(Inst.getDebugLoc()); - switch(Inst.getOpcode()) { + // We only emit constants into the entry block from here. To prevent jumpy + // debug behaviour set the line to 0. + if (const DebugLoc &DL = Inst.getDebugLoc()) + EntryBuilder->setDebugLoc( + DebugLoc::get(0, 0, DL.getScope(), DL.getInlinedAt())); + else + EntryBuilder->setDebugLoc(DebugLoc()); + + switch (Inst.getOpcode()) { #define HANDLE_INST(NUM, OPCODE, CLASS) \ case Instruction::OPCODE: \ return translate##OPCODE(Inst, *CurBuilder.get()); @@ -1622,7 +2107,7 @@ bool IRTranslator::translate(const Instruction &Inst) { } } -bool IRTranslator::translate(const Constant &C, unsigned Reg) { +bool IRTranslator::translate(const Constant &C, Register Reg) { if (auto CI = dyn_cast(&C)) EntryBuilder->buildConstant(Reg, *CI); else if (auto CF = dyn_cast(&C)) @@ -1635,7 +2120,7 @@ bool IRTranslator::translate(const Constant &C, unsigned Reg) { unsigned NullSize = DL->getTypeSizeInBits(C.getType()); auto *ZeroTy = Type::getIntNTy(C.getContext(), NullSize); auto *ZeroVal = ConstantInt::get(ZeroTy, 0); - unsigned ZeroReg = getOrCreateVReg(*ZeroVal); + Register ZeroReg = getOrCreateVReg(*ZeroVal); EntryBuilder->buildCast(Reg, ZeroReg); } else if (auto GV = dyn_cast(&C)) EntryBuilder->buildGlobalValue(Reg, GV); @@ -1645,7 +2130,7 @@ bool IRTranslator::translate(const Constant &C, unsigned Reg) { // Return the scalar if it is a <1 x Ty> vector. if (CAZ->getNumElements() == 1) return translate(*CAZ->getElementValue(0u), Reg); - SmallVector Ops; + SmallVector Ops; for (unsigned i = 0; i < CAZ->getNumElements(); ++i) { Constant &Elt = *CAZ->getElementValue(i); Ops.push_back(getOrCreateVReg(Elt)); @@ -1655,7 +2140,7 @@ bool IRTranslator::translate(const Constant &C, unsigned Reg) { // Return the scalar if it is a <1 x Ty> vector. if (CV->getNumElements() == 1) return translate(*CV->getElementAsConstant(0), Reg); - SmallVector Ops; + SmallVector Ops; for (unsigned i = 0; i < CV->getNumElements(); ++i) { Constant &Elt = *CV->getElementAsConstant(i); Ops.push_back(getOrCreateVReg(Elt)); @@ -1673,7 +2158,7 @@ bool IRTranslator::translate(const Constant &C, unsigned Reg) { } else if (auto CV = dyn_cast(&C)) { if (CV->getNumOperands() == 1) return translate(*CV->getOperand(0), Reg); - SmallVector Ops; + SmallVector Ops; for (unsigned i = 0; i < CV->getNumOperands(); ++i) { Ops.push_back(getOrCreateVReg(*CV->getOperand(i))); } @@ -1686,6 +2171,17 @@ bool IRTranslator::translate(const Constant &C, unsigned Reg) { return true; } +void IRTranslator::finalizeBasicBlock() { + for (auto &JTCase : SL->JTCases) { + // Emit header first, if it wasn't already emitted. + if (!JTCase.first.Emitted) + emitJumpTableHeader(JTCase.second, JTCase.first, JTCase.first.HeaderBB); + + emitJumpTable(JTCase.second, JTCase.second.MBB); + } + SL->JTCases.clear(); +} + void IRTranslator::finalizeFunction() { // Release the memory used by the different maps we // needed during the translation. @@ -1698,6 +2194,7 @@ void IRTranslator::finalizeFunction() { // destroying it twice (in ~IRTranslator() and ~LLVMContext()) EntryBuilder.reset(); CurBuilder.reset(); + FuncInfo.clear(); } bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) { @@ -1710,13 +2207,13 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) { // Set the CSEConfig and run the analysis. GISelCSEInfo *CSEInfo = nullptr; TPC = &getAnalysis(); - bool IsO0 = TPC->getOptLevel() == CodeGenOpt::Level::None; - // Disable CSE for O0. - bool EnableCSE = !IsO0 && EnableCSEInIRTranslator; + bool EnableCSE = EnableCSEInIRTranslator.getNumOccurrences() + ? EnableCSEInIRTranslator + : TPC->isGISelCSEEnabled(); + if (EnableCSE) { EntryBuilder = make_unique(CurMF); - std::unique_ptr Config = make_unique(); - CSEInfo = &Wrapper.get(std::move(Config)); + CSEInfo = &Wrapper.get(TPC->getCSEConfig()); EntryBuilder->setCSEInfo(CSEInfo); CurBuilder = make_unique(CurMF); CurBuilder->setCSEInfo(CSEInfo); @@ -1730,6 +2227,14 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) { MRI = &MF->getRegInfo(); DL = &F.getParent()->getDataLayout(); ORE = llvm::make_unique(&F); + FuncInfo.MF = MF; + FuncInfo.BPI = nullptr; + const auto &TLI = *MF->getSubtarget().getTargetLowering(); + const TargetMachine &TM = MF->getTarget(); + SL = make_unique(this, FuncInfo); + SL->init(TLI, TM, *DL); + + EnableOpts = TM.getOptLevel() != CodeGenOpt::None && !skipFunction(F); assert(PendingPHIs.empty() && "stale PHIs"); @@ -1749,6 +2254,10 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) { MF->push_back(EntryBB); EntryBuilder->setMBB(*EntryBB); + DebugLoc DbgLoc = F.getEntryBlock().getFirstNonPHI()->getDebugLoc(); + SwiftError.setFunction(CurMF); + SwiftError.createEntriesInEntryBlock(DbgLoc); + // Create all blocks, in IR order, to preserve the layout. for (const BasicBlock &BB: F) { auto *&MBB = BBToMBB[&BB]; @@ -1764,20 +2273,25 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) { EntryBB->addSuccessor(&getMBB(F.front())); // Lower the actual args into this basic block. - SmallVector VRegArgs; + SmallVector, 8> VRegArgs; for (const Argument &Arg: F.args()) { if (DL->getTypeStoreSize(Arg.getType()) == 0) continue; // Don't handle zero sized types. - VRegArgs.push_back( - MRI->createGenericVirtualRegister(getLLTForType(*Arg.getType(), *DL))); + ArrayRef VRegs = getOrCreateVRegs(Arg); + VRegArgs.push_back(VRegs); + + if (Arg.hasSwiftErrorAttr()) { + assert(VRegs.size() == 1 && "Too many vregs for Swift error"); + SwiftError.setCurrentVReg(EntryBB, SwiftError.getFunctionArg(), VRegs[0]); + } } // We don't currently support translating swifterror or swiftself functions. for (auto &Arg : F.args()) { - if (Arg.hasSwiftErrorAttr() || Arg.hasSwiftSelfAttr()) { + if (Arg.hasSwiftSelfAttr()) { OptimizationRemarkMissed R("gisel-irtranslator", "GISelFailure", F.getSubprogram(), &F.getEntryBlock()); - R << "unable to lower arguments due to swifterror/swiftself: " + R << "unable to lower arguments due to swiftself: " << ore::NV("Prototype", F.getType()); reportTranslationError(*MF, *TPC, *ORE, R); return false; @@ -1792,20 +2306,6 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) { return false; } - auto ArgIt = F.arg_begin(); - for (auto &VArg : VRegArgs) { - // If the argument is an unsplit scalar then don't use unpackRegs to avoid - // creating redundant copies. - if (!valueIsSplit(*ArgIt, VMap.getOffsets(*ArgIt))) { - auto &VRegs = *VMap.getVRegs(cast(*ArgIt)); - assert(VRegs.empty() && "VRegs already populated?"); - VRegs.push_back(VArg); - } else { - unpackRegs(*ArgIt, VArg, *EntryBuilder.get()); - } - ArgIt++; - } - // Need to visit defs before uses when translating instructions. GISelObserverWrapper WrapperObserver; if (EnableCSE && CSEInfo) @@ -1845,6 +2345,8 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) { reportTranslationError(*MF, *TPC, *ORE, R); return false; } + + finalizeBasicBlock(); } #ifndef NDEBUG WrapperObserver.removeObserver(&Verifier); @@ -1853,6 +2355,8 @@ bool IRTranslator::runOnMachineFunction(MachineFunction &CurMF) { finishPendingPhis(); + SwiftError.propagateVRegs(); + // Merge the argument lowering and constants block with its single // successor, the LLVM-IR entry block. We want the basic block to // be maximal. diff --git a/lib/CodeGen/GlobalISel/InstructionSelect.cpp b/lib/CodeGen/GlobalISel/InstructionSelect.cpp index c83c791327e4..70694fe6b6c8 100644 --- a/lib/CodeGen/GlobalISel/InstructionSelect.cpp +++ b/lib/CodeGen/GlobalISel/InstructionSelect.cpp @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/InstructionSelect.cpp - InstructionSelect ---==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -50,9 +49,7 @@ INITIALIZE_PASS_END(InstructionSelect, DEBUG_TYPE, "Select target instructions out of generic instructions", false, false) -InstructionSelect::InstructionSelect() : MachineFunctionPass(ID) { - initializeInstructionSelectPass(*PassRegistry::getPassRegistry()); -} +InstructionSelect::InstructionSelect() : MachineFunctionPass(ID) { } void InstructionSelect::getAnalysisUsage(AnalysisUsage &AU) const { AU.addRequired(); @@ -90,10 +87,10 @@ bool InstructionSelect::runOnMachineFunction(MachineFunction &MF) { "instruction is not legal", *MI); return false; } -#endif // FIXME: We could introduce new blocks and will need to fix the outer loop. // Until then, keep track of the number of blocks to assert that we don't. const size_t NumBlocks = MF.size(); +#endif for (MachineBasicBlock *MBB : post_order(&MF)) { if (MBB->empty()) @@ -145,8 +142,6 @@ bool InstructionSelect::runOnMachineFunction(MachineFunction &MF) { } } - const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); - for (MachineBasicBlock &MBB : MF) { if (MBB.empty()) continue; @@ -178,6 +173,8 @@ bool InstructionSelect::runOnMachineFunction(MachineFunction &MF) { } } +#ifndef NDEBUG + const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); // Now that selection is complete, there are no more generic vregs. Verify // that the size of the now-constrained vreg is unchanged and that it has a // register class. @@ -216,7 +213,7 @@ bool InstructionSelect::runOnMachineFunction(MachineFunction &MF) { reportGISelFailure(MF, TPC, MORE, R); return false; } - +#endif auto &TLI = *MF.getSubtarget().getTargetLowering(); TLI.finalizeLowering(MF); diff --git a/lib/CodeGen/GlobalISel/InstructionSelector.cpp b/lib/CodeGen/GlobalISel/InstructionSelector.cpp index 38913e4afcba..2ad35b3a72c9 100644 --- a/lib/CodeGen/GlobalISel/InstructionSelector.cpp +++ b/lib/CodeGen/GlobalISel/InstructionSelector.cpp @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/InstructionSelector.cpp --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -42,16 +41,16 @@ bool InstructionSelector::constrainOperandRegToRegClass( MachineFunction &MF = *MBB.getParent(); MachineRegisterInfo &MRI = MF.getRegInfo(); - return - constrainRegToClass(MRI, TII, RBI, I, I.getOperand(OpIdx).getReg(), RC); + return constrainOperandRegClass(MF, TRI, MRI, TII, RBI, I, RC, + I.getOperand(OpIdx), OpIdx); } bool InstructionSelector::isOperandImmEqual( const MachineOperand &MO, int64_t Value, const MachineRegisterInfo &MRI) const { if (MO.isReg() && MO.getReg()) - if (auto VRegVal = getConstantVRegVal(MO.getReg(), MRI)) - return *VRegVal == Value; + if (auto VRegVal = getConstantVRegValWithLookThrough(MO.getReg(), MRI)) + return VRegVal->Value == Value; return false; } @@ -79,6 +78,6 @@ bool InstructionSelector::isObviouslySafeToFold(MachineInstr &MI, std::next(MI.getIterator()) == IntoMI.getIterator()) return true; - return !MI.mayLoadOrStore() && !MI.hasUnmodeledSideEffects() && - empty(MI.implicit_operands()); + return !MI.mayLoadOrStore() && !MI.mayRaiseFPException() && + !MI.hasUnmodeledSideEffects() && empty(MI.implicit_operands()); } diff --git a/lib/CodeGen/GlobalISel/LegalityPredicates.cpp b/lib/CodeGen/GlobalISel/LegalityPredicates.cpp index 94eab9ae00c8..601d50e9806f 100644 --- a/lib/CodeGen/GlobalISel/LegalityPredicates.cpp +++ b/lib/CodeGen/GlobalISel/LegalityPredicates.cpp @@ -1,9 +1,8 @@ //===- lib/CodeGen/GlobalISel/LegalizerPredicates.cpp - Predicates --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -39,15 +38,19 @@ LegalityPredicate LegalityPredicates::typePairInSet( }; } -LegalityPredicate LegalityPredicates::typePairAndMemSizeInSet( +LegalityPredicate LegalityPredicates::typePairAndMemDescInSet( unsigned TypeIdx0, unsigned TypeIdx1, unsigned MMOIdx, - std::initializer_list TypesAndMemSizeInit) { - SmallVector TypesAndMemSize = TypesAndMemSizeInit; + std::initializer_list TypesAndMemDescInit) { + SmallVector TypesAndMemDesc = TypesAndMemDescInit; return [=](const LegalityQuery &Query) { - TypePairAndMemSize Match = {Query.Types[TypeIdx0], Query.Types[TypeIdx1], - Query.MMODescrs[MMOIdx].SizeInBits}; - return std::find(TypesAndMemSize.begin(), TypesAndMemSize.end(), Match) != - TypesAndMemSize.end(); + TypePairAndMemDesc Match = {Query.Types[TypeIdx0], Query.Types[TypeIdx1], + Query.MMODescrs[MMOIdx].SizeInBits, + Query.MMODescrs[MMOIdx].AlignInBits}; + return std::find_if( + TypesAndMemDesc.begin(), TypesAndMemDesc.end(), + [=](const TypePairAndMemDesc &Entry) ->bool { + return Match.isCompatible(Entry); + }) != TypesAndMemDesc.end(); }; } @@ -57,10 +60,30 @@ LegalityPredicate LegalityPredicates::isScalar(unsigned TypeIdx) { }; } +LegalityPredicate LegalityPredicates::isVector(unsigned TypeIdx) { + return [=](const LegalityQuery &Query) { + return Query.Types[TypeIdx].isVector(); + }; +} + +LegalityPredicate LegalityPredicates::isPointer(unsigned TypeIdx) { + return [=](const LegalityQuery &Query) { + return Query.Types[TypeIdx].isPointer(); + }; +} + +LegalityPredicate LegalityPredicates::isPointer(unsigned TypeIdx, + unsigned AddrSpace) { + return [=](const LegalityQuery &Query) { + LLT Ty = Query.Types[TypeIdx]; + return Ty.isPointer() && Ty.getAddressSpace() == AddrSpace; + }; +} + LegalityPredicate LegalityPredicates::narrowerThan(unsigned TypeIdx, unsigned Size) { return [=](const LegalityQuery &Query) { - const LLT &QueryTy = Query.Types[TypeIdx]; + const LLT QueryTy = Query.Types[TypeIdx]; return QueryTy.isScalar() && QueryTy.getSizeInBits() < Size; }; } @@ -68,18 +91,49 @@ LegalityPredicate LegalityPredicates::narrowerThan(unsigned TypeIdx, LegalityPredicate LegalityPredicates::widerThan(unsigned TypeIdx, unsigned Size) { return [=](const LegalityQuery &Query) { - const LLT &QueryTy = Query.Types[TypeIdx]; + const LLT QueryTy = Query.Types[TypeIdx]; return QueryTy.isScalar() && QueryTy.getSizeInBits() > Size; }; } +LegalityPredicate LegalityPredicates::scalarOrEltNarrowerThan(unsigned TypeIdx, + unsigned Size) { + return [=](const LegalityQuery &Query) { + const LLT QueryTy = Query.Types[TypeIdx]; + return QueryTy.getScalarSizeInBits() < Size; + }; +} + +LegalityPredicate LegalityPredicates::scalarOrEltWiderThan(unsigned TypeIdx, + unsigned Size) { + return [=](const LegalityQuery &Query) { + const LLT QueryTy = Query.Types[TypeIdx]; + return QueryTy.getScalarSizeInBits() > Size; + }; +} + +LegalityPredicate LegalityPredicates::scalarOrEltSizeNotPow2(unsigned TypeIdx) { + return [=](const LegalityQuery &Query) { + const LLT QueryTy = Query.Types[TypeIdx]; + return !isPowerOf2_32(QueryTy.getScalarSizeInBits()); + }; +} + LegalityPredicate LegalityPredicates::sizeNotPow2(unsigned TypeIdx) { return [=](const LegalityQuery &Query) { - const LLT &QueryTy = Query.Types[TypeIdx]; + const LLT QueryTy = Query.Types[TypeIdx]; return QueryTy.isScalar() && !isPowerOf2_32(QueryTy.getSizeInBits()); }; } +LegalityPredicate LegalityPredicates::sameSize(unsigned TypeIdx0, + unsigned TypeIdx1) { + return [=](const LegalityQuery &Query) { + return Query.Types[TypeIdx0].getSizeInBits() == + Query.Types[TypeIdx1].getSizeInBits(); + }; +} + LegalityPredicate LegalityPredicates::memSizeInBytesNotPow2(unsigned MMOIdx) { return [=](const LegalityQuery &Query) { return !isPowerOf2_32(Query.MMODescrs[MMOIdx].SizeInBits / 8); @@ -88,8 +142,8 @@ LegalityPredicate LegalityPredicates::memSizeInBytesNotPow2(unsigned MMOIdx) { LegalityPredicate LegalityPredicates::numElementsNotPow2(unsigned TypeIdx) { return [=](const LegalityQuery &Query) { - const LLT &QueryTy = Query.Types[TypeIdx]; - return QueryTy.isVector() && isPowerOf2_32(QueryTy.getNumElements()); + const LLT QueryTy = Query.Types[TypeIdx]; + return QueryTy.isVector() && !isPowerOf2_32(QueryTy.getNumElements()); }; } diff --git a/lib/CodeGen/GlobalISel/LegalizeMutations.cpp b/lib/CodeGen/GlobalISel/LegalizeMutations.cpp index a29b32ecdc03..fcbecf90a845 100644 --- a/lib/CodeGen/GlobalISel/LegalizeMutations.cpp +++ b/lib/CodeGen/GlobalISel/LegalizeMutations.cpp @@ -1,9 +1,8 @@ //===- lib/CodeGen/GlobalISel/LegalizerMutations.cpp - Mutations ----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -27,25 +26,46 @@ LegalizeMutation LegalizeMutations::changeTo(unsigned TypeIdx, }; } -LegalizeMutation LegalizeMutations::widenScalarToNextPow2(unsigned TypeIdx, - unsigned Min) { +LegalizeMutation LegalizeMutations::changeElementTo(unsigned TypeIdx, + unsigned FromTypeIdx) { return [=](const LegalityQuery &Query) { - unsigned NewSizeInBits = - 1 << Log2_32_Ceil(Query.Types[TypeIdx].getSizeInBits()); - if (NewSizeInBits < Min) - NewSizeInBits = Min; - return std::make_pair(TypeIdx, LLT::scalar(NewSizeInBits)); + const LLT OldTy = Query.Types[TypeIdx]; + const LLT NewTy = Query.Types[FromTypeIdx]; + return std::make_pair(TypeIdx, OldTy.changeElementType(NewTy)); + }; +} + +LegalizeMutation LegalizeMutations::changeElementTo(unsigned TypeIdx, + LLT NewEltTy) { + return [=](const LegalityQuery &Query) { + const LLT OldTy = Query.Types[TypeIdx]; + return std::make_pair(TypeIdx, OldTy.changeElementType(NewEltTy)); + }; +} + +LegalizeMutation LegalizeMutations::widenScalarOrEltToNextPow2(unsigned TypeIdx, + unsigned Min) { + return [=](const LegalityQuery &Query) { + const LLT Ty = Query.Types[TypeIdx]; + unsigned NewEltSizeInBits = + std::max(1u << Log2_32_Ceil(Ty.getScalarSizeInBits()), Min); + return std::make_pair(TypeIdx, Ty.changeElementSize(NewEltSizeInBits)); }; } LegalizeMutation LegalizeMutations::moreElementsToNextPow2(unsigned TypeIdx, unsigned Min) { return [=](const LegalityQuery &Query) { - const LLT &VecTy = Query.Types[TypeIdx]; - unsigned NewNumElements = 1 << Log2_32_Ceil(VecTy.getNumElements()); - if (NewNumElements < Min) - NewNumElements = Min; - return std::make_pair( - TypeIdx, LLT::vector(NewNumElements, VecTy.getScalarSizeInBits())); + const LLT VecTy = Query.Types[TypeIdx]; + unsigned NewNumElements = + std::max(1u << Log2_32_Ceil(VecTy.getNumElements()), Min); + return std::make_pair(TypeIdx, + LLT::vector(NewNumElements, VecTy.getElementType())); + }; +} + +LegalizeMutation LegalizeMutations::scalarize(unsigned TypeIdx) { + return [=](const LegalityQuery &Query) { + return std::make_pair(TypeIdx, Query.Types[TypeIdx].getElementType()); }; } diff --git a/lib/CodeGen/GlobalISel/Legalizer.cpp b/lib/CodeGen/GlobalISel/Legalizer.cpp index 84131e59948c..b5b26bff34bb 100644 --- a/lib/CodeGen/GlobalISel/Legalizer.cpp +++ b/lib/CodeGen/GlobalISel/Legalizer.cpp @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/GlobalISel/Legalizer.cpp -----------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -28,6 +27,7 @@ #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/Support/Debug.h" +#include "llvm/Target/TargetMachine.h" #include @@ -50,9 +50,7 @@ INITIALIZE_PASS_END(Legalizer, DEBUG_TYPE, "Legalize the Machine IR a function's Machine IR", false, false) -Legalizer::Legalizer() : MachineFunctionPass(ID) { - initializeLegalizerPass(*PassRegistry::getPassRegistry()); -} +Legalizer::Legalizer() : MachineFunctionPass(ID) { } void Legalizer::getAnalysisUsage(AnalysisUsage &AU) const { AU.addRequired(); @@ -77,6 +75,7 @@ static bool isArtifact(const MachineInstr &MI) { case TargetOpcode::G_UNMERGE_VALUES: case TargetOpcode::G_CONCAT_VECTORS: case TargetOpcode::G_BUILD_VECTOR: + case TargetOpcode::G_EXTRACT: return true; } } @@ -87,12 +86,15 @@ namespace { class LegalizerWorkListManager : public GISelChangeObserver { InstListTy &InstList; ArtifactListTy &ArtifactList; +#ifndef NDEBUG + SmallVector NewMIs; +#endif public: LegalizerWorkListManager(InstListTy &Insts, ArtifactListTy &Arts) : InstList(Insts), ArtifactList(Arts) {} - void createdInstr(MachineInstr &MI) override { + void createdOrChangedInstr(MachineInstr &MI) { // Only legalize pre-isel generic instructions. // Legalization process could generate Target specific pseudo // instructions with generic types. Don't record them @@ -102,7 +104,20 @@ public: else InstList.insert(&MI); } + } + + void createdInstr(MachineInstr &MI) override { LLVM_DEBUG(dbgs() << ".. .. New MI: " << MI); + LLVM_DEBUG(NewMIs.push_back(&MI)); + createdOrChangedInstr(MI); + } + + void printNewInstrs() { + LLVM_DEBUG({ + for (const auto *MI : NewMIs) + dbgs() << ".. .. New MI: " << *MI; + NewMIs.clear(); + }); } void erasingInstr(MachineInstr &MI) override { @@ -119,7 +134,7 @@ public: // When insts change, we want to revisit them to legalize them again. // We'll consider them the same as created. LLVM_DEBUG(dbgs() << ".. .. Changed MI: " << MI); - createdInstr(MI); + createdOrChangedInstr(MI); } }; } // namespace @@ -155,20 +170,22 @@ bool Legalizer::runOnMachineFunction(MachineFunction &MF) { if (!isPreISelGenericOpcode(MI.getOpcode())) continue; if (isArtifact(MI)) - ArtifactList.insert(&MI); + ArtifactList.deferred_insert(&MI); else - InstList.insert(&MI); + InstList.deferred_insert(&MI); } } + ArtifactList.finalize(); + InstList.finalize(); std::unique_ptr MIRBuilder; GISelCSEInfo *CSEInfo = nullptr; - bool IsO0 = TPC.getOptLevel() == CodeGenOpt::Level::None; - // Disable CSE for O0. - bool EnableCSE = !IsO0 && EnableCSEInLegalizer; + bool EnableCSE = EnableCSEInLegalizer.getNumOccurrences() + ? EnableCSEInLegalizer + : TPC.isGISelCSEEnabled(); + if (EnableCSE) { MIRBuilder = make_unique(); - std::unique_ptr Config = make_unique(); - CSEInfo = &Wrapper.get(std::move(Config)); + CSEInfo = &Wrapper.get(TPC.getCSEConfig()); MIRBuilder->setCSEInfo(CSEInfo); } else MIRBuilder = make_unique(); @@ -210,6 +227,7 @@ bool Legalizer::runOnMachineFunction(MachineFunction &MF) { "unable to legalize instruction", MI); return false; } + WorkListObserver.printNewInstrs(); Changed |= Res == LegalizerHelper::Legalized; } while (!ArtifactList.empty()) { @@ -222,7 +240,9 @@ bool Legalizer::runOnMachineFunction(MachineFunction &MF) { continue; } SmallVector DeadInstructions; - if (ArtCombiner.tryCombineInstruction(MI, DeadInstructions)) { + if (ArtCombiner.tryCombineInstruction(MI, DeadInstructions, + WrapperObserver)) { + WorkListObserver.printNewInstrs(); for (auto *DeadMI : DeadInstructions) { LLVM_DEBUG(dbgs() << *DeadMI << "Is dead\n"); RemoveDeadInstFromLists(DeadMI); diff --git a/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index b3fc94cdec60..f5cf7fc9bd9b 100644 --- a/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/GlobalISel/LegalizerHelper.cpp -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -30,6 +29,39 @@ using namespace llvm; using namespace LegalizeActions; +/// Try to break down \p OrigTy into \p NarrowTy sized pieces. +/// +/// Returns the number of \p NarrowTy elements needed to reconstruct \p OrigTy, +/// with any leftover piece as type \p LeftoverTy +/// +/// Returns -1 in the first element of the pair if the breakdown is not +/// satisfiable. +static std::pair +getNarrowTypeBreakDown(LLT OrigTy, LLT NarrowTy, LLT &LeftoverTy) { + assert(!LeftoverTy.isValid() && "this is an out argument"); + + unsigned Size = OrigTy.getSizeInBits(); + unsigned NarrowSize = NarrowTy.getSizeInBits(); + unsigned NumParts = Size / NarrowSize; + unsigned LeftoverSize = Size - NumParts * NarrowSize; + assert(Size > NarrowSize); + + if (LeftoverSize == 0) + return {NumParts, 0}; + + if (NarrowTy.isVector()) { + unsigned EltSize = OrigTy.getScalarSizeInBits(); + if (LeftoverSize % EltSize != 0) + return {-1, -1}; + LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize); + } else { + LeftoverTy = LLT::scalar(LeftoverSize); + } + + int NumLeftover = LeftoverSize / LeftoverTy.getSizeInBits(); + return std::make_pair(NumParts, NumLeftover); +} + LegalizerHelper::LegalizerHelper(MachineFunction &MF, GISelChangeObserver &Observer, MachineIRBuilder &Builder) @@ -50,6 +82,10 @@ LegalizerHelper::LegalizeResult LegalizerHelper::legalizeInstrStep(MachineInstr &MI) { LLVM_DEBUG(dbgs() << "Legalizing: "; MI.print(dbgs())); + if (MI.getOpcode() == TargetOpcode::G_INTRINSIC || + MI.getOpcode() == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS) + return LI.legalizeIntrinsic(MI, MRI, MIRBuilder) ? Legalized + : UnableToLegalize; auto Step = LI.getAction(MI, MRI); switch (Step.Action) { case Legal: @@ -70,6 +106,9 @@ LegalizerHelper::legalizeInstrStep(MachineInstr &MI) { case FewerElements: LLVM_DEBUG(dbgs() << ".. Reduce number of elements\n"); return fewerElementsVector(MI, Step.TypeIdx, Step.NewType); + case MoreElements: + LLVM_DEBUG(dbgs() << ".. Increase number of elements\n"); + return moreElementsVector(MI, Step.TypeIdx, Step.NewType); case Custom: LLVM_DEBUG(dbgs() << ".. Custom legalization\n"); return LI.legalizeCustom(MI, MRI, MIRBuilder, Observer) ? Legalized @@ -80,13 +119,103 @@ LegalizerHelper::legalizeInstrStep(MachineInstr &MI) { } } -void LegalizerHelper::extractParts(unsigned Reg, LLT Ty, int NumParts, - SmallVectorImpl &VRegs) { +void LegalizerHelper::extractParts(Register Reg, LLT Ty, int NumParts, + SmallVectorImpl &VRegs) { for (int i = 0; i < NumParts; ++i) VRegs.push_back(MRI.createGenericVirtualRegister(Ty)); MIRBuilder.buildUnmerge(VRegs, Reg); } +bool LegalizerHelper::extractParts(Register Reg, LLT RegTy, + LLT MainTy, LLT &LeftoverTy, + SmallVectorImpl &VRegs, + SmallVectorImpl &LeftoverRegs) { + assert(!LeftoverTy.isValid() && "this is an out argument"); + + unsigned RegSize = RegTy.getSizeInBits(); + unsigned MainSize = MainTy.getSizeInBits(); + unsigned NumParts = RegSize / MainSize; + unsigned LeftoverSize = RegSize - NumParts * MainSize; + + // Use an unmerge when possible. + if (LeftoverSize == 0) { + for (unsigned I = 0; I < NumParts; ++I) + VRegs.push_back(MRI.createGenericVirtualRegister(MainTy)); + MIRBuilder.buildUnmerge(VRegs, Reg); + return true; + } + + if (MainTy.isVector()) { + unsigned EltSize = MainTy.getScalarSizeInBits(); + if (LeftoverSize % EltSize != 0) + return false; + LeftoverTy = LLT::scalarOrVector(LeftoverSize / EltSize, EltSize); + } else { + LeftoverTy = LLT::scalar(LeftoverSize); + } + + // For irregular sizes, extract the individual parts. + for (unsigned I = 0; I != NumParts; ++I) { + Register NewReg = MRI.createGenericVirtualRegister(MainTy); + VRegs.push_back(NewReg); + MIRBuilder.buildExtract(NewReg, Reg, MainSize * I); + } + + for (unsigned Offset = MainSize * NumParts; Offset < RegSize; + Offset += LeftoverSize) { + Register NewReg = MRI.createGenericVirtualRegister(LeftoverTy); + LeftoverRegs.push_back(NewReg); + MIRBuilder.buildExtract(NewReg, Reg, Offset); + } + + return true; +} + +void LegalizerHelper::insertParts(Register DstReg, + LLT ResultTy, LLT PartTy, + ArrayRef PartRegs, + LLT LeftoverTy, + ArrayRef LeftoverRegs) { + if (!LeftoverTy.isValid()) { + assert(LeftoverRegs.empty()); + + if (!ResultTy.isVector()) { + MIRBuilder.buildMerge(DstReg, PartRegs); + return; + } + + if (PartTy.isVector()) + MIRBuilder.buildConcatVectors(DstReg, PartRegs); + else + MIRBuilder.buildBuildVector(DstReg, PartRegs); + return; + } + + unsigned PartSize = PartTy.getSizeInBits(); + unsigned LeftoverPartSize = LeftoverTy.getSizeInBits(); + + Register CurResultReg = MRI.createGenericVirtualRegister(ResultTy); + MIRBuilder.buildUndef(CurResultReg); + + unsigned Offset = 0; + for (Register PartReg : PartRegs) { + Register NewResultReg = MRI.createGenericVirtualRegister(ResultTy); + MIRBuilder.buildInsert(NewResultReg, CurResultReg, PartReg, Offset); + CurResultReg = NewResultReg; + Offset += PartSize; + } + + for (unsigned I = 0, E = LeftoverRegs.size(); I != E; ++I) { + // Use the original output register for the final insert to avoid a copy. + Register NewResultReg = (I + 1 == E) ? + DstReg : MRI.createGenericVirtualRegister(ResultTy); + + MIRBuilder.buildInsert(NewResultReg, CurResultReg, LeftoverRegs[I], Offset); + CurResultReg = NewResultReg; + Offset += LeftoverPartSize; + } +} + static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) { switch (Opcode) { case TargetOpcode::G_SDIV: @@ -116,6 +245,12 @@ static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) { case TargetOpcode::G_FDIV: assert((Size == 32 || Size == 64) && "Unsupported size"); return Size == 64 ? RTLIB::DIV_F64 : RTLIB::DIV_F32; + case TargetOpcode::G_FEXP: + assert((Size == 32 || Size == 64) && "Unsupported size"); + return Size == 64 ? RTLIB::EXP_F64 : RTLIB::EXP_F32; + case TargetOpcode::G_FEXP2: + assert((Size == 32 || Size == 64) && "Unsupported size"); + return Size == 64 ? RTLIB::EXP2_F64 : RTLIB::EXP2_F32; case TargetOpcode::G_FREM: return Size == 64 ? RTLIB::REM_F64 : RTLIB::REM_F32; case TargetOpcode::G_FPOW: @@ -123,6 +258,32 @@ static RTLIB::Libcall getRTLibDesc(unsigned Opcode, unsigned Size) { case TargetOpcode::G_FMA: assert((Size == 32 || Size == 64) && "Unsupported size"); return Size == 64 ? RTLIB::FMA_F64 : RTLIB::FMA_F32; + case TargetOpcode::G_FSIN: + assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"); + return Size == 128 ? RTLIB::SIN_F128 + : Size == 64 ? RTLIB::SIN_F64 : RTLIB::SIN_F32; + case TargetOpcode::G_FCOS: + assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"); + return Size == 128 ? RTLIB::COS_F128 + : Size == 64 ? RTLIB::COS_F64 : RTLIB::COS_F32; + case TargetOpcode::G_FLOG10: + assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"); + return Size == 128 ? RTLIB::LOG10_F128 + : Size == 64 ? RTLIB::LOG10_F64 : RTLIB::LOG10_F32; + case TargetOpcode::G_FLOG: + assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"); + return Size == 128 ? RTLIB::LOG_F128 + : Size == 64 ? RTLIB::LOG_F64 : RTLIB::LOG_F32; + case TargetOpcode::G_FLOG2: + assert((Size == 32 || Size == 64 || Size == 128) && "Unsupported size"); + return Size == 128 ? RTLIB::LOG2_F128 + : Size == 64 ? RTLIB::LOG2_F64 : RTLIB::LOG2_F32; + case TargetOpcode::G_FCEIL: + assert((Size == 32 || Size == 64) && "Unsupported size"); + return Size == 64 ? RTLIB::CEIL_F64 : RTLIB::CEIL_F32; + case TargetOpcode::G_FFLOOR: + assert((Size == 32 || Size == 64) && "Unsupported size"); + return Size == 64 ? RTLIB::FLOOR_F64 : RTLIB::FLOOR_F32; } llvm_unreachable("Unknown libcall function"); } @@ -214,7 +375,20 @@ LegalizerHelper::libcall(MachineInstr &MI) { case TargetOpcode::G_FDIV: case TargetOpcode::G_FMA: case TargetOpcode::G_FPOW: - case TargetOpcode::G_FREM: { + case TargetOpcode::G_FREM: + case TargetOpcode::G_FCOS: + case TargetOpcode::G_FSIN: + case TargetOpcode::G_FLOG10: + case TargetOpcode::G_FLOG: + case TargetOpcode::G_FLOG2: + case TargetOpcode::G_FEXP: + case TargetOpcode::G_FEXP2: + case TargetOpcode::G_FCEIL: + case TargetOpcode::G_FFLOOR: { + if (Size > 64) { + LLVM_DEBUG(dbgs() << "Size " << Size << " too large to legalize.\n"); + return UnableToLegalize; + } Type *HLTy = Size == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx); auto Status = simpleLibcall(MI, MIRBuilder, Size, HLTy); if (Status != Legalized) @@ -250,10 +424,11 @@ LegalizerHelper::libcall(MachineInstr &MI) { // FIXME: Support other types unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); - if (ToSize != 32 || (FromSize != 32 && FromSize != 64)) + if ((ToSize != 32 && ToSize != 64) || (FromSize != 32 && FromSize != 64)) return UnableToLegalize; LegalizeResult Status = conversionLibcall( - MI, MIRBuilder, Type::getInt32Ty(Ctx), + MI, MIRBuilder, + ToSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx), FromSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx)); if (Status != Legalized) return Status; @@ -264,12 +439,12 @@ LegalizerHelper::libcall(MachineInstr &MI) { // FIXME: Support other types unsigned FromSize = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); unsigned ToSize = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); - if (FromSize != 32 || (ToSize != 32 && ToSize != 64)) + if ((FromSize != 32 && FromSize != 64) || (ToSize != 32 && ToSize != 64)) return UnableToLegalize; LegalizeResult Status = conversionLibcall( MI, MIRBuilder, ToSize == 64 ? Type::getDoubleTy(Ctx) : Type::getFloatTy(Ctx), - Type::getInt32Ty(Ctx)); + FromSize == 32 ? Type::getInt32Ty(Ctx) : Type::getInt64Ty(Ctx)); if (Status != Legalized) return Status; break; @@ -283,10 +458,6 @@ LegalizerHelper::libcall(MachineInstr &MI) { LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) { - // FIXME: Don't know how to handle secondary types yet. - if (TypeIdx != 0 && MI.getOpcode() != TargetOpcode::G_EXTRACT) - return UnableToLegalize; - MIRBuilder.setInstr(MI); uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); @@ -302,12 +473,12 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI, return UnableToLegalize; int NumParts = SizeOp0 / NarrowSize; - SmallVector DstRegs; + SmallVector DstRegs; for (int i = 0; i < NumParts; ++i) DstRegs.push_back( MIRBuilder.buildUndef(NarrowTy)->getOperand(0).getReg()); - unsigned DstReg = MI.getOperand(0).getReg(); + Register DstReg = MI.getOperand(0).getReg(); if(MRI.getType(DstReg).isVector()) MIRBuilder.buildBuildVector(DstReg, DstRegs); else @@ -315,6 +486,38 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI, MI.eraseFromParent(); return Legalized; } + case TargetOpcode::G_CONSTANT: { + LLT Ty = MRI.getType(MI.getOperand(0).getReg()); + const APInt &Val = MI.getOperand(1).getCImm()->getValue(); + unsigned TotalSize = Ty.getSizeInBits(); + unsigned NarrowSize = NarrowTy.getSizeInBits(); + int NumParts = TotalSize / NarrowSize; + + SmallVector PartRegs; + for (int I = 0; I != NumParts; ++I) { + unsigned Offset = I * NarrowSize; + auto K = MIRBuilder.buildConstant(NarrowTy, + Val.lshr(Offset).trunc(NarrowSize)); + PartRegs.push_back(K.getReg(0)); + } + + LLT LeftoverTy; + unsigned LeftoverBits = TotalSize - NumParts * NarrowSize; + SmallVector LeftoverRegs; + if (LeftoverBits != 0) { + LeftoverTy = LLT::scalar(LeftoverBits); + auto K = MIRBuilder.buildConstant( + LeftoverTy, + Val.lshr(NumParts * NarrowSize).trunc(LeftoverBits)); + LeftoverRegs.push_back(K.getReg(0)); + } + + insertParts(MI.getOperand(0).getReg(), + Ty, NarrowTy, PartRegs, LeftoverTy, LeftoverRegs); + + MI.eraseFromParent(); + return Legalized; + } case TargetOpcode::G_ADD: { // FIXME: add support for when SizeOp0 isn't an exact multiple of // NarrowSize. @@ -323,16 +526,16 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI, // Expand in terms of carry-setting/consuming G_ADDE instructions. int NumParts = SizeOp0 / NarrowTy.getSizeInBits(); - SmallVector Src1Regs, Src2Regs, DstRegs; + SmallVector Src1Regs, Src2Regs, DstRegs; extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs); extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs); - unsigned CarryIn = MRI.createGenericVirtualRegister(LLT::scalar(1)); + Register CarryIn = MRI.createGenericVirtualRegister(LLT::scalar(1)); MIRBuilder.buildConstant(CarryIn, 0); for (int i = 0; i < NumParts; ++i) { - unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy); - unsigned CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1)); + Register DstReg = MRI.createGenericVirtualRegister(NarrowTy); + Register CarryOut = MRI.createGenericVirtualRegister(LLT::scalar(1)); MIRBuilder.buildUAdde(DstReg, CarryOut, Src1Regs[i], Src2Regs[i], CarryIn); @@ -340,67 +543,7 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI, DstRegs.push_back(DstReg); CarryIn = CarryOut; } - unsigned DstReg = MI.getOperand(0).getReg(); - if(MRI.getType(DstReg).isVector()) - MIRBuilder.buildBuildVector(DstReg, DstRegs); - else - MIRBuilder.buildMerge(DstReg, DstRegs); - MI.eraseFromParent(); - return Legalized; - } - case TargetOpcode::G_EXTRACT: { - if (TypeIdx != 1) - return UnableToLegalize; - - int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); - // FIXME: add support for when SizeOp1 isn't an exact multiple of - // NarrowSize. - if (SizeOp1 % NarrowSize != 0) - return UnableToLegalize; - int NumParts = SizeOp1 / NarrowSize; - - SmallVector SrcRegs, DstRegs; - SmallVector Indexes; - extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs); - - unsigned OpReg = MI.getOperand(0).getReg(); - uint64_t OpStart = MI.getOperand(2).getImm(); - uint64_t OpSize = MRI.getType(OpReg).getSizeInBits(); - for (int i = 0; i < NumParts; ++i) { - unsigned SrcStart = i * NarrowSize; - - if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) { - // No part of the extract uses this subregister, ignore it. - continue; - } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) { - // The entire subregister is extracted, forward the value. - DstRegs.push_back(SrcRegs[i]); - continue; - } - - // OpSegStart is where this destination segment would start in OpReg if it - // extended infinitely in both directions. - int64_t ExtractOffset; - uint64_t SegSize; - if (OpStart < SrcStart) { - ExtractOffset = 0; - SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart); - } else { - ExtractOffset = OpStart - SrcStart; - SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize); - } - - unsigned SegReg = SrcRegs[i]; - if (ExtractOffset != 0 || SegSize != NarrowSize) { - // A genuine extract is needed. - SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize)); - MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset); - } - - DstRegs.push_back(SegReg); - } - - unsigned DstReg = MI.getOperand(0).getReg(); + Register DstReg = MI.getOperand(0).getReg(); if(MRI.getType(DstReg).isVector()) MIRBuilder.buildBuildVector(DstReg, DstRegs); else @@ -408,178 +551,117 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI, MI.eraseFromParent(); return Legalized; } - case TargetOpcode::G_INSERT: { + case TargetOpcode::G_SUB: { // FIXME: add support for when SizeOp0 isn't an exact multiple of // NarrowSize. if (SizeOp0 % NarrowSize != 0) return UnableToLegalize; - int NumParts = SizeOp0 / NarrowSize; - - SmallVector SrcRegs, DstRegs; - SmallVector Indexes; - extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs); + int NumParts = SizeOp0 / NarrowTy.getSizeInBits(); - unsigned OpReg = MI.getOperand(2).getReg(); - uint64_t OpStart = MI.getOperand(3).getImm(); - uint64_t OpSize = MRI.getType(OpReg).getSizeInBits(); - for (int i = 0; i < NumParts; ++i) { - unsigned DstStart = i * NarrowSize; - - if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) { - // No part of the insert affects this subregister, forward the original. - DstRegs.push_back(SrcRegs[i]); - continue; - } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) { - // The entire subregister is defined by this insert, forward the new - // value. - DstRegs.push_back(OpReg); - continue; - } + SmallVector Src1Regs, Src2Regs, DstRegs; + extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs); + extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs); - // OpSegStart is where this destination segment would start in OpReg if it - // extended infinitely in both directions. - int64_t ExtractOffset, InsertOffset; - uint64_t SegSize; - if (OpStart < DstStart) { - InsertOffset = 0; - ExtractOffset = DstStart - OpStart; - SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart); - } else { - InsertOffset = OpStart - DstStart; - ExtractOffset = 0; - SegSize = - std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart); - } + Register DstReg = MRI.createGenericVirtualRegister(NarrowTy); + Register BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1)); + MIRBuilder.buildInstr(TargetOpcode::G_USUBO, {DstReg, BorrowOut}, + {Src1Regs[0], Src2Regs[0]}); + DstRegs.push_back(DstReg); + Register BorrowIn = BorrowOut; + for (int i = 1; i < NumParts; ++i) { + DstReg = MRI.createGenericVirtualRegister(NarrowTy); + BorrowOut = MRI.createGenericVirtualRegister(LLT::scalar(1)); - unsigned SegReg = OpReg; - if (ExtractOffset != 0 || SegSize != OpSize) { - // A genuine extract is needed. - SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize)); - MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset); - } + MIRBuilder.buildInstr(TargetOpcode::G_USUBE, {DstReg, BorrowOut}, + {Src1Regs[i], Src2Regs[i], BorrowIn}); - unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy); - MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset); DstRegs.push_back(DstReg); + BorrowIn = BorrowOut; } - - assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered"); - unsigned DstReg = MI.getOperand(0).getReg(); - if(MRI.getType(DstReg).isVector()) - MIRBuilder.buildBuildVector(DstReg, DstRegs); - else - MIRBuilder.buildMerge(DstReg, DstRegs); + MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs); MI.eraseFromParent(); return Legalized; } + case TargetOpcode::G_MUL: + case TargetOpcode::G_UMULH: + return narrowScalarMul(MI, NarrowTy); + case TargetOpcode::G_EXTRACT: + return narrowScalarExtract(MI, TypeIdx, NarrowTy); + case TargetOpcode::G_INSERT: + return narrowScalarInsert(MI, TypeIdx, NarrowTy); case TargetOpcode::G_LOAD: { - // FIXME: add support for when SizeOp0 isn't an exact multiple of - // NarrowSize. - if (SizeOp0 % NarrowSize != 0) - return UnableToLegalize; - const auto &MMO = **MI.memoperands_begin(); - // This implementation doesn't work for atomics. Give up instead of doing - // something invalid. - if (MMO.getOrdering() != AtomicOrdering::NotAtomic || - MMO.getFailureOrdering() != AtomicOrdering::NotAtomic) + Register DstReg = MI.getOperand(0).getReg(); + LLT DstTy = MRI.getType(DstReg); + if (DstTy.isVector()) return UnableToLegalize; - int NumParts = SizeOp0 / NarrowSize; - LLT OffsetTy = LLT::scalar( - MRI.getType(MI.getOperand(1).getReg()).getScalarSizeInBits()); - - SmallVector DstRegs; - for (int i = 0; i < NumParts; ++i) { - unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy); - unsigned SrcReg = 0; - unsigned Adjustment = i * NarrowSize / 8; - unsigned Alignment = MinAlign(MMO.getAlignment(), Adjustment); - - MachineMemOperand *SplitMMO = MIRBuilder.getMF().getMachineMemOperand( - MMO.getPointerInfo().getWithOffset(Adjustment), MMO.getFlags(), - NarrowSize / 8, Alignment, MMO.getAAInfo(), MMO.getRanges(), - MMO.getSyncScopeID(), MMO.getOrdering(), MMO.getFailureOrdering()); - - MIRBuilder.materializeGEP(SrcReg, MI.getOperand(1).getReg(), OffsetTy, - Adjustment); + if (8 * MMO.getSize() != DstTy.getSizeInBits()) { + Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy); + auto &MMO = **MI.memoperands_begin(); + MIRBuilder.buildLoad(TmpReg, MI.getOperand(1).getReg(), MMO); + MIRBuilder.buildAnyExt(DstReg, TmpReg); + MI.eraseFromParent(); + return Legalized; + } - MIRBuilder.buildLoad(DstReg, SrcReg, *SplitMMO); + return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy); + } + case TargetOpcode::G_ZEXTLOAD: + case TargetOpcode::G_SEXTLOAD: { + bool ZExt = MI.getOpcode() == TargetOpcode::G_ZEXTLOAD; + Register DstReg = MI.getOperand(0).getReg(); + Register PtrReg = MI.getOperand(1).getReg(); - DstRegs.push_back(DstReg); + Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy); + auto &MMO = **MI.memoperands_begin(); + if (MMO.getSizeInBits() == NarrowSize) { + MIRBuilder.buildLoad(TmpReg, PtrReg, MMO); + } else { + unsigned ExtLoad = ZExt ? TargetOpcode::G_ZEXTLOAD + : TargetOpcode::G_SEXTLOAD; + MIRBuilder.buildInstr(ExtLoad) + .addDef(TmpReg) + .addUse(PtrReg) + .addMemOperand(&MMO); } - unsigned DstReg = MI.getOperand(0).getReg(); - if(MRI.getType(DstReg).isVector()) - MIRBuilder.buildBuildVector(DstReg, DstRegs); + + if (ZExt) + MIRBuilder.buildZExt(DstReg, TmpReg); else - MIRBuilder.buildMerge(DstReg, DstRegs); + MIRBuilder.buildSExt(DstReg, TmpReg); + MI.eraseFromParent(); return Legalized; } case TargetOpcode::G_STORE: { - // FIXME: add support for when SizeOp0 isn't an exact multiple of - // NarrowSize. - if (SizeOp0 % NarrowSize != 0) - return UnableToLegalize; - const auto &MMO = **MI.memoperands_begin(); - // This implementation doesn't work for atomics. Give up instead of doing - // something invalid. - if (MMO.getOrdering() != AtomicOrdering::NotAtomic || - MMO.getFailureOrdering() != AtomicOrdering::NotAtomic) + + Register SrcReg = MI.getOperand(0).getReg(); + LLT SrcTy = MRI.getType(SrcReg); + if (SrcTy.isVector()) return UnableToLegalize; int NumParts = SizeOp0 / NarrowSize; - LLT OffsetTy = LLT::scalar( - MRI.getType(MI.getOperand(1).getReg()).getScalarSizeInBits()); - - SmallVector SrcRegs; - extractParts(MI.getOperand(0).getReg(), NarrowTy, NumParts, SrcRegs); - - for (int i = 0; i < NumParts; ++i) { - unsigned DstReg = 0; - unsigned Adjustment = i * NarrowSize / 8; - unsigned Alignment = MinAlign(MMO.getAlignment(), Adjustment); - - MachineMemOperand *SplitMMO = MIRBuilder.getMF().getMachineMemOperand( - MMO.getPointerInfo().getWithOffset(Adjustment), MMO.getFlags(), - NarrowSize / 8, Alignment, MMO.getAAInfo(), MMO.getRanges(), - MMO.getSyncScopeID(), MMO.getOrdering(), MMO.getFailureOrdering()); - - MIRBuilder.materializeGEP(DstReg, MI.getOperand(1).getReg(), OffsetTy, - Adjustment); - - MIRBuilder.buildStore(SrcRegs[i], DstReg, *SplitMMO); - } - MI.eraseFromParent(); - return Legalized; - } - case TargetOpcode::G_CONSTANT: { - // FIXME: add support for when SizeOp0 isn't an exact multiple of - // NarrowSize. - if (SizeOp0 % NarrowSize != 0) + unsigned HandledSize = NumParts * NarrowTy.getSizeInBits(); + unsigned LeftoverBits = SrcTy.getSizeInBits() - HandledSize; + if (SrcTy.isVector() && LeftoverBits != 0) return UnableToLegalize; - int NumParts = SizeOp0 / NarrowSize; - const APInt &Cst = MI.getOperand(1).getCImm()->getValue(); - LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext(); - SmallVector DstRegs; - for (int i = 0; i < NumParts; ++i) { - unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy); - ConstantInt *CI = - ConstantInt::get(Ctx, Cst.lshr(NarrowSize * i).trunc(NarrowSize)); - MIRBuilder.buildConstant(DstReg, *CI); - DstRegs.push_back(DstReg); + if (8 * MMO.getSize() != SrcTy.getSizeInBits()) { + Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy); + auto &MMO = **MI.memoperands_begin(); + MIRBuilder.buildTrunc(TmpReg, SrcReg); + MIRBuilder.buildStore(TmpReg, MI.getOperand(1).getReg(), MMO); + MI.eraseFromParent(); + return Legalized; } - unsigned DstReg = MI.getOperand(0).getReg(); - if(MRI.getType(DstReg).isVector()) - MIRBuilder.buildBuildVector(DstReg, DstRegs); - else - MIRBuilder.buildMerge(DstReg, DstRegs); - MI.eraseFromParent(); - return Legalized; + + return reduceLoadStoreWidth(MI, 0, NarrowTy); } + case TargetOpcode::G_SELECT: + return narrowScalarSelect(MI, TypeIdx, NarrowTy); case TargetOpcode::G_AND: case TargetOpcode::G_OR: case TargetOpcode::G_XOR: { @@ -592,44 +674,112 @@ LegalizerHelper::LegalizeResult LegalizerHelper::narrowScalar(MachineInstr &MI, // ... // AN = BinOp BN, CN // A = G_MERGE_VALUES A1, ..., AN + return narrowScalarBasic(MI, TypeIdx, NarrowTy); + } + case TargetOpcode::G_SHL: + case TargetOpcode::G_LSHR: + case TargetOpcode::G_ASHR: + return narrowScalarShift(MI, TypeIdx, NarrowTy); + case TargetOpcode::G_CTLZ: + case TargetOpcode::G_CTLZ_ZERO_UNDEF: + case TargetOpcode::G_CTTZ: + case TargetOpcode::G_CTTZ_ZERO_UNDEF: + case TargetOpcode::G_CTPOP: + if (TypeIdx != 0) + return UnableToLegalize; // TODO - // FIXME: add support for when SizeOp0 isn't an exact multiple of - // NarrowSize. - if (SizeOp0 % NarrowSize != 0) + Observer.changingInstr(MI); + narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT); + Observer.changedInstr(MI); + return Legalized; + case TargetOpcode::G_INTTOPTR: + if (TypeIdx != 1) return UnableToLegalize; - int NumParts = SizeOp0 / NarrowSize; - // List the registers where the destination will be scattered. - SmallVector DstRegs; - // List the registers where the first argument will be split. - SmallVector SrcsReg1; - // List the registers where the second argument will be split. - SmallVector SrcsReg2; - // Create all the temporary registers. - for (int i = 0; i < NumParts; ++i) { - unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy); - unsigned SrcReg1 = MRI.createGenericVirtualRegister(NarrowTy); - unsigned SrcReg2 = MRI.createGenericVirtualRegister(NarrowTy); + Observer.changingInstr(MI); + narrowScalarSrc(MI, NarrowTy, 1); + Observer.changedInstr(MI); + return Legalized; + case TargetOpcode::G_PTRTOINT: + if (TypeIdx != 0) + return UnableToLegalize; - DstRegs.push_back(DstReg); - SrcsReg1.push_back(SrcReg1); - SrcsReg2.push_back(SrcReg2); + Observer.changingInstr(MI); + narrowScalarDst(MI, NarrowTy, 0, TargetOpcode::G_ZEXT); + Observer.changedInstr(MI); + return Legalized; + case TargetOpcode::G_PHI: { + unsigned NumParts = SizeOp0 / NarrowSize; + SmallVector DstRegs; + SmallVector, 2> SrcRegs; + DstRegs.resize(NumParts); + SrcRegs.resize(MI.getNumOperands() / 2); + Observer.changingInstr(MI); + for (unsigned i = 1; i < MI.getNumOperands(); i += 2) { + MachineBasicBlock &OpMBB = *MI.getOperand(i + 1).getMBB(); + MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); + extractParts(MI.getOperand(i).getReg(), NarrowTy, NumParts, + SrcRegs[i / 2]); + } + MachineBasicBlock &MBB = *MI.getParent(); + MIRBuilder.setInsertPt(MBB, MI); + for (unsigned i = 0; i < NumParts; ++i) { + DstRegs[i] = MRI.createGenericVirtualRegister(NarrowTy); + MachineInstrBuilder MIB = + MIRBuilder.buildInstr(TargetOpcode::G_PHI).addDef(DstRegs[i]); + for (unsigned j = 1; j < MI.getNumOperands(); j += 2) + MIB.addUse(SrcRegs[j / 2][i]).add(MI.getOperand(j + 1)); } - // Explode the big arguments into smaller chunks. - MIRBuilder.buildUnmerge(SrcsReg1, MI.getOperand(1).getReg()); - MIRBuilder.buildUnmerge(SrcsReg2, MI.getOperand(2).getReg()); + MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI()); + MIRBuilder.buildMerge(MI.getOperand(0).getReg(), DstRegs); + Observer.changedInstr(MI); + MI.eraseFromParent(); + return Legalized; + } + case TargetOpcode::G_EXTRACT_VECTOR_ELT: + case TargetOpcode::G_INSERT_VECTOR_ELT: { + if (TypeIdx != 2) + return UnableToLegalize; - // Do the operation on each small part. - for (int i = 0; i < NumParts; ++i) - MIRBuilder.buildInstr(MI.getOpcode(), {DstRegs[i]}, - {SrcsReg1[i], SrcsReg2[i]}); + int OpIdx = MI.getOpcode() == TargetOpcode::G_EXTRACT_VECTOR_ELT ? 2 : 3; + Observer.changingInstr(MI); + narrowScalarSrc(MI, NarrowTy, OpIdx); + Observer.changedInstr(MI); + return Legalized; + } + case TargetOpcode::G_ICMP: { + uint64_t SrcSize = MRI.getType(MI.getOperand(2).getReg()).getSizeInBits(); + if (NarrowSize * 2 != SrcSize) + return UnableToLegalize; - // Gather the destination registers into the final destination. - unsigned DstReg = MI.getOperand(0).getReg(); - if(MRI.getType(DstReg).isVector()) - MIRBuilder.buildBuildVector(DstReg, DstRegs); - else - MIRBuilder.buildMerge(DstReg, DstRegs); + Observer.changingInstr(MI); + Register LHSL = MRI.createGenericVirtualRegister(NarrowTy); + Register LHSH = MRI.createGenericVirtualRegister(NarrowTy); + MIRBuilder.buildUnmerge({LHSL, LHSH}, MI.getOperand(2).getReg()); + + Register RHSL = MRI.createGenericVirtualRegister(NarrowTy); + Register RHSH = MRI.createGenericVirtualRegister(NarrowTy); + MIRBuilder.buildUnmerge({RHSL, RHSH}, MI.getOperand(3).getReg()); + + CmpInst::Predicate Pred = + static_cast(MI.getOperand(1).getPredicate()); + + if (Pred == CmpInst::ICMP_EQ || Pred == CmpInst::ICMP_NE) { + MachineInstrBuilder XorL = MIRBuilder.buildXor(NarrowTy, LHSL, RHSL); + MachineInstrBuilder XorH = MIRBuilder.buildXor(NarrowTy, LHSH, RHSH); + MachineInstrBuilder Or = MIRBuilder.buildOr(NarrowTy, XorL, XorH); + MachineInstrBuilder Zero = MIRBuilder.buildConstant(NarrowTy, 0); + MIRBuilder.buildICmp(Pred, MI.getOperand(0).getReg(), Or, Zero); + } else { + const LLT s1 = LLT::scalar(1); + MachineInstrBuilder CmpH = MIRBuilder.buildICmp(Pred, s1, LHSH, RHSH); + MachineInstrBuilder CmpHEQ = + MIRBuilder.buildICmp(CmpInst::Predicate::ICMP_EQ, s1, LHSH, RHSH); + MachineInstrBuilder CmpLU = MIRBuilder.buildICmp( + ICmpInst::getUnsignedPredicate(Pred), s1, LHSL, RHSL); + MIRBuilder.buildSelect(MI.getOperand(0).getReg(), CmpHEQ, CmpLU, CmpH); + } + Observer.changedInstr(MI); MI.eraseFromParent(); return Legalized; } @@ -643,43 +793,358 @@ void LegalizerHelper::widenScalarSrc(MachineInstr &MI, LLT WideTy, MO.setReg(ExtB->getOperand(0).getReg()); } +void LegalizerHelper::narrowScalarSrc(MachineInstr &MI, LLT NarrowTy, + unsigned OpIdx) { + MachineOperand &MO = MI.getOperand(OpIdx); + auto ExtB = MIRBuilder.buildInstr(TargetOpcode::G_TRUNC, {NarrowTy}, + {MO.getReg()}); + MO.setReg(ExtB->getOperand(0).getReg()); +} + void LegalizerHelper::widenScalarDst(MachineInstr &MI, LLT WideTy, unsigned OpIdx, unsigned TruncOpcode) { MachineOperand &MO = MI.getOperand(OpIdx); - unsigned DstExt = MRI.createGenericVirtualRegister(WideTy); + Register DstExt = MRI.createGenericVirtualRegister(WideTy); MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); MIRBuilder.buildInstr(TruncOpcode, {MO.getReg()}, {DstExt}); MO.setReg(DstExt); } +void LegalizerHelper::narrowScalarDst(MachineInstr &MI, LLT NarrowTy, + unsigned OpIdx, unsigned ExtOpcode) { + MachineOperand &MO = MI.getOperand(OpIdx); + Register DstTrunc = MRI.createGenericVirtualRegister(NarrowTy); + MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); + MIRBuilder.buildInstr(ExtOpcode, {MO.getReg()}, {DstTrunc}); + MO.setReg(DstTrunc); +} + +void LegalizerHelper::moreElementsVectorDst(MachineInstr &MI, LLT WideTy, + unsigned OpIdx) { + MachineOperand &MO = MI.getOperand(OpIdx); + Register DstExt = MRI.createGenericVirtualRegister(WideTy); + MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); + MIRBuilder.buildExtract(MO.getReg(), DstExt, 0); + MO.setReg(DstExt); +} + +void LegalizerHelper::moreElementsVectorSrc(MachineInstr &MI, LLT MoreTy, + unsigned OpIdx) { + MachineOperand &MO = MI.getOperand(OpIdx); + + LLT OldTy = MRI.getType(MO.getReg()); + unsigned OldElts = OldTy.getNumElements(); + unsigned NewElts = MoreTy.getNumElements(); + + unsigned NumParts = NewElts / OldElts; + + // Use concat_vectors if the result is a multiple of the number of elements. + if (NumParts * OldElts == NewElts) { + SmallVector Parts; + Parts.push_back(MO.getReg()); + + Register ImpDef = MIRBuilder.buildUndef(OldTy).getReg(0); + for (unsigned I = 1; I != NumParts; ++I) + Parts.push_back(ImpDef); + + auto Concat = MIRBuilder.buildConcatVectors(MoreTy, Parts); + MO.setReg(Concat.getReg(0)); + return; + } + + Register MoreReg = MRI.createGenericVirtualRegister(MoreTy); + Register ImpDef = MIRBuilder.buildUndef(MoreTy).getReg(0); + MIRBuilder.buildInsert(MoreReg, ImpDef, MO.getReg(), 0); + MO.setReg(MoreReg); +} + LegalizerHelper::LegalizeResult -LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { - MIRBuilder.setInstr(MI); +LegalizerHelper::widenScalarMergeValues(MachineInstr &MI, unsigned TypeIdx, + LLT WideTy) { + if (TypeIdx != 1) + return UnableToLegalize; - switch (MI.getOpcode()) { - default: + Register DstReg = MI.getOperand(0).getReg(); + LLT DstTy = MRI.getType(DstReg); + if (DstTy.isVector()) return UnableToLegalize; - case TargetOpcode::G_UADDO: - case TargetOpcode::G_USUBO: { - if (TypeIdx == 1) - return UnableToLegalize; // TODO - auto LHSZext = MIRBuilder.buildInstr(TargetOpcode::G_ZEXT, {WideTy}, - {MI.getOperand(2).getReg()}); - auto RHSZext = MIRBuilder.buildInstr(TargetOpcode::G_ZEXT, {WideTy}, - {MI.getOperand(3).getReg()}); - unsigned Opcode = MI.getOpcode() == TargetOpcode::G_UADDO - ? TargetOpcode::G_ADD - : TargetOpcode::G_SUB; - // Do the arithmetic in the larger type. - auto NewOp = MIRBuilder.buildInstr(Opcode, {WideTy}, {LHSZext, RHSZext}); - LLT OrigTy = MRI.getType(MI.getOperand(0).getReg()); - APInt Mask = APInt::getAllOnesValue(OrigTy.getSizeInBits()); - auto AndOp = MIRBuilder.buildInstr( - TargetOpcode::G_AND, {WideTy}, - {NewOp, MIRBuilder.buildConstant(WideTy, Mask.getZExtValue())}); - // There is no overflow if the AndOp is the same as NewOp. - MIRBuilder.buildICmp(CmpInst::ICMP_NE, MI.getOperand(1).getReg(), NewOp, - AndOp); + + Register Src1 = MI.getOperand(1).getReg(); + LLT SrcTy = MRI.getType(Src1); + const int DstSize = DstTy.getSizeInBits(); + const int SrcSize = SrcTy.getSizeInBits(); + const int WideSize = WideTy.getSizeInBits(); + const int NumMerge = (DstSize + WideSize - 1) / WideSize; + + unsigned NumOps = MI.getNumOperands(); + unsigned NumSrc = MI.getNumOperands() - 1; + unsigned PartSize = DstTy.getSizeInBits() / NumSrc; + + if (WideSize >= DstSize) { + // Directly pack the bits in the target type. + Register ResultReg = MIRBuilder.buildZExt(WideTy, Src1).getReg(0); + + for (unsigned I = 2; I != NumOps; ++I) { + const unsigned Offset = (I - 1) * PartSize; + + Register SrcReg = MI.getOperand(I).getReg(); + assert(MRI.getType(SrcReg) == LLT::scalar(PartSize)); + + auto ZextInput = MIRBuilder.buildZExt(WideTy, SrcReg); + + Register NextResult = I + 1 == NumOps && WideSize == DstSize ? DstReg : + MRI.createGenericVirtualRegister(WideTy); + + auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Offset); + auto Shl = MIRBuilder.buildShl(WideTy, ZextInput, ShiftAmt); + MIRBuilder.buildOr(NextResult, ResultReg, Shl); + ResultReg = NextResult; + } + + if (WideSize > DstSize) + MIRBuilder.buildTrunc(DstReg, ResultReg); + + MI.eraseFromParent(); + return Legalized; + } + + // Unmerge the original values to the GCD type, and recombine to the next + // multiple greater than the original type. + // + // %3:_(s12) = G_MERGE_VALUES %0:_(s4), %1:_(s4), %2:_(s4) -> s6 + // %4:_(s2), %5:_(s2) = G_UNMERGE_VALUES %0 + // %6:_(s2), %7:_(s2) = G_UNMERGE_VALUES %1 + // %8:_(s2), %9:_(s2) = G_UNMERGE_VALUES %2 + // %10:_(s6) = G_MERGE_VALUES %4, %5, %6 + // %11:_(s6) = G_MERGE_VALUES %7, %8, %9 + // %12:_(s12) = G_MERGE_VALUES %10, %11 + // + // Padding with undef if necessary: + // + // %2:_(s8) = G_MERGE_VALUES %0:_(s4), %1:_(s4) -> s6 + // %3:_(s2), %4:_(s2) = G_UNMERGE_VALUES %0 + // %5:_(s2), %6:_(s2) = G_UNMERGE_VALUES %1 + // %7:_(s2) = G_IMPLICIT_DEF + // %8:_(s6) = G_MERGE_VALUES %3, %4, %5 + // %9:_(s6) = G_MERGE_VALUES %6, %7, %7 + // %10:_(s12) = G_MERGE_VALUES %8, %9 + + const int GCD = greatestCommonDivisor(SrcSize, WideSize); + LLT GCDTy = LLT::scalar(GCD); + + SmallVector Parts; + SmallVector NewMergeRegs; + SmallVector Unmerges; + LLT WideDstTy = LLT::scalar(NumMerge * WideSize); + + // Decompose the original operands if they don't evenly divide. + for (int I = 1, E = MI.getNumOperands(); I != E; ++I) { + Register SrcReg = MI.getOperand(I).getReg(); + if (GCD == SrcSize) { + Unmerges.push_back(SrcReg); + } else { + auto Unmerge = MIRBuilder.buildUnmerge(GCDTy, SrcReg); + for (int J = 0, JE = Unmerge->getNumOperands() - 1; J != JE; ++J) + Unmerges.push_back(Unmerge.getReg(J)); + } + } + + // Pad with undef to the next size that is a multiple of the requested size. + if (static_cast(Unmerges.size()) != NumMerge * WideSize) { + Register UndefReg = MIRBuilder.buildUndef(GCDTy).getReg(0); + for (int I = Unmerges.size(); I != NumMerge * WideSize; ++I) + Unmerges.push_back(UndefReg); + } + + const int PartsPerGCD = WideSize / GCD; + + // Build merges of each piece. + ArrayRef Slicer(Unmerges); + for (int I = 0; I != NumMerge; ++I, Slicer = Slicer.drop_front(PartsPerGCD)) { + auto Merge = MIRBuilder.buildMerge(WideTy, Slicer.take_front(PartsPerGCD)); + NewMergeRegs.push_back(Merge.getReg(0)); + } + + // A truncate may be necessary if the requested type doesn't evenly divide the + // original result type. + if (DstTy.getSizeInBits() == WideDstTy.getSizeInBits()) { + MIRBuilder.buildMerge(DstReg, NewMergeRegs); + } else { + auto FinalMerge = MIRBuilder.buildMerge(WideDstTy, NewMergeRegs); + MIRBuilder.buildTrunc(DstReg, FinalMerge.getReg(0)); + } + + MI.eraseFromParent(); + return Legalized; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::widenScalarUnmergeValues(MachineInstr &MI, unsigned TypeIdx, + LLT WideTy) { + if (TypeIdx != 0) + return UnableToLegalize; + + unsigned NumDst = MI.getNumOperands() - 1; + Register SrcReg = MI.getOperand(NumDst).getReg(); + LLT SrcTy = MRI.getType(SrcReg); + if (!SrcTy.isScalar()) + return UnableToLegalize; + + Register Dst0Reg = MI.getOperand(0).getReg(); + LLT DstTy = MRI.getType(Dst0Reg); + if (!DstTy.isScalar()) + return UnableToLegalize; + + unsigned NewSrcSize = NumDst * WideTy.getSizeInBits(); + LLT NewSrcTy = LLT::scalar(NewSrcSize); + unsigned SizeDiff = WideTy.getSizeInBits() - DstTy.getSizeInBits(); + + auto WideSrc = MIRBuilder.buildZExt(NewSrcTy, SrcReg); + + for (unsigned I = 1; I != NumDst; ++I) { + auto ShiftAmt = MIRBuilder.buildConstant(NewSrcTy, SizeDiff * I); + auto Shl = MIRBuilder.buildShl(NewSrcTy, WideSrc, ShiftAmt); + WideSrc = MIRBuilder.buildOr(NewSrcTy, WideSrc, Shl); + } + + Observer.changingInstr(MI); + + MI.getOperand(NumDst).setReg(WideSrc->getOperand(0).getReg()); + for (unsigned I = 0; I != NumDst; ++I) + widenScalarDst(MI, WideTy, I); + + Observer.changedInstr(MI); + + return Legalized; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::widenScalarExtract(MachineInstr &MI, unsigned TypeIdx, + LLT WideTy) { + Register DstReg = MI.getOperand(0).getReg(); + Register SrcReg = MI.getOperand(1).getReg(); + LLT SrcTy = MRI.getType(SrcReg); + + LLT DstTy = MRI.getType(DstReg); + unsigned Offset = MI.getOperand(2).getImm(); + + if (TypeIdx == 0) { + if (SrcTy.isVector() || DstTy.isVector()) + return UnableToLegalize; + + SrcOp Src(SrcReg); + if (SrcTy.isPointer()) { + // Extracts from pointers can be handled only if they are really just + // simple integers. + const DataLayout &DL = MIRBuilder.getDataLayout(); + if (DL.isNonIntegralAddressSpace(SrcTy.getAddressSpace())) + return UnableToLegalize; + + LLT SrcAsIntTy = LLT::scalar(SrcTy.getSizeInBits()); + Src = MIRBuilder.buildPtrToInt(SrcAsIntTy, Src); + SrcTy = SrcAsIntTy; + } + + if (DstTy.isPointer()) + return UnableToLegalize; + + if (Offset == 0) { + // Avoid a shift in the degenerate case. + MIRBuilder.buildTrunc(DstReg, + MIRBuilder.buildAnyExtOrTrunc(WideTy, Src)); + MI.eraseFromParent(); + return Legalized; + } + + // Do a shift in the source type. + LLT ShiftTy = SrcTy; + if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) { + Src = MIRBuilder.buildAnyExt(WideTy, Src); + ShiftTy = WideTy; + } else if (WideTy.getSizeInBits() > SrcTy.getSizeInBits()) + return UnableToLegalize; + + auto LShr = MIRBuilder.buildLShr( + ShiftTy, Src, MIRBuilder.buildConstant(ShiftTy, Offset)); + MIRBuilder.buildTrunc(DstReg, LShr); + MI.eraseFromParent(); + return Legalized; + } + + if (SrcTy.isScalar()) { + Observer.changingInstr(MI); + widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); + Observer.changedInstr(MI); + return Legalized; + } + + if (!SrcTy.isVector()) + return UnableToLegalize; + + if (DstTy != SrcTy.getElementType()) + return UnableToLegalize; + + if (Offset % SrcTy.getScalarSizeInBits() != 0) + return UnableToLegalize; + + Observer.changingInstr(MI); + widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); + + MI.getOperand(2).setImm((WideTy.getSizeInBits() / SrcTy.getSizeInBits()) * + Offset); + widenScalarDst(MI, WideTy.getScalarType(), 0); + Observer.changedInstr(MI); + return Legalized; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::widenScalarInsert(MachineInstr &MI, unsigned TypeIdx, + LLT WideTy) { + if (TypeIdx != 0) + return UnableToLegalize; + Observer.changingInstr(MI); + widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); + widenScalarDst(MI, WideTy); + Observer.changedInstr(MI); + return Legalized; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { + MIRBuilder.setInstr(MI); + + switch (MI.getOpcode()) { + default: + return UnableToLegalize; + case TargetOpcode::G_EXTRACT: + return widenScalarExtract(MI, TypeIdx, WideTy); + case TargetOpcode::G_INSERT: + return widenScalarInsert(MI, TypeIdx, WideTy); + case TargetOpcode::G_MERGE_VALUES: + return widenScalarMergeValues(MI, TypeIdx, WideTy); + case TargetOpcode::G_UNMERGE_VALUES: + return widenScalarUnmergeValues(MI, TypeIdx, WideTy); + case TargetOpcode::G_UADDO: + case TargetOpcode::G_USUBO: { + if (TypeIdx == 1) + return UnableToLegalize; // TODO + auto LHSZext = MIRBuilder.buildInstr(TargetOpcode::G_ZEXT, {WideTy}, + {MI.getOperand(2).getReg()}); + auto RHSZext = MIRBuilder.buildInstr(TargetOpcode::G_ZEXT, {WideTy}, + {MI.getOperand(3).getReg()}); + unsigned Opcode = MI.getOpcode() == TargetOpcode::G_UADDO + ? TargetOpcode::G_ADD + : TargetOpcode::G_SUB; + // Do the arithmetic in the larger type. + auto NewOp = MIRBuilder.buildInstr(Opcode, {WideTy}, {LHSZext, RHSZext}); + LLT OrigTy = MRI.getType(MI.getOperand(0).getReg()); + APInt Mask = APInt::getAllOnesValue(OrigTy.getSizeInBits()); + auto AndOp = MIRBuilder.buildInstr( + TargetOpcode::G_AND, {WideTy}, + {NewOp, MIRBuilder.buildConstant(WideTy, Mask.getZExtValue())}); + // There is no overflow if the AndOp is the same as NewOp. + MIRBuilder.buildICmp(CmpInst::ICMP_NE, MI.getOperand(1).getReg(), NewOp, + AndOp); // Now trunc the NewOp to the original result. MIRBuilder.buildTrunc(MI.getOperand(0).getReg(), NewOp); MI.eraseFromParent(); @@ -690,19 +1155,28 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { case TargetOpcode::G_CTLZ: case TargetOpcode::G_CTLZ_ZERO_UNDEF: case TargetOpcode::G_CTPOP: { + if (TypeIdx == 0) { + Observer.changingInstr(MI); + widenScalarDst(MI, WideTy, 0); + Observer.changedInstr(MI); + return Legalized; + } + + Register SrcReg = MI.getOperand(1).getReg(); + // First ZEXT the input. - auto MIBSrc = MIRBuilder.buildZExt(WideTy, MI.getOperand(1).getReg()); - LLT CurTy = MRI.getType(MI.getOperand(0).getReg()); + auto MIBSrc = MIRBuilder.buildZExt(WideTy, SrcReg); + LLT CurTy = MRI.getType(SrcReg); if (MI.getOpcode() == TargetOpcode::G_CTTZ) { // The count is the same in the larger type except if the original // value was zero. This can be handled by setting the bit just off // the top of the original type. auto TopBit = APInt::getOneBitSet(WideTy.getSizeInBits(), CurTy.getSizeInBits()); - MIBSrc = MIRBuilder.buildInstr( - TargetOpcode::G_OR, {WideTy}, - {MIBSrc, MIRBuilder.buildConstant(WideTy, TopBit.getSExtValue())}); + MIBSrc = MIRBuilder.buildOr( + WideTy, MIBSrc, MIRBuilder.buildConstant(WideTy, TopBit)); } + // Perform the operation at the larger size. auto MIBNewOp = MIRBuilder.buildInstr(MI.getOpcode(), {WideTy}, {MIBSrc}); // This is already the correct result for CTPOP and CTTZs @@ -714,22 +1188,43 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { TargetOpcode::G_SUB, {WideTy}, {MIBNewOp, MIRBuilder.buildConstant(WideTy, SizeDiff)}); } - auto &TII = *MI.getMF()->getSubtarget().getInstrInfo(); - // Make the original instruction a trunc now, and update its source. + + MIRBuilder.buildZExtOrTrunc(MI.getOperand(0), MIBNewOp); + MI.eraseFromParent(); + return Legalized; + } + case TargetOpcode::G_BSWAP: { Observer.changingInstr(MI); - MI.setDesc(TII.get(TargetOpcode::G_TRUNC)); - MI.getOperand(1).setReg(MIBNewOp->getOperand(0).getReg()); + Register DstReg = MI.getOperand(0).getReg(); + + Register ShrReg = MRI.createGenericVirtualRegister(WideTy); + Register DstExt = MRI.createGenericVirtualRegister(WideTy); + Register ShiftAmtReg = MRI.createGenericVirtualRegister(WideTy); + widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); + + MI.getOperand(0).setReg(DstExt); + + MIRBuilder.setInsertPt(MIRBuilder.getMBB(), ++MIRBuilder.getInsertPt()); + + LLT Ty = MRI.getType(DstReg); + unsigned DiffBits = WideTy.getScalarSizeInBits() - Ty.getScalarSizeInBits(); + MIRBuilder.buildConstant(ShiftAmtReg, DiffBits); + MIRBuilder.buildInstr(TargetOpcode::G_LSHR) + .addDef(ShrReg) + .addUse(DstExt) + .addUse(ShiftAmtReg); + + MIRBuilder.buildTrunc(DstReg, ShrReg); Observer.changedInstr(MI); return Legalized; } - case TargetOpcode::G_ADD: case TargetOpcode::G_AND: case TargetOpcode::G_MUL: case TargetOpcode::G_OR: case TargetOpcode::G_XOR: case TargetOpcode::G_SUB: - // Perform operation at larger width (any extension is fine here, high bits + // Perform operation at larger width (any extension is fines here, high bits // don't affect the result) and then truncate the result back to the // original type. Observer.changingInstr(MI); @@ -741,16 +1236,24 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { case TargetOpcode::G_SHL: Observer.changingInstr(MI); - widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); - // The "number of bits to shift" operand must preserve its value as an - // unsigned integer: - widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); - widenScalarDst(MI, WideTy); + + if (TypeIdx == 0) { + widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); + widenScalarDst(MI, WideTy); + } else { + assert(TypeIdx == 1); + // The "number of bits to shift" operand must preserve its value as an + // unsigned integer: + widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); + } + Observer.changedInstr(MI); return Legalized; case TargetOpcode::G_SDIV: case TargetOpcode::G_SREM: + case TargetOpcode::G_SMIN: + case TargetOpcode::G_SMAX: Observer.changingInstr(MI); widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT); widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT); @@ -759,18 +1262,28 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { return Legalized; case TargetOpcode::G_ASHR: + case TargetOpcode::G_LSHR: Observer.changingInstr(MI); - widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_SEXT); - // The "number of bits to shift" operand must preserve its value as an - // unsigned integer: - widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); - widenScalarDst(MI, WideTy); + + if (TypeIdx == 0) { + unsigned CvtOp = MI.getOpcode() == TargetOpcode::G_ASHR ? + TargetOpcode::G_SEXT : TargetOpcode::G_ZEXT; + + widenScalarSrc(MI, WideTy, 1, CvtOp); + widenScalarDst(MI, WideTy); + } else { + assert(TypeIdx == 1); + // The "number of bits to shift" operand must preserve its value as an + // unsigned integer: + widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); + } + Observer.changedInstr(MI); return Legalized; - case TargetOpcode::G_UDIV: case TargetOpcode::G_UREM: - case TargetOpcode::G_LSHR: + case TargetOpcode::G_UMIN: + case TargetOpcode::G_UMAX: Observer.changingInstr(MI); widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT); widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_ZEXT); @@ -788,8 +1301,9 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { widenScalarSrc(MI, WideTy, 3, TargetOpcode::G_ANYEXT); widenScalarDst(MI, WideTy); } else { + bool IsVec = MRI.getType(MI.getOperand(1).getReg()).isVector(); // Explicit extension is required here since high bits affect the result. - widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT); + widenScalarSrc(MI, WideTy, 1, MIRBuilder.getBoolExtOp(IsVec, false)); } Observer.changedInstr(MI); return Legalized; @@ -819,23 +1333,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { Observer.changedInstr(MI); return Legalized; - case TargetOpcode::G_INSERT: - if (TypeIdx != 0) - return UnableToLegalize; - Observer.changingInstr(MI); - widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ANYEXT); - widenScalarDst(MI, WideTy); - Observer.changedInstr(MI); - return Legalized; - case TargetOpcode::G_LOAD: - // For some types like i24, we might try to widen to i32. To properly handle - // this we should be using a dedicated extending load, until then avoid - // trying to legalize. - if (alignTo(MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(), 8) != - WideTy.getSizeInBits()) - return UnableToLegalize; - LLVM_FALLTHROUGH; case TargetOpcode::G_SEXTLOAD: case TargetOpcode::G_ZEXTLOAD: Observer.changingInstr(MI); @@ -844,12 +1342,19 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { return Legalized; case TargetOpcode::G_STORE: { - if (MRI.getType(MI.getOperand(0).getReg()) != LLT::scalar(1) || - WideTy != LLT::scalar(8)) + if (TypeIdx != 0) + return UnableToLegalize; + + LLT Ty = MRI.getType(MI.getOperand(0).getReg()); + if (!isPowerOf2_32(Ty.getSizeInBits())) return UnableToLegalize; Observer.changingInstr(MI); - widenScalarSrc(MI, WideTy, 0, TargetOpcode::G_ZEXT); + + unsigned ExtType = Ty.getScalarSizeInBits() == 1 ? + TargetOpcode::G_ZEXT : TargetOpcode::G_ANYEXT; + widenScalarSrc(MI, WideTy, 0, ExtType); + Observer.changedInstr(MI); return Legalized; } @@ -871,14 +1376,19 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { bool LosesInfo; switch (WideTy.getSizeInBits()) { case 32: - Val.convert(APFloat::IEEEsingle(), APFloat::rmTowardZero, &LosesInfo); + Val.convert(APFloat::IEEEsingle(), APFloat::rmNearestTiesToEven, + &LosesInfo); break; case 64: - Val.convert(APFloat::IEEEdouble(), APFloat::rmTowardZero, &LosesInfo); + Val.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven, + &LosesInfo); break; default: - llvm_unreachable("Unhandled fp widen type"); + return UnableToLegalize; } + + assert(!LosesInfo && "extend should always be lossless"); + Observer.changingInstr(MI); SrcMO.setFPImm(ConstantFP::get(Ctx, Val)); @@ -894,7 +1404,7 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { } case TargetOpcode::G_BRCOND: Observer.changingInstr(MI); - widenScalarSrc(MI, WideTy, 0, TargetOpcode::G_ANYEXT); + widenScalarSrc(MI, WideTy, 0, MIRBuilder.getBoolExtOp(false, false)); Observer.changedInstr(MI); return Legalized; @@ -947,23 +1457,103 @@ LegalizerHelper::widenScalar(MachineInstr &MI, unsigned TypeIdx, LLT WideTy) { Observer.changedInstr(MI); return Legalized; } - case TargetOpcode::G_EXTRACT_VECTOR_ELT: + case TargetOpcode::G_EXTRACT_VECTOR_ELT: { + if (TypeIdx == 0) { + Register VecReg = MI.getOperand(1).getReg(); + LLT VecTy = MRI.getType(VecReg); + Observer.changingInstr(MI); + + widenScalarSrc(MI, LLT::vector(VecTy.getNumElements(), + WideTy.getSizeInBits()), + 1, TargetOpcode::G_SEXT); + + widenScalarDst(MI, WideTy, 0); + Observer.changedInstr(MI); + return Legalized; + } + if (TypeIdx != 2) return UnableToLegalize; Observer.changingInstr(MI); widenScalarSrc(MI, WideTy, 2, TargetOpcode::G_SEXT); Observer.changedInstr(MI); return Legalized; - + } + case TargetOpcode::G_FADD: + case TargetOpcode::G_FMUL: + case TargetOpcode::G_FSUB: + case TargetOpcode::G_FMA: + case TargetOpcode::G_FNEG: + case TargetOpcode::G_FABS: + case TargetOpcode::G_FCANONICALIZE: + case TargetOpcode::G_FMINNUM: + case TargetOpcode::G_FMAXNUM: + case TargetOpcode::G_FMINNUM_IEEE: + case TargetOpcode::G_FMAXNUM_IEEE: + case TargetOpcode::G_FMINIMUM: + case TargetOpcode::G_FMAXIMUM: + case TargetOpcode::G_FDIV: + case TargetOpcode::G_FREM: case TargetOpcode::G_FCEIL: + case TargetOpcode::G_FFLOOR: + case TargetOpcode::G_FCOS: + case TargetOpcode::G_FSIN: + case TargetOpcode::G_FLOG10: + case TargetOpcode::G_FLOG: + case TargetOpcode::G_FLOG2: + case TargetOpcode::G_FRINT: + case TargetOpcode::G_FNEARBYINT: + case TargetOpcode::G_FSQRT: + case TargetOpcode::G_FEXP: + case TargetOpcode::G_FEXP2: + case TargetOpcode::G_FPOW: + case TargetOpcode::G_INTRINSIC_TRUNC: + case TargetOpcode::G_INTRINSIC_ROUND: + assert(TypeIdx == 0); + Observer.changingInstr(MI); + + for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) + widenScalarSrc(MI, WideTy, I, TargetOpcode::G_FPEXT); + + widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); + Observer.changedInstr(MI); + return Legalized; + case TargetOpcode::G_INTTOPTR: + if (TypeIdx != 1) + return UnableToLegalize; + + Observer.changingInstr(MI); + widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_ZEXT); + Observer.changedInstr(MI); + return Legalized; + case TargetOpcode::G_PTRTOINT: if (TypeIdx != 0) return UnableToLegalize; + Observer.changingInstr(MI); - widenScalarSrc(MI, WideTy, 1, TargetOpcode::G_FPEXT); - widenScalarDst(MI, WideTy, 0, TargetOpcode::G_FPTRUNC); + widenScalarDst(MI, WideTy, 0); + Observer.changedInstr(MI); + return Legalized; + case TargetOpcode::G_BUILD_VECTOR: { + Observer.changingInstr(MI); + + const LLT WideEltTy = TypeIdx == 1 ? WideTy : WideTy.getElementType(); + for (int I = 1, E = MI.getNumOperands(); I != E; ++I) + widenScalarSrc(MI, WideEltTy, I, TargetOpcode::G_ANYEXT); + + // Avoid changing the result vector type if the source element type was + // requested. + if (TypeIdx == 1) { + auto &TII = *MI.getMF()->getSubtarget().getInstrInfo(); + MI.setDesc(TII.get(TargetOpcode::G_BUILD_VECTOR_TRUNC)); + } else { + widenScalarDst(MI, WideTy, 0); + } + Observer.changedInstr(MI); return Legalized; } + } } LegalizerHelper::LegalizeResult @@ -976,13 +1566,13 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { return UnableToLegalize; case TargetOpcode::G_SREM: case TargetOpcode::G_UREM: { - unsigned QuotReg = MRI.createGenericVirtualRegister(Ty); + Register QuotReg = MRI.createGenericVirtualRegister(Ty); MIRBuilder.buildInstr(MI.getOpcode() == G_SREM ? G_SDIV : G_UDIV) .addDef(QuotReg) .addUse(MI.getOperand(1).getReg()) .addUse(MI.getOperand(2).getReg()); - unsigned ProdReg = MRI.createGenericVirtualRegister(Ty); + Register ProdReg = MRI.createGenericVirtualRegister(Ty); MIRBuilder.buildMul(ProdReg, QuotReg, MI.getOperand(2).getReg()); MIRBuilder.buildSub(MI.getOperand(0).getReg(), MI.getOperand(1).getReg(), ProdReg); @@ -993,10 +1583,10 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { case TargetOpcode::G_UMULO: { // Generate G_UMULH/G_SMULH to check for overflow and a normal G_MUL for the // result. - unsigned Res = MI.getOperand(0).getReg(); - unsigned Overflow = MI.getOperand(1).getReg(); - unsigned LHS = MI.getOperand(2).getReg(); - unsigned RHS = MI.getOperand(3).getReg(); + Register Res = MI.getOperand(0).getReg(); + Register Overflow = MI.getOperand(1).getReg(); + Register LHS = MI.getOperand(2).getReg(); + Register RHS = MI.getOperand(3).getReg(); MIRBuilder.buildMul(Res, LHS, RHS); @@ -1004,20 +1594,20 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { ? TargetOpcode::G_SMULH : TargetOpcode::G_UMULH; - unsigned HiPart = MRI.createGenericVirtualRegister(Ty); + Register HiPart = MRI.createGenericVirtualRegister(Ty); MIRBuilder.buildInstr(Opcode) .addDef(HiPart) .addUse(LHS) .addUse(RHS); - unsigned Zero = MRI.createGenericVirtualRegister(Ty); + Register Zero = MRI.createGenericVirtualRegister(Ty); MIRBuilder.buildConstant(Zero, 0); // For *signed* multiply, overflow is detected by checking: // (hi != (lo >> bitwidth-1)) if (Opcode == TargetOpcode::G_SMULH) { - unsigned Shifted = MRI.createGenericVirtualRegister(Ty); - unsigned ShiftAmt = MRI.createGenericVirtualRegister(Ty); + Register Shifted = MRI.createGenericVirtualRegister(Ty); + Register ShiftAmt = MRI.createGenericVirtualRegister(Ty); MIRBuilder.buildConstant(ShiftAmt, Ty.getSizeInBits() - 1); MIRBuilder.buildInstr(TargetOpcode::G_ASHR) .addDef(Shifted) @@ -1035,7 +1625,7 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { // represent them. if (Ty.isVector()) return UnableToLegalize; - unsigned Res = MI.getOperand(0).getReg(); + Register Res = MI.getOperand(0).getReg(); Type *ZeroTy; LLVMContext &Ctx = MIRBuilder.getMF().getFunction().getContext(); switch (Ty.getSizeInBits()) { @@ -1057,10 +1647,10 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { ConstantFP &ZeroForNegation = *cast(ConstantFP::getZeroValueForNegation(ZeroTy)); auto Zero = MIRBuilder.buildFConstant(Ty, ZeroForNegation); - MIRBuilder.buildInstr(TargetOpcode::G_FSUB) - .addDef(Res) - .addUse(Zero->getOperand(0).getReg()) - .addUse(MI.getOperand(1).getReg()); + Register SubByReg = MI.getOperand(1).getReg(); + Register ZeroReg = Zero->getOperand(0).getReg(); + MIRBuilder.buildInstr(TargetOpcode::G_FSUB, {Res}, {ZeroReg, SubByReg}, + MI.getFlags()); MI.eraseFromParent(); return Legalized; } @@ -1070,24 +1660,21 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { // end up with an infinite loop as G_FSUB is used to legalize G_FNEG. if (LI.getAction({G_FNEG, {Ty}}).Action == Lower) return UnableToLegalize; - unsigned Res = MI.getOperand(0).getReg(); - unsigned LHS = MI.getOperand(1).getReg(); - unsigned RHS = MI.getOperand(2).getReg(); - unsigned Neg = MRI.createGenericVirtualRegister(Ty); + Register Res = MI.getOperand(0).getReg(); + Register LHS = MI.getOperand(1).getReg(); + Register RHS = MI.getOperand(2).getReg(); + Register Neg = MRI.createGenericVirtualRegister(Ty); MIRBuilder.buildInstr(TargetOpcode::G_FNEG).addDef(Neg).addUse(RHS); - MIRBuilder.buildInstr(TargetOpcode::G_FADD) - .addDef(Res) - .addUse(LHS) - .addUse(Neg); + MIRBuilder.buildInstr(TargetOpcode::G_FADD, {Res}, {LHS, Neg}, MI.getFlags()); MI.eraseFromParent(); return Legalized; } case TargetOpcode::G_ATOMIC_CMPXCHG_WITH_SUCCESS: { - unsigned OldValRes = MI.getOperand(0).getReg(); - unsigned SuccessRes = MI.getOperand(1).getReg(); - unsigned Addr = MI.getOperand(2).getReg(); - unsigned CmpVal = MI.getOperand(3).getReg(); - unsigned NewVal = MI.getOperand(4).getReg(); + Register OldValRes = MI.getOperand(0).getReg(); + Register SuccessRes = MI.getOperand(1).getReg(); + Register Addr = MI.getOperand(2).getReg(); + Register CmpVal = MI.getOperand(3).getReg(); + Register NewVal = MI.getOperand(4).getReg(); MIRBuilder.buildAtomicCmpXchg(OldValRes, Addr, CmpVal, NewVal, **MI.memoperands_begin()); MIRBuilder.buildICmp(CmpInst::ICMP_EQ, SuccessRes, OldValRes, CmpVal); @@ -1098,8 +1685,8 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { case TargetOpcode::G_SEXTLOAD: case TargetOpcode::G_ZEXTLOAD: { // Lower to a memory-width G_LOAD and a G_SEXT/G_ZEXT/G_ANYEXT - unsigned DstReg = MI.getOperand(0).getReg(); - unsigned PtrReg = MI.getOperand(1).getReg(); + Register DstReg = MI.getOperand(0).getReg(); + Register PtrReg = MI.getOperand(1).getReg(); LLT DstTy = MRI.getType(DstReg); auto &MMO = **MI.memoperands_begin(); @@ -1114,8 +1701,8 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { } if (DstTy.isScalar()) { - unsigned TmpReg = MRI.createGenericVirtualRegister( - LLT::scalar(MMO.getSize() /* in bytes */ * 8)); + Register TmpReg = + MRI.createGenericVirtualRegister(LLT::scalar(MMO.getSizeInBits())); MIRBuilder.buildLoad(TmpReg, PtrReg, MMO); switch (MI.getOpcode()) { default: @@ -1142,15 +1729,27 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { case TargetOpcode::G_CTTZ: case TargetOpcode::G_CTPOP: return lowerBitCount(MI, TypeIdx, Ty); + case G_UADDO: { + Register Res = MI.getOperand(0).getReg(); + Register CarryOut = MI.getOperand(1).getReg(); + Register LHS = MI.getOperand(2).getReg(); + Register RHS = MI.getOperand(3).getReg(); + + MIRBuilder.buildAdd(Res, LHS, RHS); + MIRBuilder.buildICmp(CmpInst::ICMP_ULT, CarryOut, Res, RHS); + + MI.eraseFromParent(); + return Legalized; + } case G_UADDE: { - unsigned Res = MI.getOperand(0).getReg(); - unsigned CarryOut = MI.getOperand(1).getReg(); - unsigned LHS = MI.getOperand(2).getReg(); - unsigned RHS = MI.getOperand(3).getReg(); - unsigned CarryIn = MI.getOperand(4).getReg(); + Register Res = MI.getOperand(0).getReg(); + Register CarryOut = MI.getOperand(1).getReg(); + Register LHS = MI.getOperand(2).getReg(); + Register RHS = MI.getOperand(3).getReg(); + Register CarryIn = MI.getOperand(4).getReg(); - unsigned TmpRes = MRI.createGenericVirtualRegister(Ty); - unsigned ZExtCarryIn = MRI.createGenericVirtualRegister(Ty); + Register TmpRes = MRI.createGenericVirtualRegister(Ty); + Register ZExtCarryIn = MRI.createGenericVirtualRegister(Ty); MIRBuilder.buildAdd(TmpRes, LHS, RHS); MIRBuilder.buildZExt(ZExtCarryIn, CarryIn); @@ -1160,113 +1759,1325 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { MI.eraseFromParent(); return Legalized; } + case G_USUBO: { + Register Res = MI.getOperand(0).getReg(); + Register BorrowOut = MI.getOperand(1).getReg(); + Register LHS = MI.getOperand(2).getReg(); + Register RHS = MI.getOperand(3).getReg(); + + MIRBuilder.buildSub(Res, LHS, RHS); + MIRBuilder.buildICmp(CmpInst::ICMP_ULT, BorrowOut, LHS, RHS); + + MI.eraseFromParent(); + return Legalized; + } + case G_USUBE: { + Register Res = MI.getOperand(0).getReg(); + Register BorrowOut = MI.getOperand(1).getReg(); + Register LHS = MI.getOperand(2).getReg(); + Register RHS = MI.getOperand(3).getReg(); + Register BorrowIn = MI.getOperand(4).getReg(); + + Register TmpRes = MRI.createGenericVirtualRegister(Ty); + Register ZExtBorrowIn = MRI.createGenericVirtualRegister(Ty); + Register LHS_EQ_RHS = MRI.createGenericVirtualRegister(LLT::scalar(1)); + Register LHS_ULT_RHS = MRI.createGenericVirtualRegister(LLT::scalar(1)); + + MIRBuilder.buildSub(TmpRes, LHS, RHS); + MIRBuilder.buildZExt(ZExtBorrowIn, BorrowIn); + MIRBuilder.buildSub(Res, TmpRes, ZExtBorrowIn); + MIRBuilder.buildICmp(CmpInst::ICMP_EQ, LHS_EQ_RHS, LHS, RHS); + MIRBuilder.buildICmp(CmpInst::ICMP_ULT, LHS_ULT_RHS, LHS, RHS); + MIRBuilder.buildSelect(BorrowOut, LHS_EQ_RHS, BorrowIn, LHS_ULT_RHS); + + MI.eraseFromParent(); + return Legalized; + } + case G_UITOFP: + return lowerUITOFP(MI, TypeIdx, Ty); + case G_SITOFP: + return lowerSITOFP(MI, TypeIdx, Ty); + case G_SMIN: + case G_SMAX: + case G_UMIN: + case G_UMAX: + return lowerMinMax(MI, TypeIdx, Ty); + case G_FCOPYSIGN: + return lowerFCopySign(MI, TypeIdx, Ty); + case G_FMINNUM: + case G_FMAXNUM: + return lowerFMinNumMaxNum(MI); } } -LegalizerHelper::LegalizeResult -LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx, - LLT NarrowTy) { - // FIXME: Don't know how to handle secondary types yet. - if (TypeIdx != 0) +LegalizerHelper::LegalizeResult LegalizerHelper::fewerElementsVectorImplicitDef( + MachineInstr &MI, unsigned TypeIdx, LLT NarrowTy) { + SmallVector DstRegs; + + unsigned NarrowSize = NarrowTy.getSizeInBits(); + Register DstReg = MI.getOperand(0).getReg(); + unsigned Size = MRI.getType(DstReg).getSizeInBits(); + int NumParts = Size / NarrowSize; + // FIXME: Don't know how to handle the situation where the small vectors + // aren't all the same size yet. + if (Size % NarrowSize != 0) return UnableToLegalize; - MIRBuilder.setInstr(MI); - switch (MI.getOpcode()) { - default: + for (int i = 0; i < NumParts; ++i) { + Register TmpReg = MRI.createGenericVirtualRegister(NarrowTy); + MIRBuilder.buildUndef(TmpReg); + DstRegs.push_back(TmpReg); + } + + if (NarrowTy.isVector()) + MIRBuilder.buildConcatVectors(DstReg, DstRegs); + else + MIRBuilder.buildBuildVector(DstReg, DstRegs); + + MI.eraseFromParent(); + return Legalized; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::fewerElementsVectorBasic(MachineInstr &MI, unsigned TypeIdx, + LLT NarrowTy) { + const unsigned Opc = MI.getOpcode(); + const unsigned NumOps = MI.getNumOperands() - 1; + const unsigned NarrowSize = NarrowTy.getSizeInBits(); + const Register DstReg = MI.getOperand(0).getReg(); + const unsigned Flags = MI.getFlags(); + const LLT DstTy = MRI.getType(DstReg); + const unsigned Size = DstTy.getSizeInBits(); + const int NumParts = Size / NarrowSize; + const LLT EltTy = DstTy.getElementType(); + const unsigned EltSize = EltTy.getSizeInBits(); + const unsigned BitsForNumParts = NarrowSize * NumParts; + + // Check if we have any leftovers. If we do, then only handle the case where + // the leftover is one element. + if (BitsForNumParts != Size && BitsForNumParts + EltSize != Size) return UnableToLegalize; - case TargetOpcode::G_IMPLICIT_DEF: { - SmallVector DstRegs; - unsigned NarrowSize = NarrowTy.getSizeInBits(); - unsigned DstReg = MI.getOperand(0).getReg(); - unsigned Size = MRI.getType(DstReg).getSizeInBits(); - int NumParts = Size / NarrowSize; - // FIXME: Don't know how to handle the situation where the small vectors - // aren't all the same size yet. - if (Size % NarrowSize != 0) - return UnableToLegalize; + if (BitsForNumParts != Size) { + Register AccumDstReg = MRI.createGenericVirtualRegister(DstTy); + MIRBuilder.buildUndef(AccumDstReg); + + // Handle the pieces which evenly divide into the requested type with + // extract/op/insert sequence. + for (unsigned Offset = 0; Offset < BitsForNumParts; Offset += NarrowSize) { + SmallVector SrcOps; + for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) { + Register PartOpReg = MRI.createGenericVirtualRegister(NarrowTy); + MIRBuilder.buildExtract(PartOpReg, MI.getOperand(I).getReg(), Offset); + SrcOps.push_back(PartOpReg); + } - for (int i = 0; i < NumParts; ++i) { - unsigned TmpReg = MRI.createGenericVirtualRegister(NarrowTy); - MIRBuilder.buildUndef(TmpReg); - DstRegs.push_back(TmpReg); + Register PartDstReg = MRI.createGenericVirtualRegister(NarrowTy); + MIRBuilder.buildInstr(Opc, {PartDstReg}, SrcOps, Flags); + + Register PartInsertReg = MRI.createGenericVirtualRegister(DstTy); + MIRBuilder.buildInsert(PartInsertReg, AccumDstReg, PartDstReg, Offset); + AccumDstReg = PartInsertReg; } - if (NarrowTy.isVector()) - MIRBuilder.buildConcatVectors(DstReg, DstRegs); - else - MIRBuilder.buildBuildVector(DstReg, DstRegs); + // Handle the remaining element sized leftover piece. + SmallVector SrcOps; + for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) { + Register PartOpReg = MRI.createGenericVirtualRegister(EltTy); + MIRBuilder.buildExtract(PartOpReg, MI.getOperand(I).getReg(), + BitsForNumParts); + SrcOps.push_back(PartOpReg); + } + Register PartDstReg = MRI.createGenericVirtualRegister(EltTy); + MIRBuilder.buildInstr(Opc, {PartDstReg}, SrcOps, Flags); + MIRBuilder.buildInsert(DstReg, AccumDstReg, PartDstReg, BitsForNumParts); MI.eraseFromParent(); + return Legalized; } - case TargetOpcode::G_ADD: { - unsigned NarrowSize = NarrowTy.getSizeInBits(); - unsigned DstReg = MI.getOperand(0).getReg(); - unsigned Size = MRI.getType(DstReg).getSizeInBits(); - int NumParts = Size / NarrowSize; + + SmallVector DstRegs, Src0Regs, Src1Regs, Src2Regs; + + extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src0Regs); + + if (NumOps >= 2) + extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src1Regs); + + if (NumOps >= 3) + extractParts(MI.getOperand(3).getReg(), NarrowTy, NumParts, Src2Regs); + + for (int i = 0; i < NumParts; ++i) { + Register DstReg = MRI.createGenericVirtualRegister(NarrowTy); + + if (NumOps == 1) + MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i]}, Flags); + else if (NumOps == 2) { + MIRBuilder.buildInstr(Opc, {DstReg}, {Src0Regs[i], Src1Regs[i]}, Flags); + } else if (NumOps == 3) { + MIRBuilder.buildInstr(Opc, {DstReg}, + {Src0Regs[i], Src1Regs[i], Src2Regs[i]}, Flags); + } + + DstRegs.push_back(DstReg); + } + + if (NarrowTy.isVector()) + MIRBuilder.buildConcatVectors(DstReg, DstRegs); + else + MIRBuilder.buildBuildVector(DstReg, DstRegs); + + MI.eraseFromParent(); + return Legalized; +} + +// Handle splitting vector operations which need to have the same number of +// elements in each type index, but each type index may have a different element +// type. +// +// e.g. <4 x s64> = G_SHL <4 x s64>, <4 x s32> -> +// <2 x s64> = G_SHL <2 x s64>, <2 x s32> +// <2 x s64> = G_SHL <2 x s64>, <2 x s32> +// +// Also handles some irregular breakdown cases, e.g. +// e.g. <3 x s64> = G_SHL <3 x s64>, <3 x s32> -> +// <2 x s64> = G_SHL <2 x s64>, <2 x s32> +// s64 = G_SHL s64, s32 +LegalizerHelper::LegalizeResult +LegalizerHelper::fewerElementsVectorMultiEltType( + MachineInstr &MI, unsigned TypeIdx, LLT NarrowTyArg) { + if (TypeIdx != 0) + return UnableToLegalize; + + const LLT NarrowTy0 = NarrowTyArg; + const unsigned NewNumElts = + NarrowTy0.isVector() ? NarrowTy0.getNumElements() : 1; + + const Register DstReg = MI.getOperand(0).getReg(); + LLT DstTy = MRI.getType(DstReg); + LLT LeftoverTy0; + + // All of the operands need to have the same number of elements, so if we can + // determine a type breakdown for the result type, we can for all of the + // source types. + int NumParts = getNarrowTypeBreakDown(DstTy, NarrowTy0, LeftoverTy0).first; + if (NumParts < 0) + return UnableToLegalize; + + SmallVector NewInsts; + + SmallVector DstRegs, LeftoverDstRegs; + SmallVector PartRegs, LeftoverRegs; + + for (unsigned I = 1, E = MI.getNumOperands(); I != E; ++I) { + LLT LeftoverTy; + Register SrcReg = MI.getOperand(I).getReg(); + LLT SrcTyI = MRI.getType(SrcReg); + LLT NarrowTyI = LLT::scalarOrVector(NewNumElts, SrcTyI.getScalarType()); + LLT LeftoverTyI; + + // Split this operand into the requested typed registers, and any leftover + // required to reproduce the original type. + if (!extractParts(SrcReg, SrcTyI, NarrowTyI, LeftoverTyI, PartRegs, + LeftoverRegs)) + return UnableToLegalize; + + if (I == 1) { + // For the first operand, create an instruction for each part and setup + // the result. + for (Register PartReg : PartRegs) { + Register PartDstReg = MRI.createGenericVirtualRegister(NarrowTy0); + NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode()) + .addDef(PartDstReg) + .addUse(PartReg)); + DstRegs.push_back(PartDstReg); + } + + for (Register LeftoverReg : LeftoverRegs) { + Register PartDstReg = MRI.createGenericVirtualRegister(LeftoverTy0); + NewInsts.push_back(MIRBuilder.buildInstrNoInsert(MI.getOpcode()) + .addDef(PartDstReg) + .addUse(LeftoverReg)); + LeftoverDstRegs.push_back(PartDstReg); + } + } else { + assert(NewInsts.size() == PartRegs.size() + LeftoverRegs.size()); + + // Add the newly created operand splits to the existing instructions. The + // odd-sized pieces are ordered after the requested NarrowTyArg sized + // pieces. + unsigned InstCount = 0; + for (unsigned J = 0, JE = PartRegs.size(); J != JE; ++J) + NewInsts[InstCount++].addUse(PartRegs[J]); + for (unsigned J = 0, JE = LeftoverRegs.size(); J != JE; ++J) + NewInsts[InstCount++].addUse(LeftoverRegs[J]); + } + + PartRegs.clear(); + LeftoverRegs.clear(); + } + + // Insert the newly built operations and rebuild the result register. + for (auto &MIB : NewInsts) + MIRBuilder.insertInstr(MIB); + + insertParts(DstReg, DstTy, NarrowTy0, DstRegs, LeftoverTy0, LeftoverDstRegs); + + MI.eraseFromParent(); + return Legalized; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::fewerElementsVectorCasts(MachineInstr &MI, unsigned TypeIdx, + LLT NarrowTy) { + if (TypeIdx != 0) + return UnableToLegalize; + + Register DstReg = MI.getOperand(0).getReg(); + Register SrcReg = MI.getOperand(1).getReg(); + LLT DstTy = MRI.getType(DstReg); + LLT SrcTy = MRI.getType(SrcReg); + + LLT NarrowTy0 = NarrowTy; + LLT NarrowTy1; + unsigned NumParts; + + if (NarrowTy.isVector()) { + // Uneven breakdown not handled. + NumParts = DstTy.getNumElements() / NarrowTy.getNumElements(); + if (NumParts * NarrowTy.getNumElements() != DstTy.getNumElements()) + return UnableToLegalize; + + NarrowTy1 = LLT::vector(NumParts, SrcTy.getElementType().getSizeInBits()); + } else { + NumParts = DstTy.getNumElements(); + NarrowTy1 = SrcTy.getElementType(); + } + + SmallVector SrcRegs, DstRegs; + extractParts(SrcReg, NarrowTy1, NumParts, SrcRegs); + + for (unsigned I = 0; I < NumParts; ++I) { + Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0); + MachineInstr *NewInst = MIRBuilder.buildInstr(MI.getOpcode()) + .addDef(DstReg) + .addUse(SrcRegs[I]); + + NewInst->setFlags(MI.getFlags()); + DstRegs.push_back(DstReg); + } + + if (NarrowTy.isVector()) + MIRBuilder.buildConcatVectors(DstReg, DstRegs); + else + MIRBuilder.buildBuildVector(DstReg, DstRegs); + + MI.eraseFromParent(); + return Legalized; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::fewerElementsVectorCmp(MachineInstr &MI, unsigned TypeIdx, + LLT NarrowTy) { + Register DstReg = MI.getOperand(0).getReg(); + Register Src0Reg = MI.getOperand(2).getReg(); + LLT DstTy = MRI.getType(DstReg); + LLT SrcTy = MRI.getType(Src0Reg); + + unsigned NumParts; + LLT NarrowTy0, NarrowTy1; + + if (TypeIdx == 0) { + unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1; + unsigned OldElts = DstTy.getNumElements(); + + NarrowTy0 = NarrowTy; + NumParts = NarrowTy.isVector() ? (OldElts / NewElts) : DstTy.getNumElements(); + NarrowTy1 = NarrowTy.isVector() ? + LLT::vector(NarrowTy.getNumElements(), SrcTy.getScalarSizeInBits()) : + SrcTy.getElementType(); + + } else { + unsigned NewElts = NarrowTy.isVector() ? NarrowTy.getNumElements() : 1; + unsigned OldElts = SrcTy.getNumElements(); + + NumParts = NarrowTy.isVector() ? (OldElts / NewElts) : + NarrowTy.getNumElements(); + NarrowTy0 = LLT::vector(NarrowTy.getNumElements(), + DstTy.getScalarSizeInBits()); + NarrowTy1 = NarrowTy; + } + + // FIXME: Don't know how to handle the situation where the small vectors + // aren't all the same size yet. + if (NarrowTy1.isVector() && + NarrowTy1.getNumElements() * NumParts != DstTy.getNumElements()) + return UnableToLegalize; + + CmpInst::Predicate Pred + = static_cast(MI.getOperand(1).getPredicate()); + + SmallVector Src1Regs, Src2Regs, DstRegs; + extractParts(MI.getOperand(2).getReg(), NarrowTy1, NumParts, Src1Regs); + extractParts(MI.getOperand(3).getReg(), NarrowTy1, NumParts, Src2Regs); + + for (unsigned I = 0; I < NumParts; ++I) { + Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0); + DstRegs.push_back(DstReg); + + if (MI.getOpcode() == TargetOpcode::G_ICMP) + MIRBuilder.buildICmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]); + else { + MachineInstr *NewCmp + = MIRBuilder.buildFCmp(Pred, DstReg, Src1Regs[I], Src2Regs[I]); + NewCmp->setFlags(MI.getFlags()); + } + } + + if (NarrowTy1.isVector()) + MIRBuilder.buildConcatVectors(DstReg, DstRegs); + else + MIRBuilder.buildBuildVector(DstReg, DstRegs); + + MI.eraseFromParent(); + return Legalized; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::fewerElementsVectorSelect(MachineInstr &MI, unsigned TypeIdx, + LLT NarrowTy) { + Register DstReg = MI.getOperand(0).getReg(); + Register CondReg = MI.getOperand(1).getReg(); + + unsigned NumParts = 0; + LLT NarrowTy0, NarrowTy1; + + LLT DstTy = MRI.getType(DstReg); + LLT CondTy = MRI.getType(CondReg); + unsigned Size = DstTy.getSizeInBits(); + + assert(TypeIdx == 0 || CondTy.isVector()); + + if (TypeIdx == 0) { + NarrowTy0 = NarrowTy; + NarrowTy1 = CondTy; + + unsigned NarrowSize = NarrowTy0.getSizeInBits(); // FIXME: Don't know how to handle the situation where the small vectors // aren't all the same size yet. if (Size % NarrowSize != 0) return UnableToLegalize; - SmallVector Src1Regs, Src2Regs, DstRegs; - extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, Src1Regs); - extractParts(MI.getOperand(2).getReg(), NarrowTy, NumParts, Src2Regs); + NumParts = Size / NarrowSize; - for (int i = 0; i < NumParts; ++i) { - unsigned DstReg = MRI.createGenericVirtualRegister(NarrowTy); - MIRBuilder.buildAdd(DstReg, Src1Regs[i], Src2Regs[i]); - DstRegs.push_back(DstReg); + // Need to break down the condition type + if (CondTy.isVector()) { + if (CondTy.getNumElements() == NumParts) + NarrowTy1 = CondTy.getElementType(); + else + NarrowTy1 = LLT::vector(CondTy.getNumElements() / NumParts, + CondTy.getScalarSizeInBits()); } + } else { + NumParts = CondTy.getNumElements(); + if (NarrowTy.isVector()) { + // TODO: Handle uneven breakdown. + if (NumParts * NarrowTy.getNumElements() != CondTy.getNumElements()) + return UnableToLegalize; + return UnableToLegalize; + } else { + NarrowTy0 = DstTy.getElementType(); + NarrowTy1 = NarrowTy; + } + } + + SmallVector DstRegs, Src0Regs, Src1Regs, Src2Regs; + if (CondTy.isVector()) + extractParts(MI.getOperand(1).getReg(), NarrowTy1, NumParts, Src0Regs); + + extractParts(MI.getOperand(2).getReg(), NarrowTy0, NumParts, Src1Regs); + extractParts(MI.getOperand(3).getReg(), NarrowTy0, NumParts, Src2Regs); + + for (unsigned i = 0; i < NumParts; ++i) { + Register DstReg = MRI.createGenericVirtualRegister(NarrowTy0); + MIRBuilder.buildSelect(DstReg, CondTy.isVector() ? Src0Regs[i] : CondReg, + Src1Regs[i], Src2Regs[i]); + DstRegs.push_back(DstReg); + } + + if (NarrowTy0.isVector()) MIRBuilder.buildConcatVectors(DstReg, DstRegs); - MI.eraseFromParent(); - return Legalized; + else + MIRBuilder.buildBuildVector(DstReg, DstRegs); + + MI.eraseFromParent(); + return Legalized; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::fewerElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, + LLT NarrowTy) { + const Register DstReg = MI.getOperand(0).getReg(); + LLT PhiTy = MRI.getType(DstReg); + LLT LeftoverTy; + + // All of the operands need to have the same number of elements, so if we can + // determine a type breakdown for the result type, we can for all of the + // source types. + int NumParts, NumLeftover; + std::tie(NumParts, NumLeftover) + = getNarrowTypeBreakDown(PhiTy, NarrowTy, LeftoverTy); + if (NumParts < 0) + return UnableToLegalize; + + SmallVector DstRegs, LeftoverDstRegs; + SmallVector NewInsts; + + const int TotalNumParts = NumParts + NumLeftover; + + // Insert the new phis in the result block first. + for (int I = 0; I != TotalNumParts; ++I) { + LLT Ty = I < NumParts ? NarrowTy : LeftoverTy; + Register PartDstReg = MRI.createGenericVirtualRegister(Ty); + NewInsts.push_back(MIRBuilder.buildInstr(TargetOpcode::G_PHI) + .addDef(PartDstReg)); + if (I < NumParts) + DstRegs.push_back(PartDstReg); + else + LeftoverDstRegs.push_back(PartDstReg); } - case TargetOpcode::G_LOAD: - case TargetOpcode::G_STORE: { - bool IsLoad = MI.getOpcode() == TargetOpcode::G_LOAD; - unsigned ValReg = MI.getOperand(0).getReg(); - unsigned AddrReg = MI.getOperand(1).getReg(); - unsigned NarrowSize = NarrowTy.getSizeInBits(); - unsigned Size = MRI.getType(ValReg).getSizeInBits(); - unsigned NumParts = Size / NarrowSize; - - SmallVector NarrowRegs; - if (!IsLoad) - extractParts(ValReg, NarrowTy, NumParts, NarrowRegs); - - const LLT OffsetTy = - LLT::scalar(MRI.getType(AddrReg).getScalarSizeInBits()); - MachineFunction &MF = *MI.getMF(); - MachineMemOperand *MMO = *MI.memoperands_begin(); - for (unsigned Idx = 0; Idx < NumParts; ++Idx) { - unsigned Adjustment = Idx * NarrowTy.getSizeInBits() / 8; - unsigned Alignment = MinAlign(MMO->getAlignment(), Adjustment); - unsigned NewAddrReg = 0; - MIRBuilder.materializeGEP(NewAddrReg, AddrReg, OffsetTy, Adjustment); - MachineMemOperand &NewMMO = *MF.getMachineMemOperand( - MMO->getPointerInfo().getWithOffset(Adjustment), MMO->getFlags(), - NarrowTy.getSizeInBits() / 8, Alignment); + + MachineBasicBlock *MBB = MI.getParent(); + MIRBuilder.setInsertPt(*MBB, MBB->getFirstNonPHI()); + insertParts(DstReg, PhiTy, NarrowTy, DstRegs, LeftoverTy, LeftoverDstRegs); + + SmallVector PartRegs, LeftoverRegs; + + // Insert code to extract the incoming values in each predecessor block. + for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { + PartRegs.clear(); + LeftoverRegs.clear(); + + Register SrcReg = MI.getOperand(I).getReg(); + MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB(); + MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); + + LLT Unused; + if (!extractParts(SrcReg, PhiTy, NarrowTy, Unused, PartRegs, + LeftoverRegs)) + return UnableToLegalize; + + // Add the newly created operand splits to the existing instructions. The + // odd-sized pieces are ordered after the requested NarrowTyArg sized + // pieces. + for (int J = 0; J != TotalNumParts; ++J) { + MachineInstrBuilder MIB = NewInsts[J]; + MIB.addUse(J < NumParts ? PartRegs[J] : LeftoverRegs[J - NumParts]); + MIB.addMBB(&OpMBB); + } + } + + MI.eraseFromParent(); + return Legalized; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::reduceLoadStoreWidth(MachineInstr &MI, unsigned TypeIdx, + LLT NarrowTy) { + // FIXME: Don't know how to handle secondary types yet. + if (TypeIdx != 0) + return UnableToLegalize; + + MachineMemOperand *MMO = *MI.memoperands_begin(); + + // This implementation doesn't work for atomics. Give up instead of doing + // something invalid. + if (MMO->getOrdering() != AtomicOrdering::NotAtomic || + MMO->getFailureOrdering() != AtomicOrdering::NotAtomic) + return UnableToLegalize; + + bool IsLoad = MI.getOpcode() == TargetOpcode::G_LOAD; + Register ValReg = MI.getOperand(0).getReg(); + Register AddrReg = MI.getOperand(1).getReg(); + LLT ValTy = MRI.getType(ValReg); + + int NumParts = -1; + int NumLeftover = -1; + LLT LeftoverTy; + SmallVector NarrowRegs, NarrowLeftoverRegs; + if (IsLoad) { + std::tie(NumParts, NumLeftover) = getNarrowTypeBreakDown(ValTy, NarrowTy, LeftoverTy); + } else { + if (extractParts(ValReg, ValTy, NarrowTy, LeftoverTy, NarrowRegs, + NarrowLeftoverRegs)) { + NumParts = NarrowRegs.size(); + NumLeftover = NarrowLeftoverRegs.size(); + } + } + + if (NumParts == -1) + return UnableToLegalize; + + const LLT OffsetTy = LLT::scalar(MRI.getType(AddrReg).getScalarSizeInBits()); + + unsigned TotalSize = ValTy.getSizeInBits(); + + // Split the load/store into PartTy sized pieces starting at Offset. If this + // is a load, return the new registers in ValRegs. For a store, each elements + // of ValRegs should be PartTy. Returns the next offset that needs to be + // handled. + auto splitTypePieces = [=](LLT PartTy, SmallVectorImpl &ValRegs, + unsigned Offset) -> unsigned { + MachineFunction &MF = MIRBuilder.getMF(); + unsigned PartSize = PartTy.getSizeInBits(); + for (unsigned Idx = 0, E = NumParts; Idx != E && Offset < TotalSize; + Offset += PartSize, ++Idx) { + unsigned ByteSize = PartSize / 8; + unsigned ByteOffset = Offset / 8; + Register NewAddrReg; + + MIRBuilder.materializeGEP(NewAddrReg, AddrReg, OffsetTy, ByteOffset); + + MachineMemOperand *NewMMO = + MF.getMachineMemOperand(MMO, ByteOffset, ByteSize); + if (IsLoad) { - unsigned Dst = MRI.createGenericVirtualRegister(NarrowTy); - NarrowRegs.push_back(Dst); - MIRBuilder.buildLoad(Dst, NewAddrReg, NewMMO); + Register Dst = MRI.createGenericVirtualRegister(PartTy); + ValRegs.push_back(Dst); + MIRBuilder.buildLoad(Dst, NewAddrReg, *NewMMO); } else { - MIRBuilder.buildStore(NarrowRegs[Idx], NewAddrReg, NewMMO); + MIRBuilder.buildStore(ValRegs[Idx], NewAddrReg, *NewMMO); } } - if (IsLoad) { - if (NarrowTy.isVector()) - MIRBuilder.buildConcatVectors(ValReg, NarrowRegs); - else - MIRBuilder.buildBuildVector(ValReg, NarrowRegs); - } + + return Offset; + }; + + unsigned HandledOffset = splitTypePieces(NarrowTy, NarrowRegs, 0); + + // Handle the rest of the register if this isn't an even type breakdown. + if (LeftoverTy.isValid()) + splitTypePieces(LeftoverTy, NarrowLeftoverRegs, HandledOffset); + + if (IsLoad) { + insertParts(ValReg, ValTy, NarrowTy, NarrowRegs, + LeftoverTy, NarrowLeftoverRegs); + } + + MI.eraseFromParent(); + return Legalized; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::fewerElementsVector(MachineInstr &MI, unsigned TypeIdx, + LLT NarrowTy) { + using namespace TargetOpcode; + + MIRBuilder.setInstr(MI); + switch (MI.getOpcode()) { + case G_IMPLICIT_DEF: + return fewerElementsVectorImplicitDef(MI, TypeIdx, NarrowTy); + case G_AND: + case G_OR: + case G_XOR: + case G_ADD: + case G_SUB: + case G_MUL: + case G_SMULH: + case G_UMULH: + case G_FADD: + case G_FMUL: + case G_FSUB: + case G_FNEG: + case G_FABS: + case G_FCANONICALIZE: + case G_FDIV: + case G_FREM: + case G_FMA: + case G_FPOW: + case G_FEXP: + case G_FEXP2: + case G_FLOG: + case G_FLOG2: + case G_FLOG10: + case G_FNEARBYINT: + case G_FCEIL: + case G_FFLOOR: + case G_FRINT: + case G_INTRINSIC_ROUND: + case G_INTRINSIC_TRUNC: + case G_FCOS: + case G_FSIN: + case G_FSQRT: + case G_BSWAP: + case G_SDIV: + case G_SMIN: + case G_SMAX: + case G_UMIN: + case G_UMAX: + case G_FMINNUM: + case G_FMAXNUM: + case G_FMINNUM_IEEE: + case G_FMAXNUM_IEEE: + case G_FMINIMUM: + case G_FMAXIMUM: + return fewerElementsVectorBasic(MI, TypeIdx, NarrowTy); + case G_SHL: + case G_LSHR: + case G_ASHR: + case G_CTLZ: + case G_CTLZ_ZERO_UNDEF: + case G_CTTZ: + case G_CTTZ_ZERO_UNDEF: + case G_CTPOP: + case G_FCOPYSIGN: + return fewerElementsVectorMultiEltType(MI, TypeIdx, NarrowTy); + case G_ZEXT: + case G_SEXT: + case G_ANYEXT: + case G_FPEXT: + case G_FPTRUNC: + case G_SITOFP: + case G_UITOFP: + case G_FPTOSI: + case G_FPTOUI: + case G_INTTOPTR: + case G_PTRTOINT: + case G_ADDRSPACE_CAST: + return fewerElementsVectorCasts(MI, TypeIdx, NarrowTy); + case G_ICMP: + case G_FCMP: + return fewerElementsVectorCmp(MI, TypeIdx, NarrowTy); + case G_SELECT: + return fewerElementsVectorSelect(MI, TypeIdx, NarrowTy); + case G_PHI: + return fewerElementsVectorPhi(MI, TypeIdx, NarrowTy); + case G_LOAD: + case G_STORE: + return reduceLoadStoreWidth(MI, TypeIdx, NarrowTy); + default: + return UnableToLegalize; + } +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::narrowScalarShiftByConstant(MachineInstr &MI, const APInt &Amt, + const LLT HalfTy, const LLT AmtTy) { + + Register InL = MRI.createGenericVirtualRegister(HalfTy); + Register InH = MRI.createGenericVirtualRegister(HalfTy); + MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1).getReg()); + + if (Amt.isNullValue()) { + MIRBuilder.buildMerge(MI.getOperand(0).getReg(), {InL, InH}); MI.eraseFromParent(); return Legalized; } + + LLT NVT = HalfTy; + unsigned NVTBits = HalfTy.getSizeInBits(); + unsigned VTBits = 2 * NVTBits; + + SrcOp Lo(Register(0)), Hi(Register(0)); + if (MI.getOpcode() == TargetOpcode::G_SHL) { + if (Amt.ugt(VTBits)) { + Lo = Hi = MIRBuilder.buildConstant(NVT, 0); + } else if (Amt.ugt(NVTBits)) { + Lo = MIRBuilder.buildConstant(NVT, 0); + Hi = MIRBuilder.buildShl(NVT, InL, + MIRBuilder.buildConstant(AmtTy, Amt - NVTBits)); + } else if (Amt == NVTBits) { + Lo = MIRBuilder.buildConstant(NVT, 0); + Hi = InL; + } else { + Lo = MIRBuilder.buildShl(NVT, InL, MIRBuilder.buildConstant(AmtTy, Amt)); + auto OrLHS = + MIRBuilder.buildShl(NVT, InH, MIRBuilder.buildConstant(AmtTy, Amt)); + auto OrRHS = MIRBuilder.buildLShr( + NVT, InL, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits)); + Hi = MIRBuilder.buildOr(NVT, OrLHS, OrRHS); + } + } else if (MI.getOpcode() == TargetOpcode::G_LSHR) { + if (Amt.ugt(VTBits)) { + Lo = Hi = MIRBuilder.buildConstant(NVT, 0); + } else if (Amt.ugt(NVTBits)) { + Lo = MIRBuilder.buildLShr(NVT, InH, + MIRBuilder.buildConstant(AmtTy, Amt - NVTBits)); + Hi = MIRBuilder.buildConstant(NVT, 0); + } else if (Amt == NVTBits) { + Lo = InH; + Hi = MIRBuilder.buildConstant(NVT, 0); + } else { + auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt); + + auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst); + auto OrRHS = MIRBuilder.buildShl( + NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits)); + + Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS); + Hi = MIRBuilder.buildLShr(NVT, InH, ShiftAmtConst); + } + } else { + if (Amt.ugt(VTBits)) { + Hi = Lo = MIRBuilder.buildAShr( + NVT, InH, MIRBuilder.buildConstant(AmtTy, NVTBits - 1)); + } else if (Amt.ugt(NVTBits)) { + Lo = MIRBuilder.buildAShr(NVT, InH, + MIRBuilder.buildConstant(AmtTy, Amt - NVTBits)); + Hi = MIRBuilder.buildAShr(NVT, InH, + MIRBuilder.buildConstant(AmtTy, NVTBits - 1)); + } else if (Amt == NVTBits) { + Lo = InH; + Hi = MIRBuilder.buildAShr(NVT, InH, + MIRBuilder.buildConstant(AmtTy, NVTBits - 1)); + } else { + auto ShiftAmtConst = MIRBuilder.buildConstant(AmtTy, Amt); + + auto OrLHS = MIRBuilder.buildLShr(NVT, InL, ShiftAmtConst); + auto OrRHS = MIRBuilder.buildShl( + NVT, InH, MIRBuilder.buildConstant(AmtTy, -Amt + NVTBits)); + + Lo = MIRBuilder.buildOr(NVT, OrLHS, OrRHS); + Hi = MIRBuilder.buildAShr(NVT, InH, ShiftAmtConst); + } + } + + MIRBuilder.buildMerge(MI.getOperand(0).getReg(), {Lo.getReg(), Hi.getReg()}); + MI.eraseFromParent(); + + return Legalized; +} + +// TODO: Optimize if constant shift amount. +LegalizerHelper::LegalizeResult +LegalizerHelper::narrowScalarShift(MachineInstr &MI, unsigned TypeIdx, + LLT RequestedTy) { + if (TypeIdx == 1) { + Observer.changingInstr(MI); + narrowScalarSrc(MI, RequestedTy, 2); + Observer.changedInstr(MI); + return Legalized; + } + + Register DstReg = MI.getOperand(0).getReg(); + LLT DstTy = MRI.getType(DstReg); + if (DstTy.isVector()) + return UnableToLegalize; + + Register Amt = MI.getOperand(2).getReg(); + LLT ShiftAmtTy = MRI.getType(Amt); + const unsigned DstEltSize = DstTy.getScalarSizeInBits(); + if (DstEltSize % 2 != 0) + return UnableToLegalize; + + // Ignore the input type. We can only go to exactly half the size of the + // input. If that isn't small enough, the resulting pieces will be further + // legalized. + const unsigned NewBitSize = DstEltSize / 2; + const LLT HalfTy = LLT::scalar(NewBitSize); + const LLT CondTy = LLT::scalar(1); + + if (const MachineInstr *KShiftAmt = + getOpcodeDef(TargetOpcode::G_CONSTANT, Amt, MRI)) { + return narrowScalarShiftByConstant( + MI, KShiftAmt->getOperand(1).getCImm()->getValue(), HalfTy, ShiftAmtTy); + } + + // TODO: Expand with known bits. + + // Handle the fully general expansion by an unknown amount. + auto NewBits = MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize); + + Register InL = MRI.createGenericVirtualRegister(HalfTy); + Register InH = MRI.createGenericVirtualRegister(HalfTy); + MIRBuilder.buildUnmerge({InL, InH}, MI.getOperand(1).getReg()); + + auto AmtExcess = MIRBuilder.buildSub(ShiftAmtTy, Amt, NewBits); + auto AmtLack = MIRBuilder.buildSub(ShiftAmtTy, NewBits, Amt); + + auto Zero = MIRBuilder.buildConstant(ShiftAmtTy, 0); + auto IsShort = MIRBuilder.buildICmp(ICmpInst::ICMP_ULT, CondTy, Amt, NewBits); + auto IsZero = MIRBuilder.buildICmp(ICmpInst::ICMP_EQ, CondTy, Amt, Zero); + + Register ResultRegs[2]; + switch (MI.getOpcode()) { + case TargetOpcode::G_SHL: { + // Short: ShAmt < NewBitSize + auto LoS = MIRBuilder.buildShl(HalfTy, InH, Amt); + + auto OrLHS = MIRBuilder.buildShl(HalfTy, InH, Amt); + auto OrRHS = MIRBuilder.buildLShr(HalfTy, InL, AmtLack); + auto HiS = MIRBuilder.buildOr(HalfTy, OrLHS, OrRHS); + + // Long: ShAmt >= NewBitSize + auto LoL = MIRBuilder.buildConstant(HalfTy, 0); // Lo part is zero. + auto HiL = MIRBuilder.buildShl(HalfTy, InL, AmtExcess); // Hi from Lo part. + + auto Lo = MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL); + auto Hi = MIRBuilder.buildSelect( + HalfTy, IsZero, InH, MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL)); + + ResultRegs[0] = Lo.getReg(0); + ResultRegs[1] = Hi.getReg(0); + break; + } + case TargetOpcode::G_LSHR: { + // Short: ShAmt < NewBitSize + auto HiS = MIRBuilder.buildLShr(HalfTy, InH, Amt); + + auto OrLHS = MIRBuilder.buildLShr(HalfTy, InL, Amt); + auto OrRHS = MIRBuilder.buildShl(HalfTy, InH, AmtLack); + auto LoS = MIRBuilder.buildOr(HalfTy, OrLHS, OrRHS); + + // Long: ShAmt >= NewBitSize + auto HiL = MIRBuilder.buildConstant(HalfTy, 0); // Hi part is zero. + auto LoL = MIRBuilder.buildLShr(HalfTy, InH, AmtExcess); // Lo from Hi part. + + auto Lo = MIRBuilder.buildSelect( + HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL)); + auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL); + + ResultRegs[0] = Lo.getReg(0); + ResultRegs[1] = Hi.getReg(0); + break; + } + case TargetOpcode::G_ASHR: { + // Short: ShAmt < NewBitSize + auto HiS = MIRBuilder.buildAShr(HalfTy, InH, Amt); + + auto OrLHS = MIRBuilder.buildLShr(HalfTy, InL, Amt); + auto OrRHS = MIRBuilder.buildLShr(HalfTy, InH, AmtLack); + auto LoS = MIRBuilder.buildOr(HalfTy, OrLHS, OrRHS); + + // Long: ShAmt >= NewBitSize + + // Sign of Hi part. + auto HiL = MIRBuilder.buildAShr( + HalfTy, InH, MIRBuilder.buildConstant(ShiftAmtTy, NewBitSize - 1)); + + auto LoL = MIRBuilder.buildAShr(HalfTy, InH, AmtExcess); // Lo from Hi part. + + auto Lo = MIRBuilder.buildSelect( + HalfTy, IsZero, InL, MIRBuilder.buildSelect(HalfTy, IsShort, LoS, LoL)); + + auto Hi = MIRBuilder.buildSelect(HalfTy, IsShort, HiS, HiL); + + ResultRegs[0] = Lo.getReg(0); + ResultRegs[1] = Hi.getReg(0); + break; + } + default: + llvm_unreachable("not a shift"); + } + + MIRBuilder.buildMerge(DstReg, ResultRegs); + MI.eraseFromParent(); + return Legalized; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::moreElementsVectorPhi(MachineInstr &MI, unsigned TypeIdx, + LLT MoreTy) { + assert(TypeIdx == 0 && "Expecting only Idx 0"); + + Observer.changingInstr(MI); + for (unsigned I = 1, E = MI.getNumOperands(); I != E; I += 2) { + MachineBasicBlock &OpMBB = *MI.getOperand(I + 1).getMBB(); + MIRBuilder.setInsertPt(OpMBB, OpMBB.getFirstTerminator()); + moreElementsVectorSrc(MI, MoreTy, I); + } + + MachineBasicBlock &MBB = *MI.getParent(); + MIRBuilder.setInsertPt(MBB, --MBB.getFirstNonPHI()); + moreElementsVectorDst(MI, MoreTy, 0); + Observer.changedInstr(MI); + return Legalized; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx, + LLT MoreTy) { + MIRBuilder.setInstr(MI); + unsigned Opc = MI.getOpcode(); + switch (Opc) { + case TargetOpcode::G_IMPLICIT_DEF: { + Observer.changingInstr(MI); + moreElementsVectorDst(MI, MoreTy, 0); + Observer.changedInstr(MI); + return Legalized; + } + case TargetOpcode::G_AND: + case TargetOpcode::G_OR: + case TargetOpcode::G_XOR: + case TargetOpcode::G_SMIN: + case TargetOpcode::G_SMAX: + case TargetOpcode::G_UMIN: + case TargetOpcode::G_UMAX: { + Observer.changingInstr(MI); + moreElementsVectorSrc(MI, MoreTy, 1); + moreElementsVectorSrc(MI, MoreTy, 2); + moreElementsVectorDst(MI, MoreTy, 0); + Observer.changedInstr(MI); + return Legalized; } + case TargetOpcode::G_EXTRACT: + if (TypeIdx != 1) + return UnableToLegalize; + Observer.changingInstr(MI); + moreElementsVectorSrc(MI, MoreTy, 1); + Observer.changedInstr(MI); + return Legalized; + case TargetOpcode::G_INSERT: + if (TypeIdx != 0) + return UnableToLegalize; + Observer.changingInstr(MI); + moreElementsVectorSrc(MI, MoreTy, 1); + moreElementsVectorDst(MI, MoreTy, 0); + Observer.changedInstr(MI); + return Legalized; + case TargetOpcode::G_SELECT: + if (TypeIdx != 0) + return UnableToLegalize; + if (MRI.getType(MI.getOperand(1).getReg()).isVector()) + return UnableToLegalize; + + Observer.changingInstr(MI); + moreElementsVectorSrc(MI, MoreTy, 2); + moreElementsVectorSrc(MI, MoreTy, 3); + moreElementsVectorDst(MI, MoreTy, 0); + Observer.changedInstr(MI); + return Legalized; + case TargetOpcode::G_PHI: + return moreElementsVectorPhi(MI, TypeIdx, MoreTy); + default: + return UnableToLegalize; + } +} + +void LegalizerHelper::multiplyRegisters(SmallVectorImpl &DstRegs, + ArrayRef Src1Regs, + ArrayRef Src2Regs, + LLT NarrowTy) { + MachineIRBuilder &B = MIRBuilder; + unsigned SrcParts = Src1Regs.size(); + unsigned DstParts = DstRegs.size(); + + unsigned DstIdx = 0; // Low bits of the result. + Register FactorSum = + B.buildMul(NarrowTy, Src1Regs[DstIdx], Src2Regs[DstIdx]).getReg(0); + DstRegs[DstIdx] = FactorSum; + + unsigned CarrySumPrevDstIdx; + SmallVector Factors; + + for (DstIdx = 1; DstIdx < DstParts; DstIdx++) { + // Collect low parts of muls for DstIdx. + for (unsigned i = DstIdx + 1 < SrcParts ? 0 : DstIdx - SrcParts + 1; + i <= std::min(DstIdx, SrcParts - 1); ++i) { + MachineInstrBuilder Mul = + B.buildMul(NarrowTy, Src1Regs[DstIdx - i], Src2Regs[i]); + Factors.push_back(Mul.getReg(0)); + } + // Collect high parts of muls from previous DstIdx. + for (unsigned i = DstIdx < SrcParts ? 0 : DstIdx - SrcParts; + i <= std::min(DstIdx - 1, SrcParts - 1); ++i) { + MachineInstrBuilder Umulh = + B.buildUMulH(NarrowTy, Src1Regs[DstIdx - 1 - i], Src2Regs[i]); + Factors.push_back(Umulh.getReg(0)); + } + // Add CarrySum from additons calculated for previous DstIdx. + if (DstIdx != 1) { + Factors.push_back(CarrySumPrevDstIdx); + } + + Register CarrySum; + // Add all factors and accumulate all carries into CarrySum. + if (DstIdx != DstParts - 1) { + MachineInstrBuilder Uaddo = + B.buildUAddo(NarrowTy, LLT::scalar(1), Factors[0], Factors[1]); + FactorSum = Uaddo.getReg(0); + CarrySum = B.buildZExt(NarrowTy, Uaddo.getReg(1)).getReg(0); + for (unsigned i = 2; i < Factors.size(); ++i) { + MachineInstrBuilder Uaddo = + B.buildUAddo(NarrowTy, LLT::scalar(1), FactorSum, Factors[i]); + FactorSum = Uaddo.getReg(0); + MachineInstrBuilder Carry = B.buildZExt(NarrowTy, Uaddo.getReg(1)); + CarrySum = B.buildAdd(NarrowTy, CarrySum, Carry).getReg(0); + } + } else { + // Since value for the next index is not calculated, neither is CarrySum. + FactorSum = B.buildAdd(NarrowTy, Factors[0], Factors[1]).getReg(0); + for (unsigned i = 2; i < Factors.size(); ++i) + FactorSum = B.buildAdd(NarrowTy, FactorSum, Factors[i]).getReg(0); + } + + CarrySumPrevDstIdx = CarrySum; + DstRegs[DstIdx] = FactorSum; + Factors.clear(); + } +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::narrowScalarMul(MachineInstr &MI, LLT NarrowTy) { + Register DstReg = MI.getOperand(0).getReg(); + Register Src1 = MI.getOperand(1).getReg(); + Register Src2 = MI.getOperand(2).getReg(); + + LLT Ty = MRI.getType(DstReg); + if (Ty.isVector()) + return UnableToLegalize; + + unsigned SrcSize = MRI.getType(Src1).getSizeInBits(); + unsigned DstSize = Ty.getSizeInBits(); + unsigned NarrowSize = NarrowTy.getSizeInBits(); + if (DstSize % NarrowSize != 0 || SrcSize % NarrowSize != 0) + return UnableToLegalize; + + unsigned NumDstParts = DstSize / NarrowSize; + unsigned NumSrcParts = SrcSize / NarrowSize; + bool IsMulHigh = MI.getOpcode() == TargetOpcode::G_UMULH; + unsigned DstTmpParts = NumDstParts * (IsMulHigh ? 2 : 1); + + SmallVector Src1Parts, Src2Parts, DstTmpRegs; + extractParts(Src1, NarrowTy, NumSrcParts, Src1Parts); + extractParts(Src2, NarrowTy, NumSrcParts, Src2Parts); + DstTmpRegs.resize(DstTmpParts); + multiplyRegisters(DstTmpRegs, Src1Parts, Src2Parts, NarrowTy); + + // Take only high half of registers if this is high mul. + ArrayRef DstRegs( + IsMulHigh ? &DstTmpRegs[DstTmpParts / 2] : &DstTmpRegs[0], NumDstParts); + MIRBuilder.buildMerge(DstReg, DstRegs); + MI.eraseFromParent(); + return Legalized; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::narrowScalarExtract(MachineInstr &MI, unsigned TypeIdx, + LLT NarrowTy) { + if (TypeIdx != 1) + return UnableToLegalize; + + uint64_t NarrowSize = NarrowTy.getSizeInBits(); + + int64_t SizeOp1 = MRI.getType(MI.getOperand(1).getReg()).getSizeInBits(); + // FIXME: add support for when SizeOp1 isn't an exact multiple of + // NarrowSize. + if (SizeOp1 % NarrowSize != 0) + return UnableToLegalize; + int NumParts = SizeOp1 / NarrowSize; + + SmallVector SrcRegs, DstRegs; + SmallVector Indexes; + extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs); + + Register OpReg = MI.getOperand(0).getReg(); + uint64_t OpStart = MI.getOperand(2).getImm(); + uint64_t OpSize = MRI.getType(OpReg).getSizeInBits(); + for (int i = 0; i < NumParts; ++i) { + unsigned SrcStart = i * NarrowSize; + + if (SrcStart + NarrowSize <= OpStart || SrcStart >= OpStart + OpSize) { + // No part of the extract uses this subregister, ignore it. + continue; + } else if (SrcStart == OpStart && NarrowTy == MRI.getType(OpReg)) { + // The entire subregister is extracted, forward the value. + DstRegs.push_back(SrcRegs[i]); + continue; + } + + // OpSegStart is where this destination segment would start in OpReg if it + // extended infinitely in both directions. + int64_t ExtractOffset; + uint64_t SegSize; + if (OpStart < SrcStart) { + ExtractOffset = 0; + SegSize = std::min(NarrowSize, OpStart + OpSize - SrcStart); + } else { + ExtractOffset = OpStart - SrcStart; + SegSize = std::min(SrcStart + NarrowSize - OpStart, OpSize); + } + + Register SegReg = SrcRegs[i]; + if (ExtractOffset != 0 || SegSize != NarrowSize) { + // A genuine extract is needed. + SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize)); + MIRBuilder.buildExtract(SegReg, SrcRegs[i], ExtractOffset); + } + + DstRegs.push_back(SegReg); + } + + Register DstReg = MI.getOperand(0).getReg(); + if(MRI.getType(DstReg).isVector()) + MIRBuilder.buildBuildVector(DstReg, DstRegs); + else + MIRBuilder.buildMerge(DstReg, DstRegs); + MI.eraseFromParent(); + return Legalized; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::narrowScalarInsert(MachineInstr &MI, unsigned TypeIdx, + LLT NarrowTy) { + // FIXME: Don't know how to handle secondary types yet. + if (TypeIdx != 0) + return UnableToLegalize; + + uint64_t SizeOp0 = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits(); + uint64_t NarrowSize = NarrowTy.getSizeInBits(); + + // FIXME: add support for when SizeOp0 isn't an exact multiple of + // NarrowSize. + if (SizeOp0 % NarrowSize != 0) + return UnableToLegalize; + + int NumParts = SizeOp0 / NarrowSize; + + SmallVector SrcRegs, DstRegs; + SmallVector Indexes; + extractParts(MI.getOperand(1).getReg(), NarrowTy, NumParts, SrcRegs); + + Register OpReg = MI.getOperand(2).getReg(); + uint64_t OpStart = MI.getOperand(3).getImm(); + uint64_t OpSize = MRI.getType(OpReg).getSizeInBits(); + for (int i = 0; i < NumParts; ++i) { + unsigned DstStart = i * NarrowSize; + + if (DstStart + NarrowSize <= OpStart || DstStart >= OpStart + OpSize) { + // No part of the insert affects this subregister, forward the original. + DstRegs.push_back(SrcRegs[i]); + continue; + } else if (DstStart == OpStart && NarrowTy == MRI.getType(OpReg)) { + // The entire subregister is defined by this insert, forward the new + // value. + DstRegs.push_back(OpReg); + continue; + } + + // OpSegStart is where this destination segment would start in OpReg if it + // extended infinitely in both directions. + int64_t ExtractOffset, InsertOffset; + uint64_t SegSize; + if (OpStart < DstStart) { + InsertOffset = 0; + ExtractOffset = DstStart - OpStart; + SegSize = std::min(NarrowSize, OpStart + OpSize - DstStart); + } else { + InsertOffset = OpStart - DstStart; + ExtractOffset = 0; + SegSize = + std::min(NarrowSize - InsertOffset, OpStart + OpSize - DstStart); + } + + Register SegReg = OpReg; + if (ExtractOffset != 0 || SegSize != OpSize) { + // A genuine extract is needed. + SegReg = MRI.createGenericVirtualRegister(LLT::scalar(SegSize)); + MIRBuilder.buildExtract(SegReg, OpReg, ExtractOffset); + } + + Register DstReg = MRI.createGenericVirtualRegister(NarrowTy); + MIRBuilder.buildInsert(DstReg, SrcRegs[i], SegReg, InsertOffset); + DstRegs.push_back(DstReg); + } + + assert(DstRegs.size() == (unsigned)NumParts && "not all parts covered"); + Register DstReg = MI.getOperand(0).getReg(); + if(MRI.getType(DstReg).isVector()) + MIRBuilder.buildBuildVector(DstReg, DstRegs); + else + MIRBuilder.buildMerge(DstReg, DstRegs); + MI.eraseFromParent(); + return Legalized; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::narrowScalarBasic(MachineInstr &MI, unsigned TypeIdx, + LLT NarrowTy) { + Register DstReg = MI.getOperand(0).getReg(); + LLT DstTy = MRI.getType(DstReg); + + assert(MI.getNumOperands() == 3 && TypeIdx == 0); + + SmallVector DstRegs, DstLeftoverRegs; + SmallVector Src0Regs, Src0LeftoverRegs; + SmallVector Src1Regs, Src1LeftoverRegs; + LLT LeftoverTy; + if (!extractParts(MI.getOperand(1).getReg(), DstTy, NarrowTy, LeftoverTy, + Src0Regs, Src0LeftoverRegs)) + return UnableToLegalize; + + LLT Unused; + if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, Unused, + Src1Regs, Src1LeftoverRegs)) + llvm_unreachable("inconsistent extractParts result"); + + for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) { + auto Inst = MIRBuilder.buildInstr(MI.getOpcode(), {NarrowTy}, + {Src0Regs[I], Src1Regs[I]}); + DstRegs.push_back(Inst->getOperand(0).getReg()); + } + + for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) { + auto Inst = MIRBuilder.buildInstr( + MI.getOpcode(), + {LeftoverTy}, {Src0LeftoverRegs[I], Src1LeftoverRegs[I]}); + DstLeftoverRegs.push_back(Inst->getOperand(0).getReg()); + } + + insertParts(DstReg, DstTy, NarrowTy, DstRegs, + LeftoverTy, DstLeftoverRegs); + + MI.eraseFromParent(); + return Legalized; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::narrowScalarSelect(MachineInstr &MI, unsigned TypeIdx, + LLT NarrowTy) { + if (TypeIdx != 0) + return UnableToLegalize; + + Register CondReg = MI.getOperand(1).getReg(); + LLT CondTy = MRI.getType(CondReg); + if (CondTy.isVector()) // TODO: Handle vselect + return UnableToLegalize; + + Register DstReg = MI.getOperand(0).getReg(); + LLT DstTy = MRI.getType(DstReg); + + SmallVector DstRegs, DstLeftoverRegs; + SmallVector Src1Regs, Src1LeftoverRegs; + SmallVector Src2Regs, Src2LeftoverRegs; + LLT LeftoverTy; + if (!extractParts(MI.getOperand(2).getReg(), DstTy, NarrowTy, LeftoverTy, + Src1Regs, Src1LeftoverRegs)) + return UnableToLegalize; + + LLT Unused; + if (!extractParts(MI.getOperand(3).getReg(), DstTy, NarrowTy, Unused, + Src2Regs, Src2LeftoverRegs)) + llvm_unreachable("inconsistent extractParts result"); + + for (unsigned I = 0, E = Src1Regs.size(); I != E; ++I) { + auto Select = MIRBuilder.buildSelect(NarrowTy, + CondReg, Src1Regs[I], Src2Regs[I]); + DstRegs.push_back(Select->getOperand(0).getReg()); + } + + for (unsigned I = 0, E = Src1LeftoverRegs.size(); I != E; ++I) { + auto Select = MIRBuilder.buildSelect( + LeftoverTy, CondReg, Src1LeftoverRegs[I], Src2LeftoverRegs[I]); + DstLeftoverRegs.push_back(Select->getOperand(0).getReg()); + } + + insertParts(DstReg, DstTy, NarrowTy, DstRegs, + LeftoverTy, DstLeftoverRegs); + + MI.eraseFromParent(); + return Legalized; } LegalizerHelper::LegalizeResult @@ -1288,9 +3099,9 @@ LegalizerHelper::lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { return Legalized; } case TargetOpcode::G_CTLZ: { - unsigned SrcReg = MI.getOperand(1).getReg(); + Register SrcReg = MI.getOperand(1).getReg(); unsigned Len = Ty.getSizeInBits(); - if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {Ty}})) { + if (isSupported({TargetOpcode::G_CTLZ_ZERO_UNDEF, {Ty, Ty}})) { // If CTLZ_ZERO_UNDEF is supported, emit that and a select for zero. auto MIBCtlzZU = MIRBuilder.buildInstr(TargetOpcode::G_CTLZ_ZERO_UNDEF, {Ty}, {SrcReg}); @@ -1314,7 +3125,7 @@ LegalizerHelper::lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { // return Len - popcount(x); // // Ref: "Hacker's Delight" by Henry Warren - unsigned Op = SrcReg; + Register Op = SrcReg; unsigned NewLen = PowerOf2Ceil(Len); for (unsigned i = 0; (1U << i) <= (NewLen / 2); ++i) { auto MIBShiftAmt = MIRBuilder.buildConstant(Ty, 1ULL << i); @@ -1338,9 +3149,9 @@ LegalizerHelper::lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { return Legalized; } case TargetOpcode::G_CTTZ: { - unsigned SrcReg = MI.getOperand(1).getReg(); + Register SrcReg = MI.getOperand(1).getReg(); unsigned Len = Ty.getSizeInBits(); - if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {Ty}})) { + if (isSupported({TargetOpcode::G_CTTZ_ZERO_UNDEF, {Ty, Ty}})) { // If CTTZ_ZERO_UNDEF is legal or custom, emit that and a select with // zero. auto MIBCttzZU = MIRBuilder.buildInstr(TargetOpcode::G_CTTZ_ZERO_UNDEF, @@ -1365,8 +3176,8 @@ LegalizerHelper::lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { TargetOpcode::G_AND, {Ty}, {MIBNot, MIRBuilder.buildInstr(TargetOpcode::G_ADD, {Ty}, {SrcReg, MIBCstNeg1})}); - if (!isSupported({TargetOpcode::G_CTPOP, {Ty}}) && - isSupported({TargetOpcode::G_CTLZ, {Ty}})) { + if (!isSupported({TargetOpcode::G_CTPOP, {Ty, Ty}}) && + isSupported({TargetOpcode::G_CTLZ, {Ty, Ty}})) { auto MIBCstLen = MIRBuilder.buildConstant(Ty, Len); MIRBuilder.buildInstr( TargetOpcode::G_SUB, {MI.getOperand(0).getReg()}, @@ -1381,3 +3192,230 @@ LegalizerHelper::lowerBitCount(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { } } } + +// Expand s32 = G_UITOFP s64 using bit operations to an IEEE float +// representation. +LegalizerHelper::LegalizeResult +LegalizerHelper::lowerU64ToF32BitOps(MachineInstr &MI) { + Register Dst = MI.getOperand(0).getReg(); + Register Src = MI.getOperand(1).getReg(); + const LLT S64 = LLT::scalar(64); + const LLT S32 = LLT::scalar(32); + const LLT S1 = LLT::scalar(1); + + assert(MRI.getType(Src) == S64 && MRI.getType(Dst) == S32); + + // unsigned cul2f(ulong u) { + // uint lz = clz(u); + // uint e = (u != 0) ? 127U + 63U - lz : 0; + // u = (u << lz) & 0x7fffffffffffffffUL; + // ulong t = u & 0xffffffffffUL; + // uint v = (e << 23) | (uint)(u >> 40); + // uint r = t > 0x8000000000UL ? 1U : (t == 0x8000000000UL ? v & 1U : 0U); + // return as_float(v + r); + // } + + auto Zero32 = MIRBuilder.buildConstant(S32, 0); + auto Zero64 = MIRBuilder.buildConstant(S64, 0); + + auto LZ = MIRBuilder.buildCTLZ_ZERO_UNDEF(S32, Src); + + auto K = MIRBuilder.buildConstant(S32, 127U + 63U); + auto Sub = MIRBuilder.buildSub(S32, K, LZ); + + auto NotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, Src, Zero64); + auto E = MIRBuilder.buildSelect(S32, NotZero, Sub, Zero32); + + auto Mask0 = MIRBuilder.buildConstant(S64, (-1ULL) >> 1); + auto ShlLZ = MIRBuilder.buildShl(S64, Src, LZ); + + auto U = MIRBuilder.buildAnd(S64, ShlLZ, Mask0); + + auto Mask1 = MIRBuilder.buildConstant(S64, 0xffffffffffULL); + auto T = MIRBuilder.buildAnd(S64, U, Mask1); + + auto UShl = MIRBuilder.buildLShr(S64, U, MIRBuilder.buildConstant(S64, 40)); + auto ShlE = MIRBuilder.buildShl(S32, E, MIRBuilder.buildConstant(S32, 23)); + auto V = MIRBuilder.buildOr(S32, ShlE, MIRBuilder.buildTrunc(S32, UShl)); + + auto C = MIRBuilder.buildConstant(S64, 0x8000000000ULL); + auto RCmp = MIRBuilder.buildICmp(CmpInst::ICMP_UGT, S1, T, C); + auto TCmp = MIRBuilder.buildICmp(CmpInst::ICMP_EQ, S1, T, C); + auto One = MIRBuilder.buildConstant(S32, 1); + + auto VTrunc1 = MIRBuilder.buildAnd(S32, V, One); + auto Select0 = MIRBuilder.buildSelect(S32, TCmp, VTrunc1, Zero32); + auto R = MIRBuilder.buildSelect(S32, RCmp, One, Select0); + MIRBuilder.buildAdd(Dst, V, R); + + return Legalized; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::lowerUITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { + Register Dst = MI.getOperand(0).getReg(); + Register Src = MI.getOperand(1).getReg(); + LLT DstTy = MRI.getType(Dst); + LLT SrcTy = MRI.getType(Src); + + if (SrcTy != LLT::scalar(64)) + return UnableToLegalize; + + if (DstTy == LLT::scalar(32)) { + // TODO: SelectionDAG has several alternative expansions to port which may + // be more reasonble depending on the available instructions. If a target + // has sitofp, does not have CTLZ, or can efficiently use f64 as an + // intermediate type, this is probably worse. + return lowerU64ToF32BitOps(MI); + } + + return UnableToLegalize; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::lowerSITOFP(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { + Register Dst = MI.getOperand(0).getReg(); + Register Src = MI.getOperand(1).getReg(); + LLT DstTy = MRI.getType(Dst); + LLT SrcTy = MRI.getType(Src); + + const LLT S64 = LLT::scalar(64); + const LLT S32 = LLT::scalar(32); + const LLT S1 = LLT::scalar(1); + + if (SrcTy != S64) + return UnableToLegalize; + + if (DstTy == S32) { + // signed cl2f(long l) { + // long s = l >> 63; + // float r = cul2f((l + s) ^ s); + // return s ? -r : r; + // } + Register L = Src; + auto SignBit = MIRBuilder.buildConstant(S64, 63); + auto S = MIRBuilder.buildAShr(S64, L, SignBit); + + auto LPlusS = MIRBuilder.buildAdd(S64, L, S); + auto Xor = MIRBuilder.buildXor(S64, LPlusS, S); + auto R = MIRBuilder.buildUITOFP(S32, Xor); + + auto RNeg = MIRBuilder.buildFNeg(S32, R); + auto SignNotZero = MIRBuilder.buildICmp(CmpInst::ICMP_NE, S1, S, + MIRBuilder.buildConstant(S64, 0)); + MIRBuilder.buildSelect(Dst, SignNotZero, RNeg, R); + return Legalized; + } + + return UnableToLegalize; +} + +static CmpInst::Predicate minMaxToCompare(unsigned Opc) { + switch (Opc) { + case TargetOpcode::G_SMIN: + return CmpInst::ICMP_SLT; + case TargetOpcode::G_SMAX: + return CmpInst::ICMP_SGT; + case TargetOpcode::G_UMIN: + return CmpInst::ICMP_ULT; + case TargetOpcode::G_UMAX: + return CmpInst::ICMP_UGT; + default: + llvm_unreachable("not in integer min/max"); + } +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::lowerMinMax(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { + Register Dst = MI.getOperand(0).getReg(); + Register Src0 = MI.getOperand(1).getReg(); + Register Src1 = MI.getOperand(2).getReg(); + + const CmpInst::Predicate Pred = minMaxToCompare(MI.getOpcode()); + LLT CmpType = MRI.getType(Dst).changeElementSize(1); + + auto Cmp = MIRBuilder.buildICmp(Pred, CmpType, Src0, Src1); + MIRBuilder.buildSelect(Dst, Cmp, Src0, Src1); + + MI.eraseFromParent(); + return Legalized; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::lowerFCopySign(MachineInstr &MI, unsigned TypeIdx, LLT Ty) { + Register Dst = MI.getOperand(0).getReg(); + Register Src0 = MI.getOperand(1).getReg(); + Register Src1 = MI.getOperand(2).getReg(); + + const LLT Src0Ty = MRI.getType(Src0); + const LLT Src1Ty = MRI.getType(Src1); + + const int Src0Size = Src0Ty.getScalarSizeInBits(); + const int Src1Size = Src1Ty.getScalarSizeInBits(); + + auto SignBitMask = MIRBuilder.buildConstant( + Src0Ty, APInt::getSignMask(Src0Size)); + + auto NotSignBitMask = MIRBuilder.buildConstant( + Src0Ty, APInt::getLowBitsSet(Src0Size, Src0Size - 1)); + + auto And0 = MIRBuilder.buildAnd(Src0Ty, Src0, NotSignBitMask); + MachineInstr *Or; + + if (Src0Ty == Src1Ty) { + auto And1 = MIRBuilder.buildAnd(Src1Ty, Src0, SignBitMask); + Or = MIRBuilder.buildOr(Dst, And0, And1); + } else if (Src0Size > Src1Size) { + auto ShiftAmt = MIRBuilder.buildConstant(Src0Ty, Src0Size - Src1Size); + auto Zext = MIRBuilder.buildZExt(Src0Ty, Src1); + auto Shift = MIRBuilder.buildShl(Src0Ty, Zext, ShiftAmt); + auto And1 = MIRBuilder.buildAnd(Src0Ty, Shift, SignBitMask); + Or = MIRBuilder.buildOr(Dst, And0, And1); + } else { + auto ShiftAmt = MIRBuilder.buildConstant(Src1Ty, Src1Size - Src0Size); + auto Shift = MIRBuilder.buildLShr(Src1Ty, Src1, ShiftAmt); + auto Trunc = MIRBuilder.buildTrunc(Src0Ty, Shift); + auto And1 = MIRBuilder.buildAnd(Src0Ty, Trunc, SignBitMask); + Or = MIRBuilder.buildOr(Dst, And0, And1); + } + + // Be careful about setting nsz/nnan/ninf on every instruction, since the + // constants are a nan and -0.0, but the final result should preserve + // everything. + if (unsigned Flags = MI.getFlags()) + Or->setFlags(Flags); + + MI.eraseFromParent(); + return Legalized; +} + +LegalizerHelper::LegalizeResult +LegalizerHelper::lowerFMinNumMaxNum(MachineInstr &MI) { + unsigned NewOp = MI.getOpcode() == TargetOpcode::G_FMINNUM ? + TargetOpcode::G_FMINNUM_IEEE : TargetOpcode::G_FMAXNUM_IEEE; + + Register Dst = MI.getOperand(0).getReg(); + Register Src0 = MI.getOperand(1).getReg(); + Register Src1 = MI.getOperand(2).getReg(); + LLT Ty = MRI.getType(Dst); + + if (!MI.getFlag(MachineInstr::FmNoNans)) { + // Insert canonicalizes if it's possible we need to quiet to get correct + // sNaN behavior. + + // Note this must be done here, and not as an optimization combine in the + // absence of a dedicate quiet-snan instruction as we're using an + // omni-purpose G_FCANONICALIZE. + if (!isKnownNeverSNaN(Src0, MRI)) + Src0 = MIRBuilder.buildFCanonicalize(Ty, Src0, MI.getFlags()).getReg(0); + + if (!isKnownNeverSNaN(Src1, MRI)) + Src1 = MIRBuilder.buildFCanonicalize(Ty, Src1, MI.getFlags()).getReg(0); + } + + // If there are no nans, it's safe to simply replace this with the non-IEEE + // version. + MIRBuilder.buildInstr(NewOp, {Dst}, {Src0, Src1}, MI.getFlags()); + MI.eraseFromParent(); + return Legalized; +} diff --git a/lib/CodeGen/GlobalISel/LegalizerInfo.cpp b/lib/CodeGen/GlobalISel/LegalizerInfo.cpp index fa36ede5b976..6e1de95b3277 100644 --- a/lib/CodeGen/GlobalISel/LegalizerInfo.cpp +++ b/lib/CodeGen/GlobalISel/LegalizerInfo.cpp @@ -1,9 +1,8 @@ //===- lib/CodeGen/GlobalISel/LegalizerInfo.cpp - Legalizer ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -43,6 +42,45 @@ cl::opt llvm::DisableGISelLegalityCheck( cl::desc("Don't verify that MIR is fully legal between GlobalISel passes"), cl::Hidden); +raw_ostream &llvm::operator<<(raw_ostream &OS, LegalizeAction Action) { + switch (Action) { + case Legal: + OS << "Legal"; + break; + case NarrowScalar: + OS << "NarrowScalar"; + break; + case WidenScalar: + OS << "WidenScalar"; + break; + case FewerElements: + OS << "FewerElements"; + break; + case MoreElements: + OS << "MoreElements"; + break; + case Lower: + OS << "Lower"; + break; + case Libcall: + OS << "Libcall"; + break; + case Custom: + OS << "Custom"; + break; + case Unsupported: + OS << "Unsupported"; + break; + case NotFound: + OS << "NotFound"; + break; + case UseLegacyRules: + OS << "UseLegacyRules"; + break; + } + return OS; +} + raw_ostream &LegalityQuery::print(raw_ostream &OS) const { OS << Opcode << ", Tys={"; for (const auto &Type : Types) { @@ -59,6 +97,86 @@ raw_ostream &LegalityQuery::print(raw_ostream &OS) const { return OS; } +#ifndef NDEBUG +// Make sure the rule won't (trivially) loop forever. +static bool hasNoSimpleLoops(const LegalizeRule &Rule, const LegalityQuery &Q, + const std::pair &Mutation) { + switch (Rule.getAction()) { + case Custom: + case Lower: + case MoreElements: + case FewerElements: + break; + default: + return Q.Types[Mutation.first] != Mutation.second; + } + return true; +} + +// Make sure the returned mutation makes sense for the match type. +static bool mutationIsSane(const LegalizeRule &Rule, + const LegalityQuery &Q, + std::pair Mutation) { + // If the user wants a custom mutation, then we can't really say much about + // it. Return true, and trust that they're doing the right thing. + if (Rule.getAction() == Custom) + return true; + + const unsigned TypeIdx = Mutation.first; + const LLT OldTy = Q.Types[TypeIdx]; + const LLT NewTy = Mutation.second; + + switch (Rule.getAction()) { + case FewerElements: + case MoreElements: { + if (!OldTy.isVector()) + return false; + + if (NewTy.isVector()) { + if (Rule.getAction() == FewerElements) { + // Make sure the element count really decreased. + if (NewTy.getNumElements() >= OldTy.getNumElements()) + return false; + } else { + // Make sure the element count really increased. + if (NewTy.getNumElements() <= OldTy.getNumElements()) + return false; + } + } + + // Make sure the element type didn't change. + return NewTy.getScalarType() == OldTy.getElementType(); + } + case NarrowScalar: + case WidenScalar: { + if (OldTy.isVector()) { + // Number of elements should not change. + if (!NewTy.isVector() || OldTy.getNumElements() != NewTy.getNumElements()) + return false; + } else { + // Both types must be vectors + if (NewTy.isVector()) + return false; + } + + if (Rule.getAction() == NarrowScalar) { + // Make sure the size really decreased. + if (NewTy.getScalarSizeInBits() >= OldTy.getScalarSizeInBits()) + return false; + } else { + // Make sure the size really increased. + if (NewTy.getScalarSizeInBits() <= OldTy.getScalarSizeInBits()) + return false; + } + + return true; + } + default: + return true; + } +} +#endif + LegalizeActionStep LegalizeRuleSet::apply(const LegalityQuery &Query) const { LLVM_DEBUG(dbgs() << "Applying legalizer ruleset to: "; Query.print(dbgs()); dbgs() << "\n"); @@ -66,17 +184,15 @@ LegalizeActionStep LegalizeRuleSet::apply(const LegalityQuery &Query) const { LLVM_DEBUG(dbgs() << ".. fallback to legacy rules (no rules defined)\n"); return {LegalizeAction::UseLegacyRules, 0, LLT{}}; } - for (const auto &Rule : Rules) { + for (const LegalizeRule &Rule : Rules) { if (Rule.match(Query)) { LLVM_DEBUG(dbgs() << ".. match\n"); std::pair Mutation = Rule.determineMutation(Query); - LLVM_DEBUG(dbgs() << ".. .. " << (unsigned)Rule.getAction() << ", " + LLVM_DEBUG(dbgs() << ".. .. " << Rule.getAction() << ", " << Mutation.first << ", " << Mutation.second << "\n"); - assert((Query.Types[Mutation.first] != Mutation.second || - Rule.getAction() == Lower || - Rule.getAction() == MoreElements || - Rule.getAction() == FewerElements) && - "Simple loop detected"); + assert(mutationIsSane(Rule, Query, Mutation) && + "legality mutation invalid for match"); + assert(hasNoSimpleLoops(Rule, Query, Mutation) && "Simple loop detected"); return {Rule.getAction(), Mutation.first, Mutation.second}; } else LLVM_DEBUG(dbgs() << ".. no match\n"); @@ -180,16 +296,14 @@ void LegalizerInfo::computeTables() { if (TypeIdx < ScalarSizeChangeStrategies[OpcodeIdx].size() && ScalarSizeChangeStrategies[OpcodeIdx][TypeIdx] != nullptr) S = ScalarSizeChangeStrategies[OpcodeIdx][TypeIdx]; - llvm::sort(ScalarSpecifiedActions.begin(), - ScalarSpecifiedActions.end()); + llvm::sort(ScalarSpecifiedActions); checkPartialSizeAndActionsVector(ScalarSpecifiedActions); setScalarAction(Opcode, TypeIdx, S(ScalarSpecifiedActions)); } // 2. Handle pointer types for (auto PointerSpecifiedActions : AddressSpace2SpecifiedActions) { - llvm::sort(PointerSpecifiedActions.second.begin(), - PointerSpecifiedActions.second.end()); + llvm::sort(PointerSpecifiedActions.second); checkPartialSizeAndActionsVector(PointerSpecifiedActions.second); // For pointer types, we assume that there isn't a meaningfull way // to change the number of bits used in the pointer. @@ -201,8 +315,7 @@ void LegalizerInfo::computeTables() { // 3. Handle vector types SizeAndActionsVec ElementSizesSeen; for (auto VectorSpecifiedActions : ElemSize2SpecifiedActions) { - llvm::sort(VectorSpecifiedActions.second.begin(), - VectorSpecifiedActions.second.end()); + llvm::sort(VectorSpecifiedActions.second); const uint16_t ElementSize = VectorSpecifiedActions.first; ElementSizesSeen.push_back({ElementSize, Legal}); checkPartialSizeAndActionsVector(VectorSpecifiedActions.second); @@ -328,9 +441,8 @@ LegalizerInfo::getAction(const LegalityQuery &Query) const { for (unsigned i = 0; i < Query.Types.size(); ++i) { auto Action = getAspectAction({Query.Opcode, i, Query.Types[i]}); if (Action.first != Legal) { - LLVM_DEBUG(dbgs() << ".. (legacy) Type " << i - << " Action=" << (unsigned)Action.first << ", " - << Action.second << "\n"); + LLVM_DEBUG(dbgs() << ".. (legacy) Type " << i << " Action=" + << Action.first << ", " << Action.second << "\n"); return {Action.first, i, Action.second}; } else LLVM_DEBUG(dbgs() << ".. (legacy) Type " << i << " Legal\n"); @@ -364,8 +476,9 @@ LegalizerInfo::getAction(const MachineInstr &MI, SmallVector MemDescrs; for (const auto &MMO : MI.memoperands()) - MemDescrs.push_back( - {MMO->getSize() /* in bytes */ * 8, MMO->getOrdering()}); + MemDescrs.push_back({8 * MMO->getSize() /* in bits */, + 8 * MMO->getAlignment(), + MMO->getOrdering()}); return getAction({MI.getOpcode(), Types, MemDescrs}); } @@ -375,6 +488,14 @@ bool LegalizerInfo::isLegal(const MachineInstr &MI, return getAction(MI, MRI).Action == Legal; } +bool LegalizerInfo::isLegalOrCustom(const MachineInstr &MI, + const MachineRegisterInfo &MRI) const { + auto Action = getAction(MI, MRI).Action; + // If the action is custom, it may not necessarily modify the instruction, + // so we have to assume it's legal. + return Action == Legal || Action == Custom; +} + bool LegalizerInfo::legalizeCustom(MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder, GISelChangeObserver &Observer) const { @@ -423,14 +544,10 @@ LegalizerInfo::findAction(const SizeAndActionsVec &Vec, const uint32_t Size) { // Find the last element in Vec that has a bitsize equal to or smaller than // the requested bit size. // That is the element just before the first element that is bigger than Size. - auto VecIt = std::upper_bound( - Vec.begin(), Vec.end(), Size, - [](const uint32_t Size, const SizeAndAction lhs) -> bool { - return Size < lhs.first; - }); - assert(VecIt != Vec.begin() && "Does Vec not start with size 1?"); - --VecIt; - int VecIdx = VecIt - Vec.begin(); + auto It = partition_point( + Vec, [=](const SizeAndAction &A) { return A.first <= Size; }); + assert(It != Vec.begin() && "Does Vec not start with size 1?"); + int VecIdx = It - Vec.begin() - 1; LegalizeAction Action = Vec[VecIdx].second; switch (Action) { @@ -541,6 +658,12 @@ LegalizerInfo::findVectorLegalAction(const InstrAspect &Aspect) const { IntermediateType.getScalarSizeInBits())}; } +bool LegalizerInfo::legalizeIntrinsic(MachineInstr &MI, + MachineRegisterInfo &MRI, + MachineIRBuilder &MIRBuilder) const { + return true; +} + /// \pre Type indices of every opcode form a dense set starting from 0. void LegalizerInfo::verify(const MCInstrInfo &MII) const { #ifndef NDEBUG @@ -584,7 +707,8 @@ const MachineInstr *llvm::machineFunctionIsIllegal(const MachineFunction &MF) { const MachineRegisterInfo &MRI = MF.getRegInfo(); for (const MachineBasicBlock &MBB : MF) for (const MachineInstr &MI : MBB) - if (isPreISelGenericOpcode(MI.getOpcode()) && !MLI->isLegal(MI, MRI)) + if (isPreISelGenericOpcode(MI.getOpcode()) && + !MLI->isLegalOrCustom(MI, MRI)) return &MI; } return nullptr; diff --git a/lib/CodeGen/GlobalISel/Localizer.cpp b/lib/CodeGen/GlobalISel/Localizer.cpp index 52b340753a50..3592409710a7 100644 --- a/lib/CodeGen/GlobalISel/Localizer.cpp +++ b/lib/CodeGen/GlobalISel/Localizer.cpp @@ -1,9 +1,8 @@ //===- Localizer.cpp ---------------------- Localize some instrs -*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -11,8 +10,8 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/GlobalISel/Localizer.h" +#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallPtrSet.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/Support/Debug.h" @@ -21,17 +20,53 @@ using namespace llvm; char Localizer::ID = 0; -INITIALIZE_PASS(Localizer, DEBUG_TYPE, - "Move/duplicate certain instructions close to their use", false, - false) +INITIALIZE_PASS_BEGIN(Localizer, DEBUG_TYPE, + "Move/duplicate certain instructions close to their use", + false, false) +INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) +INITIALIZE_PASS_END(Localizer, DEBUG_TYPE, + "Move/duplicate certain instructions close to their use", + false, false) -Localizer::Localizer() : MachineFunctionPass(ID) { - initializeLocalizerPass(*PassRegistry::getPassRegistry()); -} +Localizer::Localizer() : MachineFunctionPass(ID) { } -void Localizer::init(MachineFunction &MF) { MRI = &MF.getRegInfo(); } +void Localizer::init(MachineFunction &MF) { + MRI = &MF.getRegInfo(); + TTI = &getAnalysis().getTTI(MF.getFunction()); +} bool Localizer::shouldLocalize(const MachineInstr &MI) { + // Assuming a spill and reload of a value has a cost of 1 instruction each, + // this helper function computes the maximum number of uses we should consider + // for remat. E.g. on arm64 global addresses take 2 insts to materialize. We + // break even in terms of code size when the original MI has 2 users vs + // choosing to potentially spill. Any more than 2 users we we have a net code + // size increase. This doesn't take into account register pressure though. + auto maxUses = [](unsigned RematCost) { + // A cost of 1 means remats are basically free. + if (RematCost == 1) + return UINT_MAX; + if (RematCost == 2) + return 2U; + + // Remat is too expensive, only sink if there's one user. + if (RematCost > 2) + return 1U; + llvm_unreachable("Unexpected remat cost"); + }; + + // Helper to walk through uses and terminate if we've reached a limit. Saves + // us spending time traversing uses if all we want to know is if it's >= min. + auto isUsesAtMost = [&](unsigned Reg, unsigned MaxUses) { + unsigned NumUses = 0; + auto UI = MRI->use_instr_nodbg_begin(Reg), UE = MRI->use_instr_nodbg_end(); + for (; UI != UE && NumUses < MaxUses; ++UI) { + NumUses++; + } + // If we haven't reached the end yet then there are more than MaxUses users. + return UI == UE; + }; + switch (MI.getOpcode()) { default: return false; @@ -40,11 +75,22 @@ bool Localizer::shouldLocalize(const MachineInstr &MI) { case TargetOpcode::G_CONSTANT: case TargetOpcode::G_FCONSTANT: case TargetOpcode::G_FRAME_INDEX: + case TargetOpcode::G_INTTOPTR: return true; + case TargetOpcode::G_GLOBAL_VALUE: { + unsigned RematCost = TTI->getGISelRematGlobalCost(); + unsigned Reg = MI.getOperand(0).getReg(); + unsigned MaxUses = maxUses(RematCost); + if (MaxUses == UINT_MAX) + return true; // Remats are "free" so always localize. + bool B = isUsesAtMost(Reg, MaxUses); + return B; + } } } void Localizer::getAnalysisUsage(AnalysisUsage &AU) const { + AU.addRequired(); getSelectionDAGFallbackAnalysisUsage(AU); MachineFunctionPass::getAnalysisUsage(AU); } @@ -58,6 +104,107 @@ bool Localizer::isLocalUse(MachineOperand &MOUse, const MachineInstr &Def, return InsertMBB == Def.getParent(); } +bool Localizer::localizeInterBlock(MachineFunction &MF, + LocalizedSetVecT &LocalizedInstrs) { + bool Changed = false; + DenseMap, unsigned> MBBWithLocalDef; + + // Since the IRTranslator only emits constants into the entry block, and the + // rest of the GISel pipeline generally emits constants close to their users, + // we only localize instructions in the entry block here. This might change if + // we start doing CSE across blocks. + auto &MBB = MF.front(); + for (auto RI = MBB.rbegin(), RE = MBB.rend(); RI != RE; ++RI) { + MachineInstr &MI = *RI; + if (!shouldLocalize(MI)) + continue; + LLVM_DEBUG(dbgs() << "Should localize: " << MI); + assert(MI.getDesc().getNumDefs() == 1 && + "More than one definition not supported yet"); + unsigned Reg = MI.getOperand(0).getReg(); + // Check if all the users of MI are local. + // We are going to invalidation the list of use operands, so we + // can't use range iterator. + for (auto MOIt = MRI->use_begin(Reg), MOItEnd = MRI->use_end(); + MOIt != MOItEnd;) { + MachineOperand &MOUse = *MOIt++; + // Check if the use is already local. + MachineBasicBlock *InsertMBB; + LLVM_DEBUG(MachineInstr &MIUse = *MOUse.getParent(); + dbgs() << "Checking use: " << MIUse + << " #Opd: " << MIUse.getOperandNo(&MOUse) << '\n'); + if (isLocalUse(MOUse, MI, InsertMBB)) + continue; + LLVM_DEBUG(dbgs() << "Fixing non-local use\n"); + Changed = true; + auto MBBAndReg = std::make_pair(InsertMBB, Reg); + auto NewVRegIt = MBBWithLocalDef.find(MBBAndReg); + if (NewVRegIt == MBBWithLocalDef.end()) { + // Create the localized instruction. + MachineInstr *LocalizedMI = MF.CloneMachineInstr(&MI); + LocalizedInstrs.insert(LocalizedMI); + MachineInstr &UseMI = *MOUse.getParent(); + if (MRI->hasOneUse(Reg) && !UseMI.isPHI()) + InsertMBB->insert(InsertMBB->SkipPHIsAndLabels(UseMI), LocalizedMI); + else + InsertMBB->insert(InsertMBB->SkipPHIsAndLabels(InsertMBB->begin()), + LocalizedMI); + + // Set a new register for the definition. + unsigned NewReg = MRI->createGenericVirtualRegister(MRI->getType(Reg)); + MRI->setRegClassOrRegBank(NewReg, MRI->getRegClassOrRegBank(Reg)); + LocalizedMI->getOperand(0).setReg(NewReg); + NewVRegIt = + MBBWithLocalDef.insert(std::make_pair(MBBAndReg, NewReg)).first; + LLVM_DEBUG(dbgs() << "Inserted: " << *LocalizedMI); + } + LLVM_DEBUG(dbgs() << "Update use with: " << printReg(NewVRegIt->second) + << '\n'); + // Update the user reg. + MOUse.setReg(NewVRegIt->second); + } + } + return Changed; +} + +bool Localizer::localizeIntraBlock(LocalizedSetVecT &LocalizedInstrs) { + bool Changed = false; + + // For each already-localized instruction which has multiple users, then we + // scan the block top down from the current position until we hit one of them. + + // FIXME: Consider doing inst duplication if live ranges are very long due to + // many users, but this case may be better served by regalloc improvements. + + for (MachineInstr *MI : LocalizedInstrs) { + unsigned Reg = MI->getOperand(0).getReg(); + MachineBasicBlock &MBB = *MI->getParent(); + // All of the user MIs of this reg. + SmallPtrSet Users; + for (MachineInstr &UseMI : MRI->use_nodbg_instructions(Reg)) { + if (!UseMI.isPHI()) + Users.insert(&UseMI); + } + // If all the users were PHIs then they're not going to be in our block, + // don't try to move this instruction. + if (Users.empty()) + continue; + + MachineBasicBlock::iterator II(MI); + ++II; + while (II != MBB.end() && !Users.count(&*II)) + ++II; + + LLVM_DEBUG(dbgs() << "Intra-block: moving " << *MI << " before " << *&*II + << "\n"); + assert(II != MBB.end() && "Didn't find the user in the MBB"); + MI->removeFromParent(); + MBB.insert(II, MI); + Changed = true; + } + return Changed; +} + bool Localizer::runOnMachineFunction(MachineFunction &MF) { // If the ISel pipeline failed, do not bother running that pass. if (MF.getProperties().hasProperty( @@ -68,62 +215,10 @@ bool Localizer::runOnMachineFunction(MachineFunction &MF) { init(MF); - bool Changed = false; - // Keep track of the instructions we localized. - // We won't need to process them if we see them later in the CFG. - SmallPtrSet LocalizedInstrs; - DenseMap, unsigned> MBBWithLocalDef; - // TODO: Do bottom up traversal. - for (MachineBasicBlock &MBB : MF) { - for (MachineInstr &MI : MBB) { - if (LocalizedInstrs.count(&MI) || !shouldLocalize(MI)) - continue; - LLVM_DEBUG(dbgs() << "Should localize: " << MI); - assert(MI.getDesc().getNumDefs() == 1 && - "More than one definition not supported yet"); - unsigned Reg = MI.getOperand(0).getReg(); - // Check if all the users of MI are local. - // We are going to invalidation the list of use operands, so we - // can't use range iterator. - for (auto MOIt = MRI->use_begin(Reg), MOItEnd = MRI->use_end(); - MOIt != MOItEnd;) { - MachineOperand &MOUse = *MOIt++; - // Check if the use is already local. - MachineBasicBlock *InsertMBB; - LLVM_DEBUG(MachineInstr &MIUse = *MOUse.getParent(); - dbgs() << "Checking use: " << MIUse - << " #Opd: " << MIUse.getOperandNo(&MOUse) << '\n'); - if (isLocalUse(MOUse, MI, InsertMBB)) - continue; - LLVM_DEBUG(dbgs() << "Fixing non-local use\n"); - Changed = true; - auto MBBAndReg = std::make_pair(InsertMBB, Reg); - auto NewVRegIt = MBBWithLocalDef.find(MBBAndReg); - if (NewVRegIt == MBBWithLocalDef.end()) { - // Create the localized instruction. - MachineInstr *LocalizedMI = MF.CloneMachineInstr(&MI); - LocalizedInstrs.insert(LocalizedMI); - // Don't try to be smart for the insertion point. - // There is no guarantee that the first seen use is the first - // use in the block. - InsertMBB->insert(InsertMBB->SkipPHIsAndLabels(InsertMBB->begin()), - LocalizedMI); + // Keep track of the instructions we localized. We'll do a second pass of + // intra-block localization to further reduce live ranges. + LocalizedSetVecT LocalizedInstrs; - // Set a new register for the definition. - unsigned NewReg = - MRI->createGenericVirtualRegister(MRI->getType(Reg)); - MRI->setRegClassOrRegBank(NewReg, MRI->getRegClassOrRegBank(Reg)); - LocalizedMI->getOperand(0).setReg(NewReg); - NewVRegIt = - MBBWithLocalDef.insert(std::make_pair(MBBAndReg, NewReg)).first; - LLVM_DEBUG(dbgs() << "Inserted: " << *LocalizedMI); - } - LLVM_DEBUG(dbgs() << "Update use with: " << printReg(NewVRegIt->second) - << '\n'); - // Update the user reg. - MOUse.setReg(NewVRegIt->second); - } - } - } - return Changed; + bool Changed = localizeInterBlock(MF, LocalizedInstrs); + return Changed |= localizeIntraBlock(LocalizedInstrs); } diff --git a/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp index 1f5611061994..b7a73326b85c 100644 --- a/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ b/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/GlobalISel/MachineIRBuilder.cpp - MIBuilder--*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -17,6 +16,7 @@ #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/DebugInfo.h" @@ -87,7 +87,7 @@ MachineInstrBuilder MachineIRBuilder::insertInstr(MachineInstrBuilder MIB) { } MachineInstrBuilder -MachineIRBuilder::buildDirectDbgValue(unsigned Reg, const MDNode *Variable, +MachineIRBuilder::buildDirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr) { assert(isa(Variable) && "not a variable"); assert(cast(Expr)->isValid() && "not an expression"); @@ -100,7 +100,7 @@ MachineIRBuilder::buildDirectDbgValue(unsigned Reg, const MDNode *Variable, } MachineInstrBuilder -MachineIRBuilder::buildIndirectDbgValue(unsigned Reg, const MDNode *Variable, +MachineIRBuilder::buildIndirectDbgValue(Register Reg, const MDNode *Variable, const MDNode *Expr) { assert(isa(Variable) && "not a variable"); assert(cast(Expr)->isValid() && "not an expression"); @@ -160,23 +160,32 @@ MachineInstrBuilder MachineIRBuilder::buildDbgLabel(const MDNode *Label) { return MIB.addMetadata(Label); } -MachineInstrBuilder MachineIRBuilder::buildFrameIndex(unsigned Res, int Idx) { - assert(getMRI()->getType(Res).isPointer() && "invalid operand type"); - return buildInstr(TargetOpcode::G_FRAME_INDEX) - .addDef(Res) - .addFrameIndex(Idx); +MachineInstrBuilder MachineIRBuilder::buildFrameIndex(const DstOp &Res, + int Idx) { + assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type"); + auto MIB = buildInstr(TargetOpcode::G_FRAME_INDEX); + Res.addDefToMIB(*getMRI(), MIB); + MIB.addFrameIndex(Idx); + return MIB; } -MachineInstrBuilder MachineIRBuilder::buildGlobalValue(unsigned Res, +MachineInstrBuilder MachineIRBuilder::buildGlobalValue(const DstOp &Res, const GlobalValue *GV) { - assert(getMRI()->getType(Res).isPointer() && "invalid operand type"); - assert(getMRI()->getType(Res).getAddressSpace() == + assert(Res.getLLTTy(*getMRI()).isPointer() && "invalid operand type"); + assert(Res.getLLTTy(*getMRI()).getAddressSpace() == GV->getType()->getAddressSpace() && "address space mismatch"); - return buildInstr(TargetOpcode::G_GLOBAL_VALUE) - .addDef(Res) - .addGlobalAddress(GV); + auto MIB = buildInstr(TargetOpcode::G_GLOBAL_VALUE); + Res.addDefToMIB(*getMRI(), MIB); + MIB.addGlobalAddress(GV); + return MIB; +} + +MachineInstrBuilder MachineIRBuilder::buildJumpTable(const LLT PtrTy, + unsigned JTI) { + return buildInstr(TargetOpcode::G_JUMP_TABLE, {PtrTy}, {}) + .addJumpTableIndex(JTI); } void MachineIRBuilder::validateBinaryOp(const LLT &Res, const LLT &Op0, @@ -185,20 +194,28 @@ void MachineIRBuilder::validateBinaryOp(const LLT &Res, const LLT &Op0, assert((Res == Op0 && Res == Op1) && "type mismatch"); } -MachineInstrBuilder MachineIRBuilder::buildGEP(unsigned Res, unsigned Op0, - unsigned Op1) { - assert(getMRI()->getType(Res).isPointer() && - getMRI()->getType(Res) == getMRI()->getType(Op0) && "type mismatch"); - assert(getMRI()->getType(Op1).isScalar() && "invalid offset type"); +void MachineIRBuilder::validateShiftOp(const LLT &Res, const LLT &Op0, + const LLT &Op1) { + assert((Res.isScalar() || Res.isVector()) && "invalid operand type"); + assert((Res == Op0) && "type mismatch"); +} + +MachineInstrBuilder MachineIRBuilder::buildGEP(const DstOp &Res, + const SrcOp &Op0, + const SrcOp &Op1) { + assert(Res.getLLTTy(*getMRI()).isPointer() && + Res.getLLTTy(*getMRI()) == Op0.getLLTTy(*getMRI()) && "type mismatch"); + assert(Op1.getLLTTy(*getMRI()).isScalar() && "invalid offset type"); - return buildInstr(TargetOpcode::G_GEP) - .addDef(Res) - .addUse(Op0) - .addUse(Op1); + auto MIB = buildInstr(TargetOpcode::G_GEP); + Res.addDefToMIB(*getMRI(), MIB); + Op0.addSrcToMIB(MIB); + Op1.addSrcToMIB(MIB); + return MIB; } Optional -MachineIRBuilder::materializeGEP(unsigned &Res, unsigned Op0, +MachineIRBuilder::materializeGEP(Register &Res, Register Op0, const LLT &ValueTy, uint64_t Value) { assert(Res == 0 && "Res is a result argument"); assert(ValueTy.isScalar() && "invalid offset type"); @@ -209,32 +226,43 @@ MachineIRBuilder::materializeGEP(unsigned &Res, unsigned Op0, } Res = getMRI()->createGenericVirtualRegister(getMRI()->getType(Op0)); - unsigned TmpReg = getMRI()->createGenericVirtualRegister(ValueTy); - - buildConstant(TmpReg, Value); - return buildGEP(Res, Op0, TmpReg); + auto Cst = buildConstant(ValueTy, Value); + return buildGEP(Res, Op0, Cst.getReg(0)); } -MachineInstrBuilder MachineIRBuilder::buildPtrMask(unsigned Res, unsigned Op0, +MachineInstrBuilder MachineIRBuilder::buildPtrMask(const DstOp &Res, + const SrcOp &Op0, uint32_t NumBits) { - assert(getMRI()->getType(Res).isPointer() && - getMRI()->getType(Res) == getMRI()->getType(Op0) && "type mismatch"); + assert(Res.getLLTTy(*getMRI()).isPointer() && + Res.getLLTTy(*getMRI()) == Op0.getLLTTy(*getMRI()) && "type mismatch"); - return buildInstr(TargetOpcode::G_PTR_MASK) - .addDef(Res) - .addUse(Op0) - .addImm(NumBits); + auto MIB = buildInstr(TargetOpcode::G_PTR_MASK); + Res.addDefToMIB(*getMRI(), MIB); + Op0.addSrcToMIB(MIB); + MIB.addImm(NumBits); + return MIB; } MachineInstrBuilder MachineIRBuilder::buildBr(MachineBasicBlock &Dest) { return buildInstr(TargetOpcode::G_BR).addMBB(&Dest); } -MachineInstrBuilder MachineIRBuilder::buildBrIndirect(unsigned Tgt) { +MachineInstrBuilder MachineIRBuilder::buildBrIndirect(Register Tgt) { assert(getMRI()->getType(Tgt).isPointer() && "invalid branch destination"); return buildInstr(TargetOpcode::G_BRINDIRECT).addUse(Tgt); } +MachineInstrBuilder MachineIRBuilder::buildBrJT(Register TablePtr, + unsigned JTI, + Register IndexReg) { + assert(getMRI()->getType(TablePtr).isPointer() && + "Table reg must be a pointer"); + return buildInstr(TargetOpcode::G_BRJT) + .addUse(TablePtr) + .addJumpTableIndex(JTI) + .addUse(IndexReg); +} + MachineInstrBuilder MachineIRBuilder::buildCopy(const DstOp &Res, const SrcOp &Op) { return buildInstr(TargetOpcode::COPY, Res, Op); @@ -243,36 +271,60 @@ MachineInstrBuilder MachineIRBuilder::buildCopy(const DstOp &Res, MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res, const ConstantInt &Val) { LLT Ty = Res.getLLTTy(*getMRI()); + LLT EltTy = Ty.getScalarType(); + assert(EltTy.getScalarSizeInBits() == Val.getBitWidth() && + "creating constant with the wrong size"); + + if (Ty.isVector()) { + auto Const = buildInstr(TargetOpcode::G_CONSTANT) + .addDef(getMRI()->createGenericVirtualRegister(EltTy)) + .addCImm(&Val); + return buildSplatVector(Res, Const); + } - assert((Ty.isScalar() || Ty.isPointer()) && "invalid operand type"); - - const ConstantInt *NewVal = &Val; - if (Ty.getSizeInBits() != Val.getBitWidth()) - NewVal = ConstantInt::get(getMF().getFunction().getContext(), - Val.getValue().sextOrTrunc(Ty.getSizeInBits())); - - auto MIB = buildInstr(TargetOpcode::G_CONSTANT); - Res.addDefToMIB(*getMRI(), MIB); - MIB.addCImm(NewVal); - return MIB; + auto Const = buildInstr(TargetOpcode::G_CONSTANT); + Res.addDefToMIB(*getMRI(), Const); + Const.addCImm(&Val); + return Const; } MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res, int64_t Val) { auto IntN = IntegerType::get(getMF().getFunction().getContext(), - Res.getLLTTy(*getMRI()).getSizeInBits()); + Res.getLLTTy(*getMRI()).getScalarSizeInBits()); ConstantInt *CI = ConstantInt::get(IntN, Val, true); return buildConstant(Res, *CI); } MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res, const ConstantFP &Val) { - assert(Res.getLLTTy(*getMRI()).isScalar() && "invalid operand type"); + LLT Ty = Res.getLLTTy(*getMRI()); + LLT EltTy = Ty.getScalarType(); - auto MIB = buildInstr(TargetOpcode::G_FCONSTANT); - Res.addDefToMIB(*getMRI(), MIB); - MIB.addFPImm(&Val); - return MIB; + assert(APFloat::getSizeInBits(Val.getValueAPF().getSemantics()) + == EltTy.getSizeInBits() && + "creating fconstant with the wrong size"); + + assert(!Ty.isPointer() && "invalid operand type"); + + if (Ty.isVector()) { + auto Const = buildInstr(TargetOpcode::G_FCONSTANT) + .addDef(getMRI()->createGenericVirtualRegister(EltTy)) + .addFPImm(&Val); + + return buildSplatVector(Res, Const); + } + + auto Const = buildInstr(TargetOpcode::G_FCONSTANT); + Res.addDefToMIB(*getMRI(), Const); + Const.addFPImm(&Val); + return Const; +} + +MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res, + const APInt &Val) { + ConstantInt *CI = ConstantInt::get(getMF().getFunction().getContext(), Val); + return buildConstant(Res, *CI); } MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res, @@ -280,44 +332,62 @@ MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res, LLT DstTy = Res.getLLTTy(*getMRI()); auto &Ctx = getMF().getFunction().getContext(); auto *CFP = - ConstantFP::get(Ctx, getAPFloatFromSize(Val, DstTy.getSizeInBits())); + ConstantFP::get(Ctx, getAPFloatFromSize(Val, DstTy.getScalarSizeInBits())); return buildFConstant(Res, *CFP); } -MachineInstrBuilder MachineIRBuilder::buildBrCond(unsigned Tst, +MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res, + const APFloat &Val) { + auto &Ctx = getMF().getFunction().getContext(); + auto *CFP = ConstantFP::get(Ctx, Val); + return buildFConstant(Res, *CFP); +} + +MachineInstrBuilder MachineIRBuilder::buildBrCond(Register Tst, MachineBasicBlock &Dest) { assert(getMRI()->getType(Tst).isScalar() && "invalid operand type"); return buildInstr(TargetOpcode::G_BRCOND).addUse(Tst).addMBB(&Dest); } -MachineInstrBuilder MachineIRBuilder::buildLoad(unsigned Res, unsigned Addr, +MachineInstrBuilder MachineIRBuilder::buildLoad(const DstOp &Res, + const SrcOp &Addr, MachineMemOperand &MMO) { return buildLoadInstr(TargetOpcode::G_LOAD, Res, Addr, MMO); } MachineInstrBuilder MachineIRBuilder::buildLoadInstr(unsigned Opcode, - unsigned Res, - unsigned Addr, + const DstOp &Res, + const SrcOp &Addr, MachineMemOperand &MMO) { - assert(getMRI()->getType(Res).isValid() && "invalid operand type"); - assert(getMRI()->getType(Addr).isPointer() && "invalid operand type"); + assert(Res.getLLTTy(*getMRI()).isValid() && "invalid operand type"); + assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type"); - return buildInstr(Opcode) - .addDef(Res) - .addUse(Addr) - .addMemOperand(&MMO); + auto MIB = buildInstr(Opcode); + Res.addDefToMIB(*getMRI(), MIB); + Addr.addSrcToMIB(MIB); + MIB.addMemOperand(&MMO); + return MIB; } -MachineInstrBuilder MachineIRBuilder::buildStore(unsigned Val, unsigned Addr, +MachineInstrBuilder MachineIRBuilder::buildStore(const SrcOp &Val, + const SrcOp &Addr, MachineMemOperand &MMO) { - assert(getMRI()->getType(Val).isValid() && "invalid operand type"); - assert(getMRI()->getType(Addr).isPointer() && "invalid operand type"); + assert(Val.getLLTTy(*getMRI()).isValid() && "invalid operand type"); + assert(Addr.getLLTTy(*getMRI()).isPointer() && "invalid operand type"); - return buildInstr(TargetOpcode::G_STORE) - .addUse(Val) - .addUse(Addr) - .addMemOperand(&MMO); + auto MIB = buildInstr(TargetOpcode::G_STORE); + Val.addSrcToMIB(MIB); + Addr.addSrcToMIB(MIB); + MIB.addMemOperand(&MMO); + return MIB; +} + +MachineInstrBuilder MachineIRBuilder::buildUAddo(const DstOp &Res, + const DstOp &CarryOut, + const SrcOp &Op0, + const SrcOp &Op1) { + return buildInstr(TargetOpcode::G_UADDO, {Res, CarryOut}, {Op0, Op1}); } MachineInstrBuilder MachineIRBuilder::buildUAdde(const DstOp &Res, @@ -344,6 +414,25 @@ MachineInstrBuilder MachineIRBuilder::buildZExt(const DstOp &Res, return buildInstr(TargetOpcode::G_ZEXT, Res, Op); } +unsigned MachineIRBuilder::getBoolExtOp(bool IsVec, bool IsFP) const { + const auto *TLI = getMF().getSubtarget().getTargetLowering(); + switch (TLI->getBooleanContents(IsVec, IsFP)) { + case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: + return TargetOpcode::G_SEXT; + case TargetLoweringBase::ZeroOrOneBooleanContent: + return TargetOpcode::G_ZEXT; + default: + return TargetOpcode::G_ANYEXT; + } +} + +MachineInstrBuilder MachineIRBuilder::buildBoolExt(const DstOp &Res, + const SrcOp &Op, + bool IsFP) { + unsigned ExtOp = getBoolExtOp(getMRI()->getType(Op.getReg()).isVector(), IsFP); + return buildInstr(ExtOp, Res, Op); +} + MachineInstrBuilder MachineIRBuilder::buildExtOrTrunc(unsigned ExtOpc, const DstOp &Res, const SrcOp &Op) { @@ -403,29 +492,32 @@ MachineInstrBuilder MachineIRBuilder::buildCast(const DstOp &Dst, return buildInstr(Opcode, Dst, Src); } -MachineInstrBuilder MachineIRBuilder::buildExtract(unsigned Res, unsigned Src, +MachineInstrBuilder MachineIRBuilder::buildExtract(const DstOp &Dst, + const SrcOp &Src, uint64_t Index) { + LLT SrcTy = Src.getLLTTy(*getMRI()); + LLT DstTy = Dst.getLLTTy(*getMRI()); + #ifndef NDEBUG - assert(getMRI()->getType(Src).isValid() && "invalid operand type"); - assert(getMRI()->getType(Res).isValid() && "invalid operand type"); - assert(Index + getMRI()->getType(Res).getSizeInBits() <= - getMRI()->getType(Src).getSizeInBits() && + assert(SrcTy.isValid() && "invalid operand type"); + assert(DstTy.isValid() && "invalid operand type"); + assert(Index + DstTy.getSizeInBits() <= SrcTy.getSizeInBits() && "extracting off end of register"); #endif - if (getMRI()->getType(Res).getSizeInBits() == - getMRI()->getType(Src).getSizeInBits()) { + if (DstTy.getSizeInBits() == SrcTy.getSizeInBits()) { assert(Index == 0 && "insertion past the end of a register"); - return buildCast(Res, Src); + return buildCast(Dst, Src); } - return buildInstr(TargetOpcode::G_EXTRACT) - .addDef(Res) - .addUse(Src) - .addImm(Index); + auto Extract = buildInstr(TargetOpcode::G_EXTRACT); + Dst.addDefToMIB(*getMRI(), Extract); + Src.addSrcToMIB(Extract); + Extract.addImm(Index); + return Extract; } -void MachineIRBuilder::buildSequence(unsigned Res, ArrayRef Ops, +void MachineIRBuilder::buildSequence(Register Res, ArrayRef Ops, ArrayRef Indices) { #ifndef NDEBUG assert(Ops.size() == Indices.size() && "incompatible args"); @@ -454,11 +546,11 @@ void MachineIRBuilder::buildSequence(unsigned Res, ArrayRef Ops, return; } - unsigned ResIn = getMRI()->createGenericVirtualRegister(ResTy); + Register ResIn = getMRI()->createGenericVirtualRegister(ResTy); buildUndef(ResIn); for (unsigned i = 0; i < Ops.size(); ++i) { - unsigned ResOut = i + 1 == Ops.size() + Register ResOut = i + 1 == Ops.size() ? Res : getMRI()->createGenericVirtualRegister(ResTy); buildInsert(ResOut, ResIn, Ops[i], Indices[i]); @@ -471,11 +563,12 @@ MachineInstrBuilder MachineIRBuilder::buildUndef(const DstOp &Res) { } MachineInstrBuilder MachineIRBuilder::buildMerge(const DstOp &Res, - ArrayRef Ops) { + ArrayRef Ops) { // Unfortunately to convert from ArrayRef to ArrayRef, // we need some temporary storage for the DstOp objects. Here we use a // sufficiently large SmallVector to not go through the heap. SmallVector TmpVec(Ops.begin(), Ops.end()); + assert(TmpVec.size() > 1); return buildInstr(TargetOpcode::G_MERGE_VALUES, Res, TmpVec); } @@ -485,31 +578,48 @@ MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef Res, // we need some temporary storage for the DstOp objects. Here we use a // sufficiently large SmallVector to not go through the heap. SmallVector TmpVec(Res.begin(), Res.end()); + assert(TmpVec.size() > 1); return buildInstr(TargetOpcode::G_UNMERGE_VALUES, TmpVec, Op); } -MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef Res, +MachineInstrBuilder MachineIRBuilder::buildUnmerge(LLT Res, + const SrcOp &Op) { + unsigned NumReg = Op.getLLTTy(*getMRI()).getSizeInBits() / Res.getSizeInBits(); + SmallVector TmpVec; + for (unsigned I = 0; I != NumReg; ++I) + TmpVec.push_back(getMRI()->createGenericVirtualRegister(Res)); + return buildUnmerge(TmpVec, Op); +} + +MachineInstrBuilder MachineIRBuilder::buildUnmerge(ArrayRef Res, const SrcOp &Op) { - // Unfortunately to convert from ArrayRef to ArrayRef, + // Unfortunately to convert from ArrayRef to ArrayRef, // we need some temporary storage for the DstOp objects. Here we use a // sufficiently large SmallVector to not go through the heap. SmallVector TmpVec(Res.begin(), Res.end()); + assert(TmpVec.size() > 1); return buildInstr(TargetOpcode::G_UNMERGE_VALUES, TmpVec, Op); } MachineInstrBuilder MachineIRBuilder::buildBuildVector(const DstOp &Res, - ArrayRef Ops) { - // Unfortunately to convert from ArrayRef to ArrayRef, + ArrayRef Ops) { + // Unfortunately to convert from ArrayRef to ArrayRef, // we need some temporary storage for the DstOp objects. Here we use a // sufficiently large SmallVector to not go through the heap. SmallVector TmpVec(Ops.begin(), Ops.end()); return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec); } +MachineInstrBuilder MachineIRBuilder::buildSplatVector(const DstOp &Res, + const SrcOp &Src) { + SmallVector TmpVec(Res.getLLTTy(*getMRI()).getNumElements(), Src); + return buildInstr(TargetOpcode::G_BUILD_VECTOR, Res, TmpVec); +} + MachineInstrBuilder MachineIRBuilder::buildBuildVectorTrunc(const DstOp &Res, - ArrayRef Ops) { - // Unfortunately to convert from ArrayRef to ArrayRef, + ArrayRef Ops) { + // Unfortunately to convert from ArrayRef to ArrayRef, // we need some temporary storage for the DstOp objects. Here we use a // sufficiently large SmallVector to not go through the heap. SmallVector TmpVec(Ops.begin(), Ops.end()); @@ -517,16 +627,16 @@ MachineIRBuilder::buildBuildVectorTrunc(const DstOp &Res, } MachineInstrBuilder -MachineIRBuilder::buildConcatVectors(const DstOp &Res, ArrayRef Ops) { - // Unfortunately to convert from ArrayRef to ArrayRef, +MachineIRBuilder::buildConcatVectors(const DstOp &Res, ArrayRef Ops) { + // Unfortunately to convert from ArrayRef to ArrayRef, // we need some temporary storage for the DstOp objects. Here we use a // sufficiently large SmallVector to not go through the heap. SmallVector TmpVec(Ops.begin(), Ops.end()); return buildInstr(TargetOpcode::G_CONCAT_VECTORS, Res, TmpVec); } -MachineInstrBuilder MachineIRBuilder::buildInsert(unsigned Res, unsigned Src, - unsigned Op, unsigned Index) { +MachineInstrBuilder MachineIRBuilder::buildInsert(Register Res, Register Src, + Register Op, unsigned Index) { assert(Index + getMRI()->getType(Op).getSizeInBits() <= getMRI()->getType(Res).getSizeInBits() && "insertion past the end of a register"); @@ -544,13 +654,25 @@ MachineInstrBuilder MachineIRBuilder::buildInsert(unsigned Res, unsigned Src, } MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID, - unsigned Res, + ArrayRef ResultRegs, bool HasSideEffects) { auto MIB = buildInstr(HasSideEffects ? TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS : TargetOpcode::G_INTRINSIC); - if (Res) - MIB.addDef(Res); + for (unsigned ResultReg : ResultRegs) + MIB.addDef(ResultReg); + MIB.addIntrinsicID(ID); + return MIB; +} + +MachineInstrBuilder MachineIRBuilder::buildIntrinsic(Intrinsic::ID ID, + ArrayRef Results, + bool HasSideEffects) { + auto MIB = + buildInstr(HasSideEffects ? TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS + : TargetOpcode::G_INTRINSIC); + for (DstOp Result : Results) + Result.addDefToMIB(*getMRI(), MIB); MIB.addIntrinsicID(ID); return MIB; } @@ -601,8 +723,8 @@ MachineIRBuilder::buildExtractVectorElement(const DstOp &Res, const SrcOp &Val, } MachineInstrBuilder MachineIRBuilder::buildAtomicCmpXchgWithSuccess( - unsigned OldValRes, unsigned SuccessRes, unsigned Addr, unsigned CmpVal, - unsigned NewVal, MachineMemOperand &MMO) { + Register OldValRes, Register SuccessRes, Register Addr, Register CmpVal, + Register NewVal, MachineMemOperand &MMO) { #ifndef NDEBUG LLT OldValResTy = getMRI()->getType(OldValRes); LLT SuccessResTy = getMRI()->getType(SuccessRes); @@ -628,8 +750,8 @@ MachineInstrBuilder MachineIRBuilder::buildAtomicCmpXchgWithSuccess( } MachineInstrBuilder -MachineIRBuilder::buildAtomicCmpXchg(unsigned OldValRes, unsigned Addr, - unsigned CmpVal, unsigned NewVal, +MachineIRBuilder::buildAtomicCmpXchg(Register OldValRes, Register Addr, + Register CmpVal, Register NewVal, MachineMemOperand &MMO) { #ifndef NDEBUG LLT OldValResTy = getMRI()->getType(OldValRes); @@ -653,9 +775,9 @@ MachineIRBuilder::buildAtomicCmpXchg(unsigned OldValRes, unsigned Addr, } MachineInstrBuilder MachineIRBuilder::buildAtomicRMW(unsigned Opcode, - unsigned OldValRes, - unsigned Addr, - unsigned Val, + Register OldValRes, + Register Addr, + Register Val, MachineMemOperand &MMO) { #ifndef NDEBUG LLT OldValResTy = getMRI()->getType(OldValRes); @@ -675,75 +797,82 @@ MachineInstrBuilder MachineIRBuilder::buildAtomicRMW(unsigned Opcode, } MachineInstrBuilder -MachineIRBuilder::buildAtomicRMWXchg(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO) { +MachineIRBuilder::buildAtomicRMWXchg(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO) { return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_XCHG, OldValRes, Addr, Val, MMO); } MachineInstrBuilder -MachineIRBuilder::buildAtomicRMWAdd(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO) { +MachineIRBuilder::buildAtomicRMWAdd(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO) { return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_ADD, OldValRes, Addr, Val, MMO); } MachineInstrBuilder -MachineIRBuilder::buildAtomicRMWSub(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO) { +MachineIRBuilder::buildAtomicRMWSub(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO) { return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_SUB, OldValRes, Addr, Val, MMO); } MachineInstrBuilder -MachineIRBuilder::buildAtomicRMWAnd(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO) { +MachineIRBuilder::buildAtomicRMWAnd(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO) { return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_AND, OldValRes, Addr, Val, MMO); } MachineInstrBuilder -MachineIRBuilder::buildAtomicRMWNand(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO) { +MachineIRBuilder::buildAtomicRMWNand(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO) { return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_NAND, OldValRes, Addr, Val, MMO); } -MachineInstrBuilder MachineIRBuilder::buildAtomicRMWOr(unsigned OldValRes, - unsigned Addr, - unsigned Val, +MachineInstrBuilder MachineIRBuilder::buildAtomicRMWOr(Register OldValRes, + Register Addr, + Register Val, MachineMemOperand &MMO) { return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_OR, OldValRes, Addr, Val, MMO); } MachineInstrBuilder -MachineIRBuilder::buildAtomicRMWXor(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO) { +MachineIRBuilder::buildAtomicRMWXor(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO) { return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_XOR, OldValRes, Addr, Val, MMO); } MachineInstrBuilder -MachineIRBuilder::buildAtomicRMWMax(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO) { +MachineIRBuilder::buildAtomicRMWMax(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO) { return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_MAX, OldValRes, Addr, Val, MMO); } MachineInstrBuilder -MachineIRBuilder::buildAtomicRMWMin(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO) { +MachineIRBuilder::buildAtomicRMWMin(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO) { return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_MIN, OldValRes, Addr, Val, MMO); } MachineInstrBuilder -MachineIRBuilder::buildAtomicRMWUmax(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO) { +MachineIRBuilder::buildAtomicRMWUmax(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO) { return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_UMAX, OldValRes, Addr, Val, MMO); } MachineInstrBuilder -MachineIRBuilder::buildAtomicRMWUmin(unsigned OldValRes, unsigned Addr, - unsigned Val, MachineMemOperand &MMO) { +MachineIRBuilder::buildAtomicRMWUmin(Register OldValRes, Register Addr, + Register Val, MachineMemOperand &MMO) { return buildAtomicRMW(TargetOpcode::G_ATOMICRMW_UMIN, OldValRes, Addr, Val, MMO); } MachineInstrBuilder -MachineIRBuilder::buildBlockAddress(unsigned Res, const BlockAddress *BA) { +MachineIRBuilder::buildFence(unsigned Ordering, unsigned Scope) { + return buildInstr(TargetOpcode::G_FENCE) + .addImm(Ordering) + .addImm(Scope); +} + +MachineInstrBuilder +MachineIRBuilder::buildBlockAddress(Register Res, const BlockAddress *BA) { #ifndef NDEBUG assert(getMRI()->getType(Res).isPointer() && "invalid res type"); #endif @@ -803,17 +932,18 @@ MachineInstrBuilder MachineIRBuilder::buildInstr(unsigned Opc, } case TargetOpcode::G_ADD: case TargetOpcode::G_AND: - case TargetOpcode::G_ASHR: - case TargetOpcode::G_LSHR: case TargetOpcode::G_MUL: case TargetOpcode::G_OR: - case TargetOpcode::G_SHL: case TargetOpcode::G_SUB: case TargetOpcode::G_XOR: case TargetOpcode::G_UDIV: case TargetOpcode::G_SDIV: case TargetOpcode::G_UREM: - case TargetOpcode::G_SREM: { + case TargetOpcode::G_SREM: + case TargetOpcode::G_SMIN: + case TargetOpcode::G_SMAX: + case TargetOpcode::G_UMIN: + case TargetOpcode::G_UMAX: { // All these are binary ops. assert(DstOps.size() == 1 && "Invalid Dst"); assert(SrcOps.size() == 2 && "Invalid Srcs"); @@ -821,6 +951,17 @@ MachineInstrBuilder MachineIRBuilder::buildInstr(unsigned Opc, SrcOps[0].getLLTTy(*getMRI()), SrcOps[1].getLLTTy(*getMRI())); break; + } + case TargetOpcode::G_SHL: + case TargetOpcode::G_ASHR: + case TargetOpcode::G_LSHR: { + assert(DstOps.size() == 1 && "Invalid Dst"); + assert(SrcOps.size() == 2 && "Invalid Srcs"); + validateShiftOp(DstOps[0].getLLTTy(*getMRI()), + SrcOps[0].getLLTTy(*getMRI()), + SrcOps[1].getLLTTy(*getMRI())); + break; + } case TargetOpcode::G_SEXT: case TargetOpcode::G_ZEXT: case TargetOpcode::G_ANYEXT: @@ -830,7 +971,7 @@ MachineInstrBuilder MachineIRBuilder::buildInstr(unsigned Opc, SrcOps[0].getLLTTy(*getMRI()), true); break; case TargetOpcode::G_TRUNC: - case TargetOpcode::G_FPTRUNC: + case TargetOpcode::G_FPTRUNC: { assert(DstOps.size() == 1 && "Invalid Dst"); assert(SrcOps.size() == 1 && "Invalid Srcs"); validateTruncExt(DstOps[0].getLLTTy(*getMRI()), @@ -839,10 +980,8 @@ MachineInstrBuilder MachineIRBuilder::buildInstr(unsigned Opc, } case TargetOpcode::COPY: assert(DstOps.size() == 1 && "Invalid Dst"); - assert(SrcOps.size() == 1 && "Invalid Srcs"); - assert(DstOps[0].getLLTTy(*getMRI()) == LLT() || - SrcOps[0].getLLTTy(*getMRI()) == LLT() || - DstOps[0].getLLTTy(*getMRI()) == SrcOps[0].getLLTTy(*getMRI())); + // If the caller wants to add a subreg source it has to be done separately + // so we may not have any SrcOps at this point yet. break; case TargetOpcode::G_FCMP: case TargetOpcode::G_ICMP: { @@ -943,7 +1082,7 @@ MachineInstrBuilder MachineIRBuilder::buildInstr(unsigned Opc, "type mismatch in input list"); assert(SrcOps.size() * SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() == DstOps[0].getLLTTy(*getMRI()).getSizeInBits() && - "input scalars do not exactly cover the outpur vector register"); + "input scalars do not exactly cover the output vector register"); break; } case TargetOpcode::G_BUILD_VECTOR_TRUNC: { @@ -976,7 +1115,7 @@ MachineInstrBuilder MachineIRBuilder::buildInstr(unsigned Opc, "type mismatch in input list"); assert(SrcOps.size() * SrcOps[0].getLLTTy(*getMRI()).getSizeInBits() == DstOps[0].getLLTTy(*getMRI()).getSizeInBits() && - "input vectors do not exactly cover the outpur vector register"); + "input vectors do not exactly cover the output vector register"); break; } case TargetOpcode::G_UADDE: { diff --git a/lib/CodeGen/GlobalISel/RegBankSelect.cpp b/lib/CodeGen/GlobalISel/RegBankSelect.cpp index dcc8b7cc23c5..42be88fcf947 100644 --- a/lib/CodeGen/GlobalISel/RegBankSelect.cpp +++ b/lib/CodeGen/GlobalISel/RegBankSelect.cpp @@ -1,9 +1,8 @@ //==- llvm/CodeGen/GlobalISel/RegBankSelect.cpp - RegBankSelect --*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -72,7 +71,6 @@ INITIALIZE_PASS_END(RegBankSelect, DEBUG_TYPE, RegBankSelect::RegBankSelect(Mode RunningMode) : MachineFunctionPass(ID), OptMode(RunningMode) { - initializeRegBankSelectPass(*PassRegistry::getPassRegistry()); if (RegBankSelectMode.getNumOccurrences() != 0) { OptMode = RegBankSelectMode; if (RegBankSelectMode != RunningMode) @@ -110,7 +108,7 @@ void RegBankSelect::getAnalysisUsage(AnalysisUsage &AU) const { } bool RegBankSelect::assignmentMatch( - unsigned Reg, const RegisterBankInfo::ValueMapping &ValMapping, + Register Reg, const RegisterBankInfo::ValueMapping &ValMapping, bool &OnlyAssign) const { // By default we assume we will have to repair something. OnlyAssign = false; @@ -135,34 +133,84 @@ bool RegBankSelect::assignmentMatch( bool RegBankSelect::repairReg( MachineOperand &MO, const RegisterBankInfo::ValueMapping &ValMapping, RegBankSelect::RepairingPlacement &RepairPt, - const iterator_range::const_iterator> &NewVRegs) { - if (ValMapping.NumBreakDowns != 1 && !TPC->isGlobalISelAbortEnabled()) - return false; - assert(ValMapping.NumBreakDowns == 1 && "Not yet implemented"); + const iterator_range::const_iterator> &NewVRegs) { + + assert(ValMapping.NumBreakDowns == (unsigned)size(NewVRegs) && + "need new vreg for each breakdown"); + // An empty range of new register means no repairing. assert(!empty(NewVRegs) && "We should not have to repair"); - // Assume we are repairing a use and thus, the original reg will be - // the source of the repairing. - unsigned Src = MO.getReg(); - unsigned Dst = *NewVRegs.begin(); - - // If we repair a definition, swap the source and destination for - // the repairing. - if (MO.isDef()) - std::swap(Src, Dst); - - assert((RepairPt.getNumInsertPoints() == 1 || - TargetRegisterInfo::isPhysicalRegister(Dst)) && - "We are about to create several defs for Dst"); - - // Build the instruction used to repair, then clone it at the right - // places. Avoiding buildCopy bypasses the check that Src and Dst have the - // same types because the type is a placeholder when this function is called. - MachineInstr *MI = - MIRBuilder.buildInstrNoInsert(TargetOpcode::COPY).addDef(Dst).addUse(Src); - LLVM_DEBUG(dbgs() << "Copy: " << printReg(Src) << " to: " << printReg(Dst) - << '\n'); + MachineInstr *MI; + if (ValMapping.NumBreakDowns == 1) { + // Assume we are repairing a use and thus, the original reg will be + // the source of the repairing. + Register Src = MO.getReg(); + Register Dst = *NewVRegs.begin(); + + // If we repair a definition, swap the source and destination for + // the repairing. + if (MO.isDef()) + std::swap(Src, Dst); + + assert((RepairPt.getNumInsertPoints() == 1 || + TargetRegisterInfo::isPhysicalRegister(Dst)) && + "We are about to create several defs for Dst"); + + // Build the instruction used to repair, then clone it at the right + // places. Avoiding buildCopy bypasses the check that Src and Dst have the + // same types because the type is a placeholder when this function is called. + MI = MIRBuilder.buildInstrNoInsert(TargetOpcode::COPY) + .addDef(Dst) + .addUse(Src); + LLVM_DEBUG(dbgs() << "Copy: " << printReg(Src) << " to: " << printReg(Dst) + << '\n'); + } else { + // TODO: Support with G_IMPLICIT_DEF + G_INSERT sequence or G_EXTRACT + // sequence. + assert(ValMapping.partsAllUniform() && "irregular breakdowns not supported"); + + LLT RegTy = MRI->getType(MO.getReg()); + if (MO.isDef()) { + unsigned MergeOp; + if (RegTy.isVector()) { + if (ValMapping.NumBreakDowns == RegTy.getNumElements()) + MergeOp = TargetOpcode::G_BUILD_VECTOR; + else { + assert( + (ValMapping.BreakDown[0].Length * ValMapping.NumBreakDowns == + RegTy.getSizeInBits()) && + (ValMapping.BreakDown[0].Length % RegTy.getScalarSizeInBits() == + 0) && + "don't understand this value breakdown"); + + MergeOp = TargetOpcode::G_CONCAT_VECTORS; + } + } else + MergeOp = TargetOpcode::G_MERGE_VALUES; + + auto MergeBuilder = + MIRBuilder.buildInstrNoInsert(MergeOp) + .addDef(MO.getReg()); + + for (Register SrcReg : NewVRegs) + MergeBuilder.addUse(SrcReg); + + MI = MergeBuilder; + } else { + MachineInstrBuilder UnMergeBuilder = + MIRBuilder.buildInstrNoInsert(TargetOpcode::G_UNMERGE_VALUES); + for (Register DefReg : NewVRegs) + UnMergeBuilder.addDef(DefReg); + + UnMergeBuilder.addUse(MO.getReg()); + MI = UnMergeBuilder; + } + } + + if (RepairPt.getNumInsertPoints() != 1) + report_fatal_error("need testcase to support multiple insertion points"); + // TODO: // Check if MI is legal. if not, we need to legalize all the // instructions we are going to insert. @@ -195,7 +243,8 @@ uint64_t RegBankSelect::getRepairCost( const RegisterBank *CurRegBank = RBI->getRegBank(MO.getReg(), *MRI, *TRI); // If MO does not have a register bank, we should have just been // able to set one unless we have to break the value down. - assert((!IsSameNumOfValues || CurRegBank) && "We should not have to repair"); + assert(CurRegBank || MO.isDef()); + // Def: Val <- NewDefs // Same number of values: copy // Different number: Val = build_sequence Defs1, Defs2, ... @@ -206,6 +255,9 @@ uint64_t RegBankSelect::getRepairCost( // We should remember that this value is available somewhere else to // coalesce the value. + if (ValMapping.NumBreakDowns != 1) + return RBI->getBreakDownCost(ValMapping, CurRegBank); + if (IsSameNumOfValues) { const RegisterBank *DesiredRegBrank = ValMapping.BreakDown[0].RegBank; // If we repair a definition, swap the source and destination for @@ -345,7 +397,7 @@ void RegBankSelect::tryAvoidingSplit( // repairing. // Check if this is a physical or virtual register. - unsigned Reg = MO.getReg(); + Register Reg = MO.getReg(); if (TargetRegisterInfo::isPhysicalRegister(Reg)) { // We are going to split every outgoing edges. // Check that this is possible. @@ -416,7 +468,7 @@ RegBankSelect::MappingCost RegBankSelect::computeMapping( const MachineOperand &MO = MI.getOperand(OpIdx); if (!MO.isReg()) continue; - unsigned Reg = MO.getReg(); + Register Reg = MO.getReg(); if (!Reg) continue; LLVM_DEBUG(dbgs() << "Opd" << OpIdx << '\n'); @@ -542,7 +594,7 @@ bool RegBankSelect::applyMapping( MachineOperand &MO = MI.getOperand(OpIdx); const RegisterBankInfo::ValueMapping &ValMapping = InstrMapping.getOperandMapping(OpIdx); - unsigned Reg = MO.getReg(); + Register Reg = MO.getReg(); switch (RepairPt.getKind()) { case RepairingPlacement::Reassign: @@ -605,7 +657,7 @@ bool RegBankSelect::runOnMachineFunction(MachineFunction &MF) { LLVM_DEBUG(dbgs() << "Assign register banks for: " << MF.getName() << '\n'); const Function &F = MF.getFunction(); Mode SaveOptMode = OptMode; - if (F.hasFnAttribute(Attribute::OptimizeNone)) + if (F.hasOptNone()) OptMode = Mode::Fast; init(MF); @@ -644,8 +696,21 @@ bool RegBankSelect::runOnMachineFunction(MachineFunction &MF) { "unable to map instruction", MI); return false; } + + // It's possible the mapping changed control flow, and moved the following + // instruction to a new block, so figure out the new parent. + if (MII != End) { + MachineBasicBlock *NextInstBB = MII->getParent(); + if (NextInstBB != MBB) { + LLVM_DEBUG(dbgs() << "Instruction mapping changed control flow\n"); + MBB = NextInstBB; + MIRBuilder.setMBB(*MBB); + End = MBB->end(); + } + } } } + OptMode = SaveOptMode; return false; } @@ -692,7 +757,7 @@ RegBankSelect::RepairingPlacement::RepairingPlacement( MachineBasicBlock &Pred = *MI.getOperand(OpIdx + 1).getMBB(); // Check if we can move the insertion point prior to the // terminators of the predecessor. - unsigned Reg = MO.getReg(); + Register Reg = MO.getReg(); MachineBasicBlock::iterator It = Pred.getLastNonDebugInstr(); for (auto Begin = Pred.begin(); It != Begin && It->isTerminator(); --It) if (It->modifiesRegister(Reg, &TRI)) { diff --git a/lib/CodeGen/GlobalISel/RegisterBank.cpp b/lib/CodeGen/GlobalISel/RegisterBank.cpp index 16f67a217ce1..4e41f338934d 100644 --- a/lib/CodeGen/GlobalISel/RegisterBank.cpp +++ b/lib/CodeGen/GlobalISel/RegisterBank.cpp @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/RegisterBank.cpp - Register Bank --*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file diff --git a/lib/CodeGen/GlobalISel/RegisterBankInfo.cpp b/lib/CodeGen/GlobalISel/RegisterBankInfo.cpp index 28404e52d6ea..159422e38878 100644 --- a/lib/CodeGen/GlobalISel/RegisterBankInfo.cpp +++ b/lib/CodeGen/GlobalISel/RegisterBankInfo.cpp @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/RegisterBankInfo.cpp --------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -81,7 +80,7 @@ bool RegisterBankInfo::verify(const TargetRegisterInfo &TRI) const { } const RegisterBank * -RegisterBankInfo::getRegBank(unsigned Reg, const MachineRegisterInfo &MRI, +RegisterBankInfo::getRegBank(Register Reg, const MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI) const { if (TargetRegisterInfo::isPhysicalRegister(Reg)) return &getRegBankFromRegClass(getMinimalPhysRegClass(Reg, TRI)); @@ -96,7 +95,7 @@ RegisterBankInfo::getRegBank(unsigned Reg, const MachineRegisterInfo &MRI, } const TargetRegisterClass & -RegisterBankInfo::getMinimalPhysRegClass(unsigned Reg, +RegisterBankInfo::getMinimalPhysRegClass(Register Reg, const TargetRegisterInfo &TRI) const { assert(TargetRegisterInfo::isPhysicalRegister(Reg) && "Reg must be a physreg"); @@ -126,7 +125,7 @@ const RegisterBank *RegisterBankInfo::getRegBankFromConstraints( } const TargetRegisterClass *RegisterBankInfo::constrainGenericRegister( - unsigned Reg, const TargetRegisterClass &RC, MachineRegisterInfo &MRI) { + Register Reg, const TargetRegisterClass &RC, MachineRegisterInfo &MRI) { // If the register already has a class, fallback to MRI::constrainRegClass. auto &RegClassOrBank = MRI.getRegClassOrRegBank(Reg); @@ -181,7 +180,7 @@ RegisterBankInfo::getInstrMappingImpl(const MachineInstr &MI) const { const MachineOperand &MO = MI.getOperand(OpIdx); if (!MO.isReg()) continue; - unsigned Reg = MO.getReg(); + Register Reg = MO.getReg(); if (!Reg) continue; // The register bank of Reg is just a side effect of the current @@ -208,19 +207,49 @@ RegisterBankInfo::getInstrMappingImpl(const MachineInstr &MI) const { continue; } } - const ValueMapping *ValMapping = - &getValueMapping(0, getSizeInBits(Reg, MRI, TRI), *CurRegBank); + + unsigned Size = getSizeInBits(Reg, MRI, TRI); + const ValueMapping *ValMapping = &getValueMapping(0, Size, *CurRegBank); if (IsCopyLike) { - OperandsMapping[0] = ValMapping; + if (!OperandsMapping[0]) { + if (MI.isRegSequence()) { + // For reg_sequence, the result size does not match the input. + unsigned ResultSize = getSizeInBits(MI.getOperand(0).getReg(), + MRI, TRI); + OperandsMapping[0] = &getValueMapping(0, ResultSize, *CurRegBank); + } else { + OperandsMapping[0] = ValMapping; + } + } + + // The default handling assumes any register bank can be copied to any + // other. If this isn't the case, the target should specially deal with + // reg_sequence/phi. There may also be unsatisfiable copies. + for (; OpIdx != EndIdx; ++OpIdx) { + const MachineOperand &MO = MI.getOperand(OpIdx); + if (!MO.isReg()) + continue; + Register Reg = MO.getReg(); + if (!Reg) + continue; + + const RegisterBank *AltRegBank = getRegBank(Reg, MRI, TRI); + if (AltRegBank && + cannotCopy(*CurRegBank, *AltRegBank, getSizeInBits(Reg, MRI, TRI))) + return getInvalidInstructionMapping(); + } + CompleteMapping = true; break; } + OperandsMapping[OpIdx] = ValMapping; } - if (IsCopyLike && !CompleteMapping) + if (IsCopyLike && !CompleteMapping) { // No way to deduce the type from what we have. return getInvalidInstructionMapping(); + } assert(CompleteMapping && "Setting an uncomplete mapping"); return getInstructionMapping( @@ -363,11 +392,8 @@ RegisterBankInfo::getInstructionMappingImpl( ++NumInstructionMappingsCreated; auto &InstrMapping = MapOfInstructionMappings[Hash]; - if (IsInvalid) - InstrMapping = llvm::make_unique(); - else - InstrMapping = llvm::make_unique( - ID, Cost, OperandsMapping, NumOperands); + InstrMapping = llvm::make_unique( + ID, Cost, OperandsMapping, NumOperands); return *InstrMapping; } @@ -382,8 +408,12 @@ RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { RegisterBankInfo::InstructionMappings RegisterBankInfo::getInstrPossibleMappings(const MachineInstr &MI) const { InstructionMappings PossibleMappings; - // Put the default mapping first. - PossibleMappings.push_back(&getInstrMapping(MI)); + const auto &Mapping = getInstrMapping(MI); + if (Mapping.isValid()) { + // Put the default mapping first. + PossibleMappings.push_back(&Mapping); + } + // Then the alternative mapping, if any. InstructionMappings AltMappings = getInstrAlternativeMappings(MI); for (const InstructionMapping *AltMapping : AltMappings) @@ -424,14 +454,14 @@ void RegisterBankInfo::applyDefaultMapping(const OperandsMapper &OpdMapper) { assert(OpdMapper.getInstrMapping().getOperandMapping(OpIdx).NumBreakDowns == 1 && "This mapping is too complex for this function"); - iterator_range::const_iterator> NewRegs = + iterator_range::const_iterator> NewRegs = OpdMapper.getVRegs(OpIdx); if (empty(NewRegs)) { LLVM_DEBUG(dbgs() << " has not been repaired, nothing to be done\n"); continue; } - unsigned OrigReg = MO.getReg(); - unsigned NewReg = *NewRegs.begin(); + Register OrigReg = MO.getReg(); + Register NewReg = *NewRegs.begin(); LLVM_DEBUG(dbgs() << " changed, replace " << printReg(OrigReg, nullptr)); MO.setReg(NewReg); LLVM_DEBUG(dbgs() << " with " << printReg(NewReg, nullptr)); @@ -456,7 +486,7 @@ void RegisterBankInfo::applyDefaultMapping(const OperandsMapper &OpdMapper) { } } -unsigned RegisterBankInfo::getSizeInBits(unsigned Reg, +unsigned RegisterBankInfo::getSizeInBits(Register Reg, const MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI) const { if (TargetRegisterInfo::isPhysicalRegister(Reg)) { @@ -498,6 +528,19 @@ void RegisterBankInfo::PartialMapping::print(raw_ostream &OS) const { OS << "nullptr"; } +bool RegisterBankInfo::ValueMapping::partsAllUniform() const { + if (NumBreakDowns < 2) + return true; + + const PartialMapping *First = begin(); + for (const PartialMapping *Part = First + 1; Part != end(); ++Part) { + if (Part->Length != First->Length || Part->RegBank != First->RegBank) + return false; + } + + return true; +} + bool RegisterBankInfo::ValueMapping::verify(unsigned MeaningfulBitWidth) const { assert(NumBreakDowns && "Value mapped nowhere?!"); unsigned OrigValueBitWidth = 0; @@ -565,7 +608,7 @@ bool RegisterBankInfo::InstructionMapping::verify( "We should not care about non-reg mapping"); continue; } - unsigned Reg = MO.getReg(); + Register Reg = MO.getReg(); if (!Reg) continue; assert(getOperandMapping(Idx).isValid() && @@ -610,7 +653,7 @@ RegisterBankInfo::OperandsMapper::OperandsMapper( assert(InstrMapping.verify(MI) && "Invalid mapping for MI"); } -iterator_range::iterator> +iterator_range::iterator> RegisterBankInfo::OperandsMapper::getVRegsMem(unsigned OpIdx) { assert(OpIdx < getInstrMapping().getNumOperands() && "Out-of-bound access"); unsigned NumPartialVal = @@ -626,18 +669,18 @@ RegisterBankInfo::OperandsMapper::getVRegsMem(unsigned OpIdx) { for (unsigned i = 0; i < NumPartialVal; ++i) NewVRegs.push_back(0); } - SmallVectorImpl::iterator End = + SmallVectorImpl::iterator End = getNewVRegsEnd(StartIdx, NumPartialVal); return make_range(&NewVRegs[StartIdx], End); } -SmallVectorImpl::const_iterator +SmallVectorImpl::const_iterator RegisterBankInfo::OperandsMapper::getNewVRegsEnd(unsigned StartIdx, unsigned NumVal) const { return const_cast(this)->getNewVRegsEnd(StartIdx, NumVal); } -SmallVectorImpl::iterator +SmallVectorImpl::iterator RegisterBankInfo::OperandsMapper::getNewVRegsEnd(unsigned StartIdx, unsigned NumVal) { assert((NewVRegs.size() == StartIdx + NumVal || @@ -649,11 +692,11 @@ RegisterBankInfo::OperandsMapper::getNewVRegsEnd(unsigned StartIdx, void RegisterBankInfo::OperandsMapper::createVRegs(unsigned OpIdx) { assert(OpIdx < getInstrMapping().getNumOperands() && "Out-of-bound access"); - iterator_range::iterator> NewVRegsForOpIdx = + iterator_range::iterator> NewVRegsForOpIdx = getVRegsMem(OpIdx); const ValueMapping &ValMapping = getInstrMapping().getOperandMapping(OpIdx); const PartialMapping *PartMap = ValMapping.begin(); - for (unsigned &NewVReg : NewVRegsForOpIdx) { + for (Register &NewVReg : NewVRegsForOpIdx) { assert(PartMap != ValMapping.end() && "Out-of-bound access"); assert(NewVReg == 0 && "Register has already been created"); // The new registers are always bound to scalar with the right size. @@ -669,7 +712,7 @@ void RegisterBankInfo::OperandsMapper::createVRegs(unsigned OpIdx) { void RegisterBankInfo::OperandsMapper::setVRegs(unsigned OpIdx, unsigned PartialMapIdx, - unsigned NewVReg) { + Register NewVReg) { assert(OpIdx < getInstrMapping().getNumOperands() && "Out-of-bound access"); assert(getInstrMapping().getOperandMapping(OpIdx).NumBreakDowns > PartialMapIdx && @@ -681,7 +724,7 @@ void RegisterBankInfo::OperandsMapper::setVRegs(unsigned OpIdx, NewVRegs[OpToNewVRegIdx[OpIdx] + PartialMapIdx] = NewVReg; } -iterator_range::const_iterator> +iterator_range::const_iterator> RegisterBankInfo::OperandsMapper::getVRegs(unsigned OpIdx, bool ForDebug) const { (void)ForDebug; @@ -693,12 +736,12 @@ RegisterBankInfo::OperandsMapper::getVRegs(unsigned OpIdx, unsigned PartMapSize = getInstrMapping().getOperandMapping(OpIdx).NumBreakDowns; - SmallVectorImpl::const_iterator End = + SmallVectorImpl::const_iterator End = getNewVRegsEnd(StartIdx, PartMapSize); - iterator_range::const_iterator> Res = + iterator_range::const_iterator> Res = make_range(&NewVRegs[StartIdx], End); #ifndef NDEBUG - for (unsigned VReg : Res) + for (Register VReg : Res) assert((VReg || ForDebug) && "Some registers are uninitialized"); #endif return Res; @@ -747,7 +790,7 @@ void RegisterBankInfo::OperandsMapper::print(raw_ostream &OS, IsFirst = false; OS << '(' << printReg(getMI().getOperand(Idx).getReg(), TRI) << ", ["; bool IsFirstNewVReg = true; - for (unsigned VReg : getVRegs(Idx)) { + for (Register VReg : getVRegs(Idx)) { if (!IsFirstNewVReg) OS << ", "; IsFirstNewVReg = false; diff --git a/lib/CodeGen/GlobalISel/Utils.cpp b/lib/CodeGen/GlobalISel/Utils.cpp index 59cbf93e7cd1..766ea1d60bac 100644 --- a/lib/CodeGen/GlobalISel/Utils.cpp +++ b/lib/CodeGen/GlobalISel/Utils.cpp @@ -1,9 +1,8 @@ //===- llvm/CodeGen/GlobalISel/Utils.cpp -------------------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file This file implements the utility functions used by the GlobalISel @@ -30,20 +29,45 @@ using namespace llvm; unsigned llvm::constrainRegToClass(MachineRegisterInfo &MRI, const TargetInstrInfo &TII, - const RegisterBankInfo &RBI, - MachineInstr &InsertPt, unsigned Reg, + const RegisterBankInfo &RBI, unsigned Reg, const TargetRegisterClass &RegClass) { - if (!RBI.constrainGenericRegister(Reg, RegClass, MRI)) { - unsigned NewReg = MRI.createVirtualRegister(&RegClass); - BuildMI(*InsertPt.getParent(), InsertPt, InsertPt.getDebugLoc(), - TII.get(TargetOpcode::COPY), NewReg) - .addReg(Reg); - return NewReg; - } + if (!RBI.constrainGenericRegister(Reg, RegClass, MRI)) + return MRI.createVirtualRegister(&RegClass); return Reg; } +unsigned llvm::constrainOperandRegClass( + const MachineFunction &MF, const TargetRegisterInfo &TRI, + MachineRegisterInfo &MRI, const TargetInstrInfo &TII, + const RegisterBankInfo &RBI, MachineInstr &InsertPt, + const TargetRegisterClass &RegClass, const MachineOperand &RegMO, + unsigned OpIdx) { + unsigned Reg = RegMO.getReg(); + // Assume physical registers are properly constrained. + assert(TargetRegisterInfo::isVirtualRegister(Reg) && + "PhysReg not implemented"); + + unsigned ConstrainedReg = constrainRegToClass(MRI, TII, RBI, Reg, RegClass); + // If we created a new virtual register because the class is not compatible + // then create a copy between the new and the old register. + if (ConstrainedReg != Reg) { + MachineBasicBlock::iterator InsertIt(&InsertPt); + MachineBasicBlock &MBB = *InsertPt.getParent(); + if (RegMO.isUse()) { + BuildMI(MBB, InsertIt, InsertPt.getDebugLoc(), + TII.get(TargetOpcode::COPY), ConstrainedReg) + .addReg(Reg); + } else { + assert(RegMO.isDef() && "Must be a definition"); + BuildMI(MBB, std::next(InsertIt), InsertPt.getDebugLoc(), + TII.get(TargetOpcode::COPY), Reg) + .addReg(ConstrainedReg); + } + } + return ConstrainedReg; +} + unsigned llvm::constrainOperandRegClass( const MachineFunction &MF, const TargetRegisterInfo &TRI, MachineRegisterInfo &MRI, const TargetInstrInfo &TII, @@ -82,7 +106,8 @@ unsigned llvm::constrainOperandRegClass( // and they never reach this function. return Reg; } - return constrainRegToClass(MRI, TII, RBI, InsertPt, Reg, *RegClass); + return constrainOperandRegClass(MF, TRI, MRI, TII, RBI, InsertPt, *RegClass, + RegMO, OpIdx); } bool llvm::constrainSelectedInstRegOperands(MachineInstr &I, @@ -184,18 +209,71 @@ void llvm::reportGISelFailure(MachineFunction &MF, const TargetPassConfig &TPC, Optional llvm::getConstantVRegVal(unsigned VReg, const MachineRegisterInfo &MRI) { - MachineInstr *MI = MRI.getVRegDef(VReg); - if (MI->getOpcode() != TargetOpcode::G_CONSTANT) + Optional ValAndVReg = + getConstantVRegValWithLookThrough(VReg, MRI, /*LookThroughInstrs*/ false); + assert((!ValAndVReg || ValAndVReg->VReg == VReg) && + "Value found while looking through instrs"); + if (!ValAndVReg) + return None; + return ValAndVReg->Value; +} + +Optional llvm::getConstantVRegValWithLookThrough( + unsigned VReg, const MachineRegisterInfo &MRI, bool LookThroughInstrs) { + SmallVector, 4> SeenOpcodes; + MachineInstr *MI; + while ((MI = MRI.getVRegDef(VReg)) && + MI->getOpcode() != TargetOpcode::G_CONSTANT && LookThroughInstrs) { + switch (MI->getOpcode()) { + case TargetOpcode::G_TRUNC: + case TargetOpcode::G_SEXT: + case TargetOpcode::G_ZEXT: + SeenOpcodes.push_back(std::make_pair( + MI->getOpcode(), + MRI.getType(MI->getOperand(0).getReg()).getSizeInBits())); + VReg = MI->getOperand(1).getReg(); + break; + case TargetOpcode::COPY: + VReg = MI->getOperand(1).getReg(); + if (TargetRegisterInfo::isPhysicalRegister(VReg)) + return None; + break; + case TargetOpcode::G_INTTOPTR: + VReg = MI->getOperand(1).getReg(); + break; + default: + return None; + } + } + if (!MI || MI->getOpcode() != TargetOpcode::G_CONSTANT || + (!MI->getOperand(1).isImm() && !MI->getOperand(1).isCImm())) return None; - if (MI->getOperand(1).isImm()) - return MI->getOperand(1).getImm(); + const MachineOperand &CstVal = MI->getOperand(1); + unsigned BitWidth = MRI.getType(MI->getOperand(0).getReg()).getSizeInBits(); + APInt Val = CstVal.isImm() ? APInt(BitWidth, CstVal.getImm()) + : CstVal.getCImm()->getValue(); + assert(Val.getBitWidth() == BitWidth && + "Value bitwidth doesn't match definition type"); + while (!SeenOpcodes.empty()) { + std::pair OpcodeAndSize = SeenOpcodes.pop_back_val(); + switch (OpcodeAndSize.first) { + case TargetOpcode::G_TRUNC: + Val = Val.trunc(OpcodeAndSize.second); + break; + case TargetOpcode::G_SEXT: + Val = Val.sext(OpcodeAndSize.second); + break; + case TargetOpcode::G_ZEXT: + Val = Val.zext(OpcodeAndSize.second); + break; + } + } - if (MI->getOperand(1).isCImm() && - MI->getOperand(1).getCImm()->getBitWidth() <= 64) - return MI->getOperand(1).getCImm()->getSExtValue(); + if (Val.getBitWidth() > 64) + return None; - return None; + return ValueAndVReg{Val.getSExtValue(), VReg}; } const llvm::ConstantFP* llvm::getConstantFPVRegVal(unsigned VReg, @@ -206,8 +284,8 @@ const llvm::ConstantFP* llvm::getConstantFPVRegVal(unsigned VReg, return MI->getOperand(1).getFPImm(); } -llvm::MachineInstr *llvm::getOpcodeDef(unsigned Opcode, unsigned Reg, - const MachineRegisterInfo &MRI) { +llvm::MachineInstr *llvm::getDefIgnoringCopies(Register Reg, + const MachineRegisterInfo &MRI) { auto *DefMI = MRI.getVRegDef(Reg); auto DstTy = MRI.getType(DefMI->getOperand(0).getReg()); if (!DstTy.isValid()) @@ -219,7 +297,13 @@ llvm::MachineInstr *llvm::getOpcodeDef(unsigned Opcode, unsigned Reg, break; DefMI = MRI.getVRegDef(SrcReg); } - return DefMI->getOpcode() == Opcode ? DefMI : nullptr; + return DefMI; +} + +llvm::MachineInstr *llvm::getOpcodeDef(unsigned Opcode, Register Reg, + const MachineRegisterInfo &MRI) { + MachineInstr *DefMI = getDefIgnoringCopies(Reg, MRI); + return DefMI && DefMI->getOpcode() == Opcode ? DefMI : nullptr; } APFloat llvm::getAPFloatFromSize(double Val, unsigned Size) { @@ -286,6 +370,31 @@ Optional llvm::ConstantFoldBinOp(unsigned Opcode, const unsigned Op1, return None; } +bool llvm::isKnownNeverNaN(Register Val, const MachineRegisterInfo &MRI, + bool SNaN) { + const MachineInstr *DefMI = MRI.getVRegDef(Val); + if (!DefMI) + return false; + + if (DefMI->getFlag(MachineInstr::FmNoNans)) + return true; + + if (SNaN) { + // FP operations quiet. For now, just handle the ones inserted during + // legalization. + switch (DefMI->getOpcode()) { + case TargetOpcode::G_FPEXT: + case TargetOpcode::G_FPTRUNC: + case TargetOpcode::G_FCANONICALIZE: + return true; + default: + return false; + } + } + + return false; +} + void llvm::getSelectionDAGFallbackAnalysisUsage(AnalysisUsage &AU) { AU.addPreserved(); } diff --git a/lib/CodeGen/GlobalMerge.cpp b/lib/CodeGen/GlobalMerge.cpp index d3364952f244..09201c2e7bae 100644 --- a/lib/CodeGen/GlobalMerge.cpp +++ b/lib/CodeGen/GlobalMerge.cpp @@ -1,9 +1,8 @@ //===- GlobalMerge.cpp - Internal globals merging -------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -220,11 +219,11 @@ bool GlobalMerge::doMerge(SmallVectorImpl &Globals, Module &M, bool isConst, unsigned AddrSpace) const { auto &DL = M.getDataLayout(); // FIXME: Find better heuristics - std::stable_sort(Globals.begin(), Globals.end(), - [&DL](const GlobalVariable *GV1, const GlobalVariable *GV2) { - return DL.getTypeAllocSize(GV1->getValueType()) < - DL.getTypeAllocSize(GV2->getValueType()); - }); + llvm::stable_sort( + Globals, [&DL](const GlobalVariable *GV1, const GlobalVariable *GV2) { + return DL.getTypeAllocSize(GV1->getValueType()) < + DL.getTypeAllocSize(GV2->getValueType()); + }); // If we want to just blindly group all globals together, do so. if (!GlobalMergeGroupByUse) { @@ -331,7 +330,7 @@ bool GlobalMerge::doMerge(SmallVectorImpl &Globals, Function *ParentFn = I->getParent()->getParent(); // If we're only optimizing for size, ignore non-minsize functions. - if (OnlyOptimizeForSize && !ParentFn->optForMinSize()) + if (OnlyOptimizeForSize && !ParentFn->hasMinSize()) continue; size_t UGSIdx = GlobalUsesByFunction[ParentFn]; @@ -386,11 +385,11 @@ bool GlobalMerge::doMerge(SmallVectorImpl &Globals, // // Multiply that by the size of the set to give us a crude profitability // metric. - std::stable_sort(UsedGlobalSets.begin(), UsedGlobalSets.end(), - [](const UsedGlobalSet &UGS1, const UsedGlobalSet &UGS2) { - return UGS1.Globals.count() * UGS1.UsageCount < - UGS2.Globals.count() * UGS2.UsageCount; - }); + llvm::stable_sort(UsedGlobalSets, + [](const UsedGlobalSet &UGS1, const UsedGlobalSet &UGS2) { + return UGS1.Globals.count() * UGS1.UsageCount < + UGS2.Globals.count() * UGS2.UsageCount; + }); // We can choose to merge all globals together, but ignore globals never used // with another global. This catches the obviously non-profitable cases of diff --git a/lib/CodeGen/HardwareLoops.cpp b/lib/CodeGen/HardwareLoops.cpp new file mode 100644 index 000000000000..5f57cabbe865 --- /dev/null +++ b/lib/CodeGen/HardwareLoops.cpp @@ -0,0 +1,463 @@ +//===-- HardwareLoops.cpp - Target Independent Hardware Loops --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// Insert hardware loop intrinsics into loops which are deemed profitable by +/// the target, by querying TargetTransformInfo. A hardware loop comprises of +/// two intrinsics: one, outside the loop, to set the loop iteration count and +/// another, in the exit block, to decrement the counter. The decremented value +/// can either be carried through the loop via a phi or handled in some opaque +/// way by the target. +/// +//===----------------------------------------------------------------------===// + +#include "llvm/Pass.h" +#include "llvm/PassRegistry.h" +#include "llvm/PassSupport.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/Analysis/AssumptionCache.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/ScalarEvolution.h" +#include "llvm/Analysis/ScalarEvolutionExpander.h" +#include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/CodeGen/Passes.h" +#include "llvm/CodeGen/TargetPassConfig.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/DataLayout.h" +#include "llvm/IR/Dominators.h" +#include "llvm/IR/Constants.h" +#include "llvm/IR/IRBuilder.h" +#include "llvm/IR/Instructions.h" +#include "llvm/IR/IntrinsicInst.h" +#include "llvm/IR/Value.h" +#include "llvm/Support/Debug.h" +#include "llvm/Transforms/Scalar.h" +#include "llvm/Transforms/Utils.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" +#include "llvm/Transforms/Utils/Local.h" +#include "llvm/Transforms/Utils/LoopUtils.h" + +#define DEBUG_TYPE "hardware-loops" + +#define HW_LOOPS_NAME "Hardware Loop Insertion" + +using namespace llvm; + +static cl::opt +ForceHardwareLoops("force-hardware-loops", cl::Hidden, cl::init(false), + cl::desc("Force hardware loops intrinsics to be inserted")); + +static cl::opt +ForceHardwareLoopPHI( + "force-hardware-loop-phi", cl::Hidden, cl::init(false), + cl::desc("Force hardware loop counter to be updated through a phi")); + +static cl::opt +ForceNestedLoop("force-nested-hardware-loop", cl::Hidden, cl::init(false), + cl::desc("Force allowance of nested hardware loops")); + +static cl::opt +LoopDecrement("hardware-loop-decrement", cl::Hidden, cl::init(1), + cl::desc("Set the loop decrement value")); + +static cl::opt +CounterBitWidth("hardware-loop-counter-bitwidth", cl::Hidden, cl::init(32), + cl::desc("Set the loop counter bitwidth")); + +static cl::opt +ForceGuardLoopEntry( + "force-hardware-loop-guard", cl::Hidden, cl::init(false), + cl::desc("Force generation of loop guard intrinsic")); + +STATISTIC(NumHWLoops, "Number of loops converted to hardware loops"); + +namespace { + + using TTI = TargetTransformInfo; + + class HardwareLoops : public FunctionPass { + public: + static char ID; + + HardwareLoops() : FunctionPass(ID) { + initializeHardwareLoopsPass(*PassRegistry::getPassRegistry()); + } + + bool runOnFunction(Function &F) override; + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); + AU.addPreserved(); + AU.addRequired(); + AU.addPreserved(); + AU.addRequired(); + AU.addRequired(); + AU.addRequired(); + } + + // Try to convert the given Loop into a hardware loop. + bool TryConvertLoop(Loop *L); + + // Given that the target believes the loop to be profitable, try to + // convert it. + bool TryConvertLoop(HardwareLoopInfo &HWLoopInfo); + + private: + ScalarEvolution *SE = nullptr; + LoopInfo *LI = nullptr; + const DataLayout *DL = nullptr; + const TargetTransformInfo *TTI = nullptr; + DominatorTree *DT = nullptr; + bool PreserveLCSSA = false; + AssumptionCache *AC = nullptr; + TargetLibraryInfo *LibInfo = nullptr; + Module *M = nullptr; + bool MadeChange = false; + }; + + class HardwareLoop { + // Expand the trip count scev into a value that we can use. + Value *InitLoopCount(); + + // Insert the set_loop_iteration intrinsic. + void InsertIterationSetup(Value *LoopCountInit); + + // Insert the loop_decrement intrinsic. + void InsertLoopDec(); + + // Insert the loop_decrement_reg intrinsic. + Instruction *InsertLoopRegDec(Value *EltsRem); + + // If the target requires the counter value to be updated in the loop, + // insert a phi to hold the value. The intended purpose is for use by + // loop_decrement_reg. + PHINode *InsertPHICounter(Value *NumElts, Value *EltsRem); + + // Create a new cmp, that checks the returned value of loop_decrement*, + // and update the exit branch to use it. + void UpdateBranch(Value *EltsRem); + + public: + HardwareLoop(HardwareLoopInfo &Info, ScalarEvolution &SE, + const DataLayout &DL) : + SE(SE), DL(DL), L(Info.L), M(L->getHeader()->getModule()), + ExitCount(Info.ExitCount), + CountType(Info.CountType), + ExitBranch(Info.ExitBranch), + LoopDecrement(Info.LoopDecrement), + UsePHICounter(Info.CounterInReg), + UseLoopGuard(Info.PerformEntryTest) { } + + void Create(); + + private: + ScalarEvolution &SE; + const DataLayout &DL; + Loop *L = nullptr; + Module *M = nullptr; + const SCEV *ExitCount = nullptr; + Type *CountType = nullptr; + BranchInst *ExitBranch = nullptr; + Value *LoopDecrement = nullptr; + bool UsePHICounter = false; + bool UseLoopGuard = false; + BasicBlock *BeginBB = nullptr; + }; +} + +char HardwareLoops::ID = 0; + +bool HardwareLoops::runOnFunction(Function &F) { + if (skipFunction(F)) + return false; + + LLVM_DEBUG(dbgs() << "HWLoops: Running on " << F.getName() << "\n"); + + LI = &getAnalysis().getLoopInfo(); + SE = &getAnalysis().getSE(); + DT = &getAnalysis().getDomTree(); + TTI = &getAnalysis().getTTI(F); + DL = &F.getParent()->getDataLayout(); + auto *TLIP = getAnalysisIfAvailable(); + LibInfo = TLIP ? &TLIP->getTLI() : nullptr; + PreserveLCSSA = mustPreserveAnalysisID(LCSSAID); + AC = &getAnalysis().getAssumptionCache(F); + M = F.getParent(); + + for (LoopInfo::iterator I = LI->begin(), E = LI->end(); I != E; ++I) { + Loop *L = *I; + if (!L->getParentLoop()) + TryConvertLoop(L); + } + + return MadeChange; +} + +// Return true if the search should stop, which will be when an inner loop is +// converted and the parent loop doesn't support containing a hardware loop. +bool HardwareLoops::TryConvertLoop(Loop *L) { + // Process nested loops first. + for (Loop::iterator I = L->begin(), E = L->end(); I != E; ++I) + if (TryConvertLoop(*I)) + return true; // Stop search. + + HardwareLoopInfo HWLoopInfo(L); + if (!HWLoopInfo.canAnalyze(*LI)) + return false; + + if (TTI->isHardwareLoopProfitable(L, *SE, *AC, LibInfo, HWLoopInfo) || + ForceHardwareLoops) { + + // Allow overriding of the counter width and loop decrement value. + if (CounterBitWidth.getNumOccurrences()) + HWLoopInfo.CountType = + IntegerType::get(M->getContext(), CounterBitWidth); + + if (LoopDecrement.getNumOccurrences()) + HWLoopInfo.LoopDecrement = + ConstantInt::get(HWLoopInfo.CountType, LoopDecrement); + + MadeChange |= TryConvertLoop(HWLoopInfo); + return MadeChange && (!HWLoopInfo.IsNestingLegal && !ForceNestedLoop); + } + + return false; +} + +bool HardwareLoops::TryConvertLoop(HardwareLoopInfo &HWLoopInfo) { + + Loop *L = HWLoopInfo.L; + LLVM_DEBUG(dbgs() << "HWLoops: Try to convert profitable loop: " << *L); + + if (!HWLoopInfo.isHardwareLoopCandidate(*SE, *LI, *DT, ForceNestedLoop, + ForceHardwareLoopPHI)) + return false; + + assert( + (HWLoopInfo.ExitBlock && HWLoopInfo.ExitBranch && HWLoopInfo.ExitCount) && + "Hardware Loop must have set exit info."); + + BasicBlock *Preheader = L->getLoopPreheader(); + + // If we don't have a preheader, then insert one. + if (!Preheader) + Preheader = InsertPreheaderForLoop(L, DT, LI, nullptr, PreserveLCSSA); + if (!Preheader) + return false; + + HardwareLoop HWLoop(HWLoopInfo, *SE, *DL); + HWLoop.Create(); + ++NumHWLoops; + return true; +} + +void HardwareLoop::Create() { + LLVM_DEBUG(dbgs() << "HWLoops: Converting loop..\n"); + + Value *LoopCountInit = InitLoopCount(); + if (!LoopCountInit) + return; + + InsertIterationSetup(LoopCountInit); + + if (UsePHICounter || ForceHardwareLoopPHI) { + Instruction *LoopDec = InsertLoopRegDec(LoopCountInit); + Value *EltsRem = InsertPHICounter(LoopCountInit, LoopDec); + LoopDec->setOperand(0, EltsRem); + UpdateBranch(LoopDec); + } else + InsertLoopDec(); + + // Run through the basic blocks of the loop and see if any of them have dead + // PHIs that can be removed. + for (auto I : L->blocks()) + DeleteDeadPHIs(I); +} + +static bool CanGenerateTest(Loop *L, Value *Count) { + BasicBlock *Preheader = L->getLoopPreheader(); + if (!Preheader->getSinglePredecessor()) + return false; + + BasicBlock *Pred = Preheader->getSinglePredecessor(); + if (!isa(Pred->getTerminator())) + return false; + + auto *BI = cast(Pred->getTerminator()); + if (BI->isUnconditional() || !isa(BI->getCondition())) + return false; + + // Check that the icmp is checking for equality of Count and zero and that + // a non-zero value results in entering the loop. + auto ICmp = cast(BI->getCondition()); + LLVM_DEBUG(dbgs() << " - Found condition: " << *ICmp << "\n"); + if (!ICmp->isEquality()) + return false; + + auto IsCompareZero = [](ICmpInst *ICmp, Value *Count, unsigned OpIdx) { + if (auto *Const = dyn_cast(ICmp->getOperand(OpIdx))) + return Const->isZero() && ICmp->getOperand(OpIdx ^ 1) == Count; + return false; + }; + + if (!IsCompareZero(ICmp, Count, 0) && !IsCompareZero(ICmp, Count, 1)) + return false; + + unsigned SuccIdx = ICmp->getPredicate() == ICmpInst::ICMP_NE ? 0 : 1; + if (BI->getSuccessor(SuccIdx) != Preheader) + return false; + + return true; +} + +Value *HardwareLoop::InitLoopCount() { + LLVM_DEBUG(dbgs() << "HWLoops: Initialising loop counter value:\n"); + // Can we replace a conditional branch with an intrinsic that sets the + // loop counter and tests that is not zero? + + SCEVExpander SCEVE(SE, DL, "loopcnt"); + if (!ExitCount->getType()->isPointerTy() && + ExitCount->getType() != CountType) + ExitCount = SE.getZeroExtendExpr(ExitCount, CountType); + + ExitCount = SE.getAddExpr(ExitCount, SE.getOne(CountType)); + + // If we're trying to use the 'test and set' form of the intrinsic, we need + // to replace a conditional branch that is controlling entry to the loop. It + // is likely (guaranteed?) that the preheader has an unconditional branch to + // the loop header, so also check if it has a single predecessor. + if (SE.isLoopEntryGuardedByCond(L, ICmpInst::ICMP_NE, ExitCount, + SE.getZero(ExitCount->getType()))) { + LLVM_DEBUG(dbgs() << " - Attempting to use test.set counter.\n"); + UseLoopGuard |= ForceGuardLoopEntry; + } else + UseLoopGuard = false; + + BasicBlock *BB = L->getLoopPreheader(); + if (UseLoopGuard && BB->getSinglePredecessor() && + cast(BB->getTerminator())->isUnconditional()) + BB = BB->getSinglePredecessor(); + + if (!isSafeToExpandAt(ExitCount, BB->getTerminator(), SE)) { + LLVM_DEBUG(dbgs() << "- Bailing, unsafe to expand ExitCount " + << *ExitCount << "\n"); + return nullptr; + } + + Value *Count = SCEVE.expandCodeFor(ExitCount, CountType, + BB->getTerminator()); + + // FIXME: We've expanded Count where we hope to insert the counter setting + // intrinsic. But, in the case of the 'test and set' form, we may fallback to + // the just 'set' form and in which case the insertion block is most likely + // different. It means there will be instruction(s) in a block that possibly + // aren't needed. The isLoopEntryGuardedByCond is trying to avoid this issue, + // but it's doesn't appear to work in all cases. + + UseLoopGuard = UseLoopGuard && CanGenerateTest(L, Count); + BeginBB = UseLoopGuard ? BB : L->getLoopPreheader(); + LLVM_DEBUG(dbgs() << " - Loop Count: " << *Count << "\n" + << " - Expanded Count in " << BB->getName() << "\n" + << " - Will insert set counter intrinsic into: " + << BeginBB->getName() << "\n"); + return Count; +} + +void HardwareLoop::InsertIterationSetup(Value *LoopCountInit) { + IRBuilder<> Builder(BeginBB->getTerminator()); + Type *Ty = LoopCountInit->getType(); + Intrinsic::ID ID = UseLoopGuard ? + Intrinsic::test_set_loop_iterations : Intrinsic::set_loop_iterations; + Function *LoopIter = Intrinsic::getDeclaration(M, ID, Ty); + Value *SetCount = Builder.CreateCall(LoopIter, LoopCountInit); + + // Use the return value of the intrinsic to control the entry of the loop. + if (UseLoopGuard) { + assert((isa(BeginBB->getTerminator()) && + cast(BeginBB->getTerminator())->isConditional()) && + "Expected conditional branch"); + auto *LoopGuard = cast(BeginBB->getTerminator()); + LoopGuard->setCondition(SetCount); + if (LoopGuard->getSuccessor(0) != L->getLoopPreheader()) + LoopGuard->swapSuccessors(); + } + LLVM_DEBUG(dbgs() << "HWLoops: Inserted loop counter: " + << *SetCount << "\n"); +} + +void HardwareLoop::InsertLoopDec() { + IRBuilder<> CondBuilder(ExitBranch); + + Function *DecFunc = + Intrinsic::getDeclaration(M, Intrinsic::loop_decrement, + LoopDecrement->getType()); + Value *Ops[] = { LoopDecrement }; + Value *NewCond = CondBuilder.CreateCall(DecFunc, Ops); + Value *OldCond = ExitBranch->getCondition(); + ExitBranch->setCondition(NewCond); + + // The false branch must exit the loop. + if (!L->contains(ExitBranch->getSuccessor(0))) + ExitBranch->swapSuccessors(); + + // The old condition may be dead now, and may have even created a dead PHI + // (the original induction variable). + RecursivelyDeleteTriviallyDeadInstructions(OldCond); + + LLVM_DEBUG(dbgs() << "HWLoops: Inserted loop dec: " << *NewCond << "\n"); +} + +Instruction* HardwareLoop::InsertLoopRegDec(Value *EltsRem) { + IRBuilder<> CondBuilder(ExitBranch); + + Function *DecFunc = + Intrinsic::getDeclaration(M, Intrinsic::loop_decrement_reg, + { EltsRem->getType(), EltsRem->getType(), + LoopDecrement->getType() + }); + Value *Ops[] = { EltsRem, LoopDecrement }; + Value *Call = CondBuilder.CreateCall(DecFunc, Ops); + + LLVM_DEBUG(dbgs() << "HWLoops: Inserted loop dec: " << *Call << "\n"); + return cast(Call); +} + +PHINode* HardwareLoop::InsertPHICounter(Value *NumElts, Value *EltsRem) { + BasicBlock *Preheader = L->getLoopPreheader(); + BasicBlock *Header = L->getHeader(); + BasicBlock *Latch = ExitBranch->getParent(); + IRBuilder<> Builder(Header->getFirstNonPHI()); + PHINode *Index = Builder.CreatePHI(NumElts->getType(), 2); + Index->addIncoming(NumElts, Preheader); + Index->addIncoming(EltsRem, Latch); + LLVM_DEBUG(dbgs() << "HWLoops: PHI Counter: " << *Index << "\n"); + return Index; +} + +void HardwareLoop::UpdateBranch(Value *EltsRem) { + IRBuilder<> CondBuilder(ExitBranch); + Value *NewCond = + CondBuilder.CreateICmpNE(EltsRem, ConstantInt::get(EltsRem->getType(), 0)); + Value *OldCond = ExitBranch->getCondition(); + ExitBranch->setCondition(NewCond); + + // The false branch must exit the loop. + if (!L->contains(ExitBranch->getSuccessor(0))) + ExitBranch->swapSuccessors(); + + // The old condition may be dead now, and may have even created a dead PHI + // (the original induction variable). + RecursivelyDeleteTriviallyDeadInstructions(OldCond); +} + +INITIALIZE_PASS_BEGIN(HardwareLoops, DEBUG_TYPE, HW_LOOPS_NAME, false, false) +INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) +INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) +INITIALIZE_PASS_DEPENDENCY(ScalarEvolutionWrapperPass) +INITIALIZE_PASS_END(HardwareLoops, DEBUG_TYPE, HW_LOOPS_NAME, false, false) + +FunctionPass *llvm::createHardwareLoopsPass() { return new HardwareLoops(); } diff --git a/lib/CodeGen/IfConversion.cpp b/lib/CodeGen/IfConversion.cpp index ceeba639ee09..b17a253fe23f 100644 --- a/lib/CodeGen/IfConversion.cpp +++ b/lib/CodeGen/IfConversion.cpp @@ -1,9 +1,8 @@ //===- IfConversion.cpp - Machine code if conversion pass -----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -1317,7 +1316,7 @@ void IfConverter::AnalyzeBlocks( AnalyzeBlock(MBB, Tokens); // Sort to favor more complex ifcvt scheme. - std::stable_sort(Tokens.begin(), Tokens.end(), IfcvtTokenCmp); + llvm::stable_sort(Tokens, IfcvtTokenCmp); } /// Returns true either if ToMBB is the next block after MBB or that all the diff --git a/lib/CodeGen/ImplicitNullChecks.cpp b/lib/CodeGen/ImplicitNullChecks.cpp index f411ee6745d0..1e82ea659617 100644 --- a/lib/CodeGen/ImplicitNullChecks.cpp +++ b/lib/CodeGen/ImplicitNullChecks.cpp @@ -1,9 +1,8 @@ //===- ImplicitNullChecks.cpp - Fold null checks into memory accesses -----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -181,7 +180,8 @@ class ImplicitNullChecks : public MachineFunctionPass { /// Returns AR_NoAlias if \p MI memory operation does not alias with /// \p PrevMI, AR_MayAlias if they may alias and AR_WillAliasEverything if /// they may alias and any further memory operation may alias with \p PrevMI. - AliasResult areMemoryOpsAliased(MachineInstr &MI, MachineInstr *PrevMI); + AliasResult areMemoryOpsAliased(const MachineInstr &MI, + const MachineInstr *PrevMI) const; enum SuitabilityResult { SR_Suitable, @@ -195,7 +195,8 @@ class ImplicitNullChecks : public MachineFunctionPass { /// no sense to continue lookup due to any other instruction will not be able /// to be used. \p PrevInsts is the set of instruction seen since /// the explicit null check on \p PointerReg. - SuitabilityResult isSuitableMemoryOp(MachineInstr &MI, unsigned PointerReg, + SuitabilityResult isSuitableMemoryOp(const MachineInstr &MI, + unsigned PointerReg, ArrayRef PrevInsts); /// Return true if \p FaultingMI can be hoisted from after the @@ -228,7 +229,8 @@ public: } // end anonymous namespace bool ImplicitNullChecks::canHandle(const MachineInstr *MI) { - if (MI->isCall() || MI->hasUnmodeledSideEffects()) + if (MI->isCall() || MI->mayRaiseFPException() || + MI->hasUnmodeledSideEffects()) return false; auto IsRegMask = [](const MachineOperand &MO) { return MO.isRegMask(); }; (void)IsRegMask; @@ -319,8 +321,8 @@ static bool AnyAliasLiveIn(const TargetRegisterInfo *TRI, } ImplicitNullChecks::AliasResult -ImplicitNullChecks::areMemoryOpsAliased(MachineInstr &MI, - MachineInstr *PrevMI) { +ImplicitNullChecks::areMemoryOpsAliased(const MachineInstr &MI, + const MachineInstr *PrevMI) const { // If it is not memory access, skip the check. if (!(PrevMI->mayStore() || PrevMI->mayLoad())) return AR_NoAlias; @@ -357,10 +359,11 @@ ImplicitNullChecks::areMemoryOpsAliased(MachineInstr &MI, } ImplicitNullChecks::SuitabilityResult -ImplicitNullChecks::isSuitableMemoryOp(MachineInstr &MI, unsigned PointerReg, +ImplicitNullChecks::isSuitableMemoryOp(const MachineInstr &MI, + unsigned PointerReg, ArrayRef PrevInsts) { int64_t Offset; - MachineOperand *BaseOp; + const MachineOperand *BaseOp; if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, TRI) || !BaseOp->isReg() || BaseOp->getReg() != PointerReg) diff --git a/lib/CodeGen/IndirectBrExpandPass.cpp b/lib/CodeGen/IndirectBrExpandPass.cpp index 7b05ebf820fd..7ac093ba4a71 100644 --- a/lib/CodeGen/IndirectBrExpandPass.cpp +++ b/lib/CodeGen/IndirectBrExpandPass.cpp @@ -1,9 +1,8 @@ //===- IndirectBrExpandPass.cpp - Expand indirectbr to switch -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -149,11 +148,9 @@ bool IndirectBrExpandPass::runOnFunction(Function &F) { ConstantInt *BBIndexC = ConstantInt::get(ITy, BBIndex); // Now rewrite the blockaddress to an integer constant based on the index. - // FIXME: We could potentially preserve the uses as arguments to inline asm. - // This would allow some uses such as diagnostic information in crashes to - // have higher quality even when this transform is enabled, but would break - // users that round-trip blockaddresses through inline assembly and then - // back into an indirectbr. + // FIXME: This part doesn't properly recognize other uses of blockaddress + // expressions, for instance, where they are used to pass labels to + // asm-goto. This part of the pass needs a rework. BA->replaceAllUsesWith(ConstantExpr::getIntToPtr(BBIndexC, BA->getType())); } diff --git a/lib/CodeGen/InlineSpiller.cpp b/lib/CodeGen/InlineSpiller.cpp index 007e9283d833..41ae8061a917 100644 --- a/lib/CodeGen/InlineSpiller.cpp +++ b/lib/CodeGen/InlineSpiller.cpp @@ -1,9 +1,8 @@ //===- InlineSpiller.cpp - Insert spills and restores inline --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -76,6 +75,10 @@ STATISTIC(NumRemats, "Number of rematerialized defs for spilling"); static cl::opt DisableHoisting("disable-spill-hoist", cl::Hidden, cl::desc("Disable inline spill hoisting")); +static cl::opt +RestrictStatepointRemat("restrict-statepoint-remat", + cl::init(false), cl::Hidden, + cl::desc("Restrict remat for statepoint operands")); namespace { @@ -215,6 +218,7 @@ private: void eliminateRedundantSpills(LiveInterval &LI, VNInfo *VNI); void markValueUsed(LiveInterval*, VNInfo*); + bool canGuaranteeAssignmentAfterRemat(unsigned VReg, MachineInstr &MI); bool reMaterializeFor(LiveInterval &, MachineInstr &MI); void reMaterializeAll(); @@ -514,6 +518,28 @@ void InlineSpiller::markValueUsed(LiveInterval *LI, VNInfo *VNI) { } while (!WorkList.empty()); } +bool InlineSpiller::canGuaranteeAssignmentAfterRemat(unsigned VReg, + MachineInstr &MI) { + if (!RestrictStatepointRemat) + return true; + // Here's a quick explanation of the problem we're trying to handle here: + // * There are some pseudo instructions with more vreg uses than there are + // physical registers on the machine. + // * This is normally handled by spilling the vreg, and folding the reload + // into the user instruction. (Thus decreasing the number of used vregs + // until the remainder can be assigned to physregs.) + // * However, since we may try to spill vregs in any order, we can end up + // trying to spill each operand to the instruction, and then rematting it + // instead. When that happens, the new live intervals (for the remats) are + // expected to be trivially assignable (i.e. RS_Done). However, since we + // may have more remats than physregs, we're guaranteed to fail to assign + // one. + // At the moment, we only handle this for STATEPOINTs since they're the only + // psuedo op where we've seen this. If we start seeing other instructions + // with the same problem, we need to revisit this. + return (MI.getOpcode() != TargetOpcode::STATEPOINT); +} + /// reMaterializeFor - Attempt to rematerialize before MI instead of reloading. bool InlineSpiller::reMaterializeFor(LiveInterval &VirtReg, MachineInstr &MI) { // Analyze instruction @@ -569,6 +595,14 @@ bool InlineSpiller::reMaterializeFor(LiveInterval &VirtReg, MachineInstr &MI) { return true; } + // If we can't guarantee that we'll be able to actually assign the new vreg, + // we can't remat. + if (!canGuaranteeAssignmentAfterRemat(VirtReg.reg, MI)) { + markValueUsed(&VirtReg, ParentVNI); + LLVM_DEBUG(dbgs() << "\tcannot remat for " << UseIdx << '\t' << MI); + return false; + } + // Allocate a new register for the remat. unsigned NewVReg = Edit->createFrom(Original); @@ -799,11 +833,11 @@ foldMemoryOperand(ArrayRef> Ops, if (FoldOps.empty()) return false; - MachineInstrSpan MIS(MI); + MachineInstrSpan MIS(MI, MI->getParent()); MachineInstr *FoldMI = LoadMI ? TII.foldMemoryOperand(*MI, FoldOps, *LoadMI, &LIS) - : TII.foldMemoryOperand(*MI, FoldOps, StackSlot, &LIS); + : TII.foldMemoryOperand(*MI, FoldOps, StackSlot, &LIS, &VRM); if (!FoldMI) return false; @@ -834,6 +868,8 @@ foldMemoryOperand(ArrayRef> Ops, HSpiller.rmFromMergeableSpills(*MI, FI)) --NumSpills; LIS.ReplaceMachineInstrInMaps(*MI, *FoldMI); + if (MI->isCall()) + MI->getMF()->updateCallSiteInfo(MI, FoldMI); MI->eraseFromParent(); // Insert any new instructions other than FoldMI into the LIS maps. @@ -871,7 +907,7 @@ void InlineSpiller::insertReload(unsigned NewVReg, MachineBasicBlock::iterator MI) { MachineBasicBlock &MBB = *MI->getParent(); - MachineInstrSpan MIS(MI); + MachineInstrSpan MIS(MI, &MBB); TII.loadRegFromStackSlot(MBB, MI, NewVReg, StackSlot, MRI.getRegClass(NewVReg), &TRI); @@ -901,7 +937,7 @@ void InlineSpiller::insertSpill(unsigned NewVReg, bool isKill, MachineBasicBlock::iterator MI) { MachineBasicBlock &MBB = *MI->getParent(); - MachineInstrSpan MIS(MI); + MachineInstrSpan MIS(MI, &MBB); bool IsRealSpill = true; if (isFullUndefDef(*MI)) { // Don't spill undef value. diff --git a/lib/CodeGen/InterferenceCache.cpp b/lib/CodeGen/InterferenceCache.cpp index 82f6e8d8e234..7b50dac4cd1a 100644 --- a/lib/CodeGen/InterferenceCache.cpp +++ b/lib/CodeGen/InterferenceCache.cpp @@ -1,9 +1,8 @@ //===- InterferenceCache.cpp - Caching per-block interference -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/InterferenceCache.h b/lib/CodeGen/InterferenceCache.h index 160e2b16e294..50c6ac62d194 100644 --- a/lib/CodeGen/InterferenceCache.h +++ b/lib/CodeGen/InterferenceCache.h @@ -1,9 +1,8 @@ //===- InterferenceCache.h - Caching per-block interference ----*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/InterleavedAccessPass.cpp b/lib/CodeGen/InterleavedAccessPass.cpp index fd2ff162630a..14bc560a561c 100644 --- a/lib/CodeGen/InterleavedAccessPass.cpp +++ b/lib/CodeGen/InterleavedAccessPass.cpp @@ -1,9 +1,8 @@ //===- InterleavedAccessPass.cpp ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -164,14 +163,19 @@ static bool isDeInterleaveMaskOfFactor(ArrayRef Mask, unsigned Factor, /// <0, 2, 4, 6> (mask of index 0 to extract even elements) /// <1, 3, 5, 7> (mask of index 1 to extract odd elements) static bool isDeInterleaveMask(ArrayRef Mask, unsigned &Factor, - unsigned &Index, unsigned MaxFactor) { + unsigned &Index, unsigned MaxFactor, + unsigned NumLoadElements) { if (Mask.size() < 2) return false; // Check potential Factors. - for (Factor = 2; Factor <= MaxFactor; Factor++) + for (Factor = 2; Factor <= MaxFactor; Factor++) { + // Make sure we don't produce a load wider than the input load. + if (Mask.size() * Factor > NumLoadElements) + return false; if (isDeInterleaveMaskOfFactor(Mask, Factor, Index)) return true; + } return false; } @@ -303,9 +307,10 @@ bool InterleavedAccess::lowerInterleavedLoad( unsigned Factor, Index; + unsigned NumLoadElements = LI->getType()->getVectorNumElements(); // Check if the first shufflevector is DE-interleave shuffle. if (!isDeInterleaveMask(Shuffles[0]->getShuffleMask(), Factor, Index, - MaxFactor)) + MaxFactor, NumLoadElements)) return false; // Holds the corresponding index for each DE-interleave shuffle. diff --git a/lib/CodeGen/InterleavedLoadCombinePass.cpp b/lib/CodeGen/InterleavedLoadCombinePass.cpp index 989fa164ad2d..9525da849e2a 100644 --- a/lib/CodeGen/InterleavedLoadCombinePass.cpp +++ b/lib/CodeGen/InterleavedLoadCombinePass.cpp @@ -1,9 +1,8 @@ //===- InterleavedLoadCombine.cpp - Combine Interleaved Loads ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -961,6 +960,7 @@ public: if (!PtrTy) { Result = Polynomial(); BasePtr = nullptr; + return; } unsigned PointerBits = DL.getIndexSizeInBits(PtrTy->getPointerAddressSpace()); @@ -1219,7 +1219,7 @@ bool InterleavedLoadCombineImpl::combine(std::list &InterleavedLoad, "interleaved.wide.ptrcast"); // Create the wide load and update the MemorySSA. - auto LI = Builder.CreateAlignedLoad(CI, InsertionPoint->getAlignment(), + auto LI = Builder.CreateAlignedLoad(ILTy, CI, InsertionPoint->getAlignment(), "interleaved.wide.load"); auto MSSAU = MemorySSAUpdater(&MSSA); MemoryUse *MSSALoad = cast(MSSAU.createMemoryAccessBefore( diff --git a/lib/CodeGen/IntrinsicLowering.cpp b/lib/CodeGen/IntrinsicLowering.cpp index 707113bd973b..8cbd8bcaeabb 100644 --- a/lib/CodeGen/IntrinsicLowering.cpp +++ b/lib/CodeGen/IntrinsicLowering.cpp @@ -1,9 +1,8 @@ //===-- IntrinsicLowering.cpp - Intrinsic Lowering default implementation -===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -24,39 +23,6 @@ #include "llvm/Support/raw_ostream.h" using namespace llvm; -template -static void EnsureFunctionExists(Module &M, const char *Name, - ArgIt ArgBegin, ArgIt ArgEnd, - Type *RetTy) { - // Insert a correctly-typed definition now. - std::vector ParamTys; - for (ArgIt I = ArgBegin; I != ArgEnd; ++I) - ParamTys.push_back(I->getType()); - M.getOrInsertFunction(Name, FunctionType::get(RetTy, ParamTys, false)); -} - -static void EnsureFPIntrinsicsExist(Module &M, Function &Fn, - const char *FName, - const char *DName, const char *LDName) { - // Insert definitions for all the floating point types. - switch((int)Fn.arg_begin()->getType()->getTypeID()) { - case Type::FloatTyID: - EnsureFunctionExists(M, FName, Fn.arg_begin(), Fn.arg_end(), - Type::getFloatTy(M.getContext())); - break; - case Type::DoubleTyID: - EnsureFunctionExists(M, DName, Fn.arg_begin(), Fn.arg_end(), - Type::getDoubleTy(M.getContext())); - break; - case Type::X86_FP80TyID: - case Type::FP128TyID: - case Type::PPC_FP128TyID: - EnsureFunctionExists(M, LDName, Fn.arg_begin(), Fn.arg_end(), - Fn.arg_begin()->getType()); - break; - } -} - /// This function is used when we want to lower an intrinsic call to a call of /// an external function. This handles hard cases such as when there was already /// a prototype for the external function, but that prototype doesn't match the @@ -72,8 +38,8 @@ static CallInst *ReplaceCallWith(const char *NewFn, CallInst *CI, std::vector ParamTys; for (ArgIt I = ArgBegin; I != ArgEnd; ++I) ParamTys.push_back((*I)->getType()); - Constant* FCache = M->getOrInsertFunction(NewFn, - FunctionType::get(RetTy, ParamTys, false)); + FunctionCallee FCache = + M->getOrInsertFunction(NewFn, FunctionType::get(RetTy, ParamTys, false)); IRBuilder<> Builder(CI->getParent(), CI->getIterator()); SmallVector Args(ArgBegin, ArgEnd); @@ -92,75 +58,6 @@ static CallInst *ReplaceCallWith(const char *NewFn, CallInst *CI, # define setjmp_undefined_for_msvc #endif -void IntrinsicLowering::AddPrototypes(Module &M) { - LLVMContext &Context = M.getContext(); - for (auto &F : M) - if (F.isDeclaration() && !F.use_empty()) - switch (F.getIntrinsicID()) { - default: break; - case Intrinsic::setjmp: - EnsureFunctionExists(M, "setjmp", F.arg_begin(), F.arg_end(), - Type::getInt32Ty(M.getContext())); - break; - case Intrinsic::longjmp: - EnsureFunctionExists(M, "longjmp", F.arg_begin(), F.arg_end(), - Type::getVoidTy(M.getContext())); - break; - case Intrinsic::siglongjmp: - EnsureFunctionExists(M, "abort", F.arg_end(), F.arg_end(), - Type::getVoidTy(M.getContext())); - break; - case Intrinsic::memcpy: - M.getOrInsertFunction("memcpy", - Type::getInt8PtrTy(Context), - Type::getInt8PtrTy(Context), - Type::getInt8PtrTy(Context), - DL.getIntPtrType(Context)); - break; - case Intrinsic::memmove: - M.getOrInsertFunction("memmove", - Type::getInt8PtrTy(Context), - Type::getInt8PtrTy(Context), - Type::getInt8PtrTy(Context), - DL.getIntPtrType(Context)); - break; - case Intrinsic::memset: - M.getOrInsertFunction("memset", - Type::getInt8PtrTy(Context), - Type::getInt8PtrTy(Context), - Type::getInt32Ty(M.getContext()), - DL.getIntPtrType(Context)); - break; - case Intrinsic::sqrt: - EnsureFPIntrinsicsExist(M, F, "sqrtf", "sqrt", "sqrtl"); - break; - case Intrinsic::sin: - EnsureFPIntrinsicsExist(M, F, "sinf", "sin", "sinl"); - break; - case Intrinsic::cos: - EnsureFPIntrinsicsExist(M, F, "cosf", "cos", "cosl"); - break; - case Intrinsic::pow: - EnsureFPIntrinsicsExist(M, F, "powf", "pow", "powl"); - break; - case Intrinsic::log: - EnsureFPIntrinsicsExist(M, F, "logf", "log", "logl"); - break; - case Intrinsic::log2: - EnsureFPIntrinsicsExist(M, F, "log2f", "log2", "log2l"); - break; - case Intrinsic::log10: - EnsureFPIntrinsicsExist(M, F, "log10f", "log10", "log10l"); - break; - case Intrinsic::exp: - EnsureFPIntrinsicsExist(M, F, "expf", "exp", "expl"); - break; - case Intrinsic::exp2: - EnsureFPIntrinsicsExist(M, F, "exp2f", "exp2", "exp2l"); - break; - } -} - /// Emit the code to lower bswap of V before the specified instruction IP. static Value *LowerBSWAP(LLVMContext &Context, Value *V, Instruction *IP) { assert(V->getType()->isIntOrIntVectorTy() && "Can't bswap a non-integer type!"); @@ -601,7 +498,7 @@ bool IntrinsicLowering::LowerToByteSwap(CallInst *CI) { // Okay, we can do this xform, do so now. Module *M = CI->getModule(); - Constant *Int = Intrinsic::getDeclaration(M, Intrinsic::bswap, Ty); + Function *Int = Intrinsic::getDeclaration(M, Intrinsic::bswap, Ty); Value *Op = CI->getArgOperand(0); Op = CallInst::Create(Int, Op, CI->getName(), CI); diff --git a/lib/CodeGen/LLVMTargetMachine.cpp b/lib/CodeGen/LLVMTargetMachine.cpp index 52e832cc38c1..886ae7e94adb 100644 --- a/lib/CodeGen/LLVMTargetMachine.cpp +++ b/lib/CodeGen/LLVMTargetMachine.cpp @@ -1,9 +1,8 @@ //===-- LLVMTargetMachine.cpp - Implement the LLVMTargetMachine class -----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -202,6 +201,15 @@ bool LLVMTargetMachine::addPassesToEmitFile(PassManagerBase &PM, return true; if (!TargetPassConfig::willCompleteCodeGenPipeline()) { + if (this->getTargetTriple().isOSAIX()) { + // On AIX, we might manifest MCSymbols during SDAG lowering. For MIR + // testing to be meaningful, we need to ensure that the symbols created + // are MCSymbolXCOFF variants, which requires that + // the TargetLoweringObjectFile instance has been initialized. + MCContext &Ctx = MMI->getContext(); + const_cast(*this->getObjFileLowering()) + .Initialize(Ctx, *this); + } PM.add(createPrintMIRPass(Out)); } else if (addAsmPrinter(PM, Out, DwoOut, FileType, MMI->getContext())) return true; diff --git a/lib/CodeGen/LatencyPriorityQueue.cpp b/lib/CodeGen/LatencyPriorityQueue.cpp index f9f33a98a9d1..8a7a41d0f763 100644 --- a/lib/CodeGen/LatencyPriorityQueue.cpp +++ b/lib/CodeGen/LatencyPriorityQueue.cpp @@ -1,9 +1,8 @@ //===---- LatencyPriorityQueue.cpp - A latency-oriented priority queue ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/LazyMachineBlockFrequencyInfo.cpp b/lib/CodeGen/LazyMachineBlockFrequencyInfo.cpp index 5b52cc66a297..200ac0ba15bf 100644 --- a/lib/CodeGen/LazyMachineBlockFrequencyInfo.cpp +++ b/lib/CodeGen/LazyMachineBlockFrequencyInfo.cpp @@ -1,9 +1,8 @@ ///===- LazyMachineBlockFrequencyInfo.cpp - Lazy Machine Block Frequency --===// /// -/// The LLVM Compiler Infrastructure -/// -/// This file is distributed under the University of Illinois Open Source -/// License. See LICENSE.TXT for details. +/// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +/// See https://llvm.org/LICENSE.txt for license information. +/// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception /// ///===---------------------------------------------------------------------===// /// \file diff --git a/lib/CodeGen/LexicalScopes.cpp b/lib/CodeGen/LexicalScopes.cpp index d06821bdfcce..503821537ed9 100644 --- a/lib/CodeGen/LexicalScopes.cpp +++ b/lib/CodeGen/LexicalScopes.cpp @@ -1,9 +1,8 @@ //===- LexicalScopes.cpp - Collecting lexical scope info ------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/LiveDebugValues.cpp b/lib/CodeGen/LiveDebugValues.cpp index fc0ebea2d36c..a669e64692b9 100644 --- a/lib/CodeGen/LiveDebugValues.cpp +++ b/lib/CodeGen/LiveDebugValues.cpp @@ -1,9 +1,8 @@ //===- LiveDebugValues.cpp - Tracking Debug Value MIs ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -21,6 +20,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/SmallPtrSet.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/SparseBitVector.h" #include "llvm/ADT/Statistic.h" @@ -35,13 +35,15 @@ #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/PseudoSourceValue.h" +#include "llvm/CodeGen/RegisterScavenging.h" #include "llvm/CodeGen/TargetFrameLowering.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetLowering.h" +#include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" -#include "llvm/CodeGen/RegisterScavenging.h" #include "llvm/Config/llvm-config.h" +#include "llvm/IR/DIBuilder.h" #include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/DebugLoc.h" #include "llvm/IR/Function.h" @@ -57,6 +59,7 @@ #include #include #include +#include #include #include @@ -68,12 +71,12 @@ STATISTIC(NumInserted, "Number of DBG_VALUE instructions inserted"); // If @MI is a DBG_VALUE with debug value described by a defined // register, returns the number of this register. In the other case, returns 0. -static unsigned isDbgValueDescribedByReg(const MachineInstr &MI) { +static Register isDbgValueDescribedByReg(const MachineInstr &MI) { assert(MI.isDebugValue() && "expected a DBG_VALUE"); assert(MI.getNumOperands() == 4 && "malformed DBG_VALUE"); // If location of variable is described using a register (directly // or indirectly), this register is always a first operand. - return MI.getOperand(0).isReg() ? MI.getOperand(0).getReg() : 0; + return MI.getOperand(0).isReg() ? MI.getOperand(0).getReg() : Register(); } namespace { @@ -86,6 +89,8 @@ private: BitVector CalleeSavedRegs; LexicalScopes LS; + enum struct TransferKind { TransferCopy, TransferSpill, TransferRestore }; + /// Keeps track of lexical scopes associated with a user value's source /// location. class UserValueScopes { @@ -105,51 +110,134 @@ private: } }; - /// Based on std::pair so it can be used as an index into a DenseMap. - using DebugVariableBase = - std::pair; - /// A potentially inlined instance of a variable. - struct DebugVariable : public DebugVariableBase { - DebugVariable(const DILocalVariable *Var, const DILocation *InlinedAt) - : DebugVariableBase(Var, InlinedAt) {} - - const DILocalVariable *getVar() const { return this->first; } - const DILocation *getInlinedAt() const { return this->second; } - - bool operator<(const DebugVariable &DV) const { - if (getVar() == DV.getVar()) - return getInlinedAt() < DV.getInlinedAt(); - return getVar() < DV.getVar(); + using FragmentInfo = DIExpression::FragmentInfo; + using OptFragmentInfo = Optional; + + /// Storage for identifying a potentially inlined instance of a variable, + /// or a fragment thereof. + class DebugVariable { + const DILocalVariable *Variable; + OptFragmentInfo Fragment; + const DILocation *InlinedAt; + + /// Fragment that will overlap all other fragments. Used as default when + /// caller demands a fragment. + static const FragmentInfo DefaultFragment; + + public: + DebugVariable(const DILocalVariable *Var, OptFragmentInfo &&FragmentInfo, + const DILocation *InlinedAt) + : Variable(Var), Fragment(FragmentInfo), InlinedAt(InlinedAt) {} + + DebugVariable(const DILocalVariable *Var, OptFragmentInfo &FragmentInfo, + const DILocation *InlinedAt) + : Variable(Var), Fragment(FragmentInfo), InlinedAt(InlinedAt) {} + + DebugVariable(const DILocalVariable *Var, const DIExpression *DIExpr, + const DILocation *InlinedAt) + : DebugVariable(Var, DIExpr->getFragmentInfo(), InlinedAt) {} + + DebugVariable(const MachineInstr &MI) + : DebugVariable(MI.getDebugVariable(), + MI.getDebugExpression()->getFragmentInfo(), + MI.getDebugLoc()->getInlinedAt()) {} + + const DILocalVariable *getVar() const { return Variable; } + const OptFragmentInfo &getFragment() const { return Fragment; } + const DILocation *getInlinedAt() const { return InlinedAt; } + + const FragmentInfo getFragmentDefault() const { + return Fragment.getValueOr(DefaultFragment); + } + + static bool isFragmentDefault(FragmentInfo &F) { + return F == DefaultFragment; + } + + bool operator==(const DebugVariable &Other) const { + return std::tie(Variable, Fragment, InlinedAt) == + std::tie(Other.Variable, Other.Fragment, Other.InlinedAt); + } + + bool operator<(const DebugVariable &Other) const { + return std::tie(Variable, Fragment, InlinedAt) < + std::tie(Other.Variable, Other.Fragment, Other.InlinedAt); } }; + friend struct llvm::DenseMapInfo; + /// A pair of debug variable and value location. struct VarLoc { + // The location at which a spilled variable resides. It consists of a + // register and an offset. + struct SpillLoc { + unsigned SpillBase; + int SpillOffset; + bool operator==(const SpillLoc &Other) const { + return SpillBase == Other.SpillBase && SpillOffset == Other.SpillOffset; + } + }; + const DebugVariable Var; const MachineInstr &MI; ///< Only used for cloning a new DBG_VALUE. mutable UserValueScopes UVS; - enum { InvalidKind = 0, RegisterKind } Kind = InvalidKind; + enum VarLocKind { + InvalidKind = 0, + RegisterKind, + SpillLocKind, + ImmediateKind, + EntryValueKind + } Kind = InvalidKind; /// The value location. Stored separately to avoid repeatedly /// extracting it from MI. union { uint64_t RegNo; + SpillLoc SpillLocation; uint64_t Hash; + int64_t Immediate; + const ConstantFP *FPImm; + const ConstantInt *CImm; } Loc; - VarLoc(const MachineInstr &MI, LexicalScopes &LS) - : Var(MI.getDebugVariable(), MI.getDebugLoc()->getInlinedAt()), MI(MI), - UVS(MI.getDebugLoc(), LS) { + VarLoc(const MachineInstr &MI, LexicalScopes &LS, + VarLocKind K = InvalidKind) + : Var(MI), MI(MI), UVS(MI.getDebugLoc(), LS){ static_assert((sizeof(Loc) == sizeof(uint64_t)), "hash does not cover all members of Loc"); assert(MI.isDebugValue() && "not a DBG_VALUE"); assert(MI.getNumOperands() == 4 && "malformed DBG_VALUE"); if (int RegNo = isDbgValueDescribedByReg(MI)) { - Kind = RegisterKind; + Kind = MI.isDebugEntryValue() ? EntryValueKind : RegisterKind; Loc.RegNo = RegNo; + } else if (MI.getOperand(0).isImm()) { + Kind = ImmediateKind; + Loc.Immediate = MI.getOperand(0).getImm(); + } else if (MI.getOperand(0).isFPImm()) { + Kind = ImmediateKind; + Loc.FPImm = MI.getOperand(0).getFPImm(); + } else if (MI.getOperand(0).isCImm()) { + Kind = ImmediateKind; + Loc.CImm = MI.getOperand(0).getCImm(); } + assert((Kind != ImmediateKind || !MI.isDebugEntryValue()) && + "entry values must be register locations"); + } + + /// The constructor for spill locations. + VarLoc(const MachineInstr &MI, unsigned SpillBase, int SpillOffset, + LexicalScopes &LS) + : Var(MI), MI(MI), UVS(MI.getDebugLoc(), LS) { + assert(MI.isDebugValue() && "not a DBG_VALUE"); + assert(MI.getNumOperands() == 4 && "malformed DBG_VALUE"); + Kind = SpillLocKind; + Loc.SpillLocation = {SpillBase, SpillOffset}; } + // Is the Loc field a constant or constant object? + bool isConstant() const { return Kind == ImmediateKind; } + /// If this variable is described by a register, return it, /// otherwise return 0. unsigned isDescribedByReg() const { @@ -167,17 +255,18 @@ private: #endif bool operator==(const VarLoc &Other) const { - return Var == Other.Var && Loc.Hash == Other.Loc.Hash; + return Kind == Other.Kind && Var == Other.Var && + Loc.Hash == Other.Loc.Hash; } /// This operator guarantees that VarLocs are sorted by Variable first. bool operator<(const VarLoc &Other) const { - if (Var == Other.Var) - return Loc.Hash < Other.Loc.Hash; - return Var < Other.Var; + return std::tie(Var, Kind, Loc.Hash) < + std::tie(Other.Var, Other.Kind, Other.Loc.Hash); } }; + using DebugParamMap = SmallDenseMap; using VarLocMap = UniqueVector; using VarLocSet = SparseBitVector<>; using VarLocInMBB = SmallDenseMap; @@ -187,26 +276,35 @@ private: }; using TransferMap = SmallVector; + // Types for recording sets of variable fragments that overlap. For a given + // local variable, we record all other fragments of that variable that could + // overlap it, to reduce search time. + using FragmentOfVar = + std::pair; + using OverlapMap = + DenseMap>; + + // Helper while building OverlapMap, a map of all fragments seen for a given + // DILocalVariable. + using VarToFragments = + DenseMap>; + /// This holds the working set of currently open ranges. For fast /// access, this is done both as a set of VarLocIDs, and a map of /// DebugVariable to recent VarLocID. Note that a DBG_VALUE ends all /// previous open ranges for the same variable. class OpenRangesSet { VarLocSet VarLocs; - SmallDenseMap Vars; + SmallDenseMap Vars; + OverlapMap &OverlappingFragments; public: + OpenRangesSet(OverlapMap &_OLapMap) : OverlappingFragments(_OLapMap) {} + const VarLocSet &getVarLocs() const { return VarLocs; } /// Terminate all open ranges for Var by removing it from the set. - void erase(DebugVariable Var) { - auto It = Vars.find(Var); - if (It != Vars.end()) { - unsigned ID = It->second; - VarLocs.reset(ID); - Vars.erase(It); - } - } + void erase(DebugVariable Var); /// Terminate all open ranges listed in \c KillSet by removing /// them from the set. @@ -217,7 +315,7 @@ private: } /// Insert a new range into the set. - void insert(unsigned VarLocID, DebugVariableBase Var) { + void insert(unsigned VarLocID, DebugVariable Var) { VarLocs.set(VarLocID); Vars.insert({Var, VarLocID}); } @@ -237,24 +335,43 @@ private: bool isSpillInstruction(const MachineInstr &MI, MachineFunction *MF, unsigned &Reg); - int extractSpillBaseRegAndOffset(const MachineInstr &MI, unsigned &Reg); + /// If a given instruction is identified as a spill, return the spill location + /// and set \p Reg to the spilled register. + Optional isRestoreInstruction(const MachineInstr &MI, + MachineFunction *MF, + unsigned &Reg); + /// Given a spill instruction, extract the register and offset used to + /// address the spill location in a target independent way. + VarLoc::SpillLoc extractSpillBaseRegAndOffset(const MachineInstr &MI); void insertTransferDebugPair(MachineInstr &MI, OpenRangesSet &OpenRanges, TransferMap &Transfers, VarLocMap &VarLocIDs, - unsigned OldVarID, unsigned NewReg = 0); + unsigned OldVarID, TransferKind Kind, + unsigned NewReg = 0); void transferDebugValue(const MachineInstr &MI, OpenRangesSet &OpenRanges, VarLocMap &VarLocIDs); - void transferSpillInst(MachineInstr &MI, OpenRangesSet &OpenRanges, - VarLocMap &VarLocIDs, TransferMap &Transfers); + void transferSpillOrRestoreInst(MachineInstr &MI, OpenRangesSet &OpenRanges, + VarLocMap &VarLocIDs, TransferMap &Transfers); + void emitEntryValues(MachineInstr &MI, OpenRangesSet &OpenRanges, + VarLocMap &VarLocIDs, TransferMap &Transfers, + DebugParamMap &DebugEntryVals, + SparseBitVector<> &KillSet); void transferRegisterCopy(MachineInstr &MI, OpenRangesSet &OpenRanges, VarLocMap &VarLocIDs, TransferMap &Transfers); void transferRegisterDef(MachineInstr &MI, OpenRangesSet &OpenRanges, - const VarLocMap &VarLocIDs); + VarLocMap &VarLocIDs, TransferMap &Transfers, + DebugParamMap &DebugEntryVals); bool transferTerminatorInst(MachineInstr &MI, OpenRangesSet &OpenRanges, VarLocInMBB &OutLocs, const VarLocMap &VarLocIDs); + bool process(MachineInstr &MI, OpenRangesSet &OpenRanges, VarLocInMBB &OutLocs, VarLocMap &VarLocIDs, - TransferMap &Transfers, bool transferChanges); + TransferMap &Transfers, DebugParamMap &DebugEntryVals, + bool transferChanges, OverlapMap &OverlapFragments, + VarToFragments &SeenFragments); + + void accumulateFragmentMap(MachineInstr &MI, VarToFragments &SeenFragments, + OverlapMap &OLapMap); bool join(MachineBasicBlock &MBB, VarLocInMBB &OutLocs, VarLocInMBB &InLocs, const VarLocMap &VarLocIDs, @@ -289,10 +406,46 @@ public: } // end anonymous namespace +namespace llvm { + +template <> struct DenseMapInfo { + using DV = LiveDebugValues::DebugVariable; + using OptFragmentInfo = LiveDebugValues::OptFragmentInfo; + using FragmentInfo = LiveDebugValues::FragmentInfo; + + // Empty key: no key should be generated that has no DILocalVariable. + static inline DV getEmptyKey() { + return DV(nullptr, OptFragmentInfo(), nullptr); + } + + // Difference in tombstone is that the Optional is meaningful + static inline DV getTombstoneKey() { + return DV(nullptr, OptFragmentInfo({0, 0}), nullptr); + } + + static unsigned getHashValue(const DV &D) { + unsigned HV = 0; + const OptFragmentInfo &Fragment = D.getFragment(); + if (Fragment) + HV = DenseMapInfo::getHashValue(*Fragment); + + return hash_combine(D.getVar(), HV, D.getInlinedAt()); + } + + static bool isEqual(const DV &A, const DV &B) { return A == B; } +}; + +} // namespace llvm + //===----------------------------------------------------------------------===// // Implementation //===----------------------------------------------------------------------===// +const DIExpression::FragmentInfo + LiveDebugValues::DebugVariable::DefaultFragment = { + std::numeric_limits::max(), + std::numeric_limits::min()}; + char LiveDebugValues::ID = 0; char &llvm::LiveDebugValuesID = LiveDebugValues::ID; @@ -312,6 +465,39 @@ void LiveDebugValues::getAnalysisUsage(AnalysisUsage &AU) const { MachineFunctionPass::getAnalysisUsage(AU); } +/// Erase a variable from the set of open ranges, and additionally erase any +/// fragments that may overlap it. +void LiveDebugValues::OpenRangesSet::erase(DebugVariable Var) { + // Erasure helper. + auto DoErase = [this](DebugVariable VarToErase) { + auto It = Vars.find(VarToErase); + if (It != Vars.end()) { + unsigned ID = It->second; + VarLocs.reset(ID); + Vars.erase(It); + } + }; + + // Erase the variable/fragment that ends here. + DoErase(Var); + + // Extract the fragment. Interpret an empty fragment as one that covers all + // possible bits. + FragmentInfo ThisFragment = Var.getFragmentDefault(); + + // There may be fragments that overlap the designated fragment. Look them up + // in the pre-computed overlap map, and erase them too. + auto MapIt = OverlappingFragments.find({Var.getVar(), ThisFragment}); + if (MapIt != OverlappingFragments.end()) { + for (auto Fragment : MapIt->second) { + LiveDebugValues::OptFragmentInfo FragmentHolder; + if (!DebugVariable::isFragmentDefault(Fragment)) + FragmentHolder = LiveDebugValues::OptFragmentInfo(Fragment); + DoErase({Var.getVar(), FragmentHolder, Var.getInlinedAt()}); + } + } +} + //===----------------------------------------------------------------------===// // Debug Range Extension Implementation //===----------------------------------------------------------------------===// @@ -339,10 +525,8 @@ void LiveDebugValues::printVarLocInMBB(const MachineFunction &MF, } #endif -/// Given a spill instruction, extract the register and offset used to -/// address the spill location in a target independent way. -int LiveDebugValues::extractSpillBaseRegAndOffset(const MachineInstr &MI, - unsigned &Reg) { +LiveDebugValues::VarLoc::SpillLoc +LiveDebugValues::extractSpillBaseRegAndOffset(const MachineInstr &MI) { assert(MI.hasOneMemOperand() && "Spill instruction does not have exactly one memory operand?"); auto MMOI = MI.memoperands_begin(); @@ -351,7 +535,9 @@ int LiveDebugValues::extractSpillBaseRegAndOffset(const MachineInstr &MI, "Inconsistent memory operand in spill instruction"); int FI = cast(PVal)->getFrameIndex(); const MachineBasicBlock *MBB = MI.getParent(); - return TFI->getFrameIndexReference(*MBB->getParent(), FI, Reg); + unsigned Reg; + int Offset = TFI->getFrameIndexReference(*MBB->getParent(), FI, Reg); + return {Reg, Offset}; } /// End all previous ranges related to @MI and start a new range from @MI @@ -362,21 +548,72 @@ void LiveDebugValues::transferDebugValue(const MachineInstr &MI, if (!MI.isDebugValue()) return; const DILocalVariable *Var = MI.getDebugVariable(); + const DIExpression *Expr = MI.getDebugExpression(); const DILocation *DebugLoc = MI.getDebugLoc(); const DILocation *InlinedAt = DebugLoc->getInlinedAt(); assert(Var->isValidLocationForIntrinsic(DebugLoc) && "Expected inlined-at fields to agree"); // End all previous ranges of Var. - DebugVariable V(Var, InlinedAt); + DebugVariable V(Var, Expr, InlinedAt); OpenRanges.erase(V); // Add the VarLoc to OpenRanges from this DBG_VALUE. - // TODO: Currently handles DBG_VALUE which has only reg as location. - if (isDbgValueDescribedByReg(MI)) { + unsigned ID; + if (isDbgValueDescribedByReg(MI) || MI.getOperand(0).isImm() || + MI.getOperand(0).isFPImm() || MI.getOperand(0).isCImm()) { + // Use normal VarLoc constructor for registers and immediates. VarLoc VL(MI, LS); - unsigned ID = VarLocIDs.insert(VL); + ID = VarLocIDs.insert(VL); + OpenRanges.insert(ID, VL.Var); + } else if (MI.hasOneMemOperand()) { + // It's a stack spill -- fetch spill base and offset. + VarLoc::SpillLoc SpillLocation = extractSpillBaseRegAndOffset(MI); + VarLoc VL(MI, SpillLocation.SpillBase, SpillLocation.SpillOffset, LS); + ID = VarLocIDs.insert(VL); OpenRanges.insert(ID, VL.Var); + } else { + // This must be an undefined location. We should leave OpenRanges closed. + assert(MI.getOperand(0).isReg() && MI.getOperand(0).getReg() == 0 && + "Unexpected non-undef DBG_VALUE encountered"); + } +} + +void LiveDebugValues::emitEntryValues(MachineInstr &MI, + OpenRangesSet &OpenRanges, + VarLocMap &VarLocIDs, + TransferMap &Transfers, + DebugParamMap &DebugEntryVals, + SparseBitVector<> &KillSet) { + MachineFunction *MF = MI.getParent()->getParent(); + for (unsigned ID : KillSet) { + if (!VarLocIDs[ID].Var.getVar()->isParameter()) + continue; + + const MachineInstr *CurrDebugInstr = &VarLocIDs[ID].MI; + + // If parameter's DBG_VALUE is not in the map that means we can't + // generate parameter's entry value. + if (!DebugEntryVals.count(CurrDebugInstr->getDebugVariable())) + continue; + + auto ParamDebugInstr = DebugEntryVals[CurrDebugInstr->getDebugVariable()]; + DIExpression *NewExpr = DIExpression::prepend( + ParamDebugInstr->getDebugExpression(), DIExpression::EntryValue); + MachineInstr *EntryValDbgMI = + BuildMI(*MF, ParamDebugInstr->getDebugLoc(), ParamDebugInstr->getDesc(), + ParamDebugInstr->isIndirectDebugValue(), + ParamDebugInstr->getOperand(0).getReg(), + ParamDebugInstr->getDebugVariable(), NewExpr); + + if (ParamDebugInstr->isIndirectDebugValue()) + EntryValDbgMI->getOperand(1).setImm( + ParamDebugInstr->getOperand(1).getImm()); + + Transfers.push_back({&MI, EntryValDbgMI}); + VarLoc VL(*EntryValDbgMI, LS); + unsigned EntryValLocID = VarLocIDs.insert(VL); + OpenRanges.insert(EntryValLocID, VL.Var); } } @@ -387,51 +624,92 @@ void LiveDebugValues::transferDebugValue(const MachineInstr &MI, /// otherwise it is variable's location on the stack. void LiveDebugValues::insertTransferDebugPair( MachineInstr &MI, OpenRangesSet &OpenRanges, TransferMap &Transfers, - VarLocMap &VarLocIDs, unsigned OldVarID, unsigned NewReg) { - const MachineInstr *DMI = &VarLocIDs[OldVarID].MI; + VarLocMap &VarLocIDs, unsigned OldVarID, TransferKind Kind, + unsigned NewReg) { + const MachineInstr *DebugInstr = &VarLocIDs[OldVarID].MI; MachineFunction *MF = MI.getParent()->getParent(); - MachineInstr *NewDMI; - if (NewReg) { + MachineInstr *NewDebugInstr; + + auto ProcessVarLoc = [&MI, &OpenRanges, &Transfers, &DebugInstr, + &VarLocIDs](VarLoc &VL, MachineInstr *NewDebugInstr) { + unsigned LocId = VarLocIDs.insert(VL); + + // Close this variable's previous location range. + DebugVariable V(*DebugInstr); + OpenRanges.erase(V); + + OpenRanges.insert(LocId, VL.Var); + // The newly created DBG_VALUE instruction NewDebugInstr must be inserted + // after MI. Keep track of the pairing. + TransferDebugPair MIP = {&MI, NewDebugInstr}; + Transfers.push_back(MIP); + }; + + // End all previous ranges of Var. + OpenRanges.erase(VarLocIDs[OldVarID].Var); + switch (Kind) { + case TransferKind::TransferCopy: { + assert(NewReg && + "No register supplied when handling a copy of a debug value"); // Create a DBG_VALUE instruction to describe the Var in its new // register location. - NewDMI = BuildMI(*MF, DMI->getDebugLoc(), DMI->getDesc(), - DMI->isIndirectDebugValue(), NewReg, - DMI->getDebugVariable(), DMI->getDebugExpression()); - if (DMI->isIndirectDebugValue()) - NewDMI->getOperand(1).setImm(DMI->getOperand(1).getImm()); + NewDebugInstr = BuildMI( + *MF, DebugInstr->getDebugLoc(), DebugInstr->getDesc(), + DebugInstr->isIndirectDebugValue(), NewReg, + DebugInstr->getDebugVariable(), DebugInstr->getDebugExpression()); + if (DebugInstr->isIndirectDebugValue()) + NewDebugInstr->getOperand(1).setImm(DebugInstr->getOperand(1).getImm()); + VarLoc VL(*NewDebugInstr, LS); + ProcessVarLoc(VL, NewDebugInstr); LLVM_DEBUG(dbgs() << "Creating DBG_VALUE inst for register copy: "; - NewDMI->print(dbgs(), false, false, false, TII)); - } else { + NewDebugInstr->print(dbgs(), /*IsStandalone*/false, + /*SkipOpers*/false, /*SkipDebugLoc*/false, + /*AddNewLine*/true, TII)); + return; + } + case TransferKind::TransferSpill: { // Create a DBG_VALUE instruction to describe the Var in its spilled // location. - unsigned SpillBase; - int SpillOffset = extractSpillBaseRegAndOffset(MI, SpillBase); - auto *SpillExpr = DIExpression::prepend(DMI->getDebugExpression(), - DIExpression::NoDeref, SpillOffset); - NewDMI = BuildMI(*MF, DMI->getDebugLoc(), DMI->getDesc(), true, SpillBase, - DMI->getDebugVariable(), SpillExpr); + VarLoc::SpillLoc SpillLocation = extractSpillBaseRegAndOffset(MI); + auto *SpillExpr = DIExpression::prepend(DebugInstr->getDebugExpression(), + DIExpression::ApplyOffset, + SpillLocation.SpillOffset); + NewDebugInstr = BuildMI( + *MF, DebugInstr->getDebugLoc(), DebugInstr->getDesc(), true, + SpillLocation.SpillBase, DebugInstr->getDebugVariable(), SpillExpr); + VarLoc VL(*NewDebugInstr, SpillLocation.SpillBase, + SpillLocation.SpillOffset, LS); + ProcessVarLoc(VL, NewDebugInstr); LLVM_DEBUG(dbgs() << "Creating DBG_VALUE inst for spill: "; - NewDMI->print(dbgs(), false, false, false, TII)); + NewDebugInstr->print(dbgs(), /*IsStandalone*/false, + /*SkipOpers*/false, /*SkipDebugLoc*/false, + /*AddNewLine*/true, TII)); + return; } - - // The newly created DBG_VALUE instruction NewDMI must be inserted after - // MI. Keep track of the pairing. - TransferDebugPair MIP = {&MI, NewDMI}; - Transfers.push_back(MIP); - - // End all previous ranges of Var. - OpenRanges.erase(VarLocIDs[OldVarID].Var); - - // Add the VarLoc to OpenRanges. - VarLoc VL(*NewDMI, LS); - unsigned LocID = VarLocIDs.insert(VL); - OpenRanges.insert(LocID, VL.Var); + case TransferKind::TransferRestore: { + assert(NewReg && + "No register supplied when handling a restore of a debug value"); + MachineFunction *MF = MI.getMF(); + DIBuilder DIB(*const_cast(MF->getFunction()).getParent()); + NewDebugInstr = + BuildMI(*MF, DebugInstr->getDebugLoc(), DebugInstr->getDesc(), false, + NewReg, DebugInstr->getDebugVariable(), DIB.createExpression()); + VarLoc VL(*NewDebugInstr, LS); + ProcessVarLoc(VL, NewDebugInstr); + LLVM_DEBUG(dbgs() << "Creating DBG_VALUE inst for register restore: "; + NewDebugInstr->print(dbgs(), /*IsStandalone*/false, + /*SkipOpers*/false, /*SkipDebugLoc*/false, + /*AddNewLine*/true, TII)); + return; + } + } + llvm_unreachable("Invalid transfer kind"); } /// A definition of a register may mark the end of a range. -void LiveDebugValues::transferRegisterDef(MachineInstr &MI, - OpenRangesSet &OpenRanges, - const VarLocMap &VarLocIDs) { +void LiveDebugValues::transferRegisterDef( + MachineInstr &MI, OpenRangesSet &OpenRanges, VarLocMap &VarLocIDs, + TransferMap &Transfers, DebugParamMap &DebugEntryVals) { MachineFunction *MF = MI.getMF(); const TargetLowering *TLI = MF->getSubtarget().getTargetLowering(); unsigned SP = TLI->getStackPointerRegisterToSaveRestore(); @@ -461,6 +739,13 @@ void LiveDebugValues::transferRegisterDef(MachineInstr &MI, } } OpenRanges.erase(KillSet, VarLocIDs); + + if (auto *TPC = getAnalysisIfAvailable()) { + auto &TM = TPC->getTM(); + if (TM.Options.EnableDebugEntryValues) + emitEntryValues(MI, OpenRanges, VarLocIDs, Transfers, DebugEntryVals, + KillSet); + } } /// Decide if @MI is a spill instruction and return true if it is. We use 2 @@ -471,24 +756,15 @@ void LiveDebugValues::transferRegisterDef(MachineInstr &MI, /// other spills). We do not handle this yet (more than one memory operand). bool LiveDebugValues::isSpillInstruction(const MachineInstr &MI, MachineFunction *MF, unsigned &Reg) { - const MachineFrameInfo &FrameInfo = MF->getFrameInfo(); - int FI; SmallVector Accesses; // TODO: Handle multiple stores folded into one. if (!MI.hasOneMemOperand()) return false; - // To identify a spill instruction, use the same criteria as in AsmPrinter. - if (!((TII->isStoreToStackSlotPostFE(MI, FI) && - FrameInfo.isSpillSlotObjectIndex(FI)) || - (TII->hasStoreToStackSlot(MI, Accesses) && - llvm::any_of(Accesses, [&FrameInfo](const MachineMemOperand *MMO) { - return FrameInfo.isSpillSlotObjectIndex( - cast(MMO->getPseudoValue()) - ->getFrameIndex()); - })))) - return false; + if (!MI.getSpillSize(TII) && !MI.getFoldedSpillSize(TII)) + return false; // This is not a spill instruction, since no valid size was + // returned from either function. auto isKilledReg = [&](const MachineOperand MO, unsigned &Reg) { if (!MO.isReg() || !MO.isUse()) { @@ -525,29 +801,67 @@ bool LiveDebugValues::isSpillInstruction(const MachineInstr &MI, return false; } +Optional +LiveDebugValues::isRestoreInstruction(const MachineInstr &MI, + MachineFunction *MF, unsigned &Reg) { + if (!MI.hasOneMemOperand()) + return None; + + // FIXME: Handle folded restore instructions with more than one memory + // operand. + if (MI.getRestoreSize(TII)) { + Reg = MI.getOperand(0).getReg(); + return extractSpillBaseRegAndOffset(MI); + } + return None; +} + /// A spilled register may indicate that we have to end the current range of /// a variable and create a new one for the spill location. +/// A restored register may indicate the reverse situation. /// We don't want to insert any instructions in process(), so we just create /// the DBG_VALUE without inserting it and keep track of it in \p Transfers. /// It will be inserted into the BB when we're done iterating over the /// instructions. -void LiveDebugValues::transferSpillInst(MachineInstr &MI, - OpenRangesSet &OpenRanges, - VarLocMap &VarLocIDs, - TransferMap &Transfers) { - unsigned Reg; +void LiveDebugValues::transferSpillOrRestoreInst(MachineInstr &MI, + OpenRangesSet &OpenRanges, + VarLocMap &VarLocIDs, + TransferMap &Transfers) { MachineFunction *MF = MI.getMF(); - if (!isSpillInstruction(MI, MF, Reg)) - return; + TransferKind TKind; + unsigned Reg; + Optional Loc; - // Check if the register is the location of a debug value. + LLVM_DEBUG(dbgs() << "Examining instruction: "; MI.dump();); + + if (isSpillInstruction(MI, MF, Reg)) { + TKind = TransferKind::TransferSpill; + LLVM_DEBUG(dbgs() << "Recognized as spill: "; MI.dump();); + LLVM_DEBUG(dbgs() << "Register: " << Reg << " " << printReg(Reg, TRI) + << "\n"); + } else { + if (!(Loc = isRestoreInstruction(MI, MF, Reg))) + return; + TKind = TransferKind::TransferRestore; + LLVM_DEBUG(dbgs() << "Recognized as restore: "; MI.dump();); + LLVM_DEBUG(dbgs() << "Register: " << Reg << " " << printReg(Reg, TRI) + << "\n"); + } + // Check if the register or spill location is the location of a debug value. for (unsigned ID : OpenRanges.getVarLocs()) { - if (VarLocIDs[ID].isDescribedByReg() == Reg) { + if (TKind == TransferKind::TransferSpill && + VarLocIDs[ID].isDescribedByReg() == Reg) { LLVM_DEBUG(dbgs() << "Spilling Register " << printReg(Reg, TRI) << '(' << VarLocIDs[ID].Var.getVar()->getName() << ")\n"); - insertTransferDebugPair(MI, OpenRanges, Transfers, VarLocIDs, ID); - return; - } + } else if (TKind == TransferKind::TransferRestore && + VarLocIDs[ID].Loc.SpillLocation == *Loc) { + LLVM_DEBUG(dbgs() << "Restoring Register " << printReg(Reg, TRI) << '(' + << VarLocIDs[ID].Var.getVar()->getName() << ")\n"); + } else + continue; + insertTransferDebugPair(MI, OpenRanges, Transfers, VarLocIDs, ID, TKind, + Reg); + return; } } @@ -585,7 +899,7 @@ void LiveDebugValues::transferRegisterCopy(MachineInstr &MI, for (unsigned ID : OpenRanges.getVarLocs()) { if (VarLocIDs[ID].isDescribedByReg() == SrcReg) { insertTransferDebugPair(MI, OpenRanges, Transfers, VarLocIDs, ID, - DestReg); + TransferKind::TransferCopy, DestReg); return; } } @@ -612,20 +926,92 @@ bool LiveDebugValues::transferTerminatorInst(MachineInstr &MI, }); VarLocSet &VLS = OutLocs[CurMBB]; Changed = VLS |= OpenRanges.getVarLocs(); + // New OutLocs set may be different due to spill, restore or register + // copy instruction processing. + if (Changed) + VLS = OpenRanges.getVarLocs(); OpenRanges.clear(); return Changed; } +/// Accumulate a mapping between each DILocalVariable fragment and other +/// fragments of that DILocalVariable which overlap. This reduces work during +/// the data-flow stage from "Find any overlapping fragments" to "Check if the +/// known-to-overlap fragments are present". +/// \param MI A previously unprocessed DEBUG_VALUE instruction to analyze for +/// fragment usage. +/// \param SeenFragments Map from DILocalVariable to all fragments of that +/// Variable which are known to exist. +/// \param OverlappingFragments The overlap map being constructed, from one +/// Var/Fragment pair to a vector of fragments known to overlap. +void LiveDebugValues::accumulateFragmentMap(MachineInstr &MI, + VarToFragments &SeenFragments, + OverlapMap &OverlappingFragments) { + DebugVariable MIVar(MI); + FragmentInfo ThisFragment = MIVar.getFragmentDefault(); + + // If this is the first sighting of this variable, then we are guaranteed + // there are currently no overlapping fragments either. Initialize the set + // of seen fragments, record no overlaps for the current one, and return. + auto SeenIt = SeenFragments.find(MIVar.getVar()); + if (SeenIt == SeenFragments.end()) { + SmallSet OneFragment; + OneFragment.insert(ThisFragment); + SeenFragments.insert({MIVar.getVar(), OneFragment}); + + OverlappingFragments.insert({{MIVar.getVar(), ThisFragment}, {}}); + return; + } + + // If this particular Variable/Fragment pair already exists in the overlap + // map, it has already been accounted for. + auto IsInOLapMap = + OverlappingFragments.insert({{MIVar.getVar(), ThisFragment}, {}}); + if (!IsInOLapMap.second) + return; + + auto &ThisFragmentsOverlaps = IsInOLapMap.first->second; + auto &AllSeenFragments = SeenIt->second; + + // Otherwise, examine all other seen fragments for this variable, with "this" + // fragment being a previously unseen fragment. Record any pair of + // overlapping fragments. + for (auto &ASeenFragment : AllSeenFragments) { + // Does this previously seen fragment overlap? + if (DIExpression::fragmentsOverlap(ThisFragment, ASeenFragment)) { + // Yes: Mark the current fragment as being overlapped. + ThisFragmentsOverlaps.push_back(ASeenFragment); + // Mark the previously seen fragment as being overlapped by the current + // one. + auto ASeenFragmentsOverlaps = + OverlappingFragments.find({MIVar.getVar(), ASeenFragment}); + assert(ASeenFragmentsOverlaps != OverlappingFragments.end() && + "Previously seen var fragment has no vector of overlaps"); + ASeenFragmentsOverlaps->second.push_back(ThisFragment); + } + } + + AllSeenFragments.insert(ThisFragment); +} + /// This routine creates OpenRanges and OutLocs. bool LiveDebugValues::process(MachineInstr &MI, OpenRangesSet &OpenRanges, VarLocInMBB &OutLocs, VarLocMap &VarLocIDs, - TransferMap &Transfers, bool transferChanges) { + TransferMap &Transfers, DebugParamMap &DebugEntryVals, + bool transferChanges, + OverlapMap &OverlapFragments, + VarToFragments &SeenFragments) { bool Changed = false; transferDebugValue(MI, OpenRanges, VarLocIDs); - transferRegisterDef(MI, OpenRanges, VarLocIDs); + transferRegisterDef(MI, OpenRanges, VarLocIDs, Transfers, + DebugEntryVals); if (transferChanges) { transferRegisterCopy(MI, OpenRanges, VarLocIDs, Transfers); - transferSpillInst(MI, OpenRanges, VarLocIDs, Transfers); + transferSpillOrRestoreInst(MI, OpenRanges, VarLocIDs, Transfers); + } else { + // Build up a map of overlapping fragments on the first run through. + if (MI.isDebugValue()) + accumulateFragmentMap(MI, SeenFragments, OverlapFragments); } Changed = transferTerminatorInst(MI, OpenRanges, OutLocs, VarLocIDs); return Changed; @@ -713,13 +1099,23 @@ bool LiveDebugValues::join( // new range is started for the var from the mbb's beginning by inserting // a new DBG_VALUE. process() will end this range however appropriate. const VarLoc &DiffIt = VarLocIDs[ID]; - const MachineInstr *DMI = &DiffIt.MI; - MachineInstr *MI = - BuildMI(MBB, MBB.instr_begin(), DMI->getDebugLoc(), DMI->getDesc(), - DMI->isIndirectDebugValue(), DMI->getOperand(0).getReg(), - DMI->getDebugVariable(), DMI->getDebugExpression()); - if (DMI->isIndirectDebugValue()) - MI->getOperand(1).setImm(DMI->getOperand(1).getImm()); + const MachineInstr *DebugInstr = &DiffIt.MI; + MachineInstr *MI = nullptr; + if (DiffIt.isConstant()) { + MachineOperand MO(DebugInstr->getOperand(0)); + MI = BuildMI(MBB, MBB.instr_begin(), DebugInstr->getDebugLoc(), + DebugInstr->getDesc(), false, MO, + DebugInstr->getDebugVariable(), + DebugInstr->getDebugExpression()); + } else { + MI = BuildMI(MBB, MBB.instr_begin(), DebugInstr->getDebugLoc(), + DebugInstr->getDesc(), DebugInstr->isIndirectDebugValue(), + DebugInstr->getOperand(0).getReg(), + DebugInstr->getDebugVariable(), + DebugInstr->getDebugExpression()); + if (DebugInstr->isIndirectDebugValue()) + MI->getOperand(1).setImm(DebugInstr->getOperand(1).getImm()); + } LLVM_DEBUG(dbgs() << "Inserted: "; MI->dump();); ILS.set(ID); ++NumInserted; @@ -737,11 +1133,15 @@ bool LiveDebugValues::ExtendRanges(MachineFunction &MF) { bool OLChanged = false; bool MBBJoined = false; - VarLocMap VarLocIDs; // Map VarLoc<>unique ID for use in bitvectors. - OpenRangesSet OpenRanges; // Ranges that are open until end of bb. - VarLocInMBB OutLocs; // Ranges that exist beyond bb. - VarLocInMBB InLocs; // Ranges that are incoming after joining. - TransferMap Transfers; // DBG_VALUEs associated with spills. + VarLocMap VarLocIDs; // Map VarLoc<>unique ID for use in bitvectors. + OverlapMap OverlapFragments; // Map of overlapping variable fragments + OpenRangesSet OpenRanges(OverlapFragments); + // Ranges that are open until end of bb. + VarLocInMBB OutLocs; // Ranges that exist beyond bb. + VarLocInMBB InLocs; // Ranges that are incoming after joining. + TransferMap Transfers; // DBG_VALUEs associated with spills. + + VarToFragments SeenFragments; // Blocks which are artificial, i.e. blocks which exclusively contain // instructions without locations, or with line 0 locations. @@ -758,15 +1158,61 @@ bool LiveDebugValues::ExtendRanges(MachineFunction &MF) { enum : bool { dontTransferChanges = false, transferChanges = true }; + // Besides parameter's modification, check whether a DBG_VALUE is inlined + // in order to deduce whether the variable that it tracks comes from + // a different function. If that is the case we can't track its entry value. + auto IsUnmodifiedFuncParam = [&](const MachineInstr &MI) { + auto *DIVar = MI.getDebugVariable(); + return DIVar->isParameter() && DIVar->isNotModified() && + !MI.getDebugLoc()->getInlinedAt(); + }; + + const TargetLowering *TLI = MF.getSubtarget().getTargetLowering(); + unsigned SP = TLI->getStackPointerRegisterToSaveRestore(); + unsigned FP = TRI->getFrameRegister(MF); + auto IsRegOtherThanSPAndFP = [&](const MachineOperand &Op) -> bool { + return Op.isReg() && Op.getReg() != SP && Op.getReg() != FP; + }; + + // Working set of currently collected debug variables mapped to DBG_VALUEs + // representing candidates for production of debug entry values. + DebugParamMap DebugEntryVals; + + MachineBasicBlock &First_MBB = *(MF.begin()); + // Only in the case of entry MBB collect DBG_VALUEs representing + // function parameters in order to generate debug entry values for them. + // Currently, we generate debug entry values only for parameters that are + // unmodified throughout the function and located in a register. + // TODO: Add support for parameters that are described as fragments. + // TODO: Add support for modified arguments that can be expressed + // by using its entry value. + // TODO: Add support for local variables that are expressed in terms of + // parameters entry values. + for (auto &MI : First_MBB) + if (MI.isDebugValue() && IsUnmodifiedFuncParam(MI) && + !MI.isIndirectDebugValue() && IsRegOtherThanSPAndFP(MI.getOperand(0)) && + !DebugEntryVals.count(MI.getDebugVariable()) && + !MI.getDebugExpression()->isFragment()) + DebugEntryVals[MI.getDebugVariable()] = &MI; + // Initialize every mbb with OutLocs. // We are not looking at any spill instructions during the initial pass // over the BBs. The LiveDebugVariables pass has already created DBG_VALUE // instructions for spills of registers that are known to be user variables // within the BB in which the spill occurs. - for (auto &MBB : MF) - for (auto &MI : MBB) - process(MI, OpenRanges, OutLocs, VarLocIDs, Transfers, - dontTransferChanges); + for (auto &MBB : MF) { + for (auto &MI : MBB) { + process(MI, OpenRanges, OutLocs, VarLocIDs, Transfers, DebugEntryVals, + dontTransferChanges, OverlapFragments, SeenFragments); + } + // Add any entry DBG_VALUE instructions necessitated by parameter + // clobbering. + for (auto &TR : Transfers) { + MBB.insertAfter(MachineBasicBlock::iterator(*TR.TransferInst), + TR.DebugInst); + } + Transfers.clear(); + } auto hasNonArtificialLocation = [](const MachineInstr &MI) -> bool { if (const DebugLoc &DL = MI.getDebugLoc()) @@ -812,8 +1258,10 @@ bool LiveDebugValues::ExtendRanges(MachineFunction &MF) { // examine spill instructions to see whether they spill registers that // correspond to user variables. for (auto &MI : *MBB) - OLChanged |= process(MI, OpenRanges, OutLocs, VarLocIDs, Transfers, - transferChanges); + OLChanged |= + process(MI, OpenRanges, OutLocs, VarLocIDs, Transfers, + DebugEntryVals, transferChanges, OverlapFragments, + SeenFragments); // Add any DBG_VALUE instructions necessitated by spills. for (auto &TR : Transfers) diff --git a/lib/CodeGen/LiveDebugVariables.cpp b/lib/CodeGen/LiveDebugVariables.cpp index d0d889782a35..656ec7d4bdfd 100644 --- a/lib/CodeGen/LiveDebugVariables.cpp +++ b/lib/CodeGen/LiveDebugVariables.cpp @@ -1,9 +1,8 @@ //===- LiveDebugVariables.cpp - Tracking debug info variables -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -23,6 +22,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/IntervalMap.h" +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" @@ -71,6 +71,7 @@ EnableLDV("live-debug-variables", cl::init(true), cl::desc("Enable the live debug variables pass"), cl::Hidden); STATISTIC(NumInsertedDebugValues, "Number of DBG_VALUEs inserted"); +STATISTIC(NumInsertedDebugLabels, "Number of DBG_LABELs inserted"); char LiveDebugVariables::ID = 0; @@ -166,10 +167,6 @@ class UserValue { /// Map of slot indices where this value is live. LocMap locInts; - /// Set of interval start indexes that have been trimmed to the - /// lexical scope. - SmallSet trimmedDefs; - /// Insert a DBG_VALUE into MBB at Idx for LocNo. void insertDebugValue(MachineBasicBlock *MBB, SlotIndex StartIdx, SlotIndex StopIdx, DbgValueLocation Loc, bool Spilled, @@ -339,6 +336,37 @@ public: void print(raw_ostream &, const TargetRegisterInfo *); }; +/// A user label is a part of a debug info user label. +class UserLabel { + const DILabel *Label; ///< The debug info label we are part of. + DebugLoc dl; ///< The debug location for the label. This is + ///< used by dwarf writer to find lexical scope. + SlotIndex loc; ///< Slot used by the debug label. + + /// Insert a DBG_LABEL into MBB at Idx. + void insertDebugLabel(MachineBasicBlock *MBB, SlotIndex Idx, + LiveIntervals &LIS, const TargetInstrInfo &TII); + +public: + /// Create a new UserLabel. + UserLabel(const DILabel *label, DebugLoc L, SlotIndex Idx) + : Label(label), dl(std::move(L)), loc(Idx) {} + + /// Does this UserLabel match the parameters? + bool match(const DILabel *L, const DILocation *IA, + const SlotIndex Index) const { + return Label == L && dl->getInlinedAt() == IA && loc == Index; + } + + /// Recreate DBG_LABEL instruction from data structures. + void emitDebugLabel(LiveIntervals &LIS, const TargetInstrInfo &TII); + + /// Return DebugLoc of this UserLabel. + DebugLoc getDebugLoc() { return dl; } + + void print(raw_ostream &, const TargetRegisterInfo *); +}; + /// Implementation of the LiveDebugVariables pass. class LDVImpl { LiveDebugVariables &pass; @@ -356,6 +384,9 @@ class LDVImpl { /// All allocated UserValue instances. SmallVector, 8> userValues; + /// All allocated UserLabel instances. + SmallVector, 2> userLabels; + /// Map virtual register to eq class leader. using VRMap = DenseMap; VRMap virtRegToEqClass; @@ -379,6 +410,14 @@ class LDVImpl { /// \returns True if the DBG_VALUE instruction should be deleted. bool handleDebugValue(MachineInstr &MI, SlotIndex Idx); + /// Add DBG_LABEL instruction to UserLabel. + /// + /// \param MI DBG_LABEL instruction + /// \param Idx Last valid SlotIndex before instruction. + /// + /// \returns True if the DBG_LABEL instruction should be deleted. + bool handleDebugLabel(MachineInstr &MI, SlotIndex Idx); + /// Collect and erase all DBG_VALUE instructions, adding a UserValue def /// for each instruction. /// @@ -400,6 +439,7 @@ public: void clear() { MF = nullptr; userValues.clear(); + userLabels.clear(); virtRegToEqClass.clear(); userVarMap.clear(); // Make sure we call emitDebugValues if the machine function was modified. @@ -445,13 +485,23 @@ static void printDebugLoc(const DebugLoc &DL, raw_ostream &CommentOS, CommentOS << " ]"; } -static void printExtendedName(raw_ostream &OS, const DILocalVariable *V, +static void printExtendedName(raw_ostream &OS, const DINode *Node, const DILocation *DL) { - const LLVMContext &Ctx = V->getContext(); - StringRef Res = V->getName(); + const LLVMContext &Ctx = Node->getContext(); + StringRef Res; + unsigned Line; + if (const auto *V = dyn_cast(Node)) { + Res = V->getName(); + Line = V->getLine(); + } else if (const auto *L = dyn_cast(Node)) { + Res = L->getName(); + Line = L->getLine(); + } + if (!Res.empty()) - OS << Res << "," << V->getLine(); - if (auto *InlinedAt = DL->getInlinedAt()) { + OS << Res << "," << Line; + auto *InlinedAt = DL ? DL->getInlinedAt() : nullptr; + if (InlinedAt) { if (DebugLoc InlinedAtDL = InlinedAt) { OS << " @["; printDebugLoc(InlinedAtDL, OS, Ctx); @@ -461,9 +511,8 @@ static void printExtendedName(raw_ostream &OS, const DILocalVariable *V, } void UserValue::print(raw_ostream &OS, const TargetRegisterInfo *TRI) { - auto *DV = cast(Variable); OS << "!\""; - printExtendedName(OS, DV, dl); + printExtendedName(OS, Variable, dl); OS << "\"\t"; for (LocMap::const_iterator I = locInts.begin(); I.valid(); ++I) { @@ -483,10 +532,22 @@ void UserValue::print(raw_ostream &OS, const TargetRegisterInfo *TRI) { OS << '\n'; } +void UserLabel::print(raw_ostream &OS, const TargetRegisterInfo *TRI) { + OS << "!\""; + printExtendedName(OS, Label, dl); + + OS << "\"\t"; + OS << loc; + OS << '\n'; +} + void LDVImpl::print(raw_ostream &OS) { OS << "********** DEBUG VARIABLES **********\n"; - for (unsigned i = 0, e = userValues.size(); i != e; ++i) - userValues[i]->print(OS, TRI); + for (auto &userValue : userValues) + userValue->print(OS, TRI); + OS << "********** DEBUG LABELS **********\n"; + for (auto &userLabel : userLabels) + userLabel->print(OS, TRI); } #endif @@ -556,7 +617,7 @@ bool LDVImpl::handleDebugValue(MachineInstr &MI, SlotIndex Idx) { } else { // The DBG_VALUE is only valid if either Reg is live out from Idx, or Reg // is defined dead at Idx (where Idx is the slot index for the instruction - // preceeding the DBG_VALUE). + // preceding the DBG_VALUE). const LiveInterval &LI = LIS->getInterval(Reg); LiveQueryResult LRQ = LI.Query(Idx); if (!LRQ.valueOutOrDead()) { @@ -587,6 +648,29 @@ bool LDVImpl::handleDebugValue(MachineInstr &MI, SlotIndex Idx) { return true; } +bool LDVImpl::handleDebugLabel(MachineInstr &MI, SlotIndex Idx) { + // DBG_LABEL label + if (MI.getNumOperands() != 1 || !MI.getOperand(0).isMetadata()) { + LLVM_DEBUG(dbgs() << "Can't handle " << MI); + return false; + } + + // Get or create the UserLabel for label here. + const DILabel *Label = MI.getDebugLabel(); + const DebugLoc &DL = MI.getDebugLoc(); + bool Found = false; + for (auto const &L : userLabels) { + if (L->match(Label, DL->getInlinedAt(), Idx)) { + Found = true; + break; + } + } + if (!Found) + userLabels.push_back(llvm::make_unique(Label, DL, Idx)); + + return true; +} + bool LDVImpl::collectDebugValues(MachineFunction &mf) { bool Changed = false; for (MachineFunction::iterator MFI = mf.begin(), MFE = mf.end(); MFI != MFE; @@ -610,7 +694,8 @@ bool LDVImpl::collectDebugValues(MachineFunction &mf) { do { // Only handle DBG_VALUE in handleDebugValue(). Skip all other // kinds of debug instructions. - if (MBBI->isDebugValue() && handleDebugValue(*MBBI, Idx)) { + if ((MBBI->isDebugValue() && handleDebugValue(*MBBI, Idx)) || + (MBBI->isDebugLabel() && handleDebugLabel(*MBBI, Idx))) { MBBI = MBB->erase(MBBI); Changed = true; } else @@ -655,10 +740,8 @@ void UserValue::extendDef(SlotIndex Idx, DbgValueLocation Loc, LiveRange *LR, } // Limited by the next def. - if (I.valid() && I.start() < Stop) { + if (I.valid() && I.start() < Stop) Stop = I.start(); - ToEnd = false; - } // Limited by VNI's live range. else if (!ToEnd && Kills) Kills->push_back(Stop); @@ -826,8 +909,7 @@ void UserValue::computeIntervals(MachineRegisterInfo &MRI, ++I; // If the interval also overlaps the start of the "next" (i.e. - // current) range create a new interval for the remainder (which - // may be further trimmed). + // current) range create a new interval for the remainder if (RStart < IStop) I.insert(RStart, IStop, Loc); } @@ -837,13 +919,6 @@ void UserValue::computeIntervals(MachineRegisterInfo &MRI, if (!I.valid()) return; - if (I.start() < RStart) { - // Interval start overlaps range - trim to the scope range. - I.setStartUnchecked(RStart); - // Remember that this interval was trimmed. - trimmedDefs.insert(RStart); - } - // The end of a lexical scope range is the last instruction in the // range. To convert to an interval we need the index of the // instruction after it. @@ -1227,11 +1302,13 @@ void UserValue::insertDebugValue(MachineBasicBlock *MBB, SlotIndex StartIdx, // that the original virtual register was a pointer. Also, add the stack slot // offset for the spilled register to the expression. const DIExpression *Expr = Expression; + uint8_t DIExprFlags = DIExpression::ApplyOffset; bool IsIndirect = Loc.wasIndirect(); if (Spilled) { - auto Deref = IsIndirect ? DIExpression::WithDeref : DIExpression::NoDeref; + if (IsIndirect) + DIExprFlags |= DIExpression::DerefAfter; Expr = - DIExpression::prepend(Expr, DIExpression::NoDeref, SpillOffset, Deref); + DIExpression::prepend(Expr, DIExprFlags, SpillOffset); IsIndirect = true; } @@ -1247,6 +1324,15 @@ void UserValue::insertDebugValue(MachineBasicBlock *MBB, SlotIndex StartIdx, } while (I != MBB->end()); } +void UserLabel::insertDebugLabel(MachineBasicBlock *MBB, SlotIndex Idx, + LiveIntervals &LIS, + const TargetInstrInfo &TII) { + MachineBasicBlock::iterator I = findInsertLocation(MBB, Idx, LIS); + ++NumInsertedDebugLabels; + BuildMI(*MBB, I, getDebugLoc(), TII.get(TargetOpcode::DBG_LABEL)) + .addMetadata(Label); +} + void UserValue::emitDebugValues(VirtRegMap *VRM, LiveIntervals &LIS, const TargetInstrInfo &TII, const TargetRegisterInfo &TRI, @@ -1262,12 +1348,6 @@ void UserValue::emitDebugValues(VirtRegMap *VRM, LiveIntervals &LIS, bool Spilled = SpillIt != SpillOffsets.end(); unsigned SpillOffset = Spilled ? SpillIt->second : 0; - // If the interval start was trimmed to the lexical scope insert the - // DBG_VALUE at the previous index (otherwise it appears after the - // first instruction in the range). - if (trimmedDefs.count(Start)) - Start = Start.getPrevIndex(); - LLVM_DEBUG(dbgs() << "\t[" << Start << ';' << Stop << "):" << Loc.locNo()); MachineFunction::iterator MBB = LIS.getMBBFromIndex(Start)->getIterator(); SlotIndex MBBEnd = LIS.getMBBEndIdx(&*MBB); @@ -1295,16 +1375,31 @@ void UserValue::emitDebugValues(VirtRegMap *VRM, LiveIntervals &LIS, } } +void UserLabel::emitDebugLabel(LiveIntervals &LIS, const TargetInstrInfo &TII) { + LLVM_DEBUG(dbgs() << "\t" << loc); + MachineFunction::iterator MBB = LIS.getMBBFromIndex(loc)->getIterator(); + + LLVM_DEBUG(dbgs() << ' ' << printMBBReference(*MBB)); + insertDebugLabel(&*MBB, loc, LIS, TII); + + LLVM_DEBUG(dbgs() << '\n'); +} + void LDVImpl::emitDebugValues(VirtRegMap *VRM) { LLVM_DEBUG(dbgs() << "********** EMITTING LIVE DEBUG VARIABLES **********\n"); if (!MF) return; const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); SpillOffsetMap SpillOffsets; - for (unsigned i = 0, e = userValues.size(); i != e; ++i) { - LLVM_DEBUG(userValues[i]->print(dbgs(), TRI)); - userValues[i]->rewriteLocations(*VRM, *MF, *TII, *TRI, SpillOffsets); - userValues[i]->emitDebugValues(VRM, *LIS, *TII, *TRI, SpillOffsets); + for (auto &userValue : userValues) { + LLVM_DEBUG(userValue->print(dbgs(), TRI)); + userValue->rewriteLocations(*VRM, *MF, *TII, *TRI, SpillOffsets); + userValue->emitDebugValues(VRM, *LIS, *TII, *TRI, SpillOffsets); + } + LLVM_DEBUG(dbgs() << "********** EMITTING LIVE DEBUG LABELS **********\n"); + for (auto &userLabel : userLabels) { + LLVM_DEBUG(userLabel->print(dbgs(), TRI)); + userLabel->emitDebugLabel(*LIS, *TII); } EmitDone = true; } diff --git a/lib/CodeGen/LiveDebugVariables.h b/lib/CodeGen/LiveDebugVariables.h index 0060399c2b04..0cbe10c6a422 100644 --- a/lib/CodeGen/LiveDebugVariables.h +++ b/lib/CodeGen/LiveDebugVariables.h @@ -1,9 +1,8 @@ //===- LiveDebugVariables.h - Tracking debug info variables -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/LiveInterval.cpp b/lib/CodeGen/LiveInterval.cpp index 2340b6abd87c..70b2a77fe800 100644 --- a/lib/CodeGen/LiveInterval.cpp +++ b/lib/CodeGen/LiveInterval.cpp @@ -1,9 +1,8 @@ //===- LiveInterval.cpp - Live Interval Representation --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -297,9 +296,7 @@ private: iterator find(SlotIndex Pos) { return LR->find(Pos); } - iterator findInsertPos(Segment S) { - return std::upper_bound(LR->begin(), LR->end(), S.start); - } + iterator findInsertPos(Segment S) { return llvm::upper_bound(*LR, S.start); } }; //===----------------------------------------------------------------------===// @@ -880,8 +877,53 @@ void LiveInterval::clearSubRanges() { SubRanges = nullptr; } -void LiveInterval::refineSubRanges(BumpPtrAllocator &Allocator, - LaneBitmask LaneMask, std::function Apply) { +/// For each VNI in \p SR, check whether or not that value defines part +/// of the mask describe by \p LaneMask and if not, remove that value +/// from \p SR. +static void stripValuesNotDefiningMask(unsigned Reg, LiveInterval::SubRange &SR, + LaneBitmask LaneMask, + const SlotIndexes &Indexes, + const TargetRegisterInfo &TRI) { + // Phys reg should not be tracked at subreg level. + // Same for noreg (Reg == 0). + if (!TargetRegisterInfo::isVirtualRegister(Reg) || !Reg) + return; + // Remove the values that don't define those lanes. + SmallVector ToBeRemoved; + for (VNInfo *VNI : SR.valnos) { + if (VNI->isUnused()) + continue; + // PHI definitions don't have MI attached, so there is nothing + // we can use to strip the VNI. + if (VNI->isPHIDef()) + continue; + const MachineInstr *MI = Indexes.getInstructionFromIndex(VNI->def); + assert(MI && "Cannot find the definition of a value"); + bool hasDef = false; + for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) { + if (!MOI->isReg() || !MOI->isDef()) + continue; + if (MOI->getReg() != Reg) + continue; + if ((TRI.getSubRegIndexLaneMask(MOI->getSubReg()) & LaneMask).none()) + continue; + hasDef = true; + break; + } + + if (!hasDef) + ToBeRemoved.push_back(VNI); + } + for (VNInfo *VNI : ToBeRemoved) + SR.removeValNo(VNI); + + assert(!SR.empty() && "At least one value should be defined by this mask"); +} + +void LiveInterval::refineSubRanges( + BumpPtrAllocator &Allocator, LaneBitmask LaneMask, + std::function Apply, + const SlotIndexes &Indexes, const TargetRegisterInfo &TRI) { LaneBitmask ToApply = LaneMask; for (SubRange &SR : subranges()) { LaneBitmask SRMask = SR.LaneMask; @@ -899,6 +941,10 @@ void LiveInterval::refineSubRanges(BumpPtrAllocator &Allocator, SR.LaneMask = SRMask & ~Matching; // Create a new subrange for the matching part MatchingRange = createSubRangeFrom(Allocator, Matching, SR); + // Now that the subrange is split in half, make sure we + // only keep in the subranges the VNIs that touch the related half. + stripValuesNotDefiningMask(reg, *MatchingRange, Matching, Indexes, TRI); + stripValuesNotDefiningMask(reg, SR, SR.LaneMask, Indexes, TRI); } Apply(*MatchingRange); ToApply &= ~Matching; diff --git a/lib/CodeGen/LiveIntervalUnion.cpp b/lib/CodeGen/LiveIntervalUnion.cpp index 36428e0335f9..43fa8f2d7157 100644 --- a/lib/CodeGen/LiveIntervalUnion.cpp +++ b/lib/CodeGen/LiveIntervalUnion.cpp @@ -1,9 +1,8 @@ //===- LiveIntervalUnion.cpp - Live interval union data structure ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/LiveIntervals.cpp b/lib/CodeGen/LiveIntervals.cpp index 471775f8706b..aa85569063b3 100644 --- a/lib/CodeGen/LiveIntervals.cpp +++ b/lib/CodeGen/LiveIntervals.cpp @@ -1,9 +1,8 @@ //===- LiveIntervals.cpp - Live Interval Analysis -------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -901,8 +900,7 @@ bool LiveIntervals::checkRegMaskInterference(LiveInterval &LI, // We are going to enumerate all the register mask slots contained in LI. // Start with a binary search of RegMaskSlots to find a starting point. - ArrayRef::iterator SlotI = - std::lower_bound(Slots.begin(), Slots.end(), LiveI->start); + ArrayRef::iterator SlotI = llvm::lower_bound(Slots, LiveI->start); ArrayRef::iterator SlotE = Slots.end(); // No slots in range, LI begins after the last call. @@ -1371,8 +1369,7 @@ private: void updateRegMaskSlots() { SmallVectorImpl::iterator RI = - std::lower_bound(LIS.RegMaskSlots.begin(), LIS.RegMaskSlots.end(), - OldIdx); + llvm::lower_bound(LIS.RegMaskSlots, OldIdx); assert(RI != LIS.RegMaskSlots.end() && *RI == OldIdx.getRegSlot() && "No RegMask at OldIdx."); *RI = NewIdx.getRegSlot(); diff --git a/lib/CodeGen/LivePhysRegs.cpp b/lib/CodeGen/LivePhysRegs.cpp index 619643acb6d3..cd3d248ac878 100644 --- a/lib/CodeGen/LivePhysRegs.cpp +++ b/lib/CodeGen/LivePhysRegs.cpp @@ -1,9 +1,8 @@ //===--- LivePhysRegs.cpp - Live Physical Register Set --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/LiveRangeCalc.cpp b/lib/CodeGen/LiveRangeCalc.cpp index 70e135ab1aff..d670f28df6ba 100644 --- a/lib/CodeGen/LiveRangeCalc.cpp +++ b/lib/CodeGen/LiveRangeCalc.cpp @@ -1,9 +1,8 @@ //===- LiveRangeCalc.cpp - Calculate live ranges --------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -96,10 +95,11 @@ void LiveRangeCalc::calculate(LiveInterval &LI, bool TrackSubRegs) { } LI.refineSubRanges(*Alloc, SubMask, - [&MO, this](LiveInterval::SubRange &SR) { - if (MO.isDef()) - createDeadDef(*Indexes, *Alloc, SR, MO); - }); + [&MO, this](LiveInterval::SubRange &SR) { + if (MO.isDef()) + createDeadDef(*Indexes, *Alloc, SR, MO); + }, + *Indexes, TRI); } // Create the def in the main liverange. We do not have to do this if diff --git a/lib/CodeGen/LiveRangeCalc.h b/lib/CodeGen/LiveRangeCalc.h index 9f226b154a67..11aea5a3b016 100644 --- a/lib/CodeGen/LiveRangeCalc.h +++ b/lib/CodeGen/LiveRangeCalc.h @@ -1,9 +1,8 @@ //===- LiveRangeCalc.h - Calculate live ranges ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/LiveRangeEdit.cpp b/lib/CodeGen/LiveRangeEdit.cpp index 8dfe8b68c3af..882e562ba95c 100644 --- a/lib/CodeGen/LiveRangeEdit.cpp +++ b/lib/CodeGen/LiveRangeEdit.cpp @@ -1,9 +1,8 @@ //===-- LiveRangeEdit.cpp - Basic tools for editing a register live range -===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -232,6 +231,8 @@ bool LiveRangeEdit::foldAsLoad(LiveInterval *LI, return false; LLVM_DEBUG(dbgs() << " folded: " << *FoldMI); LIS.ReplaceMachineInstrInMaps(*UseMI, *FoldMI); + if (UseMI->isCall()) + UseMI->getMF()->updateCallSiteInfo(UseMI, FoldMI); UseMI->eraseFromParent(); DefMI->addRegisterDead(LI->reg, nullptr); Dead.push_back(DefMI); diff --git a/lib/CodeGen/LiveRangeShrink.cpp b/lib/CodeGen/LiveRangeShrink.cpp index f75d513c89f5..8818f1ce0ad9 100644 --- a/lib/CodeGen/LiveRangeShrink.cpp +++ b/lib/CodeGen/LiveRangeShrink.cpp @@ -1,9 +1,8 @@ //===- LiveRangeShrink.cpp - Move instructions to shrink live range -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // ///===---------------------------------------------------------------------===// /// diff --git a/lib/CodeGen/LiveRangeUtils.h b/lib/CodeGen/LiveRangeUtils.h index bd57609c3d84..0e6bfeb0d4a5 100644 --- a/lib/CodeGen/LiveRangeUtils.h +++ b/lib/CodeGen/LiveRangeUtils.h @@ -1,9 +1,8 @@ //===-- LiveRangeUtils.h - Live Range modification utilities ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/LiveRegMatrix.cpp b/lib/CodeGen/LiveRegMatrix.cpp index e72977b02675..ce99e5535c25 100644 --- a/lib/CodeGen/LiveRegMatrix.cpp +++ b/lib/CodeGen/LiveRegMatrix.cpp @@ -1,9 +1,8 @@ //===- LiveRegMatrix.cpp - Track register interference --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/LiveRegUnits.cpp b/lib/CodeGen/LiveRegUnits.cpp index c22681385492..6afb7fb7aa11 100644 --- a/lib/CodeGen/LiveRegUnits.cpp +++ b/lib/CodeGen/LiveRegUnits.cpp @@ -1,9 +1,8 @@ //===- LiveRegUnits.cpp - Register Unit Set -------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -126,13 +125,15 @@ void LiveRegUnits::addPristines(const MachineFunction &MF) { void LiveRegUnits::addLiveOuts(const MachineBasicBlock &MBB) { const MachineFunction &MF = *MBB.getParent(); - if (!MBB.succ_empty()) { - addPristines(MF); - // To get the live-outs we simply merge the live-ins of all successors. - for (const MachineBasicBlock *Succ : MBB.successors()) - addBlockLiveIns(*this, *Succ); - } else if (MBB.isReturnBlock()) { - // For the return block: Add all callee saved registers. + + addPristines(MF); + + // To get the live-outs we simply merge the live-ins of all successors. + for (const MachineBasicBlock *Succ : MBB.successors()) + addBlockLiveIns(*this, *Succ); + + // For the return block: Add all callee saved registers. + if (MBB.isReturnBlock()) { const MachineFrameInfo &MFI = MF.getFrameInfo(); if (MFI.isCalleeSavedInfoValid()) addCalleeSavedRegs(*this, MF); diff --git a/lib/CodeGen/LiveStacks.cpp b/lib/CodeGen/LiveStacks.cpp index 80ecfdb7a507..f55977d72723 100644 --- a/lib/CodeGen/LiveStacks.cpp +++ b/lib/CodeGen/LiveStacks.cpp @@ -1,9 +1,8 @@ //===-- LiveStacks.cpp - Live Stack Slot Analysis -------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/LiveVariables.cpp b/lib/CodeGen/LiveVariables.cpp index 0b92eab83806..aaff982ef1b0 100644 --- a/lib/CodeGen/LiveVariables.cpp +++ b/lib/CodeGen/LiveVariables.cpp @@ -1,9 +1,8 @@ //===-- LiveVariables.cpp - Live Variable Analysis for Machine Code -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -401,7 +400,7 @@ bool LiveVariables::HandlePhysRegKill(unsigned Reg, MachineInstr *MI) { true/*IsImp*/, true/*IsKill*/)); else { MachineOperand *MO = - LastRefOrPartRef->findRegisterDefOperand(Reg, false, TRI); + LastRefOrPartRef->findRegisterDefOperand(Reg, false, false, TRI); bool NeedEC = MO->isEarlyClobber() && MO->getReg() != Reg; // If the last reference is the last def, then it's not used at all. // That is, unless we are currently processing the last reference itself. diff --git a/lib/CodeGen/LocalStackSlotAllocation.cpp b/lib/CodeGen/LocalStackSlotAllocation.cpp index 795028e97929..b14d76a585f7 100644 --- a/lib/CodeGen/LocalStackSlotAllocation.cpp +++ b/lib/CodeGen/LocalStackSlotAllocation.cpp @@ -1,9 +1,8 @@ //===- LocalStackSlotAllocation.cpp - Pre-allocate locals to stack slots --===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -200,19 +199,27 @@ void LocalStackSlotPass::calculateFrameObjectOffsets(MachineFunction &Fn) { // Make sure that the stack protector comes before the local variables on the // stack. SmallSet ProtectedObjs; - if (MFI.getStackProtectorIndex() >= 0) { + if (MFI.hasStackProtectorIndex()) { + int StackProtectorFI = MFI.getStackProtectorIndex(); + + // We need to make sure we didn't pre-allocate the stack protector when + // doing this. + // If we already have a stack protector, this will re-assign it to a slot + // that is **not** covering the protected objects. + assert(!MFI.isObjectPreAllocated(StackProtectorFI) && + "Stack protector pre-allocated in LocalStackSlotAllocation"); + StackObjSet LargeArrayObjs; StackObjSet SmallArrayObjs; StackObjSet AddrOfObjs; - AdjustStackOffset(MFI, MFI.getStackProtectorIndex(), Offset, - StackGrowsDown, MaxAlign); + AdjustStackOffset(MFI, StackProtectorFI, Offset, StackGrowsDown, MaxAlign); // Assign large stack objects first. for (unsigned i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) { if (MFI.isDeadObjectIndex(i)) continue; - if (MFI.getStackProtectorIndex() == (int)i) + if (StackProtectorFI == (int)i) continue; switch (MFI.getObjectSSPLayout(i)) { diff --git a/lib/CodeGen/LoopTraversal.cpp b/lib/CodeGen/LoopTraversal.cpp index a02d10e09d7d..9490dfc40a82 100644 --- a/lib/CodeGen/LoopTraversal.cpp +++ b/lib/CodeGen/LoopTraversal.cpp @@ -1,9 +1,8 @@ //===- LoopTraversal.cpp - Optimal basic block traversal order --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/LowLevelType.cpp b/lib/CodeGen/LowLevelType.cpp index 1c682e72fa49..ca0daa14fedf 100644 --- a/lib/CodeGen/LowLevelType.cpp +++ b/lib/CodeGen/LowLevelType.cpp @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/LowLevelType.cpp -------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/LowerEmuTLS.cpp b/lib/CodeGen/LowerEmuTLS.cpp index 36c1d358a9bd..c8cf6abda4fc 100644 --- a/lib/CodeGen/LowerEmuTLS.cpp +++ b/lib/CodeGen/LowerEmuTLS.cpp @@ -1,9 +1,8 @@ //===- LowerEmuTLS.cpp - Add __emutls_[vt].* variables --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/MIRCanonicalizerPass.cpp b/lib/CodeGen/MIRCanonicalizerPass.cpp index f17c23619ed5..f49bc854e23f 100644 --- a/lib/CodeGen/MIRCanonicalizerPass.cpp +++ b/lib/CodeGen/MIRCanonicalizerPass.cpp @@ -1,9 +1,8 @@ //===-------------- MIRCanonicalizer.cpp - MIR Canonicalizer --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -105,6 +104,8 @@ INITIALIZE_PASS_END(MIRCanonicalizer, "mir-canonicalizer", "Rename Register Operands Canonically", false, false) static std::vector GetRPOList(MachineFunction &MF) { + if (MF.empty()) + return {}; ReversePostOrderTraversal RPOT(&*MF.begin()); std::vector RPOList; for (auto MBB : RPOT) { @@ -179,6 +180,8 @@ static bool rescheduleCanonically(unsigned &PseudoIdempotentInstCount, } std::map> MultiUsers; + std::map MultiUserLookup; + unsigned UseToBringDefCloserToCount = 0; std::vector PseudoIdempotentInstructions; std::vector PhysRegDefs; for (auto *II : Instructions) { @@ -254,6 +257,7 @@ static bool rescheduleCanonically(unsigned &PseudoIdempotentInstCount, if (Delta < Distance) { Distance = Delta; UseToBringDefCloserTo = UseInst; + MultiUserLookup[UseToBringDefCloserToCount++] = UseToBringDefCloserTo; } } @@ -293,11 +297,11 @@ static bool rescheduleCanonically(unsigned &PseudoIdempotentInstCount, } // Sort the defs for users of multiple defs lexographically. - for (const auto &E : MultiUsers) { + for (const auto &E : MultiUserLookup) { auto UseI = std::find_if(MBB->instr_begin(), MBB->instr_end(), - [&](MachineInstr &MI) -> bool { return &MI == E.first; }); + [&](MachineInstr &MI) -> bool { return &MI == E.second; }); if (UseI == MBB->instr_end()) continue; @@ -305,7 +309,8 @@ static bool rescheduleCanonically(unsigned &PseudoIdempotentInstCount, LLVM_DEBUG( dbgs() << "Rescheduling Multi-Use Instructions Lexographically.";); Changed |= rescheduleLexographically( - E.second, MBB, [&]() -> MachineBasicBlock::iterator { return UseI; }); + MultiUsers[E.second], MBB, + [&]() -> MachineBasicBlock::iterator { return UseI; }); } PseudoIdempotentInstCount = PseudoIdempotentInstructions.size(); @@ -342,15 +347,23 @@ static bool propagateLocalCopies(MachineBasicBlock *MBB) { continue; if (!TargetRegisterInfo::isVirtualRegister(Src)) continue; + // Not folding COPY instructions if regbankselect has not set the RCs. + // Why are we only considering Register Classes? Because the verifier + // sometimes gets upset if the register classes don't match even if the + // types do. A future patch might add COPY folding for matching types in + // pre-registerbankselect code. + if (!MRI.getRegClassOrNull(Dst)) + continue; if (MRI.getRegClass(Dst) != MRI.getRegClass(Src)) continue; - for (auto UI = MRI.use_begin(Dst); UI != MRI.use_end(); ++UI) { - MachineOperand *MO = &*UI; + std::vector Uses; + for (auto UI = MRI.use_begin(Dst); UI != MRI.use_end(); ++UI) + Uses.push_back(&*UI); + for (auto *MO : Uses) MO->setReg(Src); - Changed = true; - } + Changed = true; MI->eraseFromParent(); } @@ -474,18 +487,14 @@ class NamedVRegCursor { unsigned virtualVRegNumber; public: - NamedVRegCursor(MachineRegisterInfo &MRI) : MRI(MRI) { - unsigned VRegGapIndex = 0; - const unsigned VR_GAP = (++VRegGapIndex * 1000); - - unsigned I = MRI.createIncompleteVirtualRegister(); - const unsigned E = (((I + VR_GAP) / VR_GAP) + 1) * VR_GAP; - - virtualVRegNumber = E; - } + NamedVRegCursor(MachineRegisterInfo &MRI) : MRI(MRI), virtualVRegNumber(0) {} void SkipVRegs() { unsigned VRegGapIndex = 1; + if (!virtualVRegNumber) { + VRegGapIndex = 0; + virtualVRegNumber = MRI.createIncompleteVirtualRegister(); + } const unsigned VR_GAP = (++VRegGapIndex * 1000); unsigned I = virtualVRegNumber; @@ -501,14 +510,17 @@ public: return virtualVRegNumber; } - unsigned createVirtualRegister(const TargetRegisterClass *RC) { + unsigned createVirtualRegister(unsigned VReg) { + if (!virtualVRegNumber) + SkipVRegs(); std::string S; raw_string_ostream OS(S); OS << "namedVReg" << (virtualVRegNumber & ~0x80000000); OS.flush(); virtualVRegNumber++; - - return MRI.createVirtualRegister(RC, OS.str()); + if (auto RC = MRI.getRegClassOrNull(VReg)) + return MRI.createVirtualRegister(RC, OS.str()); + return MRI.createGenericVirtualRegister(MRI.getType(VReg), OS.str()); } }; } // namespace @@ -558,7 +570,7 @@ GetVRegRenameMap(const std::vector &VRegs, continue; } - auto Rename = NVC.createVirtualRegister(MRI.getRegClass(Reg)); + auto Rename = NVC.createVirtualRegister(Reg); if (VRegRenameMap.find(Reg) == VRegRenameMap.end()) { LLVM_DEBUG(dbgs() << "Mapping vreg ";); @@ -735,14 +747,15 @@ static bool runOnBasicBlock(MachineBasicBlock *MBB, // of the MachineBasicBlock so that they are named in the order that we sorted // them alphabetically. Eventually we wont need SkipVRegs because we will use // named vregs instead. - NVC.SkipVRegs(); + if (IdempotentInstCount) + NVC.SkipVRegs(); auto MII = MBB->begin(); for (unsigned i = 0; i < IdempotentInstCount && MII != MBB->end(); ++i) { MachineInstr &MI = *MII++; Changed = true; unsigned vRegToRename = MI.getOperand(0).getReg(); - auto Rename = NVC.createVirtualRegister(MRI.getRegClass(vRegToRename)); + auto Rename = NVC.createVirtualRegister(vRegToRename); std::vector RenameMOs; for (auto &MO : MRI.reg_operands(vRegToRename)) { diff --git a/lib/CodeGen/MIRParser/MILexer.cpp b/lib/CodeGen/MIRParser/MILexer.cpp index 265877c2f5b4..4899bd3f5811 100644 --- a/lib/CodeGen/MIRParser/MILexer.cpp +++ b/lib/CodeGen/MIRParser/MILexer.cpp @@ -1,9 +1,8 @@ //===- MILexer.cpp - Machine instructions lexer implementation ------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -205,6 +204,7 @@ static MIToken::TokenKind getIdentifierKind(StringRef Identifier) { .Case("nuw" , MIToken::kw_nuw) .Case("nsw" , MIToken::kw_nsw) .Case("exact" , MIToken::kw_exact) + .Case("fpexcept", MIToken::kw_fpexcept) .Case("debug-location", MIToken::kw_debug_location) .Case("same_value", MIToken::kw_cfi_same_value) .Case("offset", MIToken::kw_cfi_offset) diff --git a/lib/CodeGen/MIRParser/MILexer.h b/lib/CodeGen/MIRParser/MILexer.h index ceff79087d81..0fe3f9f706db 100644 --- a/lib/CodeGen/MIRParser/MILexer.h +++ b/lib/CodeGen/MIRParser/MILexer.h @@ -1,9 +1,8 @@ //===- MILexer.h - Lexer for machine instructions ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -74,6 +73,7 @@ struct MIToken { kw_nuw, kw_nsw, kw_exact, + kw_fpexcept, kw_debug_location, kw_cfi_same_value, kw_cfi_offset, diff --git a/lib/CodeGen/MIRParser/MIParser.cpp b/lib/CodeGen/MIRParser/MIParser.cpp index 6f2d8bb53ac8..c0b800a0b870 100644 --- a/lib/CodeGen/MIRParser/MIParser.cpp +++ b/lib/CodeGen/MIRParser/MIParser.cpp @@ -1,9 +1,8 @@ //===- MIParser.cpp - Machine instructions parser implementation ----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -11,7 +10,7 @@ // //===----------------------------------------------------------------------===// -#include "MIParser.h" +#include "llvm/CodeGen/MIRParser/MIParser.h" #include "MILexer.h" #include "llvm/ADT/APInt.h" #include "llvm/ADT/APSInt.h" @@ -27,6 +26,8 @@ #include "llvm/Analysis/MemoryLocation.h" #include "llvm/AsmParser/Parser.h" #include "llvm/AsmParser/SlotMapping.h" +#include "llvm/CodeGen/GlobalISel/RegisterBank.h" +#include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h" #include "llvm/CodeGen/MIRPrinter.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFrameInfo.h" @@ -81,12 +82,242 @@ using namespace llvm; +void PerTargetMIParsingState::setTarget( + const TargetSubtargetInfo &NewSubtarget) { + + // If the subtarget changed, over conservatively assume everything is invalid. + if (&Subtarget == &NewSubtarget) + return; + + Names2InstrOpCodes.clear(); + Names2Regs.clear(); + Names2RegMasks.clear(); + Names2SubRegIndices.clear(); + Names2TargetIndices.clear(); + Names2DirectTargetFlags.clear(); + Names2BitmaskTargetFlags.clear(); + Names2MMOTargetFlags.clear(); + + initNames2RegClasses(); + initNames2RegBanks(); +} + +void PerTargetMIParsingState::initNames2Regs() { + if (!Names2Regs.empty()) + return; + + // The '%noreg' register is the register 0. + Names2Regs.insert(std::make_pair("noreg", 0)); + const auto *TRI = Subtarget.getRegisterInfo(); + assert(TRI && "Expected target register info"); + + for (unsigned I = 0, E = TRI->getNumRegs(); I < E; ++I) { + bool WasInserted = + Names2Regs.insert(std::make_pair(StringRef(TRI->getName(I)).lower(), I)) + .second; + (void)WasInserted; + assert(WasInserted && "Expected registers to be unique case-insensitively"); + } +} + +bool PerTargetMIParsingState::getRegisterByName(StringRef RegName, + unsigned &Reg) { + initNames2Regs(); + auto RegInfo = Names2Regs.find(RegName); + if (RegInfo == Names2Regs.end()) + return true; + Reg = RegInfo->getValue(); + return false; +} + +void PerTargetMIParsingState::initNames2InstrOpCodes() { + if (!Names2InstrOpCodes.empty()) + return; + const auto *TII = Subtarget.getInstrInfo(); + assert(TII && "Expected target instruction info"); + for (unsigned I = 0, E = TII->getNumOpcodes(); I < E; ++I) + Names2InstrOpCodes.insert(std::make_pair(StringRef(TII->getName(I)), I)); +} + +bool PerTargetMIParsingState::parseInstrName(StringRef InstrName, + unsigned &OpCode) { + initNames2InstrOpCodes(); + auto InstrInfo = Names2InstrOpCodes.find(InstrName); + if (InstrInfo == Names2InstrOpCodes.end()) + return true; + OpCode = InstrInfo->getValue(); + return false; +} + +void PerTargetMIParsingState::initNames2RegMasks() { + if (!Names2RegMasks.empty()) + return; + const auto *TRI = Subtarget.getRegisterInfo(); + assert(TRI && "Expected target register info"); + ArrayRef RegMasks = TRI->getRegMasks(); + ArrayRef RegMaskNames = TRI->getRegMaskNames(); + assert(RegMasks.size() == RegMaskNames.size()); + for (size_t I = 0, E = RegMasks.size(); I < E; ++I) + Names2RegMasks.insert( + std::make_pair(StringRef(RegMaskNames[I]).lower(), RegMasks[I])); +} + +const uint32_t *PerTargetMIParsingState::getRegMask(StringRef Identifier) { + initNames2RegMasks(); + auto RegMaskInfo = Names2RegMasks.find(Identifier); + if (RegMaskInfo == Names2RegMasks.end()) + return nullptr; + return RegMaskInfo->getValue(); +} + +void PerTargetMIParsingState::initNames2SubRegIndices() { + if (!Names2SubRegIndices.empty()) + return; + const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); + for (unsigned I = 1, E = TRI->getNumSubRegIndices(); I < E; ++I) + Names2SubRegIndices.insert( + std::make_pair(TRI->getSubRegIndexName(I), I)); +} + +unsigned PerTargetMIParsingState::getSubRegIndex(StringRef Name) { + initNames2SubRegIndices(); + auto SubRegInfo = Names2SubRegIndices.find(Name); + if (SubRegInfo == Names2SubRegIndices.end()) + return 0; + return SubRegInfo->getValue(); +} + +void PerTargetMIParsingState::initNames2TargetIndices() { + if (!Names2TargetIndices.empty()) + return; + const auto *TII = Subtarget.getInstrInfo(); + assert(TII && "Expected target instruction info"); + auto Indices = TII->getSerializableTargetIndices(); + for (const auto &I : Indices) + Names2TargetIndices.insert(std::make_pair(StringRef(I.second), I.first)); +} + +bool PerTargetMIParsingState::getTargetIndex(StringRef Name, int &Index) { + initNames2TargetIndices(); + auto IndexInfo = Names2TargetIndices.find(Name); + if (IndexInfo == Names2TargetIndices.end()) + return true; + Index = IndexInfo->second; + return false; +} + +void PerTargetMIParsingState::initNames2DirectTargetFlags() { + if (!Names2DirectTargetFlags.empty()) + return; + + const auto *TII = Subtarget.getInstrInfo(); + assert(TII && "Expected target instruction info"); + auto Flags = TII->getSerializableDirectMachineOperandTargetFlags(); + for (const auto &I : Flags) + Names2DirectTargetFlags.insert( + std::make_pair(StringRef(I.second), I.first)); +} + +bool PerTargetMIParsingState::getDirectTargetFlag(StringRef Name, + unsigned &Flag) { + initNames2DirectTargetFlags(); + auto FlagInfo = Names2DirectTargetFlags.find(Name); + if (FlagInfo == Names2DirectTargetFlags.end()) + return true; + Flag = FlagInfo->second; + return false; +} + +void PerTargetMIParsingState::initNames2BitmaskTargetFlags() { + if (!Names2BitmaskTargetFlags.empty()) + return; + + const auto *TII = Subtarget.getInstrInfo(); + assert(TII && "Expected target instruction info"); + auto Flags = TII->getSerializableBitmaskMachineOperandTargetFlags(); + for (const auto &I : Flags) + Names2BitmaskTargetFlags.insert( + std::make_pair(StringRef(I.second), I.first)); +} + +bool PerTargetMIParsingState::getBitmaskTargetFlag(StringRef Name, + unsigned &Flag) { + initNames2BitmaskTargetFlags(); + auto FlagInfo = Names2BitmaskTargetFlags.find(Name); + if (FlagInfo == Names2BitmaskTargetFlags.end()) + return true; + Flag = FlagInfo->second; + return false; +} + +void PerTargetMIParsingState::initNames2MMOTargetFlags() { + if (!Names2MMOTargetFlags.empty()) + return; + + const auto *TII = Subtarget.getInstrInfo(); + assert(TII && "Expected target instruction info"); + auto Flags = TII->getSerializableMachineMemOperandTargetFlags(); + for (const auto &I : Flags) + Names2MMOTargetFlags.insert(std::make_pair(StringRef(I.second), I.first)); +} + +bool PerTargetMIParsingState::getMMOTargetFlag(StringRef Name, + MachineMemOperand::Flags &Flag) { + initNames2MMOTargetFlags(); + auto FlagInfo = Names2MMOTargetFlags.find(Name); + if (FlagInfo == Names2MMOTargetFlags.end()) + return true; + Flag = FlagInfo->second; + return false; +} + +void PerTargetMIParsingState::initNames2RegClasses() { + if (!Names2RegClasses.empty()) + return; + + const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo(); + for (unsigned I = 0, E = TRI->getNumRegClasses(); I < E; ++I) { + const auto *RC = TRI->getRegClass(I); + Names2RegClasses.insert( + std::make_pair(StringRef(TRI->getRegClassName(RC)).lower(), RC)); + } +} + +void PerTargetMIParsingState::initNames2RegBanks() { + if (!Names2RegBanks.empty()) + return; + + const RegisterBankInfo *RBI = Subtarget.getRegBankInfo(); + // If the target does not support GlobalISel, we may not have a + // register bank info. + if (!RBI) + return; + + for (unsigned I = 0, E = RBI->getNumRegBanks(); I < E; ++I) { + const auto &RegBank = RBI->getRegBank(I); + Names2RegBanks.insert( + std::make_pair(StringRef(RegBank.getName()).lower(), &RegBank)); + } +} + +const TargetRegisterClass * +PerTargetMIParsingState::getRegClass(StringRef Name) { + auto RegClassInfo = Names2RegClasses.find(Name); + if (RegClassInfo == Names2RegClasses.end()) + return nullptr; + return RegClassInfo->getValue(); +} + +const RegisterBank *PerTargetMIParsingState::getRegBank(StringRef Name) { + auto RegBankInfo = Names2RegBanks.find(Name); + if (RegBankInfo == Names2RegBanks.end()) + return nullptr; + return RegBankInfo->getValue(); +} + PerFunctionMIParsingState::PerFunctionMIParsingState(MachineFunction &MF, - SourceMgr &SM, const SlotMapping &IRSlots, - const Name2RegClassMap &Names2RegClasses, - const Name2RegBankMap &Names2RegBanks) - : MF(MF), SM(&SM), IRSlots(IRSlots), Names2RegClasses(Names2RegClasses), - Names2RegBanks(Names2RegBanks) { + SourceMgr &SM, const SlotMapping &IRSlots, PerTargetMIParsingState &T) + : MF(MF), SM(&SM), IRSlots(IRSlots), Target(T) { } VRegInfo &PerFunctionMIParsingState::getVRegInfo(unsigned Num) { @@ -137,26 +368,10 @@ class MIParser { StringRef Source, CurrentSource; MIToken Token; PerFunctionMIParsingState &PFS; - /// Maps from instruction names to op codes. - StringMap Names2InstrOpCodes; - /// Maps from register names to registers. - StringMap Names2Regs; - /// Maps from register mask names to register masks. - StringMap Names2RegMasks; - /// Maps from subregister names to subregister indices. - StringMap Names2SubRegIndices; /// Maps from slot numbers to function's unnamed basic blocks. DenseMap Slots2BasicBlocks; /// Maps from slot numbers to function's unnamed values. DenseMap Slots2Values; - /// Maps from target index names to target indices. - StringMap Names2TargetIndices; - /// Maps from direct target flag names to the direct target flag values. - StringMap Names2DirectTargetFlags; - /// Maps from direct target flag names to the bitmask target flag values. - StringMap Names2BitmaskTargetFlags; - /// Maps from MMO target flag names to MMO target flag values. - StringMap Names2MMOTargetFlags; public: MIParser(PerFunctionMIParsingState &PFS, SMDiagnostic &Error, @@ -281,12 +496,6 @@ private: /// Otherwise return false. bool consumeIfPresent(MIToken::TokenKind TokenKind); - void initNames2InstrOpCodes(); - - /// Try to convert an instruction name to an opcode. Return true if the - /// instruction name is invalid. - bool parseInstrName(StringRef InstrName, unsigned &OpCode); - bool parseInstruction(unsigned &OpCode, unsigned &Flags); bool assignRegisterTies(MachineInstr &MI, @@ -295,62 +504,11 @@ private: bool verifyImplicitOperands(ArrayRef Operands, const MCInstrDesc &MCID); - void initNames2Regs(); - - /// Try to convert a register name to a register number. Return true if the - /// register name is invalid. - bool getRegisterByName(StringRef RegName, unsigned &Reg); - - void initNames2RegMasks(); - - /// Check if the given identifier is a name of a register mask. - /// - /// Return null if the identifier isn't a register mask. - const uint32_t *getRegMask(StringRef Identifier); - - void initNames2SubRegIndices(); - - /// Check if the given identifier is a name of a subregister index. - /// - /// Return 0 if the name isn't a subregister index class. - unsigned getSubRegIndex(StringRef Name); - const BasicBlock *getIRBlock(unsigned Slot); const BasicBlock *getIRBlock(unsigned Slot, const Function &F); const Value *getIRValue(unsigned Slot); - void initNames2TargetIndices(); - - /// Try to convert a name of target index to the corresponding target index. - /// - /// Return true if the name isn't a name of a target index. - bool getTargetIndex(StringRef Name, int &Index); - - void initNames2DirectTargetFlags(); - - /// Try to convert a name of a direct target flag to the corresponding - /// target flag. - /// - /// Return true if the name isn't a name of a direct flag. - bool getDirectTargetFlag(StringRef Name, unsigned &Flag); - - void initNames2BitmaskTargetFlags(); - - /// Try to convert a name of a bitmask target flag to the corresponding - /// target flag. - /// - /// Return true if the name isn't a name of a bitmask target flag. - bool getBitmaskTargetFlag(StringRef Name, unsigned &Flag); - - void initNames2MMOTargetFlags(); - - /// Try to convert a name of a MachineMemOperand target flag to the - /// corresponding target flag. - /// - /// Return true if the name isn't a name of a target MMO flag. - bool getMMOTargetFlag(StringRef Name, MachineMemOperand::Flags &Flag); - /// Get or create an MCSymbol for a given name. MCSymbol *getOrCreateMCSymbol(StringRef Name); @@ -978,7 +1136,8 @@ bool MIParser::parseInstruction(unsigned &OpCode, unsigned &Flags) { Token.is(MIToken::kw_reassoc) || Token.is(MIToken::kw_nuw) || Token.is(MIToken::kw_nsw) || - Token.is(MIToken::kw_exact)) { + Token.is(MIToken::kw_exact) || + Token.is(MIToken::kw_fpexcept)) { // Mine frame and fast math flags if (Token.is(MIToken::kw_frame_setup)) Flags |= MachineInstr::FrameSetup; @@ -1004,13 +1163,15 @@ bool MIParser::parseInstruction(unsigned &OpCode, unsigned &Flags) { Flags |= MachineInstr::NoSWrap; if (Token.is(MIToken::kw_exact)) Flags |= MachineInstr::IsExact; + if (Token.is(MIToken::kw_fpexcept)) + Flags |= MachineInstr::FPExcept; lex(); } if (Token.isNot(MIToken::Identifier)) return error("expected a machine instruction"); StringRef InstrName = Token.stringValue(); - if (parseInstrName(InstrName, OpCode)) + if (PFS.Target.parseInstrName(InstrName, OpCode)) return error(Twine("unknown machine instruction name '") + InstrName + "'"); lex(); return false; @@ -1019,7 +1180,7 @@ bool MIParser::parseInstruction(unsigned &OpCode, unsigned &Flags) { bool MIParser::parseNamedRegister(unsigned &Reg) { assert(Token.is(MIToken::NamedRegister) && "Needs NamedRegister token"); StringRef Name = Token.stringValue(); - if (getRegisterByName(Name, Reg)) + if (PFS.Target.getRegisterByName(Name, Reg)) return error(Twine("unknown register name '") + Name + "'"); return false; } @@ -1070,21 +1231,20 @@ bool MIParser::parseRegisterClassOrBank(VRegInfo &RegInfo) { StringRef Name = Token.stringValue(); // Was it a register class? - auto RCNameI = PFS.Names2RegClasses.find(Name); - if (RCNameI != PFS.Names2RegClasses.end()) { + const TargetRegisterClass *RC = PFS.Target.getRegClass(Name); + if (RC) { lex(); - const TargetRegisterClass &RC = *RCNameI->getValue(); switch (RegInfo.Kind) { case VRegInfo::UNKNOWN: case VRegInfo::NORMAL: RegInfo.Kind = VRegInfo::NORMAL; - if (RegInfo.Explicit && RegInfo.D.RC != &RC) { + if (RegInfo.Explicit && RegInfo.D.RC != RC) { const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); return error(Loc, Twine("conflicting register classes, previously: ") + Twine(TRI.getRegClassName(RegInfo.D.RC))); } - RegInfo.D.RC = &RC; + RegInfo.D.RC = RC; RegInfo.Explicit = true; return false; @@ -1098,10 +1258,9 @@ bool MIParser::parseRegisterClassOrBank(VRegInfo &RegInfo) { // Should be a register bank or a generic register. const RegisterBank *RegBank = nullptr; if (Name != "_") { - auto RBNameI = PFS.Names2RegBanks.find(Name); - if (RBNameI == PFS.Names2RegBanks.end()) + RegBank = PFS.Target.getRegBank(Name); + if (!RegBank) return error(Loc, "expected '_', register class, or register bank name"); - RegBank = RBNameI->getValue(); } lex(); @@ -1173,7 +1332,7 @@ bool MIParser::parseSubRegisterIndex(unsigned &SubReg) { if (Token.isNot(MIToken::Identifier)) return error("expected a subregister index after '.'"); auto Name = Token.stringValue(); - SubReg = getSubRegIndex(Name); + SubReg = PFS.Target.getSubRegIndex(Name); if (!SubReg) return error(Twine("use of unknown subregister index '") + Name + "'"); lex(); @@ -1341,6 +1500,19 @@ bool MIParser::parseIRConstant(StringRef::iterator Loc, const Constant *&C) { return false; } +// See LLT implemntation for bit size limits. +static bool verifyScalarSize(uint64_t Size) { + return Size != 0 && isUInt<16>(Size); +} + +static bool verifyVectorElementCount(uint64_t NumElts) { + return NumElts != 0 && isUInt<16>(NumElts); +} + +static bool verifyAddrSpace(uint64_t AddrSpace) { + return isUInt<24>(AddrSpace); +} + bool MIParser::parseLowLevelType(StringRef::iterator Loc, LLT &Ty) { if (Token.range().front() == 's' || Token.range().front() == 'p') { StringRef SizeStr = Token.range().drop_front(); @@ -1349,12 +1521,19 @@ bool MIParser::parseLowLevelType(StringRef::iterator Loc, LLT &Ty) { } if (Token.range().front() == 's') { - Ty = LLT::scalar(APSInt(Token.range().drop_front()).getZExtValue()); + auto ScalarSize = APSInt(Token.range().drop_front()).getZExtValue(); + if (!verifyScalarSize(ScalarSize)) + return error("invalid size for scalar type"); + + Ty = LLT::scalar(ScalarSize); lex(); return false; } else if (Token.range().front() == 'p') { const DataLayout &DL = MF.getDataLayout(); - unsigned AS = APSInt(Token.range().drop_front()).getZExtValue(); + uint64_t AS = APSInt(Token.range().drop_front()).getZExtValue(); + if (!verifyAddrSpace(AS)) + return error("invalid address space number"); + Ty = LLT::pointer(AS, DL.getPointerSizeInBits(AS)); lex(); return false; @@ -1369,6 +1548,9 @@ bool MIParser::parseLowLevelType(StringRef::iterator Loc, LLT &Ty) { if (Token.isNot(MIToken::IntegerLiteral)) return error(Loc, "expected or for vector type"); uint64_t NumElements = Token.integerValue().getZExtValue(); + if (!verifyVectorElementCount(NumElements)) + return error("invalid number of vector elements"); + lex(); if (Token.isNot(MIToken::Identifier) || Token.stringValue() != "x") @@ -1381,11 +1563,17 @@ bool MIParser::parseLowLevelType(StringRef::iterator Loc, LLT &Ty) { if (SizeStr.size() == 0 || !llvm::all_of(SizeStr, isdigit)) return error("expected integers after 's'/'p' type character"); - if (Token.range().front() == 's') - Ty = LLT::scalar(APSInt(Token.range().drop_front()).getZExtValue()); - else if (Token.range().front() == 'p') { + if (Token.range().front() == 's') { + auto ScalarSize = APSInt(Token.range().drop_front()).getZExtValue(); + if (!verifyScalarSize(ScalarSize)) + return error("invalid size for scalar type"); + Ty = LLT::scalar(ScalarSize); + } else if (Token.range().front() == 'p') { const DataLayout &DL = MF.getDataLayout(); - unsigned AS = APSInt(Token.range().drop_front()).getZExtValue(); + uint64_t AS = APSInt(Token.range().drop_front()).getZExtValue(); + if (!verifyAddrSpace(AS)) + return error("invalid address space number"); + Ty = LLT::pointer(AS, DL.getPointerSizeInBits(AS)); } else return error(Loc, "expected or for vector type"); @@ -1625,7 +1813,7 @@ bool MIParser::parseMCSymbolOperand(MachineOperand &Dest) { bool MIParser::parseSubRegisterIndexOperand(MachineOperand &Dest) { assert(Token.is(MIToken::SubRegisterIndex)); StringRef Name = Token.stringValue(); - unsigned SubRegIndex = getSubRegIndex(Token.stringValue()); + unsigned SubRegIndex = PFS.Target.getSubRegIndex(Token.stringValue()); if (SubRegIndex == 0) return error(Twine("unknown subregister index '") + Name + "'"); lex(); @@ -1669,6 +1857,11 @@ bool MIParser::parseDIExpression(MDNode *&Expr) { Elements.push_back(Op); continue; } + if (unsigned Enc = dwarf::getAttributeEncoding(Token.stringValue())) { + lex(); + Elements.push_back(Enc); + continue; + } return error(Twine("invalid DWARF op '") + Token.stringValue() + "'"); } @@ -2100,7 +2293,7 @@ bool MIParser::parseTargetIndexOperand(MachineOperand &Dest) { if (Token.isNot(MIToken::Identifier)) return error("expected the name of the target index"); int Index = 0; - if (getTargetIndex(Token.stringValue(), Index)) + if (PFS.Target.getTargetIndex(Token.stringValue(), Index)) return error("use of undefined target index '" + Token.stringValue() + "'"); lex(); if (expectAndConsume(MIToken::rparen)) @@ -2242,7 +2435,7 @@ bool MIParser::parseMachineOperand(MachineOperand &Dest, case MIToken::Error: return true; case MIToken::Identifier: - if (const auto *RegMask = getRegMask(Token.stringValue())) { + if (const auto *RegMask = PFS.Target.getRegMask(Token.stringValue())) { Dest = MachineOperand::CreateRegMask(RegMask); lex(); break; @@ -2268,8 +2461,8 @@ bool MIParser::parseMachineOperandAndTargetFlags( return true; if (Token.isNot(MIToken::Identifier)) return error("expected the name of the target flag"); - if (getDirectTargetFlag(Token.stringValue(), TF)) { - if (getBitmaskTargetFlag(Token.stringValue(), TF)) + if (PFS.Target.getDirectTargetFlag(Token.stringValue(), TF)) { + if (PFS.Target.getBitmaskTargetFlag(Token.stringValue(), TF)) return error("use of undefined target flag '" + Token.stringValue() + "'"); } @@ -2279,7 +2472,7 @@ bool MIParser::parseMachineOperandAndTargetFlags( if (Token.isNot(MIToken::Identifier)) return error("expected the name of the target flag"); unsigned BitFlag = 0; - if (getBitmaskTargetFlag(Token.stringValue(), BitFlag)) + if (PFS.Target.getBitmaskTargetFlag(Token.stringValue(), BitFlag)) return error("use of undefined target flag '" + Token.stringValue() + "'"); // TODO: Report an error when using a duplicate bit target flag. @@ -2325,6 +2518,10 @@ bool MIParser::parseAlignment(unsigned &Alignment) { if (getUnsigned(Alignment)) return true; lex(); + + if (!isPowerOf2_32(Alignment)) + return error("expected a power-of-2 literal after 'align'"); + return false; } @@ -2436,7 +2633,7 @@ bool MIParser::parseMemoryOperandFlag(MachineMemOperand::Flags &Flags) { break; case MIToken::StringConstant: { MachineMemOperand::Flags TF; - if (getMMOTargetFlag(Token.stringValue(), TF)) + if (PFS.Target.getMMOTargetFlag(Token.stringValue(), TF)) return error("use of undefined target MMO flag '" + Token.stringValue() + "'"); Flags |= TF; @@ -2711,87 +2908,6 @@ bool MIParser::parsePreOrPostInstrSymbol(MCSymbol *&Symbol) { return false; } -void MIParser::initNames2InstrOpCodes() { - if (!Names2InstrOpCodes.empty()) - return; - const auto *TII = MF.getSubtarget().getInstrInfo(); - assert(TII && "Expected target instruction info"); - for (unsigned I = 0, E = TII->getNumOpcodes(); I < E; ++I) - Names2InstrOpCodes.insert(std::make_pair(StringRef(TII->getName(I)), I)); -} - -bool MIParser::parseInstrName(StringRef InstrName, unsigned &OpCode) { - initNames2InstrOpCodes(); - auto InstrInfo = Names2InstrOpCodes.find(InstrName); - if (InstrInfo == Names2InstrOpCodes.end()) - return true; - OpCode = InstrInfo->getValue(); - return false; -} - -void MIParser::initNames2Regs() { - if (!Names2Regs.empty()) - return; - // The '%noreg' register is the register 0. - Names2Regs.insert(std::make_pair("noreg", 0)); - const auto *TRI = MF.getSubtarget().getRegisterInfo(); - assert(TRI && "Expected target register info"); - for (unsigned I = 0, E = TRI->getNumRegs(); I < E; ++I) { - bool WasInserted = - Names2Regs.insert(std::make_pair(StringRef(TRI->getName(I)).lower(), I)) - .second; - (void)WasInserted; - assert(WasInserted && "Expected registers to be unique case-insensitively"); - } -} - -bool MIParser::getRegisterByName(StringRef RegName, unsigned &Reg) { - initNames2Regs(); - auto RegInfo = Names2Regs.find(RegName); - if (RegInfo == Names2Regs.end()) - return true; - Reg = RegInfo->getValue(); - return false; -} - -void MIParser::initNames2RegMasks() { - if (!Names2RegMasks.empty()) - return; - const auto *TRI = MF.getSubtarget().getRegisterInfo(); - assert(TRI && "Expected target register info"); - ArrayRef RegMasks = TRI->getRegMasks(); - ArrayRef RegMaskNames = TRI->getRegMaskNames(); - assert(RegMasks.size() == RegMaskNames.size()); - for (size_t I = 0, E = RegMasks.size(); I < E; ++I) - Names2RegMasks.insert( - std::make_pair(StringRef(RegMaskNames[I]).lower(), RegMasks[I])); -} - -const uint32_t *MIParser::getRegMask(StringRef Identifier) { - initNames2RegMasks(); - auto RegMaskInfo = Names2RegMasks.find(Identifier); - if (RegMaskInfo == Names2RegMasks.end()) - return nullptr; - return RegMaskInfo->getValue(); -} - -void MIParser::initNames2SubRegIndices() { - if (!Names2SubRegIndices.empty()) - return; - const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); - for (unsigned I = 1, E = TRI->getNumSubRegIndices(); I < E; ++I) - Names2SubRegIndices.insert( - std::make_pair(StringRef(TRI->getSubRegIndexName(I)).lower(), I)); -} - -unsigned MIParser::getSubRegIndex(StringRef Name) { - initNames2SubRegIndices(); - auto SubRegInfo = Names2SubRegIndices.find(Name); - if (SubRegInfo == Names2SubRegIndices.end()) - return 0; - return SubRegInfo->getValue(); -} - static void initSlots2BasicBlocks( const Function &F, DenseMap &Slots2BasicBlocks) { @@ -2861,86 +2977,6 @@ const Value *MIParser::getIRValue(unsigned Slot) { return ValueInfo->second; } -void MIParser::initNames2TargetIndices() { - if (!Names2TargetIndices.empty()) - return; - const auto *TII = MF.getSubtarget().getInstrInfo(); - assert(TII && "Expected target instruction info"); - auto Indices = TII->getSerializableTargetIndices(); - for (const auto &I : Indices) - Names2TargetIndices.insert(std::make_pair(StringRef(I.second), I.first)); -} - -bool MIParser::getTargetIndex(StringRef Name, int &Index) { - initNames2TargetIndices(); - auto IndexInfo = Names2TargetIndices.find(Name); - if (IndexInfo == Names2TargetIndices.end()) - return true; - Index = IndexInfo->second; - return false; -} - -void MIParser::initNames2DirectTargetFlags() { - if (!Names2DirectTargetFlags.empty()) - return; - const auto *TII = MF.getSubtarget().getInstrInfo(); - assert(TII && "Expected target instruction info"); - auto Flags = TII->getSerializableDirectMachineOperandTargetFlags(); - for (const auto &I : Flags) - Names2DirectTargetFlags.insert( - std::make_pair(StringRef(I.second), I.first)); -} - -bool MIParser::getDirectTargetFlag(StringRef Name, unsigned &Flag) { - initNames2DirectTargetFlags(); - auto FlagInfo = Names2DirectTargetFlags.find(Name); - if (FlagInfo == Names2DirectTargetFlags.end()) - return true; - Flag = FlagInfo->second; - return false; -} - -void MIParser::initNames2BitmaskTargetFlags() { - if (!Names2BitmaskTargetFlags.empty()) - return; - const auto *TII = MF.getSubtarget().getInstrInfo(); - assert(TII && "Expected target instruction info"); - auto Flags = TII->getSerializableBitmaskMachineOperandTargetFlags(); - for (const auto &I : Flags) - Names2BitmaskTargetFlags.insert( - std::make_pair(StringRef(I.second), I.first)); -} - -bool MIParser::getBitmaskTargetFlag(StringRef Name, unsigned &Flag) { - initNames2BitmaskTargetFlags(); - auto FlagInfo = Names2BitmaskTargetFlags.find(Name); - if (FlagInfo == Names2BitmaskTargetFlags.end()) - return true; - Flag = FlagInfo->second; - return false; -} - -void MIParser::initNames2MMOTargetFlags() { - if (!Names2MMOTargetFlags.empty()) - return; - const auto *TII = MF.getSubtarget().getInstrInfo(); - assert(TII && "Expected target instruction info"); - auto Flags = TII->getSerializableMachineMemOperandTargetFlags(); - for (const auto &I : Flags) - Names2MMOTargetFlags.insert( - std::make_pair(StringRef(I.second), I.first)); -} - -bool MIParser::getMMOTargetFlag(StringRef Name, - MachineMemOperand::Flags &Flag) { - initNames2MMOTargetFlags(); - auto FlagInfo = Names2MMOTargetFlags.find(Name); - if (FlagInfo == Names2MMOTargetFlags.end()) - return true; - Flag = FlagInfo->second; - return false; -} - MCSymbol *MIParser::getOrCreateMCSymbol(StringRef Name) { // FIXME: Currently we can't recognize temporary or local symbols and call all // of the appropriate forms to create them. However, this handles basic cases diff --git a/lib/CodeGen/MIRParser/MIParser.h b/lib/CodeGen/MIRParser/MIParser.h deleted file mode 100644 index b06ceb21b740..000000000000 --- a/lib/CodeGen/MIRParser/MIParser.h +++ /dev/null @@ -1,125 +0,0 @@ -//===- MIParser.h - Machine Instructions Parser -----------------*- C++ -*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file declares the function that parses the machine instructions. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_LIB_CODEGEN_MIRPARSER_MIPARSER_H -#define LLVM_LIB_CODEGEN_MIRPARSER_MIPARSER_H - -#include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/StringMap.h" -#include "llvm/Support/Allocator.h" - -namespace llvm { - -class MachineBasicBlock; -class MachineFunction; -class MDNode; -class RegisterBank; -struct SlotMapping; -class SMDiagnostic; -class SourceMgr; -class StringRef; -class TargetRegisterClass; - -struct VRegInfo { - enum uint8_t { - UNKNOWN, NORMAL, GENERIC, REGBANK - } Kind = UNKNOWN; - bool Explicit = false; ///< VReg was explicitly specified in the .mir file. - union { - const TargetRegisterClass *RC; - const RegisterBank *RegBank; - } D; - unsigned VReg; - unsigned PreferredReg = 0; -}; - -using Name2RegClassMap = StringMap; -using Name2RegBankMap = StringMap; - -struct PerFunctionMIParsingState { - BumpPtrAllocator Allocator; - MachineFunction &MF; - SourceMgr *SM; - const SlotMapping &IRSlots; - const Name2RegClassMap &Names2RegClasses; - const Name2RegBankMap &Names2RegBanks; - - DenseMap MBBSlots; - DenseMap VRegInfos; - StringMap VRegInfosNamed; - DenseMap FixedStackObjectSlots; - DenseMap StackObjectSlots; - DenseMap ConstantPoolSlots; - DenseMap JumpTableSlots; - - PerFunctionMIParsingState(MachineFunction &MF, SourceMgr &SM, - const SlotMapping &IRSlots, - const Name2RegClassMap &Names2RegClasses, - const Name2RegBankMap &Names2RegBanks); - - VRegInfo &getVRegInfo(unsigned Num); - VRegInfo &getVRegInfoNamed(StringRef RegName); -}; - -/// Parse the machine basic block definitions, and skip the machine -/// instructions. -/// -/// This function runs the first parsing pass on the machine function's body. -/// It parses only the machine basic block definitions and creates the machine -/// basic blocks in the given machine function. -/// -/// The machine instructions aren't parsed during the first pass because all -/// the machine basic blocks aren't defined yet - this makes it impossible to -/// resolve the machine basic block references. -/// -/// Return true if an error occurred. -bool parseMachineBasicBlockDefinitions(PerFunctionMIParsingState &PFS, - StringRef Src, SMDiagnostic &Error); - -/// Parse the machine instructions. -/// -/// This function runs the second parsing pass on the machine function's body. -/// It skips the machine basic block definitions and parses only the machine -/// instructions and basic block attributes like liveins and successors. -/// -/// The second parsing pass assumes that the first parsing pass already ran -/// on the given source string. -/// -/// Return true if an error occurred. -bool parseMachineInstructions(PerFunctionMIParsingState &PFS, StringRef Src, - SMDiagnostic &Error); - -bool parseMBBReference(PerFunctionMIParsingState &PFS, - MachineBasicBlock *&MBB, StringRef Src, - SMDiagnostic &Error); - -bool parseRegisterReference(PerFunctionMIParsingState &PFS, - unsigned &Reg, StringRef Src, - SMDiagnostic &Error); - -bool parseNamedRegisterReference(PerFunctionMIParsingState &PFS, unsigned &Reg, - StringRef Src, SMDiagnostic &Error); - -bool parseVirtualRegisterReference(PerFunctionMIParsingState &PFS, - VRegInfo *&Info, StringRef Src, - SMDiagnostic &Error); - -bool parseStackObjectReference(PerFunctionMIParsingState &PFS, int &FI, - StringRef Src, SMDiagnostic &Error); - -bool parseMDNode(PerFunctionMIParsingState &PFS, MDNode *&Node, StringRef Src, - SMDiagnostic &Error); - -} // end namespace llvm - -#endif // LLVM_LIB_CODEGEN_MIRPARSER_MIPARSER_H diff --git a/lib/CodeGen/MIRParser/MIRParser.cpp b/lib/CodeGen/MIRParser/MIRParser.cpp index 00da92a92ec6..b242934def80 100644 --- a/lib/CodeGen/MIRParser/MIRParser.cpp +++ b/lib/CodeGen/MIRParser/MIRParser.cpp @@ -1,9 +1,8 @@ //===- MIRParser.cpp - MIR serialization format parser implementation -----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -13,7 +12,6 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/MIRParser/MIRParser.h" -#include "MIParser.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringMap.h" @@ -22,12 +20,14 @@ #include "llvm/AsmParser/SlotMapping.h" #include "llvm/CodeGen/GlobalISel/RegisterBank.h" #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h" +#include "llvm/CodeGen/MIRParser/MIParser.h" #include "llvm/CodeGen/MIRYamlMapping.h" #include "llvm/CodeGen/MachineConstantPool.h" #include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/TargetFrameLowering.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/DebugInfo.h" #include "llvm/IR/DiagnosticInfo.h" @@ -40,6 +40,7 @@ #include "llvm/Support/SMLoc.h" #include "llvm/Support/SourceMgr.h" #include "llvm/Support/YAMLTraits.h" +#include "llvm/Target/TargetMachine.h" #include using namespace llvm; @@ -54,10 +55,8 @@ class MIRParserImpl { StringRef Filename; LLVMContext &Context; SlotMapping IRSlots; - /// Maps from register class names to register classes. - Name2RegClassMap Names2RegClasses; - /// Maps from register bank names to register banks. - Name2RegBankMap Names2RegBanks; + std::unique_ptr Target; + /// True when the MIR file doesn't have LLVM IR. Dummy IR functions are /// created and inserted into the given module when this is true. bool NoLLVMIR = false; @@ -117,6 +116,9 @@ public: bool initializeFrameInfo(PerFunctionMIParsingState &PFS, const yaml::MachineFunction &YamlMF); + bool initializeCallSiteInfo(PerFunctionMIParsingState &PFS, + const yaml::MachineFunction &YamlMF); + bool parseCalleeSavedRegister(PerFunctionMIParsingState &PFS, std::vector &CSIInfo, const yaml::StringValue &RegisterSource, @@ -151,20 +153,6 @@ private: SMDiagnostic diagFromBlockStringDiag(const SMDiagnostic &Error, SMRange SourceRange); - void initNames2RegClasses(const MachineFunction &MF); - void initNames2RegBanks(const MachineFunction &MF); - - /// Check if the given identifier is a name of a register class. - /// - /// Return null if the name isn't a register class. - const TargetRegisterClass *getRegClass(const MachineFunction &MF, - StringRef Name); - - /// Check if the given identifier is a name of a register bank. - /// - /// Return null if the name isn't a register bank. - const RegisterBank *getRegBank(const MachineFunction &MF, StringRef Name); - void computeFunctionProperties(MachineFunction &MF); }; @@ -271,8 +259,9 @@ bool MIRParserImpl::parseMachineFunctions(Module &M, MachineModuleInfo &MMI) { /// Create an empty function with the given name. static Function *createDummyFunction(StringRef Name, Module &M) { auto &Context = M.getContext(); - Function *F = cast(M.getOrInsertFunction( - Name, FunctionType::get(Type::getVoidTy(Context), false))); + Function *F = + Function::Create(FunctionType::get(Type::getVoidTy(Context), false), + Function::ExternalLinkage, Name, M); BasicBlock *BB = BasicBlock::Create(Context, "entry", F); new UnreachableInst(Context, BB); return F; @@ -282,6 +271,11 @@ bool MIRParserImpl::parseMachineFunction(Module &M, MachineModuleInfo &MMI) { // Parse the yaml. yaml::MachineFunction YamlMF; yaml::EmptyContext Ctx; + + const LLVMTargetMachine &TM = MMI.getTarget(); + YamlMF.MachineFuncInfo = std::unique_ptr( + TM.createDefaultFuncInfoYAML()); + yaml::yamlize(In, YamlMF, false, Ctx); if (In.error()) return true; @@ -346,12 +340,58 @@ void MIRParserImpl::computeFunctionProperties(MachineFunction &MF) { Properties.set(MachineFunctionProperties::Property::NoVRegs); } +bool MIRParserImpl::initializeCallSiteInfo( + PerFunctionMIParsingState &PFS, const yaml::MachineFunction &YamlMF) { + MachineFunction &MF = PFS.MF; + SMDiagnostic Error; + const LLVMTargetMachine &TM = MF.getTarget(); + for (auto YamlCSInfo : YamlMF.CallSitesInfo) { + yaml::CallSiteInfo::MachineInstrLoc MILoc = YamlCSInfo.CallLocation; + if (MILoc.BlockNum >= MF.size()) + return error(Twine(MF.getName()) + + Twine(" call instruction block out of range.") + + " Unable to reference bb:" + Twine(MILoc.BlockNum)); + auto CallB = std::next(MF.begin(), MILoc.BlockNum); + if (MILoc.Offset >= CallB->size()) + return error(Twine(MF.getName()) + + Twine(" call instruction offset out of range.") + + "Unable to reference instruction at bb: " + + Twine(MILoc.BlockNum) + " at offset:" + Twine(MILoc.Offset)); + auto CallI = std::next(CallB->begin(), MILoc.Offset); + if (!CallI->isCall()) + return error(Twine(MF.getName()) + + Twine(" call site info should reference call " + "instruction. Instruction at bb:") + + Twine(MILoc.BlockNum) + " at offset:" + Twine(MILoc.Offset) + + " is not a call instruction"); + MachineFunction::CallSiteInfo CSInfo; + for (auto ArgRegPair : YamlCSInfo.ArgForwardingRegs) { + unsigned Reg = 0; + if (parseNamedRegisterReference(PFS, Reg, ArgRegPair.Reg.Value, Error)) + return error(Error, ArgRegPair.Reg.SourceRange); + CSInfo.emplace_back(Reg, ArgRegPair.ArgNo); + } + + if (TM.Options.EnableDebugEntryValues) + MF.addCallArgsForwardingRegs(&*CallI, std::move(CSInfo)); + } + + if (YamlMF.CallSitesInfo.size() && !TM.Options.EnableDebugEntryValues) + return error(Twine("Call site info provided but not used")); + return false; +} + bool MIRParserImpl::initializeMachineFunction(const yaml::MachineFunction &YamlMF, MachineFunction &MF) { // TODO: Recreate the machine function. - initNames2RegClasses(MF); - initNames2RegBanks(MF); + if (Target) { + // Avoid clearing state if we're using the same subtarget again. + Target->setTarget(MF.getSubtarget()); + } else { + Target.reset(new PerTargetMIParsingState(MF.getSubtarget())); + } + if (YamlMF.Alignment) MF.setAlignment(YamlMF.Alignment); MF.setExposesReturnsTwice(YamlMF.ExposesReturnsTwice); @@ -367,8 +407,7 @@ MIRParserImpl::initializeMachineFunction(const yaml::MachineFunction &YamlMF, if (YamlMF.FailedISel) MF.getProperties().set(MachineFunctionProperties::Property::FailedISel); - PerFunctionMIParsingState PFS(MF, SM, IRSlots, Names2RegClasses, - Names2RegBanks); + PerFunctionMIParsingState PFS(MF, SM, IRSlots, *Target); if (parseRegisterInfo(PFS, YamlMF)) return true; if (!YamlMF.Constants.empty()) { @@ -419,8 +458,32 @@ MIRParserImpl::initializeMachineFunction(const yaml::MachineFunction &YamlMF, if (setupRegisterInfo(PFS, YamlMF)) return true; + if (YamlMF.MachineFuncInfo) { + const LLVMTargetMachine &TM = MF.getTarget(); + // Note this is called after the initial constructor of the + // MachineFunctionInfo based on the MachineFunction, which may depend on the + // IR. + + SMRange SrcRange; + if (TM.parseMachineFunctionInfo(*YamlMF.MachineFuncInfo, PFS, Error, + SrcRange)) { + return error(Error, SrcRange); + } + } + + // Set the reserved registers after parsing MachineFuncInfo. The target may + // have been recording information used to select the reserved registers + // there. + // FIXME: This is a temporary workaround until the reserved registers can be + // serialized. + MachineRegisterInfo &MRI = MF.getRegInfo(); + MRI.freezeReservedRegs(MF); + computeFunctionProperties(MF); + if (initializeCallSiteInfo(PFS, YamlMF)) + return false; + MF.getSubtarget().mirFileLoaded(MF); MF.verify(); @@ -449,12 +512,12 @@ bool MIRParserImpl::parseRegisterInfo(PerFunctionMIParsingState &PFS, Info.Kind = VRegInfo::GENERIC; Info.D.RegBank = nullptr; } else { - const auto *RC = getRegClass(MF, VReg.Class.Value); + const auto *RC = Target->getRegClass(VReg.Class.Value); if (RC) { Info.Kind = VRegInfo::NORMAL; Info.D.RC = RC; } else { - const RegisterBank *RegBank = getRegBank(MF, VReg.Class.Value); + const RegisterBank *RegBank = Target->getRegBank(VReg.Class.Value); if (!RegBank) return error( VReg.Class.SourceRange.Start, @@ -557,9 +620,6 @@ bool MIRParserImpl::setupRegisterInfo(const PerFunctionMIParsingState &PFS, } } - // FIXME: This is a temporary workaround until the reserved registers can be - // serialized. - MRI.freezeReservedRegs(MF); return Error; } @@ -567,6 +627,7 @@ bool MIRParserImpl::initializeFrameInfo(PerFunctionMIParsingState &PFS, const yaml::MachineFunction &YamlMF) { MachineFunction &MF = PFS.MF; MachineFrameInfo &MFI = MF.getFrameInfo(); + const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering(); const Function &F = MF.getFunction(); const yaml::MachineFrameInfo &YamlMFI = YamlMF.FrameInfo; MFI.setFrameAddressIsTaken(YamlMFI.IsFrameAddressTaken); @@ -608,8 +669,12 @@ bool MIRParserImpl::initializeFrameInfo(PerFunctionMIParsingState &PFS, Object.IsImmutable, Object.IsAliased); else ObjectIdx = MFI.CreateFixedSpillStackObject(Object.Size, Object.Offset); - MFI.setObjectAlignment(ObjectIdx, Object.Alignment); + + if (!TFI->isSupportedStackID(Object.StackID)) + return error(Object.ID.SourceRange.Start, + Twine("StackID is not supported by target")); MFI.setStackID(ObjectIdx, Object.StackID); + MFI.setObjectAlignment(ObjectIdx, Object.Alignment); if (!PFS.FixedStackObjectSlots.insert(std::make_pair(Object.ID.Value, ObjectIdx)) .second) @@ -637,14 +702,17 @@ bool MIRParserImpl::initializeFrameInfo(PerFunctionMIParsingState &PFS, "' isn't defined in the function '" + F.getName() + "'"); } + if (!TFI->isSupportedStackID(Object.StackID)) + return error(Object.ID.SourceRange.Start, + Twine("StackID is not supported by target")); if (Object.Type == yaml::MachineStackObject::VariableSized) ObjectIdx = MFI.CreateVariableSizedObject(Object.Alignment, Alloca); else ObjectIdx = MFI.CreateStackObject( Object.Size, Object.Alignment, - Object.Type == yaml::MachineStackObject::SpillSlot, Alloca); + Object.Type == yaml::MachineStackObject::SpillSlot, Alloca, + Object.StackID); MFI.setObjectOffset(ObjectIdx, Object.Offset); - MFI.setStackID(ObjectIdx, Object.StackID); if (!PFS.StackObjectSlots.insert(std::make_pair(Object.ID.Value, ObjectIdx)) .second) @@ -844,48 +912,6 @@ SMDiagnostic MIRParserImpl::diagFromBlockStringDiag(const SMDiagnostic &Error, Error.getFixIts()); } -void MIRParserImpl::initNames2RegClasses(const MachineFunction &MF) { - if (!Names2RegClasses.empty()) - return; - const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); - for (unsigned I = 0, E = TRI->getNumRegClasses(); I < E; ++I) { - const auto *RC = TRI->getRegClass(I); - Names2RegClasses.insert( - std::make_pair(StringRef(TRI->getRegClassName(RC)).lower(), RC)); - } -} - -void MIRParserImpl::initNames2RegBanks(const MachineFunction &MF) { - if (!Names2RegBanks.empty()) - return; - const RegisterBankInfo *RBI = MF.getSubtarget().getRegBankInfo(); - // If the target does not support GlobalISel, we may not have a - // register bank info. - if (!RBI) - return; - for (unsigned I = 0, E = RBI->getNumRegBanks(); I < E; ++I) { - const auto &RegBank = RBI->getRegBank(I); - Names2RegBanks.insert( - std::make_pair(StringRef(RegBank.getName()).lower(), &RegBank)); - } -} - -const TargetRegisterClass *MIRParserImpl::getRegClass(const MachineFunction &MF, - StringRef Name) { - auto RegClassInfo = Names2RegClasses.find(Name); - if (RegClassInfo == Names2RegClasses.end()) - return nullptr; - return RegClassInfo->getValue(); -} - -const RegisterBank *MIRParserImpl::getRegBank(const MachineFunction &MF, - StringRef Name) { - auto RegBankInfo = Names2RegBanks.find(Name); - if (RegBankInfo == Names2RegBanks.end()) - return nullptr; - return RegBankInfo->getValue(); -} - MIRParser::MIRParser(std::unique_ptr Impl) : Impl(std::move(Impl)) {} diff --git a/lib/CodeGen/MIRPrinter.cpp b/lib/CodeGen/MIRPrinter.cpp index d9dcc428943f..0a95a0ced0f5 100644 --- a/lib/CodeGen/MIRPrinter.cpp +++ b/lib/CodeGen/MIRPrinter.cpp @@ -1,9 +1,8 @@ //===- MIRPrinter.cpp - MIR serialization format printer ------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -36,6 +35,7 @@ #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" +#include "llvm/CodeGen/TargetFrameLowering.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Constants.h" #include "llvm/IR/DebugInfo.h" @@ -129,6 +129,9 @@ public: const MachineJumpTableInfo &JTI); void convertStackObjects(yaml::MachineFunction &YMF, const MachineFunction &MF, ModuleSlotTracker &MST); + void convertCallSiteObjects(yaml::MachineFunction &YMF, + const MachineFunction &MF, + ModuleSlotTracker &MST); private: void initRegisterMaskIds(const MachineFunction &MF); @@ -212,10 +215,16 @@ void MIRPrinter::print(const MachineFunction &MF) { MST.incorporateFunction(MF.getFunction()); convert(MST, YamlMF.FrameInfo, MF.getFrameInfo()); convertStackObjects(YamlMF, MF, MST); + convertCallSiteObjects(YamlMF, MF, MST); if (const auto *ConstantPool = MF.getConstantPool()) convert(YamlMF, *ConstantPool); if (const auto *JumpTableInfo = MF.getJumpTableInfo()) convert(MST, YamlMF.JumpTableInfo, *JumpTableInfo); + + const TargetMachine &TM = MF.getTarget(); + YamlMF.MachineFuncInfo = + std::unique_ptr(TM.convertFuncInfoToYAML(MF)); + raw_string_ostream StrOS(YamlMF.Body.Value.Value); bool IsNewlineNeeded = false; for (const auto &MBB : MF) { @@ -352,7 +361,7 @@ void MIRPrinter::convertStackObjects(yaml::MachineFunction &YMF, const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); // Process fixed stack objects. unsigned ID = 0; - for (int I = MFI.getObjectIndexBegin(); I < 0; ++I) { + for (int I = MFI.getObjectIndexBegin(); I < 0; ++I, ++ID) { if (MFI.isDeadObjectIndex(I)) continue; @@ -364,17 +373,17 @@ void MIRPrinter::convertStackObjects(yaml::MachineFunction &YMF, YamlObject.Offset = MFI.getObjectOffset(I); YamlObject.Size = MFI.getObjectSize(I); YamlObject.Alignment = MFI.getObjectAlignment(I); - YamlObject.StackID = MFI.getStackID(I); + YamlObject.StackID = (TargetStackID::Value)MFI.getStackID(I); YamlObject.IsImmutable = MFI.isImmutableObjectIndex(I); YamlObject.IsAliased = MFI.isAliasedObjectIndex(I); YMF.FixedStackObjects.push_back(YamlObject); StackObjectOperandMapping.insert( - std::make_pair(I, FrameIndexOperand::createFixed(ID++))); + std::make_pair(I, FrameIndexOperand::createFixed(ID))); } // Process ordinary stack objects. ID = 0; - for (int I = 0, E = MFI.getObjectIndexEnd(); I < E; ++I) { + for (int I = 0, E = MFI.getObjectIndexEnd(); I < E; ++I, ++ID) { if (MFI.isDeadObjectIndex(I)) continue; @@ -391,14 +400,17 @@ void MIRPrinter::convertStackObjects(yaml::MachineFunction &YMF, YamlObject.Offset = MFI.getObjectOffset(I); YamlObject.Size = MFI.getObjectSize(I); YamlObject.Alignment = MFI.getObjectAlignment(I); - YamlObject.StackID = MFI.getStackID(I); + YamlObject.StackID = (TargetStackID::Value)MFI.getStackID(I); YMF.StackObjects.push_back(YamlObject); StackObjectOperandMapping.insert(std::make_pair( - I, FrameIndexOperand::create(YamlObject.Name.Value, ID++))); + I, FrameIndexOperand::create(YamlObject.Name.Value, ID))); } for (const auto &CSInfo : MFI.getCalleeSavedInfo()) { + if (!CSInfo.isSpilledToReg() && MFI.isDeadObjectIndex(CSInfo.getFrameIdx())) + continue; + yaml::StringValue Reg; printRegMIR(CSInfo.getReg(), Reg, TRI); if (!CSInfo.isSpilledToReg()) { @@ -452,6 +464,39 @@ void MIRPrinter::convertStackObjects(yaml::MachineFunction &YMF, } } +void MIRPrinter::convertCallSiteObjects(yaml::MachineFunction &YMF, + const MachineFunction &MF, + ModuleSlotTracker &MST) { + const auto *TRI = MF.getSubtarget().getRegisterInfo(); + for (auto CSInfo : MF.getCallSitesInfo()) { + yaml::CallSiteInfo YmlCS; + yaml::CallSiteInfo::MachineInstrLoc CallLocation; + + // Prepare instruction position. + MachineBasicBlock::const_iterator CallI = CSInfo.first->getIterator(); + CallLocation.BlockNum = CallI->getParent()->getNumber(); + // Get call instruction offset from the beginning of block. + CallLocation.Offset = std::distance(CallI->getParent()->begin(), CallI); + YmlCS.CallLocation = CallLocation; + // Construct call arguments and theirs forwarding register info. + for (auto ArgReg : CSInfo.second) { + yaml::CallSiteInfo::ArgRegPair YmlArgReg; + YmlArgReg.ArgNo = ArgReg.ArgNo; + printRegMIR(ArgReg.Reg, YmlArgReg.Reg, TRI); + YmlCS.ArgForwardingRegs.emplace_back(YmlArgReg); + } + YMF.CallSitesInfo.push_back(YmlCS); + } + + // Sort call info by position of call instructions. + llvm::sort(YMF.CallSitesInfo.begin(), YMF.CallSitesInfo.end(), + [](yaml::CallSiteInfo A, yaml::CallSiteInfo B) { + if (A.CallLocation.BlockNum == B.CallLocation.BlockNum) + return A.CallLocation.Offset < B.CallLocation.Offset; + return A.CallLocation.BlockNum < B.CallLocation.BlockNum; + }); +} + void MIRPrinter::convert(yaml::MachineFunction &MF, const MachineConstantPool &ConstantPool) { unsigned ID = 0; @@ -706,6 +751,8 @@ void MIPrinter::print(const MachineInstr &MI) { OS << "nsw "; if (MI.getFlag(MachineInstr::IsExact)) OS << "exact "; + if (MI.getFlag(MachineInstr::FPExcept)) + OS << "fpexcept "; OS << TII->getName(MI.getOpcode()); if (I < E) diff --git a/lib/CodeGen/MIRPrintingPass.cpp b/lib/CodeGen/MIRPrintingPass.cpp index 1a8427430ea0..e032fffd658c 100644 --- a/lib/CodeGen/MIRPrintingPass.cpp +++ b/lib/CodeGen/MIRPrintingPass.cpp @@ -1,9 +1,8 @@ //===- MIRPrintingPass.cpp - Pass that prints out using the MIR format ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/MachineBasicBlock.cpp b/lib/CodeGen/MachineBasicBlock.cpp index 03771bc5dae1..4d29e883d879 100644 --- a/lib/CodeGen/MachineBasicBlock.cpp +++ b/lib/CodeGen/MachineBasicBlock.cpp @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/MachineBasicBlock.cpp ----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -133,8 +132,12 @@ void ilist_traits::transferNodesFromList(ilist_traits &FromList, instr_iterator First, instr_iterator Last) { assert(Parent->getParent() == FromList.Parent->getParent() && - "MachineInstr parent mismatch!"); - assert(this != &FromList && "Called without a real transfer..."); + "cannot transfer MachineInstrs between MachineFunctions"); + + // If it's within the same BB, there's nothing to do. + if (this == &FromList) + return; + assert(Parent != FromList.Parent && "Two lists have the same parent?"); // If splicing between two blocks within the same function, just update the @@ -995,7 +998,7 @@ MachineBasicBlock *MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, while (!KilledRegs.empty()) { unsigned Reg = KilledRegs.pop_back_val(); for (instr_iterator I = instr_end(), E = instr_begin(); I != E;) { - if (!(--I)->addRegisterKilled(Reg, TRI, /* addIfNotFound= */ false)) + if (!(--I)->addRegisterKilled(Reg, TRI, /* AddIfNotFound= */ false)) continue; if (TargetRegisterInfo::isVirtualRegister(Reg)) LV->getVarInfo(Reg).Kills.push_back(&*I); diff --git a/lib/CodeGen/MachineBlockFrequencyInfo.cpp b/lib/CodeGen/MachineBlockFrequencyInfo.cpp index 3459a9f71a73..53a35b7e89c2 100644 --- a/lib/CodeGen/MachineBlockFrequencyInfo.cpp +++ b/lib/CodeGen/MachineBlockFrequencyInfo.cpp @@ -1,9 +1,8 @@ //===- MachineBlockFrequencyInfo.cpp - MBB Frequency Analysis -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/MachineBlockPlacement.cpp b/lib/CodeGen/MachineBlockPlacement.cpp index 4fee9c4ea027..639b588766a1 100644 --- a/lib/CodeGen/MachineBlockPlacement.cpp +++ b/lib/CodeGen/MachineBlockPlacement.cpp @@ -1,9 +1,8 @@ //===- MachineBlockPlacement.cpp - Basic Block Code Layout optimization ---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -452,15 +451,28 @@ class MachineBlockPlacement : public MachineFunctionPass { void buildChain(const MachineBasicBlock *BB, BlockChain &Chain, BlockFilterSet *BlockFilter = nullptr); + bool canMoveBottomBlockToTop(const MachineBasicBlock *BottomBlock, + const MachineBasicBlock *OldTop); + bool hasViableTopFallthrough(const MachineBasicBlock *Top, + const BlockFilterSet &LoopBlockSet); + BlockFrequency TopFallThroughFreq(const MachineBasicBlock *Top, + const BlockFilterSet &LoopBlockSet); + BlockFrequency FallThroughGains(const MachineBasicBlock *NewTop, + const MachineBasicBlock *OldTop, + const MachineBasicBlock *ExitBB, + const BlockFilterSet &LoopBlockSet); + MachineBasicBlock *findBestLoopTopHelper(MachineBasicBlock *OldTop, + const MachineLoop &L, const BlockFilterSet &LoopBlockSet); MachineBasicBlock *findBestLoopTop( const MachineLoop &L, const BlockFilterSet &LoopBlockSet); MachineBasicBlock *findBestLoopExit( - const MachineLoop &L, const BlockFilterSet &LoopBlockSet); + const MachineLoop &L, const BlockFilterSet &LoopBlockSet, + BlockFrequency &ExitFreq); BlockFilterSet collectLoopBlockSet(const MachineLoop &L); void buildLoopChains(const MachineLoop &L); void rotateLoop( BlockChain &LoopChain, const MachineBasicBlock *ExitingBB, - const BlockFilterSet &LoopBlockSet); + BlockFrequency ExitFreq, const BlockFilterSet &LoopBlockSet); void rotateLoopWithProfile( BlockChain &LoopChain, const MachineLoop &L, const BlockFilterSet &LoopBlockSet); @@ -938,8 +950,8 @@ MachineBlockPlacement::getBestNonConflictingEdges( // Sort for highest frequency. auto Cmp = [](WeightedEdge A, WeightedEdge B) { return A.Weight > B.Weight; }; - std::stable_sort(Edges[0].begin(), Edges[0].end(), Cmp); - std::stable_sort(Edges[1].begin(), Edges[1].end(), Cmp); + llvm::stable_sort(Edges[0], Cmp); + llvm::stable_sort(Edges[1], Cmp); auto BestA = Edges[0].begin(); auto BestB = Edges[1].begin(); // Arrange for the correct answer to be in BestA and BestB @@ -1527,15 +1539,12 @@ MachineBlockPlacement::selectBestSuccessor( // profitable than BestSucc. Position is important because we preserve it and // prefer first best match. Here we aren't comparing in order, so we capture // the position instead. - if (DupCandidates.size() != 0) { - auto cmp = - [](const std::tuple &a, - const std::tuple &b) { - return std::get<0>(a) > std::get<0>(b); - }; - std::stable_sort(DupCandidates.begin(), DupCandidates.end(), cmp); - } - for(auto &Tup : DupCandidates) { + llvm::stable_sort(DupCandidates, + [](std::tuple L, + std::tuple R) { + return std::get<0>(L) > std::get<0>(R); + }); + for (auto &Tup : DupCandidates) { BranchProbability DupProb; MachineBasicBlock *Succ; std::tie(DupProb, Succ) = Tup; @@ -1757,63 +1766,238 @@ void MachineBlockPlacement::buildChain( << getBlockName(*Chain.begin()) << "\n"); } -/// Find the best loop top block for layout. +// If bottom of block BB has only one successor OldTop, in most cases it is +// profitable to move it before OldTop, except the following case: +// +// -->OldTop<- +// | . | +// | . | +// | . | +// ---Pred | +// | | +// BB----- +// +// If BB is moved before OldTop, Pred needs a taken branch to BB, and it can't +// layout the other successor below it, so it can't reduce taken branch. +// In this case we keep its original layout. +bool +MachineBlockPlacement::canMoveBottomBlockToTop( + const MachineBasicBlock *BottomBlock, + const MachineBasicBlock *OldTop) { + if (BottomBlock->pred_size() != 1) + return true; + MachineBasicBlock *Pred = *BottomBlock->pred_begin(); + if (Pred->succ_size() != 2) + return true; + + MachineBasicBlock *OtherBB = *Pred->succ_begin(); + if (OtherBB == BottomBlock) + OtherBB = *Pred->succ_rbegin(); + if (OtherBB == OldTop) + return false; + + return true; +} + +// Find out the possible fall through frequence to the top of a loop. +BlockFrequency +MachineBlockPlacement::TopFallThroughFreq( + const MachineBasicBlock *Top, + const BlockFilterSet &LoopBlockSet) { + BlockFrequency MaxFreq = 0; + for (MachineBasicBlock *Pred : Top->predecessors()) { + BlockChain *PredChain = BlockToChain[Pred]; + if (!LoopBlockSet.count(Pred) && + (!PredChain || Pred == *std::prev(PredChain->end()))) { + // Found a Pred block can be placed before Top. + // Check if Top is the best successor of Pred. + auto TopProb = MBPI->getEdgeProbability(Pred, Top); + bool TopOK = true; + for (MachineBasicBlock *Succ : Pred->successors()) { + auto SuccProb = MBPI->getEdgeProbability(Pred, Succ); + BlockChain *SuccChain = BlockToChain[Succ]; + // Check if Succ can be placed after Pred. + // Succ should not be in any chain, or it is the head of some chain. + if (!LoopBlockSet.count(Succ) && (SuccProb > TopProb) && + (!SuccChain || Succ == *SuccChain->begin())) { + TopOK = false; + break; + } + } + if (TopOK) { + BlockFrequency EdgeFreq = MBFI->getBlockFreq(Pred) * + MBPI->getEdgeProbability(Pred, Top); + if (EdgeFreq > MaxFreq) + MaxFreq = EdgeFreq; + } + } + } + return MaxFreq; +} + +// Compute the fall through gains when move NewTop before OldTop. +// +// In following diagram, edges marked as "-" are reduced fallthrough, edges +// marked as "+" are increased fallthrough, this function computes +// +// SUM(increased fallthrough) - SUM(decreased fallthrough) +// +// | +// | - +// V +// --->OldTop +// | . +// | . +// +| . + +// | Pred ---> +// | |- +// | V +// --- NewTop <--- +// |- +// V +// +BlockFrequency +MachineBlockPlacement::FallThroughGains( + const MachineBasicBlock *NewTop, + const MachineBasicBlock *OldTop, + const MachineBasicBlock *ExitBB, + const BlockFilterSet &LoopBlockSet) { + BlockFrequency FallThrough2Top = TopFallThroughFreq(OldTop, LoopBlockSet); + BlockFrequency FallThrough2Exit = 0; + if (ExitBB) + FallThrough2Exit = MBFI->getBlockFreq(NewTop) * + MBPI->getEdgeProbability(NewTop, ExitBB); + BlockFrequency BackEdgeFreq = MBFI->getBlockFreq(NewTop) * + MBPI->getEdgeProbability(NewTop, OldTop); + + // Find the best Pred of NewTop. + MachineBasicBlock *BestPred = nullptr; + BlockFrequency FallThroughFromPred = 0; + for (MachineBasicBlock *Pred : NewTop->predecessors()) { + if (!LoopBlockSet.count(Pred)) + continue; + BlockChain *PredChain = BlockToChain[Pred]; + if (!PredChain || Pred == *std::prev(PredChain->end())) { + BlockFrequency EdgeFreq = MBFI->getBlockFreq(Pred) * + MBPI->getEdgeProbability(Pred, NewTop); + if (EdgeFreq > FallThroughFromPred) { + FallThroughFromPred = EdgeFreq; + BestPred = Pred; + } + } + } + + // If NewTop is not placed after Pred, another successor can be placed + // after Pred. + BlockFrequency NewFreq = 0; + if (BestPred) { + for (MachineBasicBlock *Succ : BestPred->successors()) { + if ((Succ == NewTop) || (Succ == BestPred) || !LoopBlockSet.count(Succ)) + continue; + if (ComputedEdges.find(Succ) != ComputedEdges.end()) + continue; + BlockChain *SuccChain = BlockToChain[Succ]; + if ((SuccChain && (Succ != *SuccChain->begin())) || + (SuccChain == BlockToChain[BestPred])) + continue; + BlockFrequency EdgeFreq = MBFI->getBlockFreq(BestPred) * + MBPI->getEdgeProbability(BestPred, Succ); + if (EdgeFreq > NewFreq) + NewFreq = EdgeFreq; + } + BlockFrequency OrigEdgeFreq = MBFI->getBlockFreq(BestPred) * + MBPI->getEdgeProbability(BestPred, NewTop); + if (NewFreq > OrigEdgeFreq) { + // If NewTop is not the best successor of Pred, then Pred doesn't + // fallthrough to NewTop. So there is no FallThroughFromPred and + // NewFreq. + NewFreq = 0; + FallThroughFromPred = 0; + } + } + + BlockFrequency Result = 0; + BlockFrequency Gains = BackEdgeFreq + NewFreq; + BlockFrequency Lost = FallThrough2Top + FallThrough2Exit + + FallThroughFromPred; + if (Gains > Lost) + Result = Gains - Lost; + return Result; +} + +/// Helper function of findBestLoopTop. Find the best loop top block +/// from predecessors of old top. +/// +/// Look for a block which is strictly better than the old top for laying +/// out before the old top of the loop. This looks for only two patterns: +/// +/// 1. a block has only one successor, the old loop top +/// +/// Because such a block will always result in an unconditional jump, +/// rotating it in front of the old top is always profitable. +/// +/// 2. a block has two successors, one is old top, another is exit +/// and it has more than one predecessors /// -/// Look for a block which is strictly better than the loop header for laying -/// out at the top of the loop. This looks for one and only one pattern: -/// a latch block with no conditional exit. This block will cause a conditional -/// jump around it or will be the bottom of the loop if we lay it out in place, -/// but if it it doesn't end up at the bottom of the loop for any reason, -/// rotation alone won't fix it. Because such a block will always result in an -/// unconditional jump (for the backedge) rotating it in front of the loop -/// header is always profitable. +/// If it is below one of its predecessors P, only P can fall through to +/// it, all other predecessors need a jump to it, and another conditional +/// jump to loop header. If it is moved before loop header, all its +/// predecessors jump to it, then fall through to loop header. So all its +/// predecessors except P can reduce one taken branch. +/// At the same time, move it before old top increases the taken branch +/// to loop exit block, so the reduced taken branch will be compared with +/// the increased taken branch to the loop exit block. MachineBasicBlock * -MachineBlockPlacement::findBestLoopTop(const MachineLoop &L, - const BlockFilterSet &LoopBlockSet) { - // Placing the latch block before the header may introduce an extra branch - // that skips this block the first time the loop is executed, which we want - // to avoid when optimising for size. - // FIXME: in theory there is a case that does not introduce a new branch, - // i.e. when the layout predecessor does not fallthrough to the loop header. - // In practice this never happens though: there always seems to be a preheader - // that can fallthrough and that is also placed before the header. - if (F->getFunction().optForSize()) - return L.getHeader(); - +MachineBlockPlacement::findBestLoopTopHelper( + MachineBasicBlock *OldTop, + const MachineLoop &L, + const BlockFilterSet &LoopBlockSet) { // Check that the header hasn't been fused with a preheader block due to // crazy branches. If it has, we need to start with the header at the top to // prevent pulling the preheader into the loop body. - BlockChain &HeaderChain = *BlockToChain[L.getHeader()]; + BlockChain &HeaderChain = *BlockToChain[OldTop]; if (!LoopBlockSet.count(*HeaderChain.begin())) - return L.getHeader(); + return OldTop; - LLVM_DEBUG(dbgs() << "Finding best loop top for: " - << getBlockName(L.getHeader()) << "\n"); + LLVM_DEBUG(dbgs() << "Finding best loop top for: " << getBlockName(OldTop) + << "\n"); - BlockFrequency BestPredFreq; + BlockFrequency BestGains = 0; MachineBasicBlock *BestPred = nullptr; - for (MachineBasicBlock *Pred : L.getHeader()->predecessors()) { + for (MachineBasicBlock *Pred : OldTop->predecessors()) { if (!LoopBlockSet.count(Pred)) continue; - LLVM_DEBUG(dbgs() << " header pred: " << getBlockName(Pred) << ", has " + if (Pred == L.getHeader()) + continue; + LLVM_DEBUG(dbgs() << " old top pred: " << getBlockName(Pred) << ", has " << Pred->succ_size() << " successors, "; MBFI->printBlockFreq(dbgs(), Pred) << " freq\n"); - if (Pred->succ_size() > 1) + if (Pred->succ_size() > 2) continue; - BlockFrequency PredFreq = MBFI->getBlockFreq(Pred); - if (!BestPred || PredFreq > BestPredFreq || - (!(PredFreq < BestPredFreq) && - Pred->isLayoutSuccessor(L.getHeader()))) { + MachineBasicBlock *OtherBB = nullptr; + if (Pred->succ_size() == 2) { + OtherBB = *Pred->succ_begin(); + if (OtherBB == OldTop) + OtherBB = *Pred->succ_rbegin(); + } + + if (!canMoveBottomBlockToTop(Pred, OldTop)) + continue; + + BlockFrequency Gains = FallThroughGains(Pred, OldTop, OtherBB, + LoopBlockSet); + if ((Gains > 0) && (Gains > BestGains || + ((Gains == BestGains) && Pred->isLayoutSuccessor(OldTop)))) { BestPred = Pred; - BestPredFreq = PredFreq; + BestGains = Gains; } } // If no direct predecessor is fine, just use the loop header. if (!BestPred) { LLVM_DEBUG(dbgs() << " final top unchanged\n"); - return L.getHeader(); + return OldTop; } // Walk backwards through any straight line of predecessors. @@ -1826,6 +2010,34 @@ MachineBlockPlacement::findBestLoopTop(const MachineLoop &L, return BestPred; } +/// Find the best loop top block for layout. +/// +/// This function iteratively calls findBestLoopTopHelper, until no new better +/// BB can be found. +MachineBasicBlock * +MachineBlockPlacement::findBestLoopTop(const MachineLoop &L, + const BlockFilterSet &LoopBlockSet) { + // Placing the latch block before the header may introduce an extra branch + // that skips this block the first time the loop is executed, which we want + // to avoid when optimising for size. + // FIXME: in theory there is a case that does not introduce a new branch, + // i.e. when the layout predecessor does not fallthrough to the loop header. + // In practice this never happens though: there always seems to be a preheader + // that can fallthrough and that is also placed before the header. + if (F->getFunction().hasOptSize()) + return L.getHeader(); + + MachineBasicBlock *OldTop = nullptr; + MachineBasicBlock *NewTop = L.getHeader(); + while (NewTop != OldTop) { + OldTop = NewTop; + NewTop = findBestLoopTopHelper(OldTop, L, LoopBlockSet); + if (NewTop != OldTop) + ComputedEdges[NewTop] = { OldTop, false }; + } + return NewTop; +} + /// Find the best loop exiting block for layout. /// /// This routine implements the logic to analyze the loop looking for the best @@ -1833,7 +2045,8 @@ MachineBlockPlacement::findBestLoopTop(const MachineLoop &L, /// fallthrough opportunities. MachineBasicBlock * MachineBlockPlacement::findBestLoopExit(const MachineLoop &L, - const BlockFilterSet &LoopBlockSet) { + const BlockFilterSet &LoopBlockSet, + BlockFrequency &ExitFreq) { // We don't want to layout the loop linearly in all cases. If the loop header // is just a normal basic block in the loop, we want to look for what block // within the loop is the best one to layout at the top. However, if the loop @@ -1944,9 +2157,43 @@ MachineBlockPlacement::findBestLoopExit(const MachineLoop &L, LLVM_DEBUG(dbgs() << " Best exiting block: " << getBlockName(ExitingBB) << "\n"); + ExitFreq = BestExitEdgeFreq; return ExitingBB; } +/// Check if there is a fallthrough to loop header Top. +/// +/// 1. Look for a Pred that can be layout before Top. +/// 2. Check if Top is the most possible successor of Pred. +bool +MachineBlockPlacement::hasViableTopFallthrough( + const MachineBasicBlock *Top, + const BlockFilterSet &LoopBlockSet) { + for (MachineBasicBlock *Pred : Top->predecessors()) { + BlockChain *PredChain = BlockToChain[Pred]; + if (!LoopBlockSet.count(Pred) && + (!PredChain || Pred == *std::prev(PredChain->end()))) { + // Found a Pred block can be placed before Top. + // Check if Top is the best successor of Pred. + auto TopProb = MBPI->getEdgeProbability(Pred, Top); + bool TopOK = true; + for (MachineBasicBlock *Succ : Pred->successors()) { + auto SuccProb = MBPI->getEdgeProbability(Pred, Succ); + BlockChain *SuccChain = BlockToChain[Succ]; + // Check if Succ can be placed after Pred. + // Succ should not be in any chain, or it is the head of some chain. + if ((!SuccChain || Succ == *SuccChain->begin()) && SuccProb > TopProb) { + TopOK = false; + break; + } + } + if (TopOK) + return true; + } + } + return false; +} + /// Attempt to rotate an exiting block to the bottom of the loop. /// /// Once we have built a chain, try to rotate it to line up the hot exit block @@ -1955,6 +2202,7 @@ MachineBlockPlacement::findBestLoopExit(const MachineLoop &L, /// of its bottom already, don't rotate it. void MachineBlockPlacement::rotateLoop(BlockChain &LoopChain, const MachineBasicBlock *ExitingBB, + BlockFrequency ExitFreq, const BlockFilterSet &LoopBlockSet) { if (!ExitingBB) return; @@ -1966,15 +2214,7 @@ void MachineBlockPlacement::rotateLoop(BlockChain &LoopChain, if (Bottom == ExitingBB) return; - bool ViableTopFallthrough = false; - for (MachineBasicBlock *Pred : Top->predecessors()) { - BlockChain *PredChain = BlockToChain[Pred]; - if (!LoopBlockSet.count(Pred) && - (!PredChain || Pred == *std::prev(PredChain->end()))) { - ViableTopFallthrough = true; - break; - } - } + bool ViableTopFallthrough = hasViableTopFallthrough(Top, LoopBlockSet); // If the header has viable fallthrough, check whether the current loop // bottom is a viable exiting block. If so, bail out as rotating will @@ -1986,6 +2226,12 @@ void MachineBlockPlacement::rotateLoop(BlockChain &LoopChain, (!SuccChain || Succ == *SuccChain->begin())) return; } + + // Rotate will destroy the top fallthrough, we need to ensure the new exit + // frequency is larger than top fallthrough. + BlockFrequency FallThrough2Top = TopFallThroughFreq(Top, LoopBlockSet); + if (FallThrough2Top >= ExitFreq) + return; } BlockChain::iterator ExitIt = llvm::find(LoopChain, ExitingBB); @@ -2041,8 +2287,6 @@ void MachineBlockPlacement::rotateLoop(BlockChain &LoopChain, void MachineBlockPlacement::rotateLoopWithProfile( BlockChain &LoopChain, const MachineLoop &L, const BlockFilterSet &LoopBlockSet) { - auto HeaderBB = L.getHeader(); - auto HeaderIter = llvm::find(LoopChain, HeaderBB); auto RotationPos = LoopChain.end(); BlockFrequency SmallestRotationCost = BlockFrequency::getMaxFrequency(); @@ -2062,12 +2306,13 @@ void MachineBlockPlacement::rotateLoopWithProfile( // chain head is not the loop header. As we only consider natural loops with // single header, this computation can be done only once. BlockFrequency HeaderFallThroughCost(0); - for (auto *Pred : HeaderBB->predecessors()) { + MachineBasicBlock *ChainHeaderBB = *LoopChain.begin(); + for (auto *Pred : ChainHeaderBB->predecessors()) { BlockChain *PredChain = BlockToChain[Pred]; if (!LoopBlockSet.count(Pred) && (!PredChain || Pred == *std::prev(PredChain->end()))) { - auto EdgeFreq = - MBFI->getBlockFreq(Pred) * MBPI->getEdgeProbability(Pred, HeaderBB); + auto EdgeFreq = MBFI->getBlockFreq(Pred) * + MBPI->getEdgeProbability(Pred, ChainHeaderBB); auto FallThruCost = ScaleBlockFrequency(EdgeFreq, MisfetchCost); // If the predecessor has only an unconditional jump to the header, we // need to consider the cost of this jump. @@ -2117,7 +2362,7 @@ void MachineBlockPlacement::rotateLoopWithProfile( // If the current BB is the loop header, we need to take into account the // cost of the missed fall through edge from outside of the loop to the // header. - if (Iter != HeaderIter) + if (Iter != LoopChain.begin()) Cost += HeaderFallThroughCost; // Collect the loop exit cost by summing up frequencies of all exit edges @@ -2238,9 +2483,7 @@ void MachineBlockPlacement::buildLoopChains(const MachineLoop &L) { // loop. This will default to the header, but may end up as one of the // predecessors to the header if there is one which will result in strictly // fewer branches in the loop body. - // When we use profile data to rotate the loop, this is unnecessary. - MachineBasicBlock *LoopTop = - RotateLoopWithProfile ? L.getHeader() : findBestLoopTop(L, LoopBlockSet); + MachineBasicBlock *LoopTop = findBestLoopTop(L, LoopBlockSet); // If we selected just the header for the loop top, look for a potentially // profitable exit block in the event that rotating the loop can eliminate @@ -2249,8 +2492,9 @@ void MachineBlockPlacement::buildLoopChains(const MachineLoop &L) { // Loops are processed innermost to uttermost, make sure we clear // PreferredLoopExit before processing a new loop. PreferredLoopExit = nullptr; + BlockFrequency ExitFreq; if (!RotateLoopWithProfile && LoopTop == L.getHeader()) - PreferredLoopExit = findBestLoopExit(L, LoopBlockSet); + PreferredLoopExit = findBestLoopExit(L, LoopBlockSet, ExitFreq); BlockChain &LoopChain = *BlockToChain[LoopTop]; @@ -2270,7 +2514,7 @@ void MachineBlockPlacement::buildLoopChains(const MachineLoop &L) { if (RotateLoopWithProfile) rotateLoopWithProfile(LoopChain, L, LoopBlockSet); else - rotateLoop(LoopChain, PreferredLoopExit, LoopBlockSet); + rotateLoop(LoopChain, PreferredLoopExit, ExitFreq, LoopBlockSet); LLVM_DEBUG({ // Crash at the end so we get all of the debugging output first. @@ -2497,8 +2741,8 @@ void MachineBlockPlacement::alignBlocks() { // exclusively on the loop info here so that we can align backedges in // unnatural CFGs and backedges that were introduced purely because of the // loop rotations done during this layout pass. - if (F->getFunction().optForMinSize() || - (F->getFunction().optForSize() && !TLI->alignLoopsWithOptSize())) + if (F->getFunction().hasMinSize() || + (F->getFunction().hasOptSize() && !TLI->alignLoopsWithOptSize())) return; BlockChain &FunctionChain = *BlockToChain[&F->front()]; if (FunctionChain.begin() == FunctionChain.end()) @@ -2773,7 +3017,7 @@ bool MachineBlockPlacement::runOnMachineFunction(MachineFunction &MF) { if (allowTailDupPlacement()) { MPDT = &getAnalysis(); - if (MF.getFunction().optForSize()) + if (MF.getFunction().hasOptSize()) TailDupSize = 1; bool PreRegAlloc = false; TailDup.initMF(MF, PreRegAlloc, MBPI, /* LayoutMode */ true, TailDupSize); @@ -2796,7 +3040,7 @@ bool MachineBlockPlacement::runOnMachineFunction(MachineFunction &MF) { if (BF.OptimizeFunction(MF, TII, MF.getSubtarget().getRegisterInfo(), getAnalysisIfAvailable(), MLI, - /*AfterBlockPlacement=*/true)) { + /*AfterPlacement=*/true)) { // Redo the layout if tail merging creates/removes/moves blocks. BlockToChain.clear(); ComputedEdges.clear(); diff --git a/lib/CodeGen/MachineBranchProbabilityInfo.cpp b/lib/CodeGen/MachineBranchProbabilityInfo.cpp index e4952aaaba06..d2277ce51746 100644 --- a/lib/CodeGen/MachineBranchProbabilityInfo.cpp +++ b/lib/CodeGen/MachineBranchProbabilityInfo.cpp @@ -1,9 +1,8 @@ //===- MachineBranchProbabilityInfo.cpp - Machine Branch Probability Info -===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/MachineCSE.cpp b/lib/CodeGen/MachineCSE.cpp index 6ee8571c28aa..2df6d40d9293 100644 --- a/lib/CodeGen/MachineCSE.cpp +++ b/lib/CodeGen/MachineCSE.cpp @@ -1,9 +1,8 @@ //===- MachineCSE.cpp - Machine Common Subexpression Elimination Pass -----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -20,6 +19,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/AliasAnalysis.h" +#include "llvm/Analysis/CFG.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineDominators.h" #include "llvm/CodeGen/MachineFunction.h" @@ -50,6 +50,8 @@ using namespace llvm; STATISTIC(NumCoalesces, "Number of copies coalesced"); STATISTIC(NumCSEs, "Number of common subexpression eliminated"); +STATISTIC(NumPREs, "Number of partial redundant expression" + " transformed to fully redundant"); STATISTIC(NumPhysCSEs, "Number of physreg referencing common subexpr eliminated"); STATISTIC(NumCrossBBCSEs, @@ -85,6 +87,7 @@ namespace { void releaseMemory() override { ScopeMap.clear(); + PREMap.clear(); Exps.clear(); } @@ -95,9 +98,12 @@ namespace { ScopedHashTable; using ScopeType = ScopedHTType::ScopeTy; + using PhysDefVector = SmallVector, 2>; unsigned LookAheadLimit = 0; DenseMap ScopeMap; + DenseMap + PREMap; ScopedHTType VNT; SmallVector Exps; unsigned CurrVN = 0; @@ -109,22 +115,24 @@ namespace { MachineBasicBlock::const_iterator E) const; bool hasLivePhysRegDefUses(const MachineInstr *MI, const MachineBasicBlock *MBB, - SmallSet &PhysRefs, - SmallVectorImpl &PhysDefs, - bool &PhysUseDef) const; + SmallSet &PhysRefs, + PhysDefVector &PhysDefs, bool &PhysUseDef) const; bool PhysRegDefsReach(MachineInstr *CSMI, MachineInstr *MI, - SmallSet &PhysRefs, - SmallVectorImpl &PhysDefs, - bool &NonLocal) const; + SmallSet &PhysRefs, + PhysDefVector &PhysDefs, bool &NonLocal) const; bool isCSECandidate(MachineInstr *MI); bool isProfitableToCSE(unsigned CSReg, unsigned Reg, - MachineInstr *CSMI, MachineInstr *MI); + MachineBasicBlock *CSBB, MachineInstr *MI); void EnterScope(MachineBasicBlock *MBB); void ExitScope(MachineBasicBlock *MBB); - bool ProcessBlock(MachineBasicBlock *MBB); + bool ProcessBlockCSE(MachineBasicBlock *MBB); void ExitScopeIfDone(MachineDomTreeNode *Node, DenseMap &OpenChildren); bool PerformCSE(MachineDomTreeNode *Node); + + bool isPRECandidate(MachineInstr *MI); + bool ProcessBlockPRE(MachineDominatorTree *MDT, MachineBasicBlock *MBB); + bool PerformSimplePRE(MachineDominatorTree *DT); }; } // end anonymous namespace @@ -256,9 +264,9 @@ static bool isCallerPreservedOrConstPhysReg(unsigned Reg, /// instruction does not uses a physical register. bool MachineCSE::hasLivePhysRegDefUses(const MachineInstr *MI, const MachineBasicBlock *MBB, - SmallSet &PhysRefs, - SmallVectorImpl &PhysDefs, - bool &PhysUseDef) const{ + SmallSet &PhysRefs, + PhysDefVector &PhysDefs, + bool &PhysUseDef) const { // First, add all uses to PhysRefs. for (const MachineOperand &MO : MI->operands()) { if (!MO.isReg() || MO.isDef()) @@ -278,7 +286,8 @@ bool MachineCSE::hasLivePhysRegDefUses(const MachineInstr *MI, // (which currently contains only uses), set the PhysUseDef flag. PhysUseDef = false; MachineBasicBlock::const_iterator I = MI; I = std::next(I); - for (const MachineOperand &MO : MI->operands()) { + for (const auto &MOP : llvm::enumerate(MI->operands())) { + const MachineOperand &MO = MOP.value(); if (!MO.isReg() || !MO.isDef()) continue; unsigned Reg = MO.getReg(); @@ -293,20 +302,21 @@ bool MachineCSE::hasLivePhysRegDefUses(const MachineInstr *MI, // common since this pass is run before livevariables. We can scan // forward a few instructions and check if it is obviously dead. if (!MO.isDead() && !isPhysDefTriviallyDead(Reg, I, MBB->end())) - PhysDefs.push_back(Reg); + PhysDefs.push_back(std::make_pair(MOP.index(), Reg)); } // Finally, add all defs to PhysRefs as well. for (unsigned i = 0, e = PhysDefs.size(); i != e; ++i) - for (MCRegAliasIterator AI(PhysDefs[i], TRI, true); AI.isValid(); ++AI) + for (MCRegAliasIterator AI(PhysDefs[i].second, TRI, true); AI.isValid(); + ++AI) PhysRefs.insert(*AI); return !PhysRefs.empty(); } bool MachineCSE::PhysRegDefsReach(MachineInstr *CSMI, MachineInstr *MI, - SmallSet &PhysRefs, - SmallVectorImpl &PhysDefs, + SmallSet &PhysRefs, + PhysDefVector &PhysDefs, bool &NonLocal) const { // For now conservatively returns false if the common subexpression is // not in the same basic block as the given instruction. The only exception @@ -320,7 +330,8 @@ bool MachineCSE::PhysRegDefsReach(MachineInstr *CSMI, MachineInstr *MI, return false; for (unsigned i = 0, e = PhysDefs.size(); i != e; ++i) { - if (MRI->isAllocatable(PhysDefs[i]) || MRI->isReserved(PhysDefs[i])) + if (MRI->isAllocatable(PhysDefs[i].second) || + MRI->isReserved(PhysDefs[i].second)) // Avoid extending live range of physical registers if they are //allocatable or reserved. return false; @@ -381,7 +392,7 @@ bool MachineCSE::isCSECandidate(MachineInstr *MI) { // Ignore stuff that we obviously can't move. if (MI->mayStore() || MI->isCall() || MI->isTerminator() || - MI->hasUnmodeledSideEffects()) + MI->mayRaiseFPException() || MI->hasUnmodeledSideEffects()) return false; if (MI->mayLoad()) { @@ -404,9 +415,10 @@ bool MachineCSE::isCSECandidate(MachineInstr *MI) { } /// isProfitableToCSE - Return true if it's profitable to eliminate MI with a -/// common expression that defines Reg. +/// common expression that defines Reg. CSBB is basic block where CSReg is +/// defined. bool MachineCSE::isProfitableToCSE(unsigned CSReg, unsigned Reg, - MachineInstr *CSMI, MachineInstr *MI) { + MachineBasicBlock *CSBB, MachineInstr *MI) { // FIXME: Heuristics that works around the lack the live range splitting. // If CSReg is used at all uses of Reg, CSE should not increase register @@ -432,7 +444,6 @@ bool MachineCSE::isProfitableToCSE(unsigned CSReg, unsigned Reg, // an immediate predecessor. We don't want to increase register pressure and // end up causing other computation to be spilled. if (TII->isAsCheapAsAMove(*MI)) { - MachineBasicBlock *CSBB = CSMI->getParent(); MachineBasicBlock *BB = MI->getParent(); if (CSBB != BB && !CSBB->isSuccessor(BB)) return false; @@ -487,7 +498,7 @@ void MachineCSE::ExitScope(MachineBasicBlock *MBB) { ScopeMap.erase(SI); } -bool MachineCSE::ProcessBlock(MachineBasicBlock *MBB) { +bool MachineCSE::ProcessBlockCSE(MachineBasicBlock *MBB) { bool Changed = false; SmallVector, 8> CSEPairs; @@ -536,7 +547,7 @@ bool MachineCSE::ProcessBlock(MachineBasicBlock *MBB) { // It's also not safe if the instruction uses physical registers. bool CrossMBBPhysDef = false; SmallSet PhysRefs; - SmallVector PhysDefs; + PhysDefVector PhysDefs; bool PhysUseDef = false; if (FoundCSE && hasLivePhysRegDefUses(MI, MBB, PhysRefs, PhysDefs, PhysUseDef)) { @@ -597,7 +608,7 @@ bool MachineCSE::ProcessBlock(MachineBasicBlock *MBB) { TargetRegisterInfo::isVirtualRegister(NewReg) && "Do not CSE physical register defs!"); - if (!isProfitableToCSE(NewReg, OldReg, CSMI, MI)) { + if (!isProfitableToCSE(NewReg, OldReg, CSMI->getParent(), MI)) { LLVM_DEBUG(dbgs() << "*** Not profitable, avoid CSE!\n"); DoCSE = false; break; @@ -635,6 +646,9 @@ bool MachineCSE::ProcessBlock(MachineBasicBlock *MBB) { // we should make sure it is not dead at CSMI. for (unsigned ImplicitDefToUpdate : ImplicitDefsToUpdate) CSMI->getOperand(ImplicitDefToUpdate).setIsDead(false); + for (auto PhysDef : PhysDefs) + if (!MI->getOperand(PhysDef.first).isDead()) + CSMI->getOperand(PhysDef.first).setIsDead(false); // Go through implicit defs of CSMI and MI, and clear the kill flags on // their uses in all the instructions between CSMI and MI. @@ -663,9 +677,9 @@ bool MachineCSE::ProcessBlock(MachineBasicBlock *MBB) { // Add physical register defs now coming in from a predecessor to MBB // livein list. while (!PhysDefs.empty()) { - unsigned LiveIn = PhysDefs.pop_back_val(); - if (!MBB->isLiveIn(LiveIn)) - MBB->addLiveIn(LiveIn); + auto LiveIn = PhysDefs.pop_back_val(); + if (!MBB->isLiveIn(LiveIn.second)) + MBB->addLiveIn(LiveIn.second); } ++NumCrossBBCSEs; } @@ -734,7 +748,7 @@ bool MachineCSE::PerformCSE(MachineDomTreeNode *Node) { for (MachineDomTreeNode *Node : Scopes) { MachineBasicBlock *MBB = Node->getBlock(); EnterScope(MBB); - Changed |= ProcessBlock(MBB); + Changed |= ProcessBlockCSE(MBB); // If it's a leaf node, it's done. Traverse upwards to pop ancestors. ExitScopeIfDone(Node, OpenChildren); } @@ -742,6 +756,104 @@ bool MachineCSE::PerformCSE(MachineDomTreeNode *Node) { return Changed; } +// We use stronger checks for PRE candidate rather than for CSE ones to embrace +// checks inside ProcessBlockCSE(), not only inside isCSECandidate(). This helps +// to exclude instrs created by PRE that won't be CSEed later. +bool MachineCSE::isPRECandidate(MachineInstr *MI) { + if (!isCSECandidate(MI) || + MI->isNotDuplicable() || + MI->mayLoad() || + MI->isAsCheapAsAMove() || + MI->getNumDefs() != 1 || + MI->getNumExplicitDefs() != 1) + return false; + + for (auto def : MI->defs()) + if (!TRI->isVirtualRegister(def.getReg())) + return false; + + for (auto use : MI->uses()) + if (use.isReg() && !TRI->isVirtualRegister(use.getReg())) + return false; + + return true; +} + +bool MachineCSE::ProcessBlockPRE(MachineDominatorTree *DT, + MachineBasicBlock *MBB) { + bool Changed = false; + for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); I != E;) { + MachineInstr *MI = &*I; + ++I; + + if (!isPRECandidate(MI)) + continue; + + if (!PREMap.count(MI)) { + PREMap[MI] = MBB; + continue; + } + + auto MBB1 = PREMap[MI]; + assert( + !DT->properlyDominates(MBB, MBB1) && + "MBB cannot properly dominate MBB1 while DFS through dominators tree!"); + auto CMBB = DT->findNearestCommonDominator(MBB, MBB1); + if (!CMBB->isLegalToHoistInto()) + continue; + + // Two instrs are partial redundant if their basic blocks are reachable + // from one to another but one doesn't dominate another. + if (CMBB != MBB1) { + auto BB = MBB->getBasicBlock(), BB1 = MBB1->getBasicBlock(); + if (BB != nullptr && BB1 != nullptr && + (isPotentiallyReachable(BB1, BB) || + isPotentiallyReachable(BB, BB1))) { + + assert(MI->getOperand(0).isDef() && + "First operand of instr with one explicit def must be this def"); + unsigned VReg = MI->getOperand(0).getReg(); + unsigned NewReg = MRI->cloneVirtualRegister(VReg); + if (!isProfitableToCSE(NewReg, VReg, CMBB, MI)) + continue; + MachineInstr &NewMI = + TII->duplicate(*CMBB, CMBB->getFirstTerminator(), *MI); + NewMI.getOperand(0).setReg(NewReg); + + PREMap[MI] = CMBB; + ++NumPREs; + Changed = true; + } + } + } + return Changed; +} + +// This simple PRE (partial redundancy elimination) pass doesn't actually +// eliminate partial redundancy but transforms it to full redundancy, +// anticipating that the next CSE step will eliminate this created redundancy. +// If CSE doesn't eliminate this, than created instruction will remain dead +// and eliminated later by Remove Dead Machine Instructions pass. +bool MachineCSE::PerformSimplePRE(MachineDominatorTree *DT) { + SmallVector BBs; + + PREMap.clear(); + bool Changed = false; + BBs.push_back(DT->getRootNode()); + do { + auto Node = BBs.pop_back_val(); + const std::vector &Children = Node->getChildren(); + for (MachineDomTreeNode *Child : Children) + BBs.push_back(Child); + + MachineBasicBlock *MBB = Node->getBlock(); + Changed |= ProcessBlockPRE(DT, MBB); + + } while (!BBs.empty()); + + return Changed; +} + bool MachineCSE::runOnMachineFunction(MachineFunction &MF) { if (skipFunction(MF.getFunction())) return false; @@ -752,5 +864,8 @@ bool MachineCSE::runOnMachineFunction(MachineFunction &MF) { AA = &getAnalysis().getAAResults(); DT = &getAnalysis(); LookAheadLimit = TII->getMachineCSELookAheadLimit(); - return PerformCSE(DT->getRootNode()); + bool ChangedPRE, ChangedCSE; + ChangedPRE = PerformSimplePRE(DT); + ChangedCSE = PerformCSE(DT->getRootNode()); + return ChangedPRE || ChangedCSE; } diff --git a/lib/CodeGen/MachineCombiner.cpp b/lib/CodeGen/MachineCombiner.cpp index f51b482e20e3..0584ec0bd2b3 100644 --- a/lib/CodeGen/MachineCombiner.cpp +++ b/lib/CodeGen/MachineCombiner.cpp @@ -1,9 +1,8 @@ //===---- MachineCombiner.cpp - Instcombining on SSA form machine code ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -559,16 +558,15 @@ bool MachineCombiner::combineInstructions(MachineBasicBlock *MBB) { continue; LLVM_DEBUG(if (dump_intrs) { - dbgs() << "\tFor the Pattern (" << (int)P << ") these instructions could be removed\n"; - for (auto const *InstrPtr : DelInstrs) { - dbgs() << "\t\t" << STI->getSchedInfoStr(*InstrPtr) << ": "; - InstrPtr->print(dbgs(), false, false, false, TII); - } + dbgs() << "\tFor the Pattern (" << (int)P + << ") these instructions could be removed\n"; + for (auto const *InstrPtr : DelInstrs) + InstrPtr->print(dbgs(), /*IsStandalone*/false, /*SkipOpers*/false, + /*SkipDebugLoc*/false, /*AddNewLine*/true, TII); dbgs() << "\tThese instructions could replace the removed ones\n"; - for (auto const *InstrPtr : InsInstrs) { - dbgs() << "\t\t" << STI->getSchedInfoStr(*InstrPtr) << ": "; - InstrPtr->print(dbgs(), false, false, false, TII); - } + for (auto const *InstrPtr : InsInstrs) + InstrPtr->print(dbgs(), /*IsStandalone*/false, /*SkipOpers*/false, + /*SkipDebugLoc*/false, /*AddNewLine*/true, TII); }); bool SubstituteAlways = false; @@ -641,7 +639,7 @@ bool MachineCombiner::runOnMachineFunction(MachineFunction &MF) { MLI = &getAnalysis(); Traces = &getAnalysis(); MinInstr = nullptr; - OptSize = MF.getFunction().optForSize(); + OptSize = MF.getFunction().hasOptSize(); LLVM_DEBUG(dbgs() << getPassName() << ": " << MF.getName() << '\n'); if (!TII->useMachineCombiner()) { diff --git a/lib/CodeGen/MachineCopyPropagation.cpp b/lib/CodeGen/MachineCopyPropagation.cpp index 19879fe89007..9fc12ac89e12 100644 --- a/lib/CodeGen/MachineCopyPropagation.cpp +++ b/lib/CodeGen/MachineCopyPropagation.cpp @@ -1,9 +1,8 @@ //===- MachineCopyPropagation.cpp - Machine Copy Propagation Pass ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/MachineDominanceFrontier.cpp b/lib/CodeGen/MachineDominanceFrontier.cpp index b559e4e513a6..6704298c17d6 100644 --- a/lib/CodeGen/MachineDominanceFrontier.cpp +++ b/lib/CodeGen/MachineDominanceFrontier.cpp @@ -1,9 +1,8 @@ //===- MachineDominanceFrontier.cpp ---------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/MachineDominators.cpp b/lib/CodeGen/MachineDominators.cpp index 6b2802626456..1dfba8638c22 100644 --- a/lib/CodeGen/MachineDominators.cpp +++ b/lib/CodeGen/MachineDominators.cpp @@ -1,9 +1,8 @@ //===- MachineDominators.cpp - Machine Dominator Calculation --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/MachineFrameInfo.cpp b/lib/CodeGen/MachineFrameInfo.cpp index 0b316871dbdf..bae3a4333bda 100644 --- a/lib/CodeGen/MachineFrameInfo.cpp +++ b/lib/CodeGen/MachineFrameInfo.cpp @@ -1,9 +1,8 @@ //===-- MachineFrameInfo.cpp ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -57,7 +56,8 @@ int MachineFrameInfo::CreateStackObject(uint64_t Size, unsigned Alignment, !IsSpillSlot, StackID)); int Index = (int)Objects.size() - NumFixedObjects - 1; assert(Index >= 0 && "Bad frame index!"); - ensureMaxAlignment(Alignment); + if (StackID == 0) + ensureMaxAlignment(Alignment); return Index; } @@ -92,7 +92,7 @@ int MachineFrameInfo::CreateFixedObject(uint64_t Size, int64_t SPOffset, Alignment = clampStackAlignment(!StackRealignable, Alignment, StackAlignment); Objects.insert(Objects.begin(), StackObject(Size, Alignment, SPOffset, IsImmutable, - /*isSpillSlot=*/false, /*Alloca=*/nullptr, + /*IsSpillSlot=*/false, /*Alloca=*/nullptr, IsAliased)); return -++NumFixedObjects; } @@ -142,11 +142,15 @@ unsigned MachineFrameInfo::estimateStackSize(const MachineFunction &MF) const { // should keep in mind that there's tight coupling between the two. for (int i = getObjectIndexBegin(); i != 0; ++i) { + // Only estimate stack size of default stack. + if (getStackID(i) != TargetStackID::Default) + continue; int FixedOff = -getObjectOffset(i); if (FixedOff > Offset) Offset = FixedOff; } for (unsigned i = 0, e = getObjectIndexEnd(); i != e; ++i) { - if (isDeadObjectIndex(i)) + // Only estimate stack size of live objects on default stack. + if (isDeadObjectIndex(i) || getStackID(i) != TargetStackID::Default) continue; Offset += getObjectSize(i); unsigned Align = getObjectAlignment(i); diff --git a/lib/CodeGen/MachineFunction.cpp b/lib/CodeGen/MachineFunction.cpp index 3495319670a5..4df5ce2dcedc 100644 --- a/lib/CodeGen/MachineFunction.cpp +++ b/lib/CodeGen/MachineFunction.cpp @@ -1,9 +1,8 @@ //===- MachineFunction.cpp ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -44,6 +43,7 @@ #include "llvm/IR/BasicBlock.h" #include "llvm/IR/Constant.h" #include "llvm/IR/DataLayout.h" +#include "llvm/IR/DebugInfoMetadata.h" #include "llvm/IR/DerivedTypes.h" #include "llvm/IR/Function.h" #include "llvm/IR/GlobalValue.h" @@ -165,7 +165,7 @@ void MachineFunction::init() { !F.hasFnAttribute("no-realign-stack"); FrameInfo = new (Allocator) MachineFrameInfo( getFnStackAlignment(STI, F), /*StackRealignable=*/CanRealignSP, - /*ForceRealign=*/CanRealignSP && + /*ForcedRealign=*/CanRealignSP && F.hasFnAttribute(Attribute::StackAlignment)); if (F.hasFnAttribute(Attribute::StackAlignment)) @@ -175,7 +175,7 @@ void MachineFunction::init() { Alignment = STI->getTargetLowering()->getMinFunctionAlignment(); // FIXME: Shouldn't use pref alignment if explicit alignment is set on F. - // FIXME: Use Function::optForSize(). + // FIXME: Use Function::hasOptSize(). if (!F.hasFnAttribute(Attribute::OptimizeForSize)) Alignment = std::max(Alignment, STI->getTargetLowering()->getPrefFunctionAlignment()); @@ -274,6 +274,12 @@ bool MachineFunction::shouldSplitStack() const { return getFunction().hasFnAttribute("split-stack"); } +LLVM_NODISCARD unsigned +MachineFunction::addFrameInst(const MCCFIInstruction &Inst) { + FrameInstructions.push_back(Inst); + return FrameInstructions.size() - 1; +} + /// This discards all of the MachineBasicBlock numbers and recomputes them. /// This guarantees that the MBB numbers are sequential, dense, and match the /// ordering of the blocks within the function. If a specific MachineBasicBlock @@ -357,6 +363,13 @@ MachineInstr &MachineFunction::CloneMachineInstrBundle(MachineBasicBlock &MBB, /// ~MachineInstr() destructor must be empty. void MachineFunction::DeleteMachineInstr(MachineInstr *MI) { + // Verify that a call site info is at valid state. This assertion should + // be triggered during the implementation of support for the + // call site info of a new architecture. If the assertion is triggered, + // back trace will tell where to insert a call to updateCallSiteInfo(). + assert((!MI->isCall(MachineInstr::IgnoreBundle) || + CallSitesInfo.find(MI) == CallSitesInfo.end()) && + "Call site info was not updated!"); // Strip it for parts. The operand array and the MI object itself are // independently recyclable. if (MI->Operands) @@ -396,19 +409,18 @@ MachineMemOperand *MachineFunction::getMachineMemOperand( MachineMemOperand * MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO, int64_t Offset, uint64_t Size) { - if (MMO->getValue()) - return new (Allocator) - MachineMemOperand(MachinePointerInfo(MMO->getValue(), - MMO->getOffset()+Offset), - MMO->getFlags(), Size, MMO->getBaseAlignment(), - AAMDNodes(), nullptr, MMO->getSyncScopeID(), - MMO->getOrdering(), MMO->getFailureOrdering()); + const MachinePointerInfo &PtrInfo = MMO->getPointerInfo(); + + // If there is no pointer value, the offset isn't tracked so we need to adjust + // the base alignment. + unsigned Align = PtrInfo.V.isNull() + ? MinAlign(MMO->getBaseAlignment(), Offset) + : MMO->getBaseAlignment(); + return new (Allocator) - MachineMemOperand(MachinePointerInfo(MMO->getPseudoValue(), - MMO->getOffset()+Offset), - MMO->getFlags(), Size, MMO->getBaseAlignment(), - AAMDNodes(), nullptr, MMO->getSyncScopeID(), - MMO->getOrdering(), MMO->getFailureOrdering()); + MachineMemOperand(PtrInfo.getWithOffset(Offset), MMO->getFlags(), Size, + Align, AAMDNodes(), nullptr, MMO->getSyncScopeID(), + MMO->getOrdering(), MMO->getFailureOrdering()); } MachineMemOperand * @@ -425,6 +437,15 @@ MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO, MMO->getOrdering(), MMO->getFailureOrdering()); } +MachineMemOperand * +MachineFunction::getMachineMemOperand(const MachineMemOperand *MMO, + MachineMemOperand::Flags Flags) { + return new (Allocator) MachineMemOperand( + MMO->getPointerInfo(), Flags, MMO->getSize(), MMO->getBaseAlignment(), + MMO->getAAInfo(), MMO->getRanges(), MMO->getSyncScopeID(), + MMO->getOrdering(), MMO->getFailureOrdering()); +} + MachineInstr::ExtraInfo * MachineFunction::createMIExtraInfo(ArrayRef MMOs, MCSymbol *PreInstrSymbol, @@ -802,6 +823,32 @@ try_next:; return FilterID; } +void MachineFunction::addCodeViewHeapAllocSite(MachineInstr *I, MDNode *MD) { + MCSymbol *BeginLabel = Ctx.createTempSymbol("heapallocsite", true); + MCSymbol *EndLabel = Ctx.createTempSymbol("heapallocsite", true); + I->setPreInstrSymbol(*this, BeginLabel); + I->setPostInstrSymbol(*this, EndLabel); + + DIType *DI = dyn_cast(MD); + CodeViewHeapAllocSites.push_back(std::make_tuple(BeginLabel, EndLabel, DI)); +} + +void MachineFunction::updateCallSiteInfo(const MachineInstr *Old, + const MachineInstr *New) { + if (!Target.Options.EnableDebugEntryValues || Old == New) + return; + + assert(Old->isCall() && (!New || New->isCall()) && + "Call site info referes only to call instructions!"); + CallSiteInfoMap::iterator CSIt = CallSitesInfo.find(Old); + if (CSIt == CallSitesInfo.end()) + return; + CallSiteInfo CSInfo = std::move(CSIt->second); + CallSitesInfo.erase(CSIt); + if (New) + CallSitesInfo[New] = CSInfo; +} + /// \} //===----------------------------------------------------------------------===// @@ -888,9 +935,11 @@ void MachineJumpTableInfo::print(raw_ostream &OS) const { OS << "Jump Tables:\n"; for (unsigned i = 0, e = JumpTables.size(); i != e; ++i) { - OS << printJumpTableEntryReference(i) << ": "; + OS << printJumpTableEntryReference(i) << ':'; for (unsigned j = 0, f = JumpTables[i].MBBs.size(); j != f; ++j) OS << ' ' << printMBBReference(*JumpTables[i].MBBs[j]); + if (i != e) + OS << '\n'; } OS << '\n'; diff --git a/lib/CodeGen/MachineFunctionPass.cpp b/lib/CodeGen/MachineFunctionPass.cpp index 5db4e299fa70..0da4cf3fc90c 100644 --- a/lib/CodeGen/MachineFunctionPass.cpp +++ b/lib/CodeGen/MachineFunctionPass.cpp @@ -1,9 +1,8 @@ //===-- MachineFunctionPass.cpp -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/MachineFunctionPrinterPass.cpp b/lib/CodeGen/MachineFunctionPrinterPass.cpp index 9c96ba748778..0ea8975cc74c 100644 --- a/lib/CodeGen/MachineFunctionPrinterPass.cpp +++ b/lib/CodeGen/MachineFunctionPrinterPass.cpp @@ -1,9 +1,8 @@ //===-- MachineFunctionPrinterPass.cpp ------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/MachineInstr.cpp b/lib/CodeGen/MachineInstr.cpp index 764a84c7e132..e5c398a2d10c 100644 --- a/lib/CodeGen/MachineInstr.cpp +++ b/lib/CodeGen/MachineInstr.cpp @@ -1,9 +1,8 @@ //===- lib/CodeGen/MachineInstr.cpp ---------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -26,6 +25,7 @@ #include "llvm/Analysis/MemoryLocation.h" #include "llvm/CodeGen/GlobalISel/RegisterBank.h" #include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineFrameInfo.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineInstrBundle.h" @@ -50,9 +50,9 @@ #include "llvm/IR/Metadata.h" #include "llvm/IR/Module.h" #include "llvm/IR/ModuleSlotTracker.h" +#include "llvm/IR/Operator.h" #include "llvm/IR/Type.h" #include "llvm/IR/Value.h" -#include "llvm/IR/Operator.h" #include "llvm/MC/MCInstrDesc.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCSymbol.h" @@ -225,12 +225,13 @@ void MachineInstr::addOperand(MachineFunction &MF, const MachineOperand &Op) { } #ifndef NDEBUG - bool isMetaDataOp = Op.getType() == MachineOperand::MO_Metadata; + bool isDebugOp = Op.getType() == MachineOperand::MO_Metadata || + Op.getType() == MachineOperand::MO_MCSymbol; // OpNo now points as the desired insertion point. Unless this is a variadic // instruction, only implicit regs are allowed beyond MCID->getNumOperands(). // RegMask operands go between the explicit and implicit operands. assert((isImpReg || Op.isRegMask() || MCID->isVariadic() || - OpNo < MCID->getNumOperands() || isMetaDataOp) && + OpNo < MCID->getNumOperands() || isDebugOp) && "Trying to add an operand to a machine instr that is already done!"); #endif @@ -512,45 +513,65 @@ void MachineInstr::setPostInstrSymbol(MachineFunction &MF, MCSymbol *Symbol) { MF.createMIExtraInfo(memoperands(), getPreInstrSymbol(), Symbol)); } +void MachineInstr::cloneInstrSymbols(MachineFunction &MF, + const MachineInstr &MI) { + if (this == &MI) + // Nothing to do for a self-clone! + return; + + assert(&MF == MI.getMF() && + "Invalid machine functions when cloning instruction symbols!"); + + setPreInstrSymbol(MF, MI.getPreInstrSymbol()); + setPostInstrSymbol(MF, MI.getPostInstrSymbol()); +} + uint16_t MachineInstr::mergeFlagsWith(const MachineInstr &Other) const { // For now, the just return the union of the flags. If the flags get more // complicated over time, we might need more logic here. return getFlags() | Other.getFlags(); } -void MachineInstr::copyIRFlags(const Instruction &I) { +uint16_t MachineInstr::copyFlagsFromInstruction(const Instruction &I) { + uint16_t MIFlags = 0; // Copy the wrapping flags. if (const OverflowingBinaryOperator *OB = dyn_cast(&I)) { if (OB->hasNoSignedWrap()) - setFlag(MachineInstr::MIFlag::NoSWrap); + MIFlags |= MachineInstr::MIFlag::NoSWrap; if (OB->hasNoUnsignedWrap()) - setFlag(MachineInstr::MIFlag::NoUWrap); + MIFlags |= MachineInstr::MIFlag::NoUWrap; } // Copy the exact flag. if (const PossiblyExactOperator *PE = dyn_cast(&I)) if (PE->isExact()) - setFlag(MachineInstr::MIFlag::IsExact); + MIFlags |= MachineInstr::MIFlag::IsExact; // Copy the fast-math flags. if (const FPMathOperator *FP = dyn_cast(&I)) { const FastMathFlags Flags = FP->getFastMathFlags(); if (Flags.noNaNs()) - setFlag(MachineInstr::MIFlag::FmNoNans); + MIFlags |= MachineInstr::MIFlag::FmNoNans; if (Flags.noInfs()) - setFlag(MachineInstr::MIFlag::FmNoInfs); + MIFlags |= MachineInstr::MIFlag::FmNoInfs; if (Flags.noSignedZeros()) - setFlag(MachineInstr::MIFlag::FmNsz); + MIFlags |= MachineInstr::MIFlag::FmNsz; if (Flags.allowReciprocal()) - setFlag(MachineInstr::MIFlag::FmArcp); + MIFlags |= MachineInstr::MIFlag::FmArcp; if (Flags.allowContract()) - setFlag(MachineInstr::MIFlag::FmContract); + MIFlags |= MachineInstr::MIFlag::FmContract; if (Flags.approxFunc()) - setFlag(MachineInstr::MIFlag::FmAfn); + MIFlags |= MachineInstr::MIFlag::FmAfn; if (Flags.allowReassoc()) - setFlag(MachineInstr::MIFlag::FmReassoc); + MIFlags |= MachineInstr::MIFlag::FmReassoc; } + + return MIFlags; +} + +void MachineInstr::copyIRFlags(const Instruction &I) { + Flags = copyFlagsFromInstruction(I); } bool MachineInstr::hasPropertyInBundle(uint64_t Mask, QueryType Type) const { @@ -1157,7 +1178,7 @@ bool MachineInstr::isSafeToMove(AliasAnalysis *AA, bool &SawStore) const { } if (isPosition() || isDebugInstr() || isTerminator() || - hasUnmodeledSideEffects()) + mayRaiseFPException() || hasUnmodeledSideEffects()) return false; // See if this instruction does a load. If so, we have to guarantee that the @@ -1173,8 +1194,8 @@ bool MachineInstr::isSafeToMove(AliasAnalysis *AA, bool &SawStore) const { return true; } -bool MachineInstr::mayAlias(AliasAnalysis *AA, MachineInstr &Other, - bool UseTBAA) { +bool MachineInstr::mayAlias(AliasAnalysis *AA, const MachineInstr &Other, + bool UseTBAA) const { const MachineFunction *MF = getMF(); const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); const MachineFrameInfo &MFI = MF->getFrameInfo(); @@ -1304,7 +1325,11 @@ bool MachineInstr::isDereferenceableInvariantLoad(AliasAnalysis *AA) const { const MachineFrameInfo &MFI = getParent()->getParent()->getFrameInfo(); for (MachineMemOperand *MMO : memoperands()) { - if (MMO->isVolatile()) return false; + if (!MMO->isUnordered()) + // If the memory operand has ordering side effects, we can't move the + // instruction. Such an instruction is technically an invariant load, + // but the caller code would need updated to expect that. + return false; if (MMO->isStore()) return false; if (MMO->isInvariant() && MMO->isDereferenceable()) continue; @@ -1447,7 +1472,7 @@ void MachineInstr::print(raw_ostream &OS, bool IsStandalone, bool SkipOpers, ModuleSlotTracker MST(M); if (F) MST.incorporateFunction(*F); - print(OS, MST, IsStandalone, SkipOpers, SkipDebugLoc, TII); + print(OS, MST, IsStandalone, SkipOpers, SkipDebugLoc, AddNewLine, TII); } void MachineInstr::print(raw_ostream &OS, ModuleSlotTracker &MST, @@ -1519,6 +1544,8 @@ void MachineInstr::print(raw_ostream &OS, ModuleSlotTracker &MST, OS << "nsw "; if (getFlag(MachineInstr::IsExact)) OS << "exact "; + if (getFlag(MachineInstr::FPExcept)) + OS << "fpexcept "; // Print the opcode name. if (TII) @@ -1905,7 +1932,7 @@ void MachineInstr::setRegisterDefReadUndef(unsigned Reg, bool IsUndef) { void MachineInstr::addRegisterDefined(unsigned Reg, const TargetRegisterInfo *RegInfo) { if (TargetRegisterInfo::isPhysicalRegister(Reg)) { - MachineOperand *MO = findRegisterDefOperand(Reg, false, RegInfo); + MachineOperand *MO = findRegisterDefOperand(Reg, false, false, RegInfo); if (MO) return; } else { @@ -2050,7 +2077,7 @@ static const DIExpression *computeExprForSpill(const MachineInstr &MI) { const DIExpression *Expr = MI.getDebugExpression(); if (MI.isIndirectDebugValue()) { assert(MI.getOperand(1).getImm() == 0 && "DBG_VALUE with nonzero offset"); - Expr = DIExpression::prepend(Expr, DIExpression::WithDeref); + Expr = DIExpression::prepend(Expr, DIExpression::DerefBefore); } return Expr; } @@ -2100,3 +2127,54 @@ void MachineInstr::changeDebugValuesDefReg(unsigned Reg) { for (auto *DBI : DbgValues) DBI->getOperand(0).setReg(Reg); } + +using MMOList = SmallVector; + +static unsigned getSpillSlotSize(MMOList &Accesses, + const MachineFrameInfo &MFI) { + unsigned Size = 0; + for (auto A : Accesses) + if (MFI.isSpillSlotObjectIndex( + cast(A->getPseudoValue()) + ->getFrameIndex())) + Size += A->getSize(); + return Size; +} + +Optional +MachineInstr::getSpillSize(const TargetInstrInfo *TII) const { + int FI; + if (TII->isStoreToStackSlotPostFE(*this, FI)) { + const MachineFrameInfo &MFI = getMF()->getFrameInfo(); + if (MFI.isSpillSlotObjectIndex(FI)) + return (*memoperands_begin())->getSize(); + } + return None; +} + +Optional +MachineInstr::getFoldedSpillSize(const TargetInstrInfo *TII) const { + MMOList Accesses; + if (TII->hasStoreToStackSlot(*this, Accesses)) + return getSpillSlotSize(Accesses, getMF()->getFrameInfo()); + return None; +} + +Optional +MachineInstr::getRestoreSize(const TargetInstrInfo *TII) const { + int FI; + if (TII->isLoadFromStackSlotPostFE(*this, FI)) { + const MachineFrameInfo &MFI = getMF()->getFrameInfo(); + if (MFI.isSpillSlotObjectIndex(FI)) + return (*memoperands_begin())->getSize(); + } + return None; +} + +Optional +MachineInstr::getFoldedRestoreSize(const TargetInstrInfo *TII) const { + MMOList Accesses; + if (TII->hasLoadFromStackSlot(*this, Accesses)) + return getSpillSlotSize(Accesses, getMF()->getFrameInfo()); + return None; +} diff --git a/lib/CodeGen/MachineInstrBundle.cpp b/lib/CodeGen/MachineInstrBundle.cpp index ae378cc8c464..32e266e9401e 100644 --- a/lib/CodeGen/MachineInstrBundle.cpp +++ b/lib/CodeGen/MachineInstrBundle.cpp @@ -1,9 +1,8 @@ //===-- lib/CodeGen/MachineInstrBundle.cpp --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/MachineLICM.cpp b/lib/CodeGen/MachineLICM.cpp index 58fd1f238420..1107e609c258 100644 --- a/lib/CodeGen/MachineLICM.cpp +++ b/lib/CodeGen/MachineLICM.cpp @@ -1,9 +1,8 @@ //===- MachineLICM.cpp - Machine Loop Invariant Code Motion Pass ----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/MachineLoopInfo.cpp b/lib/CodeGen/MachineLoopInfo.cpp index 2bce59235057..3b8b430d1b0f 100644 --- a/lib/CodeGen/MachineLoopInfo.cpp +++ b/lib/CodeGen/MachineLoopInfo.cpp @@ -1,9 +1,8 @@ //===- MachineLoopInfo.cpp - Natural Loop Calculator ----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/MachineModuleInfo.cpp b/lib/CodeGen/MachineModuleInfo.cpp index 6ef8de88f8b1..aadcd7319799 100644 --- a/lib/CodeGen/MachineModuleInfo.cpp +++ b/lib/CodeGen/MachineModuleInfo.cpp @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/MachineModuleInfo.cpp ----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -206,11 +205,11 @@ MachineModuleInfo::~MachineModuleInfo() = default; bool MachineModuleInfo::doInitialization(Module &M) { ObjFileMMI = nullptr; CurCallSite = 0; - UsesVAFloatArgument = UsesMorestackAddr = false; + UsesMSVCFloatingPoint = UsesMorestackAddr = false; HasSplitStack = HasNosplitStack = false; AddrLabelSymbols = nullptr; TheModule = &M; - DbgInfoAvailable = !empty(M.debug_compile_units()); + DbgInfoAvailable = !llvm::empty(M.debug_compile_units()); return false; } @@ -328,22 +327,3 @@ char FreeMachineFunction::ID; FunctionPass *llvm::createFreeMachineFunctionPass() { return new FreeMachineFunction(); } - -//===- MMI building helpers -----------------------------------------------===// - -void llvm::computeUsesVAFloatArgument(const CallInst &I, - MachineModuleInfo &MMI) { - FunctionType *FT = - cast(I.getCalledValue()->getType()->getContainedType(0)); - if (FT->isVarArg() && !MMI.usesVAFloatArgument()) { - for (unsigned i = 0, e = I.getNumArgOperands(); i != e; ++i) { - Type *T = I.getArgOperand(i)->getType(); - for (auto i : post_order(T)) { - if (i->isFloatingPointTy()) { - MMI.setUsesVAFloatArgument(true); - return; - } - } - } - } -} diff --git a/lib/CodeGen/MachineModuleInfoImpls.cpp b/lib/CodeGen/MachineModuleInfoImpls.cpp index 7b4f64bfe60d..16d24880ebe4 100644 --- a/lib/CodeGen/MachineModuleInfoImpls.cpp +++ b/lib/CodeGen/MachineModuleInfoImpls.cpp @@ -1,9 +1,8 @@ //===- llvm/CodeGen/MachineModuleInfoImpls.cpp ----------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/MachineOperand.cpp b/lib/CodeGen/MachineOperand.cpp index 05e51e1873cf..4fa4ea7f6cf5 100644 --- a/lib/CodeGen/MachineOperand.cpp +++ b/lib/CodeGen/MachineOperand.cpp @@ -1,9 +1,8 @@ //===- lib/CodeGen/MachineOperand.cpp -------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -25,6 +24,7 @@ #include "llvm/IR/Constants.h" #include "llvm/IR/IRPrintingPasses.h" #include "llvm/IR/ModuleSlotTracker.h" +#include "llvm/MC/MCDwarf.h" #include "llvm/Target/TargetIntrinsicInfo.h" #include "llvm/Target/TargetMachine.h" @@ -181,6 +181,19 @@ void MachineOperand::ChangeToES(const char *SymName, setTargetFlags(TargetFlags); } +void MachineOperand::ChangeToGA(const GlobalValue *GV, int64_t Offset, + unsigned char TargetFlags) { + assert((!isReg() || !isTied()) && + "Cannot change a tied operand into a global address"); + + removeRegFromUses(); + + OpKind = MO_GlobalAddress; + Contents.OffsetedInfo.Val.GV = GV; + setOffset(Offset); + setTargetFlags(TargetFlags); +} + void MachineOperand::ChangeToMCSymbol(MCSymbol *Sym) { assert((!isReg() || !isTied()) && "Cannot change a tied operand into an MCSymbol"); @@ -329,7 +342,7 @@ hash_code llvm::hash_value(const MachineOperand &MO) { switch (MO.getType()) { case MachineOperand::MO_Register: // Register operands don't have target flags. - return hash_combine(MO.getType(), MO.getReg(), MO.getSubReg(), MO.isDef()); + return hash_combine(MO.getType(), (unsigned)MO.getReg(), MO.getSubReg(), MO.isDef()); case MachineOperand::MO_Immediate: return hash_combine(MO.getType(), MO.getTargetFlags(), MO.getImm()); case MachineOperand::MO_CImmediate: @@ -348,7 +361,7 @@ hash_code llvm::hash_value(const MachineOperand &MO) { return hash_combine(MO.getType(), MO.getTargetFlags(), MO.getIndex()); case MachineOperand::MO_ExternalSymbol: return hash_combine(MO.getType(), MO.getTargetFlags(), MO.getOffset(), - MO.getSymbolName()); + StringRef(MO.getSymbolName())); case MachineOperand::MO_GlobalAddress: return hash_combine(MO.getType(), MO.getTargetFlags(), MO.getGlobal(), MO.getOffset()); @@ -994,7 +1007,7 @@ MachineMemOperand::MachineMemOperand(MachinePointerInfo ptrinfo, Flags f, assert((PtrInfo.V.isNull() || PtrInfo.V.is() || isa(PtrInfo.V.get()->getType())) && "invalid pointer value"); - assert(getBaseAlignment() == a && "Alignment is not a power of 2!"); + assert(getBaseAlignment() == a && a != 0 && "Alignment is not a power of 2!"); assert((isLoad() || isStore()) && "Not a load/store!"); AtomicInfo.SSID = static_cast(SSID); @@ -1125,7 +1138,7 @@ void MachineMemOperand::print(raw_ostream &OS, ModuleSlotTracker &MST, printLLVMNameWithoutPrefix( OS, cast(PVal)->getSymbol()); break; - case PseudoSourceValue::TargetCustom: + default: // FIXME: This is not necessarily the correct MIR serialization format for // a custom pseudo source value, but at least it allows // -print-machineinstrs to work on a target with custom pseudo source diff --git a/lib/CodeGen/MachineOptimizationRemarkEmitter.cpp b/lib/CodeGen/MachineOptimizationRemarkEmitter.cpp index 906d5560d568..27db9106b337 100644 --- a/lib/CodeGen/MachineOptimizationRemarkEmitter.cpp +++ b/lib/CodeGen/MachineOptimizationRemarkEmitter.cpp @@ -1,9 +1,8 @@ ///===- MachineOptimizationRemarkEmitter.cpp - Opt Diagnostic -*- C++ -*---===// /// -/// The LLVM Compiler Infrastructure -/// -/// This file is distributed under the University of Illinois Open Source -/// License. See LICENSE.TXT for details. +/// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +/// See https://llvm.org/LICENSE.txt for license information. +/// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception /// ///===---------------------------------------------------------------------===// /// \file diff --git a/lib/CodeGen/MachineOutliner.cpp b/lib/CodeGen/MachineOutliner.cpp index ad96c0e579e4..80a235aeaa5c 100644 --- a/lib/CodeGen/MachineOutliner.cpp +++ b/lib/CodeGen/MachineOutliner.cpp @@ -1,9 +1,8 @@ //===---- MachineOutliner.cpp - Outline instructions -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -74,8 +73,6 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" #include -#include -#include #include #include @@ -1095,19 +1092,15 @@ MachineOutliner::createOutlinedFunction(Module &M, OutlinedFunction &OF, InstructionMapper &Mapper, unsigned Name) { - // Create the function name. This should be unique. For now, just hash the - // module name and include it in the function name plus the number of this - // function. - std::ostringstream NameStream; + // Create the function name. This should be unique. // FIXME: We should have a better naming scheme. This should be stable, // regardless of changes to the outliner's cost model/traversal order. - NameStream << "OUTLINED_FUNCTION_" << Name; + std::string FunctionName = ("OUTLINED_FUNCTION_" + Twine(Name)).str(); // Create the function using an IR-level function. LLVMContext &C = M.getContext(); - Function *F = dyn_cast( - M.getOrInsertFunction(NameStream.str(), Type::getVoidTy(C))); - assert(F && "Function was null!"); + Function *F = Function::Create(FunctionType::get(Type::getVoidTy(C), false), + Function::ExternalLinkage, FunctionName, M); // NOTE: If this is linkonceodr, then we can take advantage of linker deduping // which gives us better results when we outline from linkonceodr functions. @@ -1205,11 +1198,10 @@ bool MachineOutliner::outline(Module &M, unsigned OutlinedFunctionNum = 0; // Sort by benefit. The most beneficial functions should be outlined first. - std::stable_sort( - FunctionList.begin(), FunctionList.end(), - [](const OutlinedFunction &LHS, const OutlinedFunction &RHS) { - return LHS.getBenefit() > RHS.getBenefit(); - }); + llvm::stable_sort(FunctionList, [](const OutlinedFunction &LHS, + const OutlinedFunction &RHS) { + return LHS.getBenefit() > RHS.getBenefit(); + }); // Walk over each function, outlining them as we go along. Functions are // outlined greedily, based off the sort above. @@ -1253,8 +1245,9 @@ bool MachineOutliner::outline(Module &M, if (MBB.getParent()->getProperties().hasProperty( MachineFunctionProperties::Property::TracksLiveness)) { // Helper lambda for adding implicit def operands to the call - // instruction. - auto CopyDefs = [&CallInst](MachineInstr &MI) { + // instruction. It also updates call site information for moved + // code. + auto CopyDefsAndUpdateCalls = [&CallInst](MachineInstr &MI) { for (MachineOperand &MOP : MI.operands()) { // Skip over anything that isn't a register. if (!MOP.isReg()) @@ -1266,13 +1259,16 @@ bool MachineOutliner::outline(Module &M, MOP.getReg(), true, /* isDef = true */ true /* isImp = true */)); } + if (MI.isCall()) + MI.getMF()->updateCallSiteInfo(&MI); }; // Copy over the defs in the outlined range. // First inst in outlined range <-- Anything that's defined in this // ... .. range has to be added as an // implicit Last inst in outlined range <-- def to the call - // instruction. - std::for_each(CallInst, std::next(EndIt), CopyDefs); + // instruction. Also remove call site information for outlined block + // of code. + std::for_each(CallInst, std::next(EndIt), CopyDefsAndUpdateCalls); } // Erase from the point after where the call was inserted up to, and diff --git a/lib/CodeGen/MachinePipeliner.cpp b/lib/CodeGen/MachinePipeliner.cpp index 4d451bdd7f69..54df522d371a 100644 --- a/lib/CodeGen/MachinePipeliner.cpp +++ b/lib/CodeGen/MachinePipeliner.cpp @@ -1,9 +1,8 @@ //===- MachinePipeliner.cpp - Machine Software Pipeliner Pass -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -97,6 +96,14 @@ using namespace llvm; STATISTIC(NumTrytoPipeline, "Number of loops that we attempt to pipeline"); STATISTIC(NumPipelined, "Number of loops software pipelined"); STATISTIC(NumNodeOrderIssues, "Number of node order issues found"); +STATISTIC(NumFailBranch, "Pipeliner abort due to unknown branch"); +STATISTIC(NumFailLoop, "Pipeliner abort due to unsupported loop"); +STATISTIC(NumFailPreheader, "Pipeliner abort due to missing preheader"); +STATISTIC(NumFailLargeMaxMII, "Pipeliner abort due to MaxMII too large"); +STATISTIC(NumFailZeroMII, "Pipeliner abort due to zero MII"); +STATISTIC(NumFailNoSchedule, "Pipeliner abort due to no schedule found"); +STATISTIC(NumFailZeroStage, "Pipeliner abort due to zero stage"); +STATISTIC(NumFailLargeMaxStage, "Pipeliner abort due to too many stages"); /// A command line option to turn software pipelining on or off. static cl::opt EnableSWP("enable-pipeliner", cl::Hidden, cl::init(true), @@ -141,6 +148,11 @@ static cl::opt SwpIgnoreRecMII("pipeliner-ignore-recmii", cl::ReallyHidden, cl::init(false), cl::ZeroOrMore, cl::desc("Ignore RecMII")); +static cl::opt SwpShowResMask("pipeliner-show-mask", cl::Hidden, + cl::init(false)); +static cl::opt SwpDebugResource("pipeliner-dbg-res", cl::Hidden, + cl::init(false)); + namespace llvm { // A command line option to enable the CopyToPhi DAG mutation. @@ -180,6 +192,16 @@ bool MachinePipeliner::runOnMachineFunction(MachineFunction &mf) { !EnableSWPOptSize.getPosition()) return false; + if (!mf.getSubtarget().enableMachinePipeliner()) + return false; + + // Cannot pipeline loops without instruction itineraries if we are using + // DFA for the pipeliner. + if (mf.getSubtarget().useDFAforSMS() && + (!mf.getSubtarget().getInstrItineraryData() || + mf.getSubtarget().getInstrItineraryData()->isEmpty())) + return false; + MF = &mf; MLI = &getAnalysis(); MDT = &getAnalysis(); @@ -211,8 +233,11 @@ bool MachinePipeliner::scheduleLoop(MachineLoop &L) { } #endif - if (!canPipelineLoop(L)) + setPragmaPipelineOptions(L); + if (!canPipelineLoop(L)) { + LLVM_DEBUG(dbgs() << "\n!!! Can not pipeline loop.\n"); return Changed; + } ++NumTrytoPipeline; @@ -221,6 +246,50 @@ bool MachinePipeliner::scheduleLoop(MachineLoop &L) { return Changed; } +void MachinePipeliner::setPragmaPipelineOptions(MachineLoop &L) { + MachineBasicBlock *LBLK = L.getTopBlock(); + + if (LBLK == nullptr) + return; + + const BasicBlock *BBLK = LBLK->getBasicBlock(); + if (BBLK == nullptr) + return; + + const Instruction *TI = BBLK->getTerminator(); + if (TI == nullptr) + return; + + MDNode *LoopID = TI->getMetadata(LLVMContext::MD_loop); + if (LoopID == nullptr) + return; + + assert(LoopID->getNumOperands() > 0 && "requires atleast one operand"); + assert(LoopID->getOperand(0) == LoopID && "invalid loop"); + + for (unsigned i = 1, e = LoopID->getNumOperands(); i < e; ++i) { + MDNode *MD = dyn_cast(LoopID->getOperand(i)); + + if (MD == nullptr) + continue; + + MDString *S = dyn_cast(MD->getOperand(0)); + + if (S == nullptr) + continue; + + if (S->getString() == "llvm.loop.pipeline.initiationinterval") { + assert(MD->getNumOperands() == 2 && + "Pipeline initiation interval hint metadata should have two operands."); + II_setByPragma = + mdconst::extract(MD->getOperand(1))->getZExtValue(); + assert(II_setByPragma >= 1 && "Pipeline initiation interval must be positive."); + } else if (S->getString() == "llvm.loop.pipeline.disable") { + disabledByPragma = true; + } + } +} + /// Return true if the loop can be software pipelined. The algorithm is /// restricted to loops with a single basic block. Make sure that the /// branch in the loop can be analyzed. @@ -228,21 +297,36 @@ bool MachinePipeliner::canPipelineLoop(MachineLoop &L) { if (L.getNumBlocks() != 1) return false; + if (disabledByPragma) + return false; + // Check if the branch can't be understood because we can't do pipelining // if that's the case. LI.TBB = nullptr; LI.FBB = nullptr; LI.BrCond.clear(); - if (TII->analyzeBranch(*L.getHeader(), LI.TBB, LI.FBB, LI.BrCond)) + if (TII->analyzeBranch(*L.getHeader(), LI.TBB, LI.FBB, LI.BrCond)) { + LLVM_DEBUG( + dbgs() << "Unable to analyzeBranch, can NOT pipeline current Loop\n"); + NumFailBranch++; return false; + } LI.LoopInductionVar = nullptr; LI.LoopCompare = nullptr; - if (TII->analyzeLoop(L, LI.LoopInductionVar, LI.LoopCompare)) + if (TII->analyzeLoop(L, LI.LoopInductionVar, LI.LoopCompare)) { + LLVM_DEBUG( + dbgs() << "Unable to analyzeLoop, can NOT pipeline current Loop\n"); + NumFailLoop++; return false; + } - if (!L.getLoopPreheader()) + if (!L.getLoopPreheader()) { + LLVM_DEBUG( + dbgs() << "Preheader not found, can NOT pipeline current Loop\n"); + NumFailPreheader++; return false; + } // Remove any subregisters from inputs to phi nodes. preprocessPhiNodes(*L.getHeader()); @@ -286,7 +370,8 @@ void MachinePipeliner::preprocessPhiNodes(MachineBasicBlock &B) { bool MachinePipeliner::swingModuloScheduler(MachineLoop &L) { assert(L.getBlocks().size() == 1 && "SMS works on single blocks only."); - SwingSchedulerDAG SMS(*this, L, getAnalysis(), RegClassInfo); + SwingSchedulerDAG SMS(*this, L, getAnalysis(), RegClassInfo, + II_setByPragma); MachineBasicBlock *MBB = L.getHeader(); // The kernel should not include any terminator instructions. These @@ -309,6 +394,20 @@ bool MachinePipeliner::swingModuloScheduler(MachineLoop &L) { return SMS.hasNewSchedule(); } +void SwingSchedulerDAG::setMII(unsigned ResMII, unsigned RecMII) { + if (II_setByPragma > 0) + MII = II_setByPragma; + else + MII = std::max(ResMII, RecMII); +} + +void SwingSchedulerDAG::setMAX_II() { + if (II_setByPragma > 0) + MAX_II = II_setByPragma; + else + MAX_II = MII + 10; +} + /// We override the schedule function in ScheduleDAGInstrs to implement the /// scheduling part of the Swing Modulo Scheduling algorithm. void SwingSchedulerDAG::schedule() { @@ -335,17 +434,28 @@ void SwingSchedulerDAG::schedule() { if (SwpIgnoreRecMII) RecMII = 0; - MII = std::max(ResMII, RecMII); - LLVM_DEBUG(dbgs() << "MII = " << MII << " (rec=" << RecMII - << ", res=" << ResMII << ")\n"); + setMII(ResMII, RecMII); + setMAX_II(); + + LLVM_DEBUG(dbgs() << "MII = " << MII << " MAX_II = " << MAX_II + << " (rec=" << RecMII << ", res=" << ResMII << ")\n"); // Can't schedule a loop without a valid MII. - if (MII == 0) + if (MII == 0) { + LLVM_DEBUG( + dbgs() + << "0 is not a valid Minimal Initiation Interval, can NOT schedule\n"); + NumFailZeroMII++; return; + } // Don't pipeline large loops. - if (SwpMaxMii != -1 && (int)MII > SwpMaxMii) + if (SwpMaxMii != -1 && (int)MII > SwpMaxMii) { + LLVM_DEBUG(dbgs() << "MII > " << SwpMaxMii + << ", we don't pipleline large loops\n"); + NumFailLargeMaxMII++; return; + } computeNodeFunctions(NodeSets); @@ -362,7 +472,7 @@ void SwingSchedulerDAG::schedule() { } }); - std::stable_sort(NodeSets.begin(), NodeSets.end(), std::greater()); + llvm::stable_sort(NodeSets, std::greater()); groupRemainingNodes(NodeSets); @@ -383,17 +493,27 @@ void SwingSchedulerDAG::schedule() { SMSchedule Schedule(Pass.MF); Scheduled = schedulePipeline(Schedule); - if (!Scheduled) + if (!Scheduled){ + LLVM_DEBUG(dbgs() << "No schedule found, return\n"); + NumFailNoSchedule++; return; + } unsigned numStages = Schedule.getMaxStageCount(); // No need to generate pipeline if there are no overlapped iterations. - if (numStages == 0) + if (numStages == 0) { + LLVM_DEBUG( + dbgs() << "No overlapped iterations, no need to generate pipeline\n"); + NumFailZeroStage++; return; - + } // Check that the maximum stage count is less than user-defined limit. - if (SwpMaxStages > -1 && (int)numStages > SwpMaxStages) + if (SwpMaxStages > -1 && (int)numStages > SwpMaxStages) { + LLVM_DEBUG(dbgs() << "numStages:" << numStages << ">" << SwpMaxStages + << " : too many stages, abort\n"); + NumFailLargeMaxStage++; return; + } generatePipelinedLoop(Schedule); ++NumPipelined; @@ -467,7 +587,8 @@ static bool isSuccOrder(SUnit *SUa, SUnit *SUb) { /// Return true if the instruction causes a chain between memory /// references before and after it. static bool isDependenceBarrier(MachineInstr &MI, AliasAnalysis *AA) { - return MI.isCall() || MI.hasUnmodeledSideEffects() || + return MI.isCall() || MI.mayRaiseFPException() || + MI.hasUnmodeledSideEffects() || (MI.hasOrderedMemoryRef() && (!MI.mayLoad() || !MI.isDereferenceableInvariantLoad(AA))); } @@ -475,16 +596,16 @@ static bool isDependenceBarrier(MachineInstr &MI, AliasAnalysis *AA) { /// Return the underlying objects for the memory references of an instruction. /// This function calls the code in ValueTracking, but first checks that the /// instruction has a memory operand. -static void getUnderlyingObjects(MachineInstr *MI, - SmallVectorImpl &Objs, +static void getUnderlyingObjects(const MachineInstr *MI, + SmallVectorImpl &Objs, const DataLayout &DL) { if (!MI->hasOneMemOperand()) return; MachineMemOperand *MM = *MI->memoperands_begin(); if (!MM->getValue()) return; - GetUnderlyingObjects(const_cast(MM->getValue()), Objs, DL); - for (Value *V : Objs) { + GetUnderlyingObjects(MM->getValue(), Objs, DL); + for (const Value *V : Objs) { if (!isIdentifiedObject(V)) { Objs.clear(); return; @@ -498,7 +619,7 @@ static void getUnderlyingObjects(MachineInstr *MI, /// dependence. This code is very similar to the code in ScheduleDAGInstrs /// but that code doesn't create loop carried dependences. void SwingSchedulerDAG::addLoopCarriedDependences(AliasAnalysis *AA) { - MapVector> PendingLoads; + MapVector> PendingLoads; Value *UnknownValue = UndefValue::get(Type::getVoidTy(MF.getFunction().getContext())); for (auto &SU : SUnits) { @@ -506,7 +627,7 @@ void SwingSchedulerDAG::addLoopCarriedDependences(AliasAnalysis *AA) { if (isDependenceBarrier(MI, AA)) PendingLoads.clear(); else if (MI.mayLoad()) { - SmallVector Objs; + SmallVector Objs; getUnderlyingObjects(&MI, Objs, MF.getDataLayout()); if (Objs.empty()) Objs.push_back(UnknownValue); @@ -515,12 +636,12 @@ void SwingSchedulerDAG::addLoopCarriedDependences(AliasAnalysis *AA) { SUs.push_back(&SU); } } else if (MI.mayStore()) { - SmallVector Objs; + SmallVector Objs; getUnderlyingObjects(&MI, Objs, MF.getDataLayout()); if (Objs.empty()) Objs.push_back(UnknownValue); for (auto V : Objs) { - MapVector>::iterator I = + MapVector>::iterator I = PendingLoads.find(V); if (I == PendingLoads.end()) continue; @@ -531,7 +652,7 @@ void SwingSchedulerDAG::addLoopCarriedDependences(AliasAnalysis *AA) { // First, perform the cheaper check that compares the base register. // If they are the same and the load offset is less than the store // offset, then mark the dependence as loop carried potentially. - MachineOperand *BaseOp1, *BaseOp2; + const MachineOperand *BaseOp1, *BaseOp2; int64_t Offset1, Offset2; if (TII->getMemOperandWithOffset(LdMI, BaseOp1, Offset1, TRI) && TII->getMemOperandWithOffset(MI, BaseOp2, Offset2, TRI)) { @@ -744,27 +865,55 @@ namespace { // the number of functional unit choices. struct FuncUnitSorter { const InstrItineraryData *InstrItins; + const MCSubtargetInfo *STI; DenseMap Resources; - FuncUnitSorter(const InstrItineraryData *IID) : InstrItins(IID) {} + FuncUnitSorter(const TargetSubtargetInfo &TSI) + : InstrItins(TSI.getInstrItineraryData()), STI(&TSI) {} // Compute the number of functional unit alternatives needed // at each stage, and take the minimum value. We prioritize the // instructions by the least number of choices first. unsigned minFuncUnits(const MachineInstr *Inst, unsigned &F) const { - unsigned schedClass = Inst->getDesc().getSchedClass(); + unsigned SchedClass = Inst->getDesc().getSchedClass(); unsigned min = UINT_MAX; - for (const InstrStage *IS = InstrItins->beginStage(schedClass), - *IE = InstrItins->endStage(schedClass); - IS != IE; ++IS) { - unsigned funcUnits = IS->getUnits(); - unsigned numAlternatives = countPopulation(funcUnits); - if (numAlternatives < min) { - min = numAlternatives; - F = funcUnits; + if (InstrItins && !InstrItins->isEmpty()) { + for (const InstrStage &IS : + make_range(InstrItins->beginStage(SchedClass), + InstrItins->endStage(SchedClass))) { + unsigned funcUnits = IS.getUnits(); + unsigned numAlternatives = countPopulation(funcUnits); + if (numAlternatives < min) { + min = numAlternatives; + F = funcUnits; + } } + return min; + } + if (STI && STI->getSchedModel().hasInstrSchedModel()) { + const MCSchedClassDesc *SCDesc = + STI->getSchedModel().getSchedClassDesc(SchedClass); + if (!SCDesc->isValid()) + // No valid Schedule Class Desc for schedClass, should be + // Pseudo/PostRAPseudo + return min; + + for (const MCWriteProcResEntry &PRE : + make_range(STI->getWriteProcResBegin(SCDesc), + STI->getWriteProcResEnd(SCDesc))) { + if (!PRE.Cycles) + continue; + const MCProcResourceDesc *ProcResource = + STI->getSchedModel().getProcResource(PRE.ProcResourceIdx); + unsigned NumUnits = ProcResource->NumUnits; + if (NumUnits < min) { + min = NumUnits; + F = PRE.ProcResourceIdx; + } + } + return min; } - return min; + llvm_unreachable("Should have non-empty InstrItins or hasInstrSchedModel!"); } // Compute the critical resources needed by the instruction. This @@ -774,13 +923,34 @@ struct FuncUnitSorter { // the same, highly used, functional unit have high priority. void calcCriticalResources(MachineInstr &MI) { unsigned SchedClass = MI.getDesc().getSchedClass(); - for (const InstrStage *IS = InstrItins->beginStage(SchedClass), - *IE = InstrItins->endStage(SchedClass); - IS != IE; ++IS) { - unsigned FuncUnits = IS->getUnits(); - if (countPopulation(FuncUnits) == 1) - Resources[FuncUnits]++; + if (InstrItins && !InstrItins->isEmpty()) { + for (const InstrStage &IS : + make_range(InstrItins->beginStage(SchedClass), + InstrItins->endStage(SchedClass))) { + unsigned FuncUnits = IS.getUnits(); + if (countPopulation(FuncUnits) == 1) + Resources[FuncUnits]++; + } + return; + } + if (STI && STI->getSchedModel().hasInstrSchedModel()) { + const MCSchedClassDesc *SCDesc = + STI->getSchedModel().getSchedClassDesc(SchedClass); + if (!SCDesc->isValid()) + // No valid Schedule Class Desc for schedClass, should be + // Pseudo/PostRAPseudo + return; + + for (const MCWriteProcResEntry &PRE : + make_range(STI->getWriteProcResBegin(SCDesc), + STI->getWriteProcResEnd(SCDesc))) { + if (!PRE.Cycles) + continue; + Resources[PRE.ProcResourceIdx]++; + } + return; } + llvm_unreachable("Should have non-empty InstrItins or hasInstrSchedModel!"); } /// Return true if IS1 has less priority than IS2. @@ -803,14 +973,15 @@ struct FuncUnitSorter { /// to add it to each existing DFA, until a legal space is found. If the /// instruction cannot be reserved in an existing DFA, we create a new one. unsigned SwingSchedulerDAG::calculateResMII() { - SmallVector Resources; + + LLVM_DEBUG(dbgs() << "calculateResMII:\n"); + SmallVector Resources; MachineBasicBlock *MBB = Loop.getHeader(); - Resources.push_back(TII->CreateTargetScheduleState(MF.getSubtarget())); + Resources.push_back(new ResourceManager(&MF.getSubtarget())); // Sort the instructions by the number of available choices for scheduling, // least to most. Use the number of critical resources as the tie breaker. - FuncUnitSorter FUS = - FuncUnitSorter(MF.getSubtarget().getInstrItineraryData()); + FuncUnitSorter FUS = FuncUnitSorter(MF.getSubtarget()); for (MachineBasicBlock::iterator I = MBB->getFirstNonPHI(), E = MBB->getFirstTerminator(); I != E; ++I) @@ -832,33 +1003,40 @@ unsigned SwingSchedulerDAG::calculateResMII() { // DFA is needed for each cycle. unsigned NumCycles = getSUnit(MI)->Latency; unsigned ReservedCycles = 0; - SmallVectorImpl::iterator RI = Resources.begin(); - SmallVectorImpl::iterator RE = Resources.end(); + SmallVectorImpl::iterator RI = Resources.begin(); + SmallVectorImpl::iterator RE = Resources.end(); + LLVM_DEBUG({ + dbgs() << "Trying to reserve resource for " << NumCycles + << " cycles for \n"; + MI->dump(); + }); for (unsigned C = 0; C < NumCycles; ++C) while (RI != RE) { - if ((*RI++)->canReserveResources(*MI)) { + if ((*RI)->canReserveResources(*MI)) { + (*RI)->reserveResources(*MI); ++ReservedCycles; break; } + RI++; } - // Start reserving resources using existing DFAs. - for (unsigned C = 0; C < ReservedCycles; ++C) { - --RI; - (*RI)->reserveResources(*MI); - } + LLVM_DEBUG(dbgs() << "ReservedCycles:" << ReservedCycles + << ", NumCycles:" << NumCycles << "\n"); // Add new DFAs, if needed, to reserve resources. for (unsigned C = ReservedCycles; C < NumCycles; ++C) { - DFAPacketizer *NewResource = - TII->CreateTargetScheduleState(MF.getSubtarget()); + LLVM_DEBUG(if (SwpDebugResource) dbgs() + << "NewResource created to reserve resources" + << "\n"); + ResourceManager *NewResource = new ResourceManager(&MF.getSubtarget()); assert(NewResource->canReserveResources(*MI) && "Reserve error."); NewResource->reserveResources(*MI); Resources.push_back(NewResource); } } int Resmii = Resources.size(); + LLVM_DEBUG(dbgs() << "Retrun Res MII:" << Resmii << "\n"); // Delete the memory for each of the DFAs that were created earlier. - for (DFAPacketizer *RI : Resources) { - DFAPacketizer *D = RI; + for (ResourceManager *RI : Resources) { + ResourceManager *D = RI; delete D; } Resources.clear(); @@ -1517,7 +1695,7 @@ void SwingSchedulerDAG::groupRemainingNodes(NodeSetType &NodeSets) { } } -/// Add the node to the set, and add all is its connected nodes to the set. +/// Add the node to the set, and add all of its connected nodes to the set. void SwingSchedulerDAG::addConnectedNodes(SUnit *SU, NodeSet &NewSet, SetVector &NodesAdded) { NewSet.insert(SU); @@ -1741,12 +1919,16 @@ void SwingSchedulerDAG::computeNodeOrder(NodeSetType &NodeSets) { /// Process the nodes in the computed order and create the pipelined schedule /// of the instructions, if possible. Return true if a schedule is found. bool SwingSchedulerDAG::schedulePipeline(SMSchedule &Schedule) { - if (NodeOrder.empty()) + + if (NodeOrder.empty()){ + LLVM_DEBUG(dbgs() << "NodeOrder is empty! abort scheduling\n" ); return false; + } bool scheduleFound = false; + unsigned II = 0; // Keep increasing II until a valid schedule is found. - for (unsigned II = MII; II < MII + 10 && !scheduleFound; ++II) { + for (II = MII; II <= MAX_II && !scheduleFound; ++II) { Schedule.reset(); Schedule.setInitiationInterval(II); LLVM_DEBUG(dbgs() << "Try to schedule with " << II << "\n"); @@ -1767,13 +1949,14 @@ bool SwingSchedulerDAG::schedulePipeline(SMSchedule &Schedule) { Schedule.computeStart(SU, &EarlyStart, &LateStart, &SchedEnd, &SchedStart, II, this); LLVM_DEBUG({ + dbgs() << "\n"; dbgs() << "Inst (" << SU->NodeNum << ") "; SU->getInstr()->dump(); dbgs() << "\n"; }); LLVM_DEBUG({ - dbgs() << "\tes: " << EarlyStart << " ls: " << LateStart - << " me: " << SchedEnd << " ms: " << SchedStart << "\n"; + dbgs() << format("\tes: %8x ls: %8x me: %8x ms: %8x\n", EarlyStart, + LateStart, SchedEnd, SchedStart); }); if (EarlyStart > LateStart || SchedEnd < EarlyStart || @@ -1818,7 +2001,8 @@ bool SwingSchedulerDAG::schedulePipeline(SMSchedule &Schedule) { scheduleFound = Schedule.isValidSchedule(this); } - LLVM_DEBUG(dbgs() << "Schedule Found? " << scheduleFound << "\n"); + LLVM_DEBUG(dbgs() << "Schedule Found? " << scheduleFound << " (II=" << II + << ")\n"); if (scheduleFound) Schedule.finalizeSchedule(this); @@ -1847,6 +2031,10 @@ void SwingSchedulerDAG::generatePipelinedLoop(SMSchedule &Schedule) { InstrMapTy InstrMap; SmallVector PrologBBs; + + MachineBasicBlock *PreheaderBB = MLI->getLoopFor(BB)->getLoopPreheader(); + assert(PreheaderBB != nullptr && + "Need to add code to handle loops w/o preheader"); // Generate the prolog instructions that set up the pipeline. generateProlog(Schedule, MaxStageCount, KernelBB, VRMap, PrologBBs); MF.insert(BB->getIterator(), KernelBB); @@ -1903,7 +2091,7 @@ void SwingSchedulerDAG::generatePipelinedLoop(SMSchedule &Schedule) { removeDeadInstructions(KernelBB, EpilogBBs); // Add branches between prolog and epilog blocks. - addBranches(PrologBBs, KernelBB, EpilogBBs, Schedule, VRMap); + addBranches(*PreheaderBB, PrologBBs, KernelBB, EpilogBBs, Schedule, VRMap); // Remove the original loop since it's no longer referenced. for (auto &I : *BB) @@ -2242,7 +2430,7 @@ void SwingSchedulerDAG::generateExistingPhis( // Use the value defined by the Phi, unless we're generating the first // epilog and the Phi refers to a Phi in a different stage. else if (VRMap[PrevStage - np].count(Def) && - (!LoopDefIsPhi || PrevStage != LastStageNum)) + (!LoopDefIsPhi || (PrevStage != LastStageNum) || (LoopValStage == StageScheduled))) PhiOp2 = VRMap[PrevStage - np][Def]; } @@ -2588,7 +2776,8 @@ static void removePhis(MachineBasicBlock *BB, MachineBasicBlock *Incoming) { /// Create branches from each prolog basic block to the appropriate epilog /// block. These edges are needed if the loop ends before reaching the /// kernel. -void SwingSchedulerDAG::addBranches(MBBVectorTy &PrologBBs, +void SwingSchedulerDAG::addBranches(MachineBasicBlock &PreheaderBB, + MBBVectorTy &PrologBBs, MachineBasicBlock *KernelBB, MBBVectorTy &EpilogBBs, SMSchedule &Schedule, ValueMapTy *VRMap) { @@ -2615,8 +2804,8 @@ void SwingSchedulerDAG::addBranches(MBBVectorTy &PrologBBs, // Check if the LOOP0 has already been removed. If so, then there is no need // to reduce the trip count. if (LC != 0) - LC = TII->reduceLoopCount(*Prolog, IndVar, *Cmp, Cond, PrevInsts, j, - MaxIter); + LC = TII->reduceLoopCount(*Prolog, PreheaderBB, IndVar, *Cmp, Cond, + PrevInsts, j, MaxIter); // Record the value of the first trip count, which is used to determine if // branches and blocks can be removed for constant trip counts. @@ -2657,7 +2846,7 @@ void SwingSchedulerDAG::addBranches(MBBVectorTy &PrologBBs, /// during each iteration. Set Delta to the amount of the change. bool SwingSchedulerDAG::computeDelta(MachineInstr &MI, unsigned &Delta) { const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); - MachineOperand *BaseOp; + const MachineOperand *BaseOp; int64_t Offset; if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, TRI)) return false; @@ -2698,7 +2887,9 @@ void SwingSchedulerDAG::updateMemOperands(MachineInstr &NewMI, return; SmallVector NewMMOs; for (MachineMemOperand *MMO : NewMI.memoperands()) { - if (MMO->isVolatile() || (MMO->isInvariant() && MMO->isDereferenceable()) || + // TODO: Figure out whether isAtomic is really necessary (see D57601). + if (MMO->isVolatile() || MMO->isAtomic() || + (MMO->isInvariant() && MMO->isDereferenceable()) || (!MMO->getValue())) { NewMMOs.push_back(MMO); continue; @@ -3058,6 +3249,7 @@ bool SwingSchedulerDAG::isLoopCarriedDep(SUnit *Source, const SDep &Dep, // Assume ordered loads and stores may have a loop carried dependence. if (SI->hasUnmodeledSideEffects() || DI->hasUnmodeledSideEffects() || + SI->mayRaiseFPException() || DI->mayRaiseFPException() || SI->hasOrderedMemoryRef() || DI->hasOrderedMemoryRef()) return true; @@ -3069,7 +3261,7 @@ bool SwingSchedulerDAG::isLoopCarriedDep(SUnit *Source, const SDep &Dep, if (!computeDelta(*SI, DeltaS) || !computeDelta(*DI, DeltaD)) return true; - MachineOperand *BaseOpS, *BaseOpD; + const MachineOperand *BaseOpS, *BaseOpD; int64_t OffsetS, OffsetD; const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); if (!TII->getMemOperandWithOffset(*SI, BaseOpS, OffsetS, TRI) || @@ -3097,12 +3289,14 @@ bool SwingSchedulerDAG::isLoopCarriedDep(SUnit *Source, const SDep &Dep, // This is the main test, which checks the offset values and the loop // increment value to determine if the accesses may be loop carried. - if (OffsetS >= OffsetD) - return OffsetS + AccessSizeS > DeltaS; - else - return OffsetD + AccessSizeD > DeltaD; + if (AccessSizeS == MemoryLocation::UnknownSize || + AccessSizeD == MemoryLocation::UnknownSize) + return true; - return true; + if (DeltaS != DeltaD || DeltaS < AccessSizeS || DeltaD < AccessSizeD) + return true; + + return (OffsetS + (int64_t)AccessSizeS < OffsetD + (int64_t)AccessSizeD); } void SwingSchedulerDAG::postprocessDAG() { @@ -3117,6 +3311,10 @@ void SwingSchedulerDAG::postprocessDAG() { /// the relative values of StartCycle and EndCycle. bool SMSchedule::insert(SUnit *SU, int StartCycle, int EndCycle, int II) { bool forward = true; + LLVM_DEBUG({ + dbgs() << "Trying to insert node between " << StartCycle << " and " + << EndCycle << " II: " << II << "\n"; + }); if (StartCycle > EndCycle) forward = false; @@ -3125,8 +3323,9 @@ bool SMSchedule::insert(SUnit *SU, int StartCycle, int EndCycle, int II) { for (int curCycle = StartCycle; curCycle != termCycle; forward ? ++curCycle : --curCycle) { - // Add the already scheduled instructions at the specified cycle to the DFA. - Resources->clearResources(); + // Add the already scheduled instructions at the specified cycle to the + // DFA. + ProcItinResources.clearResources(); for (int checkCycle = FirstCycle + ((curCycle - FirstCycle) % II); checkCycle <= LastCycle; checkCycle += II) { std::deque &cycleInstrs = ScheduledInstrs[checkCycle]; @@ -3136,13 +3335,13 @@ bool SMSchedule::insert(SUnit *SU, int StartCycle, int EndCycle, int II) { I != E; ++I) { if (ST.getInstrInfo()->isZeroCost((*I)->getInstr()->getOpcode())) continue; - assert(Resources->canReserveResources(*(*I)->getInstr()) && + assert(ProcItinResources.canReserveResources(*(*I)->getInstr()) && "These instructions have already been scheduled."); - Resources->reserveResources(*(*I)->getInstr()); + ProcItinResources.reserveResources(*(*I)->getInstr()); } } if (ST.getInstrInfo()->isZeroCost(SU->getInstr()->getOpcode()) || - Resources->canReserveResources(*SU->getInstr())) { + ProcItinResources.canReserveResources(*SU->getInstr())) { LLVM_DEBUG({ dbgs() << "\tinsert at cycle " << curCycle << " "; SU->getInstr()->dump(); @@ -3360,6 +3559,14 @@ void SMSchedule::orderDependence(SwingSchedulerDAG *SSD, SUnit *SU, if (Pos < MoveUse) MoveUse = Pos; } + // We did not handle HW dependences in previous for loop, + // and we normally set Latency = 0 for Anti deps, + // so may have nodes in same cycle with Anti denpendent on HW regs. + else if (S.getKind() == SDep::Anti && stageScheduled(*I) == StageInst1) { + OrderBeforeUse = true; + if ((MoveUse == 0) || (Pos < MoveUse)) + MoveUse = Pos; + } } for (auto &P : SU->Preds) { if (P.getSUnit() != *I) @@ -3523,9 +3730,8 @@ void SwingSchedulerDAG::checkValidNodeOrder(const NodeSetType &Circuits) const { for (SDep &PredEdge : SU->Preds) { SUnit *PredSU = PredEdge.getSUnit(); - unsigned PredIndex = - std::get<1>(*std::lower_bound(Indices.begin(), Indices.end(), - std::make_pair(PredSU, 0), CompareKey)); + unsigned PredIndex = std::get<1>( + *llvm::lower_bound(Indices, std::make_pair(PredSU, 0), CompareKey)); if (!PredSU->getInstr()->isPHI() && PredIndex < Index) { PredBefore = true; Pred = PredSU; @@ -3535,9 +3741,13 @@ void SwingSchedulerDAG::checkValidNodeOrder(const NodeSetType &Circuits) const { for (SDep &SuccEdge : SU->Succs) { SUnit *SuccSU = SuccEdge.getSUnit(); - unsigned SuccIndex = - std::get<1>(*std::lower_bound(Indices.begin(), Indices.end(), - std::make_pair(SuccSU, 0), CompareKey)); + // Do not process a boundary node, it was not included in NodeOrder, + // hence not in Indices either, call to std::lower_bound() below will + // return Indices.end(). + if (SuccSU->isBoundaryNode()) + continue; + unsigned SuccIndex = std::get<1>( + *llvm::lower_bound(Indices, std::make_pair(SuccSU, 0), CompareKey)); if (!SuccSU->getInstr()->isPHI() && SuccIndex < Index) { SuccBefore = true; Succ = SuccSU; @@ -3548,9 +3758,8 @@ void SwingSchedulerDAG::checkValidNodeOrder(const NodeSetType &Circuits) const { if (PredBefore && SuccBefore && !SU->getInstr()->isPHI()) { // instructions in circuits are allowed to be scheduled // after both a successor and predecessor. - bool InCircuit = std::any_of( - Circuits.begin(), Circuits.end(), - [SU](const NodeSet &Circuit) { return Circuit.count(SU); }); + bool InCircuit = llvm::any_of( + Circuits, [SU](const NodeSet &Circuit) { return Circuit.count(SU); }); if (InCircuit) LLVM_DEBUG(dbgs() << "In a circuit, predecessor ";); else { @@ -3740,5 +3949,140 @@ LLVM_DUMP_METHOD void NodeSet::dump() const { print(dbgs()); } #endif +void ResourceManager::initProcResourceVectors( + const MCSchedModel &SM, SmallVectorImpl &Masks) { + unsigned ProcResourceID = 0; + + // We currently limit the resource kinds to 64 and below so that we can use + // uint64_t for Masks + assert(SM.getNumProcResourceKinds() < 64 && + "Too many kinds of resources, unsupported"); + // Create a unique bitmask for every processor resource unit. + // Skip resource at index 0, since it always references 'InvalidUnit'. + Masks.resize(SM.getNumProcResourceKinds()); + for (unsigned I = 1, E = SM.getNumProcResourceKinds(); I < E; ++I) { + const MCProcResourceDesc &Desc = *SM.getProcResource(I); + if (Desc.SubUnitsIdxBegin) + continue; + Masks[I] = 1ULL << ProcResourceID; + ProcResourceID++; + } + // Create a unique bitmask for every processor resource group. + for (unsigned I = 1, E = SM.getNumProcResourceKinds(); I < E; ++I) { + const MCProcResourceDesc &Desc = *SM.getProcResource(I); + if (!Desc.SubUnitsIdxBegin) + continue; + Masks[I] = 1ULL << ProcResourceID; + for (unsigned U = 0; U < Desc.NumUnits; ++U) + Masks[I] |= Masks[Desc.SubUnitsIdxBegin[U]]; + ProcResourceID++; + } + LLVM_DEBUG({ + if (SwpShowResMask) { + dbgs() << "ProcResourceDesc:\n"; + for (unsigned I = 1, E = SM.getNumProcResourceKinds(); I < E; ++I) { + const MCProcResourceDesc *ProcResource = SM.getProcResource(I); + dbgs() << format(" %16s(%2d): Mask: 0x%08x, NumUnits:%2d\n", + ProcResource->Name, I, Masks[I], + ProcResource->NumUnits); + } + dbgs() << " -----------------\n"; + } + }); +} + +bool ResourceManager::canReserveResources(const MCInstrDesc *MID) const { + + LLVM_DEBUG({ + if (SwpDebugResource) + dbgs() << "canReserveResources:\n"; + }); + if (UseDFA) + return DFAResources->canReserveResources(MID); + + unsigned InsnClass = MID->getSchedClass(); + const MCSchedClassDesc *SCDesc = SM.getSchedClassDesc(InsnClass); + if (!SCDesc->isValid()) { + LLVM_DEBUG({ + dbgs() << "No valid Schedule Class Desc for schedClass!\n"; + dbgs() << "isPseduo:" << MID->isPseudo() << "\n"; + }); + return true; + } + + const MCWriteProcResEntry *I = STI->getWriteProcResBegin(SCDesc); + const MCWriteProcResEntry *E = STI->getWriteProcResEnd(SCDesc); + for (; I != E; ++I) { + if (!I->Cycles) + continue; + const MCProcResourceDesc *ProcResource = + SM.getProcResource(I->ProcResourceIdx); + unsigned NumUnits = ProcResource->NumUnits; + LLVM_DEBUG({ + if (SwpDebugResource) + dbgs() << format(" %16s(%2d): Count: %2d, NumUnits:%2d, Cycles:%2d\n", + ProcResource->Name, I->ProcResourceIdx, + ProcResourceCount[I->ProcResourceIdx], NumUnits, + I->Cycles); + }); + if (ProcResourceCount[I->ProcResourceIdx] >= NumUnits) + return false; + } + LLVM_DEBUG(if (SwpDebugResource) dbgs() << "return true\n\n";); + return true; +} + +void ResourceManager::reserveResources(const MCInstrDesc *MID) { + LLVM_DEBUG({ + if (SwpDebugResource) + dbgs() << "reserveResources:\n"; + }); + if (UseDFA) + return DFAResources->reserveResources(MID); + unsigned InsnClass = MID->getSchedClass(); + const MCSchedClassDesc *SCDesc = SM.getSchedClassDesc(InsnClass); + if (!SCDesc->isValid()) { + LLVM_DEBUG({ + dbgs() << "No valid Schedule Class Desc for schedClass!\n"; + dbgs() << "isPseduo:" << MID->isPseudo() << "\n"; + }); + return; + } + for (const MCWriteProcResEntry &PRE : + make_range(STI->getWriteProcResBegin(SCDesc), + STI->getWriteProcResEnd(SCDesc))) { + if (!PRE.Cycles) + continue; + ++ProcResourceCount[PRE.ProcResourceIdx]; + LLVM_DEBUG({ + if (SwpDebugResource) { + const MCProcResourceDesc *ProcResource = + SM.getProcResource(PRE.ProcResourceIdx); + dbgs() << format(" %16s(%2d): Count: %2d, NumUnits:%2d, Cycles:%2d\n", + ProcResource->Name, PRE.ProcResourceIdx, + ProcResourceCount[PRE.ProcResourceIdx], + ProcResource->NumUnits, PRE.Cycles); + } + }); + } + LLVM_DEBUG({ + if (SwpDebugResource) + dbgs() << "reserveResources: done!\n\n"; + }); +} + +bool ResourceManager::canReserveResources(const MachineInstr &MI) const { + return canReserveResources(&MI.getDesc()); +} + +void ResourceManager::reserveResources(const MachineInstr &MI) { + return reserveResources(&MI.getDesc()); +} + +void ResourceManager::clearResources() { + if (UseDFA) + return DFAResources->clearResources(); + std::fill(ProcResourceCount.begin(), ProcResourceCount.end(), 0); +} diff --git a/lib/CodeGen/MachinePostDominators.cpp b/lib/CodeGen/MachinePostDominators.cpp index 488377998cb3..7f220ed1fd8f 100644 --- a/lib/CodeGen/MachinePostDominators.cpp +++ b/lib/CodeGen/MachinePostDominators.cpp @@ -1,9 +1,8 @@ //===- MachinePostDominators.cpp -Machine Post Dominator Calculation ------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/MachineRegionInfo.cpp b/lib/CodeGen/MachineRegionInfo.cpp index 2619d8f78276..2961d456be0d 100644 --- a/lib/CodeGen/MachineRegionInfo.cpp +++ b/lib/CodeGen/MachineRegionInfo.cpp @@ -1,9 +1,8 @@ //===- lib/Codegen/MachineRegionInfo.cpp ----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/MachineRegisterInfo.cpp b/lib/CodeGen/MachineRegisterInfo.cpp index 6e5ca45d5e5e..f0fd0405d69d 100644 --- a/lib/CodeGen/MachineRegisterInfo.cpp +++ b/lib/CodeGen/MachineRegisterInfo.cpp @@ -1,9 +1,8 @@ //===- lib/Codegen/MachineRegisterInfo.cpp --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -155,7 +154,7 @@ unsigned MachineRegisterInfo::createIncompleteVirtualRegister(StringRef Name) { /// createVirtualRegister - Create and return a new virtual register in the /// function with the specified register class. /// -unsigned +Register MachineRegisterInfo::createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name) { assert(RegClass && "Cannot create register without RegClass!"); @@ -170,7 +169,7 @@ MachineRegisterInfo::createVirtualRegister(const TargetRegisterClass *RegClass, return Reg; } -unsigned MachineRegisterInfo::cloneVirtualRegister(unsigned VReg, +Register MachineRegisterInfo::cloneVirtualRegister(Register VReg, StringRef Name) { unsigned Reg = createIncompleteVirtualRegister(Name); VRegInfo[Reg].first = VRegInfo[VReg].first; @@ -185,7 +184,7 @@ void MachineRegisterInfo::setType(unsigned VReg, LLT Ty) { VRegToType[VReg] = Ty; } -unsigned +Register MachineRegisterInfo::createGenericVirtualRegister(LLT Ty, StringRef Name) { // New virtual register number. unsigned Reg = createIncompleteVirtualRegister(Name); @@ -424,6 +423,13 @@ bool MachineRegisterInfo::hasOneNonDBGUse(unsigned RegNo) const { return ++UI == use_nodbg_end(); } +bool MachineRegisterInfo::hasOneNonDBGUser(unsigned RegNo) const { + use_instr_nodbg_iterator UI = use_instr_nodbg_begin(RegNo); + if (UI == use_instr_nodbg_end()) + return false; + return ++UI == use_instr_nodbg_end(); +} + /// clearKillFlags - Iterate over all the uses of the given register and /// clear the kill flag from the MachineOperand. This function is used by /// optimization passes which extend register lifetimes and need only diff --git a/lib/CodeGen/MachineSSAUpdater.cpp b/lib/CodeGen/MachineSSAUpdater.cpp index 542491eabbf2..e8b42047b49f 100644 --- a/lib/CodeGen/MachineSSAUpdater.cpp +++ b/lib/CodeGen/MachineSSAUpdater.cpp @@ -1,9 +1,8 @@ //===- MachineSSAUpdater.cpp - Unstructured SSA Update Tool ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/MachineScheduler.cpp b/lib/CodeGen/MachineScheduler.cpp index 90dad9d399fe..ae1170ad1be6 100644 --- a/lib/CodeGen/MachineScheduler.cpp +++ b/lib/CodeGen/MachineScheduler.cpp @@ -1,9 +1,8 @@ //===- MachineScheduler.cpp - Machine Instruction Scheduler ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -487,13 +486,17 @@ getSchedRegions(MachineBasicBlock *MBB, MachineInstr &MI = *std::prev(I); if (isSchedBoundary(&MI, &*MBB, MF, TII)) break; - if (!MI.isDebugInstr()) + if (!MI.isDebugInstr()) { // MBB::size() uses instr_iterator to count. Here we need a bundle to // count as a single instruction. ++NumRegionInstrs; + } } - Regions.push_back(SchedRegion(I, RegionEnd, NumRegionInstrs)); + // It's possible we found a scheduling region that only has debug + // instructions. Don't bother scheduling these. + if (NumRegionInstrs != 0) + Regions.push_back(SchedRegion(I, RegionEnd, NumRegionInstrs)); } if (RegionsTopDown) @@ -605,23 +608,6 @@ LLVM_DUMP_METHOD void ReadyQueue::dump() const { // Provide a vtable anchor. ScheduleDAGMI::~ScheduleDAGMI() = default; -bool ScheduleDAGMI::canAddEdge(SUnit *SuccSU, SUnit *PredSU) { - return SuccSU == &ExitSU || !Topo.IsReachable(PredSU, SuccSU); -} - -bool ScheduleDAGMI::addEdge(SUnit *SuccSU, const SDep &PredDep) { - if (SuccSU != &ExitSU) { - // Do not use WillCreateCycle, it assumes SD scheduling. - // If Pred is reachable from Succ, then the edge creates a cycle. - if (Topo.IsReachable(PredDep.getSUnit(), SuccSU)) - return false; - Topo.AddPred(SuccSU, PredDep.getSUnit()); - } - SuccSU->addPred(PredDep, /*Required=*/!PredDep.isArtificial()); - // Return true regardless of whether a new edge needed to be inserted. - return true; -} - /// ReleaseSucc - Decrement the NumPredsLeft count of a successor. When /// NumPredsLeft reaches zero, release the successor node. /// @@ -762,8 +748,6 @@ void ScheduleDAGMI::schedule() { // Build the DAG. buildSchedGraph(AA); - Topo.InitDAGTopologicalSorting(); - postprocessDAG(); SmallVector TopRoots, BotRoots; @@ -1212,8 +1196,6 @@ void ScheduleDAGMILive::schedule() { LLVM_DEBUG(SchedImpl->dumpPolicy()); buildDAGWithRegPressure(); - Topo.InitDAGTopologicalSorting(); - postprocessDAG(); SmallVector TopRoots, BotRoots; @@ -1484,10 +1466,10 @@ namespace { class BaseMemOpClusterMutation : public ScheduleDAGMutation { struct MemOpInfo { SUnit *SU; - MachineOperand *BaseOp; + const MachineOperand *BaseOp; int64_t Offset; - MemOpInfo(SUnit *su, MachineOperand *Op, int64_t ofs) + MemOpInfo(SUnit *su, const MachineOperand *Op, int64_t ofs) : SU(su), BaseOp(Op), Offset(ofs) {} bool operator<(const MemOpInfo &RHS) const { @@ -1533,7 +1515,7 @@ public: void apply(ScheduleDAGInstrs *DAGInstrs) override; protected: - void clusterNeighboringMemOps(ArrayRef MemOps, ScheduleDAGMI *DAG); + void clusterNeighboringMemOps(ArrayRef MemOps, ScheduleDAGInstrs *DAG); }; class StoreClusterMutation : public BaseMemOpClusterMutation { @@ -1570,10 +1552,10 @@ createStoreClusterDAGMutation(const TargetInstrInfo *TII, } // end namespace llvm void BaseMemOpClusterMutation::clusterNeighboringMemOps( - ArrayRef MemOps, ScheduleDAGMI *DAG) { + ArrayRef MemOps, ScheduleDAGInstrs *DAG) { SmallVector MemOpRecords; for (SUnit *SU : MemOps) { - MachineOperand *BaseOp; + const MachineOperand *BaseOp; int64_t Offset; if (TII->getMemOperandWithOffset(*SU->getInstr(), BaseOp, Offset, TRI)) MemOpRecords.push_back(MemOpInfo(SU, BaseOp, Offset)); @@ -1610,9 +1592,7 @@ void BaseMemOpClusterMutation::clusterNeighboringMemOps( } /// Callback from DAG postProcessing to create cluster edges for loads. -void BaseMemOpClusterMutation::apply(ScheduleDAGInstrs *DAGInstrs) { - ScheduleDAGMI *DAG = static_cast(DAGInstrs); - +void BaseMemOpClusterMutation::apply(ScheduleDAGInstrs *DAG) { // Map DAG NodeNum to store chain ID. DenseMap StoreChainIDs; // Map each store chain to a set of dependent MemOps. @@ -1857,9 +1837,15 @@ SchedBoundary::~SchedBoundary() { delete HazardRec; } /// Given a Count of resource usage and a Latency value, return true if a /// SchedBoundary becomes resource limited. +/// If we are checking after scheduling a node, we should return true when +/// we just reach the resource limit. static bool checkResourceLimit(unsigned LFactor, unsigned Count, - unsigned Latency) { - return (int)(Count - (Latency * LFactor)) > (int)LFactor; + unsigned Latency, bool AfterSchedNode) { + int ResCntFactor = (int)(Count - (Latency * LFactor)); + if (AfterSchedNode) + return ResCntFactor >= (int)LFactor; + else + return ResCntFactor > (int)LFactor; } void SchedBoundary::reset() { @@ -1883,6 +1869,7 @@ void SchedBoundary::reset() { ZoneCritResIdx = 0; IsResourceLimited = false; ReservedCycles.clear(); + ReservedCyclesIndex.clear(); #ifndef NDEBUG // Track the maximum number of stall cycles that could arise either from the // latency of a DAG edge or the number of cycles that a processor resource is @@ -1921,8 +1908,17 @@ init(ScheduleDAGMI *dag, const TargetSchedModel *smodel, SchedRemainder *rem) { SchedModel = smodel; Rem = rem; if (SchedModel->hasInstrSchedModel()) { - ExecutedResCounts.resize(SchedModel->getNumProcResourceKinds()); - ReservedCycles.resize(SchedModel->getNumProcResourceKinds(), InvalidCycle); + unsigned ResourceCount = SchedModel->getNumProcResourceKinds(); + ReservedCyclesIndex.resize(ResourceCount); + ExecutedResCounts.resize(ResourceCount); + unsigned NumUnits = 0; + + for (unsigned i = 0; i < ResourceCount; ++i) { + ReservedCyclesIndex[i] = NumUnits; + NumUnits += SchedModel->getProcResource(i)->NumUnits; + } + + ReservedCycles.resize(NumUnits, InvalidCycle); } } @@ -1943,11 +1939,11 @@ unsigned SchedBoundary::getLatencyStallCycles(SUnit *SU) { return 0; } -/// Compute the next cycle at which the given processor resource can be -/// scheduled. -unsigned SchedBoundary:: -getNextResourceCycle(unsigned PIdx, unsigned Cycles) { - unsigned NextUnreserved = ReservedCycles[PIdx]; +/// Compute the next cycle at which the given processor resource unit +/// can be scheduled. +unsigned SchedBoundary::getNextResourceCycleByInstance(unsigned InstanceIdx, + unsigned Cycles) { + unsigned NextUnreserved = ReservedCycles[InstanceIdx]; // If this resource has never been used, always return cycle zero. if (NextUnreserved == InvalidCycle) return 0; @@ -1957,6 +1953,29 @@ getNextResourceCycle(unsigned PIdx, unsigned Cycles) { return NextUnreserved; } +/// Compute the next cycle at which the given processor resource can be +/// scheduled. Returns the next cycle and the index of the processor resource +/// instance in the reserved cycles vector. +std::pair +SchedBoundary::getNextResourceCycle(unsigned PIdx, unsigned Cycles) { + unsigned MinNextUnreserved = InvalidCycle; + unsigned InstanceIdx = 0; + unsigned StartIndex = ReservedCyclesIndex[PIdx]; + unsigned NumberOfInstances = SchedModel->getProcResource(PIdx)->NumUnits; + assert(NumberOfInstances > 0 && + "Cannot have zero instances of a ProcResource"); + + for (unsigned I = StartIndex, End = StartIndex + NumberOfInstances; I < End; + ++I) { + unsigned NextUnreserved = getNextResourceCycleByInstance(I, Cycles); + if (MinNextUnreserved > NextUnreserved) { + InstanceIdx = I; + MinNextUnreserved = NextUnreserved; + } + } + return std::make_pair(MinNextUnreserved, InstanceIdx); +} + /// Does this SU have a hazard within the current instruction group. /// /// The scheduler supports two modes of hazard recognition. The first is the @@ -1998,14 +2017,16 @@ bool SchedBoundary::checkHazard(SUnit *SU) { SchedModel->getWriteProcResEnd(SC))) { unsigned ResIdx = PE.ProcResourceIdx; unsigned Cycles = PE.Cycles; - unsigned NRCycle = getNextResourceCycle(ResIdx, Cycles); + unsigned NRCycle, InstanceIdx; + std::tie(NRCycle, InstanceIdx) = getNextResourceCycle(ResIdx, Cycles); if (NRCycle > CurrCycle) { #ifndef NDEBUG MaxObservedStall = std::max(Cycles, MaxObservedStall); #endif LLVM_DEBUG(dbgs() << " SU(" << SU->NodeNum << ") " - << SchedModel->getResourceName(ResIdx) << "=" - << NRCycle << "c\n"); + << SchedModel->getResourceName(ResIdx) + << '[' << InstanceIdx - ReservedCyclesIndex[ResIdx] << ']' + << "=" << NRCycle << "c\n"); return true; } } @@ -2119,7 +2140,7 @@ void SchedBoundary::bumpCycle(unsigned NextCycle) { CheckPending = true; IsResourceLimited = checkResourceLimit(SchedModel->getLatencyFactor(), getCriticalCount(), - getScheduledLatency()); + getScheduledLatency(), true); LLVM_DEBUG(dbgs() << "Cycle: " << CurrCycle << ' ' << Available.getName() << '\n'); @@ -2160,10 +2181,12 @@ countResource(unsigned PIdx, unsigned Cycles, unsigned NextCycle) { << "c\n"); } // For reserved resources, record the highest cycle using the resource. - unsigned NextAvailable = getNextResourceCycle(PIdx, Cycles); + unsigned NextAvailable, InstanceIdx; + std::tie(NextAvailable, InstanceIdx) = getNextResourceCycle(PIdx, Cycles); if (NextAvailable > CurrCycle) { LLVM_DEBUG(dbgs() << " Resource conflict: " - << SchedModel->getProcResource(PIdx)->Name + << SchedModel->getResourceName(PIdx) + << '[' << InstanceIdx - ReservedCyclesIndex[PIdx] << ']' << " reserved until @" << NextAvailable << "\n"); } return NextAvailable; @@ -2179,6 +2202,8 @@ void SchedBoundary::bumpNode(SUnit *SU) { HazardRec->Reset(); } HazardRec->EmitInstruction(SU); + // Scheduling an instruction may have made pending instructions available. + CheckPending = true; } // checkHazard should prevent scheduling multiple instructions per cycle that // exceed the issue width. @@ -2251,12 +2276,13 @@ void SchedBoundary::bumpNode(SUnit *SU) { PE = SchedModel->getWriteProcResEnd(SC); PI != PE; ++PI) { unsigned PIdx = PI->ProcResourceIdx; if (SchedModel->getProcResource(PIdx)->BufferSize == 0) { + unsigned ReservedUntil, InstanceIdx; + std::tie(ReservedUntil, InstanceIdx) = getNextResourceCycle(PIdx, 0); if (isTop()) { - ReservedCycles[PIdx] = - std::max(getNextResourceCycle(PIdx, 0), NextCycle + PI->Cycles); - } - else - ReservedCycles[PIdx] = NextCycle; + ReservedCycles[InstanceIdx] = + std::max(ReservedUntil, NextCycle + PI->Cycles); + } else + ReservedCycles[InstanceIdx] = NextCycle; } } } @@ -2282,7 +2308,7 @@ void SchedBoundary::bumpNode(SUnit *SU) { // resource limited. If a stall occurred, bumpCycle does this. IsResourceLimited = checkResourceLimit(SchedModel->getLatencyFactor(), getCriticalCount(), - getScheduledLatency()); + getScheduledLatency(), true); // Update CurrMOps after calling bumpCycle to handle stalls, since bumpCycle // resets CurrMOps. Loop to handle instructions with more MOps than issue in @@ -2501,7 +2527,7 @@ void GenericSchedulerBase::setPolicy(CandPolicy &Policy, bool IsPostRA, RemLatency = computeRemLatency(CurrZone); RemLatencyComputed = true; OtherResLimited = checkResourceLimit(SchedModel->getLatencyFactor(), - OtherCount, RemLatency); + OtherCount, RemLatency, false); } // Schedule aggressively for latency in PostRA mode. We don't check for @@ -2741,8 +2767,10 @@ void GenericScheduler::initPolicy(MachineBasicBlock::iterator Begin, MF.getSubtarget().overrideSchedPolicy(RegionPolicy, NumRegionInstrs); // After subtarget overrides, apply command line options. - if (!EnableRegPressure) + if (!EnableRegPressure) { RegionPolicy.ShouldTrackPressure = false; + RegionPolicy.ShouldTrackLaneMasks = false; + } // Check -misched-topdown/bottomup can force or unforce scheduling direction. // e.g. -misched-bottomup=false allows scheduling in both directions. diff --git a/lib/CodeGen/MachineSink.cpp b/lib/CodeGen/MachineSink.cpp index cdc597db6401..41db2c88ce50 100644 --- a/lib/CodeGen/MachineSink.cpp +++ b/lib/CodeGen/MachineSink.cpp @@ -1,9 +1,8 @@ //===- MachineSink.cpp - Sinking for machine instructions -----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -585,9 +584,8 @@ MachineSinking::GetAllSortedSuccessors(MachineInstr &MI, MachineBasicBlock *MBB, AllSuccs.push_back(DTChild->getBlock()); // Sort Successors according to their loop depth or block frequency info. - std::stable_sort( - AllSuccs.begin(), AllSuccs.end(), - [this](const MachineBasicBlock *L, const MachineBasicBlock *R) { + llvm::stable_sort( + AllSuccs, [this](const MachineBasicBlock *L, const MachineBasicBlock *R) { uint64_t LHSFreq = MBFI ? MBFI->getBlockFreq(L).getFrequency() : 0; uint64_t RHSFreq = MBFI ? MBFI->getBlockFreq(R).getFrequency() : 0; bool HasBlockFreq = LHSFreq != 0 && RHSFreq != 0; @@ -716,7 +714,7 @@ static bool SinkingPreventsImplicitNullCheck(MachineInstr &MI, !PredBB->getTerminator()->getMetadata(LLVMContext::MD_make_implicit)) return false; - MachineOperand *BaseOp; + const MachineOperand *BaseOp; int64_t Offset; if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, TRI)) return false; @@ -1203,6 +1201,9 @@ bool PostRAMachineSinking::tryToSinkCopy(MachineBasicBlock &CurBB, } bool PostRAMachineSinking::runOnMachineFunction(MachineFunction &MF) { + if (skipFunction(MF.getFunction())) + return false; + bool Changed = false; const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); diff --git a/lib/CodeGen/MachineTraceMetrics.cpp b/lib/CodeGen/MachineTraceMetrics.cpp index e62ed3094651..f9505df4e7f4 100644 --- a/lib/CodeGen/MachineTraceMetrics.cpp +++ b/lib/CodeGen/MachineTraceMetrics.cpp @@ -1,9 +1,8 @@ //===- lib/CodeGen/MachineTraceMetrics.cpp --------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/MachineVerifier.cpp b/lib/CodeGen/MachineVerifier.cpp index 534d3699db29..0ad792ac62cf 100644 --- a/lib/CodeGen/MachineVerifier.cpp +++ b/lib/CodeGen/MachineVerifier.cpp @@ -1,9 +1,8 @@ //===- MachineVerifier.cpp - Machine Code Verifier ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -219,7 +218,7 @@ namespace { bool isAllocatable(unsigned Reg) const { return Reg < TRI->getNumRegs() && TRI->isInAllocatableClass(Reg) && - !regsReserved.test(Reg); + !regsReserved.test(Reg); } // Analysis information if available @@ -231,6 +230,9 @@ namespace { void visitMachineFunctionBefore(); void visitMachineBasicBlockBefore(const MachineBasicBlock *MBB); void visitMachineBundleBefore(const MachineInstr *MI); + + bool verifyVectorElementMatch(LLT Ty0, LLT Ty1, const MachineInstr *MI); + void verifyPreISelGenericInstruction(const MachineInstr *MI); void visitMachineInstrBefore(const MachineInstr *MI); void visitMachineOperand(const MachineOperand *MO, unsigned MONum); void visitMachineInstrAfter(const MachineInstr *MI); @@ -838,7 +840,7 @@ void MachineVerifier::visitMachineBundleBefore(const MachineInstr *MI) { if (MI->isTerminator() && !TII->isPredicated(*MI)) { if (!FirstTerminator) FirstTerminator = MI; - } else if (FirstTerminator) { + } else if (FirstTerminator && !MI->isDebugEntryValue()) { report("Non-terminator instruction after the first terminator", MI); errs() << "First terminator was:\t" << *FirstTerminator; } @@ -889,109 +891,150 @@ void MachineVerifier::verifyInlineAsm(const MachineInstr *MI) { } } -void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) { - const MCInstrDesc &MCID = MI->getDesc(); - if (MI->getNumOperands() < MCID.getNumOperands()) { - report("Too few operands", MI); - errs() << MCID.getNumOperands() << " operands expected, but " - << MI->getNumOperands() << " given.\n"; +/// Check that types are consistent when two operands need to have the same +/// number of vector elements. +/// \return true if the types are valid. +bool MachineVerifier::verifyVectorElementMatch(LLT Ty0, LLT Ty1, + const MachineInstr *MI) { + if (Ty0.isVector() != Ty1.isVector()) { + report("operand types must be all-vector or all-scalar", MI); + // Generally we try to report as many issues as possible at once, but in + // this case it's not clear what should we be comparing the size of the + // scalar with: the size of the whole vector or its lane. Instead of + // making an arbitrary choice and emitting not so helpful message, let's + // avoid the extra noise and stop here. + return false; } - if (MI->isPHI()) { - if (MF->getProperties().hasProperty( - MachineFunctionProperties::Property::NoPHIs)) - report("Found PHI instruction with NoPHIs property set", MI); + if (Ty0.isVector() && Ty0.getNumElements() != Ty1.getNumElements()) { + report("operand types must preserve number of vector elements", MI); + return false; + } - if (FirstNonPHI) - report("Found PHI instruction after non-PHI", MI); - } else if (FirstNonPHI == nullptr) - FirstNonPHI = MI; + return true; +} - // Check the tied operands. - if (MI->isInlineAsm()) - verifyInlineAsm(MI); +void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) { + if (isFunctionSelected) + report("Unexpected generic instruction in a Selected function", MI); - // Check the MachineMemOperands for basic consistency. - for (MachineInstr::mmo_iterator I = MI->memoperands_begin(), - E = MI->memoperands_end(); + const MCInstrDesc &MCID = MI->getDesc(); + unsigned NumOps = MI->getNumOperands(); + + // Check types. + SmallVector Types; + for (unsigned I = 0, E = std::min(MCID.getNumOperands(), NumOps); I != E; ++I) { - if ((*I)->isLoad() && !MI->mayLoad()) - report("Missing mayLoad flag", MI); - if ((*I)->isStore() && !MI->mayStore()) - report("Missing mayStore flag", MI); - } + if (!MCID.OpInfo[I].isGenericType()) + continue; + // Generic instructions specify type equality constraints between some of + // their operands. Make sure these are consistent. + size_t TypeIdx = MCID.OpInfo[I].getGenericTypeIndex(); + Types.resize(std::max(TypeIdx + 1, Types.size())); + + const MachineOperand *MO = &MI->getOperand(I); + if (!MO->isReg()) { + report("generic instruction must use register operands", MI); + continue; + } - // Debug values must not have a slot index. - // Other instructions must have one, unless they are inside a bundle. - if (LiveInts) { - bool mapped = !LiveInts->isNotInMIMap(*MI); - if (MI->isDebugInstr()) { - if (mapped) - report("Debug instruction has a slot index", MI); - } else if (MI->isInsideBundle()) { - if (mapped) - report("Instruction inside bundle has a slot index", MI); + LLT OpTy = MRI->getType(MO->getReg()); + // Don't report a type mismatch if there is no actual mismatch, only a + // type missing, to reduce noise: + if (OpTy.isValid()) { + // Only the first valid type for a type index will be printed: don't + // overwrite it later so it's always clear which type was expected: + if (!Types[TypeIdx].isValid()) + Types[TypeIdx] = OpTy; + else if (Types[TypeIdx] != OpTy) + report("Type mismatch in generic instruction", MO, I, OpTy); } else { - if (!mapped) - report("Missing slot index", MI); + // Generic instructions must have types attached to their operands. + report("Generic instruction is missing a virtual register type", MO, I); } } - if (isPreISelGenericOpcode(MCID.getOpcode())) { - if (isFunctionSelected) - report("Unexpected generic instruction in a Selected function", MI); - - // Check types. - SmallVector Types; - for (unsigned I = 0; I < MCID.getNumOperands(); ++I) { - if (!MCID.OpInfo[I].isGenericType()) - continue; - // Generic instructions specify type equality constraints between some of - // their operands. Make sure these are consistent. - size_t TypeIdx = MCID.OpInfo[I].getGenericTypeIndex(); - Types.resize(std::max(TypeIdx + 1, Types.size())); - - const MachineOperand *MO = &MI->getOperand(I); - LLT OpTy = MRI->getType(MO->getReg()); - // Don't report a type mismatch if there is no actual mismatch, only a - // type missing, to reduce noise: - if (OpTy.isValid()) { - // Only the first valid type for a type index will be printed: don't - // overwrite it later so it's always clear which type was expected: - if (!Types[TypeIdx].isValid()) - Types[TypeIdx] = OpTy; - else if (Types[TypeIdx] != OpTy) - report("Type mismatch in generic instruction", MO, I, OpTy); - } else { - // Generic instructions must have types attached to their operands. - report("Generic instruction is missing a virtual register type", MO, I); - } - } - - // Generic opcodes must not have physical register operands. - for (unsigned I = 0; I < MI->getNumOperands(); ++I) { - const MachineOperand *MO = &MI->getOperand(I); - if (MO->isReg() && TargetRegisterInfo::isPhysicalRegister(MO->getReg())) - report("Generic instruction cannot have physical register", MO, I); - } + // Generic opcodes must not have physical register operands. + for (unsigned I = 0; I < MI->getNumOperands(); ++I) { + const MachineOperand *MO = &MI->getOperand(I); + if (MO->isReg() && TargetRegisterInfo::isPhysicalRegister(MO->getReg())) + report("Generic instruction cannot have physical register", MO, I); } + // Avoid out of bounds in checks below. This was already reported earlier. + if (MI->getNumOperands() < MCID.getNumOperands()) + return; + StringRef ErrorInfo; if (!TII->verifyInstruction(*MI, ErrorInfo)) report(ErrorInfo.data(), MI); // Verify properties of various specific instruction types - switch(MI->getOpcode()) { - default: + switch (MI->getOpcode()) { + case TargetOpcode::G_CONSTANT: + case TargetOpcode::G_FCONSTANT: { + if (MI->getNumOperands() < MCID.getNumOperands()) + break; + + LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); + if (DstTy.isVector()) + report("Instruction cannot use a vector result type", MI); + + if (MI->getOpcode() == TargetOpcode::G_CONSTANT) { + if (!MI->getOperand(1).isCImm()) { + report("G_CONSTANT operand must be cimm", MI); + break; + } + + const ConstantInt *CI = MI->getOperand(1).getCImm(); + if (CI->getBitWidth() != DstTy.getSizeInBits()) + report("inconsistent constant size", MI); + } else { + if (!MI->getOperand(1).isFPImm()) { + report("G_FCONSTANT operand must be fpimm", MI); + break; + } + const ConstantFP *CF = MI->getOperand(1).getFPImm(); + + if (APFloat::getSizeInBits(CF->getValueAPF().getSemantics()) != + DstTy.getSizeInBits()) { + report("inconsistent constant size", MI); + } + } + break; + } case TargetOpcode::G_LOAD: case TargetOpcode::G_STORE: + case TargetOpcode::G_ZEXTLOAD: + case TargetOpcode::G_SEXTLOAD: { + LLT ValTy = MRI->getType(MI->getOperand(0).getReg()); + LLT PtrTy = MRI->getType(MI->getOperand(1).getReg()); + if (!PtrTy.isPointer()) + report("Generic memory instruction must access a pointer", MI); + // Generic loads and stores must have a single MachineMemOperand // describing that access. - if (!MI->hasOneMemOperand()) + if (!MI->hasOneMemOperand()) { report("Generic instruction accessing memory must have one mem operand", MI); + } else { + const MachineMemOperand &MMO = **MI->memoperands_begin(); + if (MI->getOpcode() == TargetOpcode::G_ZEXTLOAD || + MI->getOpcode() == TargetOpcode::G_SEXTLOAD) { + if (MMO.getSizeInBits() >= ValTy.getSizeInBits()) + report("Generic extload must have a narrower memory type", MI); + } else if (MI->getOpcode() == TargetOpcode::G_LOAD) { + if (MMO.getSize() > ValTy.getSizeInBytes()) + report("load memory size cannot exceed result size", MI); + } else if (MI->getOpcode() == TargetOpcode::G_STORE) { + if (ValTy.getSizeInBytes() < MMO.getSize()) + report("store memory size cannot exceed value size", MI); + } + } + break; + } case TargetOpcode::G_PHI: { LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); if (!DstTy.isValid() || @@ -1009,6 +1052,70 @@ void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) { MI); break; } + case TargetOpcode::G_BITCAST: { + LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); + LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); + if (!DstTy.isValid() || !SrcTy.isValid()) + break; + + if (SrcTy.isPointer() != DstTy.isPointer()) + report("bitcast cannot convert between pointers and other types", MI); + + if (SrcTy.getSizeInBits() != DstTy.getSizeInBits()) + report("bitcast sizes must match", MI); + break; + } + case TargetOpcode::G_INTTOPTR: + case TargetOpcode::G_PTRTOINT: + case TargetOpcode::G_ADDRSPACE_CAST: { + LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); + LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); + if (!DstTy.isValid() || !SrcTy.isValid()) + break; + + verifyVectorElementMatch(DstTy, SrcTy, MI); + + DstTy = DstTy.getScalarType(); + SrcTy = SrcTy.getScalarType(); + + if (MI->getOpcode() == TargetOpcode::G_INTTOPTR) { + if (!DstTy.isPointer()) + report("inttoptr result type must be a pointer", MI); + if (SrcTy.isPointer()) + report("inttoptr source type must not be a pointer", MI); + } else if (MI->getOpcode() == TargetOpcode::G_PTRTOINT) { + if (!SrcTy.isPointer()) + report("ptrtoint source type must be a pointer", MI); + if (DstTy.isPointer()) + report("ptrtoint result type must not be a pointer", MI); + } else { + assert(MI->getOpcode() == TargetOpcode::G_ADDRSPACE_CAST); + if (!SrcTy.isPointer() || !DstTy.isPointer()) + report("addrspacecast types must be pointers", MI); + else { + if (SrcTy.getAddressSpace() == DstTy.getAddressSpace()) + report("addrspacecast must convert different address spaces", MI); + } + } + + break; + } + case TargetOpcode::G_GEP: { + LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); + LLT PtrTy = MRI->getType(MI->getOperand(1).getReg()); + LLT OffsetTy = MRI->getType(MI->getOperand(2).getReg()); + if (!DstTy.isValid() || !PtrTy.isValid() || !OffsetTy.isValid()) + break; + + if (!PtrTy.getScalarType().isPointer()) + report("gep first operand must be a pointer", MI); + + if (OffsetTy.getScalarType().isPointer()) + report("gep offset operand must not be a pointer", MI); + + // TODO: Is the offset allowed to be a scalar with a vector? + break; + } case TargetOpcode::G_SEXT: case TargetOpcode::G_ZEXT: case TargetOpcode::G_ANYEXT: @@ -1021,30 +1128,18 @@ void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) { // instructions aren't guaranteed to have the right number of operands or // types attached to them at this point assert(MCID.getNumOperands() == 2 && "Expected 2 operands G_*{EXT,TRUNC}"); - if (MI->getNumOperands() < MCID.getNumOperands()) - break; LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); if (!DstTy.isValid() || !SrcTy.isValid()) break; - LLT DstElTy = DstTy.isVector() ? DstTy.getElementType() : DstTy; - LLT SrcElTy = SrcTy.isVector() ? SrcTy.getElementType() : SrcTy; + LLT DstElTy = DstTy.getScalarType(); + LLT SrcElTy = SrcTy.getScalarType(); if (DstElTy.isPointer() || SrcElTy.isPointer()) report("Generic extend/truncate can not operate on pointers", MI); - if (DstTy.isVector() != SrcTy.isVector()) { - report("Generic extend/truncate must be all-vector or all-scalar", MI); - // Generally we try to report as many issues as possible at once, but in - // this case it's not clear what should we be comparing the size of the - // scalar with: the size of the whole vector or its lane. Instead of - // making an arbitrary choice and emitting not so helpful message, let's - // avoid the extra noise and stop here. - break; - } - if (DstTy.isVector() && DstTy.getNumElements() != SrcTy.getNumElements()) - report("Generic vector extend/truncate must preserve number of lanes", - MI); + verifyVectorElementMatch(DstTy, SrcTy, MI); + unsigned DstSize = DstElTy.getSizeInBits(); unsigned SrcSize = SrcElTy.getSizeInBits(); switch (MI->getOpcode()) { @@ -1061,6 +1156,17 @@ void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) { } break; } + case TargetOpcode::G_SELECT: { + LLT SelTy = MRI->getType(MI->getOperand(0).getReg()); + LLT CondTy = MRI->getType(MI->getOperand(1).getReg()); + if (!SelTy.isValid() || !CondTy.isValid()) + break; + + // Scalar condition select on a vector is valid. + if (CondTy.isVector()) + verifyVectorElementMatch(SelTy, CondTy, MI); + break; + } case TargetOpcode::G_MERGE_VALUES: { // G_MERGE_VALUES should only be used to merge scalars into a larger scalar, // e.g. s2N = MERGE sN, sN @@ -1070,6 +1176,16 @@ void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) { LLT SrcTy = MRI->getType(MI->getOperand(1).getReg()); if (DstTy.isVector() || SrcTy.isVector()) report("G_MERGE_VALUES cannot operate on vectors", MI); + + const unsigned NumOps = MI->getNumOperands(); + if (DstTy.getSizeInBits() != SrcTy.getSizeInBits() * (NumOps - 1)) + report("G_MERGE_VALUES result size is inconsistent", MI); + + for (unsigned I = 2; I != NumOps; ++I) { + if (MRI->getType(MI->getOperand(I).getReg()) != SrcTy) + report("G_MERGE_VALUES source types do not match", MI); + } + break; } case TargetOpcode::G_UNMERGE_VALUES: { @@ -1092,18 +1208,23 @@ void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) { // must match the dest vector size. LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); LLT SrcEltTy = MRI->getType(MI->getOperand(1).getReg()); - if (!DstTy.isVector() || SrcEltTy.isVector()) + if (!DstTy.isVector() || SrcEltTy.isVector()) { report("G_BUILD_VECTOR must produce a vector from scalar operands", MI); + break; + } + + if (DstTy.getElementType() != SrcEltTy) + report("G_BUILD_VECTOR result element type must match source type", MI); + + if (DstTy.getNumElements() != MI->getNumOperands() - 1) + report("G_BUILD_VECTOR must have an operand for each elemement", MI); + for (unsigned i = 2; i < MI->getNumOperands(); ++i) { if (MRI->getType(MI->getOperand(1).getReg()) != MRI->getType(MI->getOperand(i).getReg())) report("G_BUILD_VECTOR source operand types are not homogeneous", MI); } - if (DstTy.getSizeInBits() != - SrcEltTy.getSizeInBits() * (MI->getNumOperands() - 1)) - report("G_BUILD_VECTOR src operands total size don't match dest " - "size.", - MI); + break; } case TargetOpcode::G_BUILD_VECTOR_TRUNC: { @@ -1144,6 +1265,176 @@ void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) { report("G_CONCAT_VECTOR num dest and source elements should match", MI); break; } + case TargetOpcode::G_ICMP: + case TargetOpcode::G_FCMP: { + LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); + LLT SrcTy = MRI->getType(MI->getOperand(2).getReg()); + + if ((DstTy.isVector() != SrcTy.isVector()) || + (DstTy.isVector() && DstTy.getNumElements() != SrcTy.getNumElements())) + report("Generic vector icmp/fcmp must preserve number of lanes", MI); + + break; + } + case TargetOpcode::G_EXTRACT: { + const MachineOperand &SrcOp = MI->getOperand(1); + if (!SrcOp.isReg()) { + report("extract source must be a register", MI); + break; + } + + const MachineOperand &OffsetOp = MI->getOperand(2); + if (!OffsetOp.isImm()) { + report("extract offset must be a constant", MI); + break; + } + + unsigned DstSize = MRI->getType(MI->getOperand(0).getReg()).getSizeInBits(); + unsigned SrcSize = MRI->getType(SrcOp.getReg()).getSizeInBits(); + if (SrcSize == DstSize) + report("extract source must be larger than result", MI); + + if (DstSize + OffsetOp.getImm() > SrcSize) + report("extract reads past end of register", MI); + break; + } + case TargetOpcode::G_INSERT: { + const MachineOperand &SrcOp = MI->getOperand(2); + if (!SrcOp.isReg()) { + report("insert source must be a register", MI); + break; + } + + const MachineOperand &OffsetOp = MI->getOperand(3); + if (!OffsetOp.isImm()) { + report("insert offset must be a constant", MI); + break; + } + + unsigned DstSize = MRI->getType(MI->getOperand(0).getReg()).getSizeInBits(); + unsigned SrcSize = MRI->getType(SrcOp.getReg()).getSizeInBits(); + + if (DstSize <= SrcSize) + report("inserted size must be smaller than total register", MI); + + if (SrcSize + OffsetOp.getImm() > DstSize) + report("insert writes past end of register", MI); + + break; + } + case TargetOpcode::G_JUMP_TABLE: { + if (!MI->getOperand(1).isJTI()) + report("G_JUMP_TABLE source operand must be a jump table index", MI); + LLT DstTy = MRI->getType(MI->getOperand(0).getReg()); + if (!DstTy.isPointer()) + report("G_JUMP_TABLE dest operand must have a pointer type", MI); + break; + } + case TargetOpcode::G_BRJT: { + if (!MRI->getType(MI->getOperand(0).getReg()).isPointer()) + report("G_BRJT src operand 0 must be a pointer type", MI); + + if (!MI->getOperand(1).isJTI()) + report("G_BRJT src operand 1 must be a jump table index", MI); + + const auto &IdxOp = MI->getOperand(2); + if (!IdxOp.isReg() || MRI->getType(IdxOp.getReg()).isPointer()) + report("G_BRJT src operand 2 must be a scalar reg type", MI); + break; + } + case TargetOpcode::G_INTRINSIC: + case TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS: { + // TODO: Should verify number of def and use operands, but the current + // interface requires passing in IR types for mangling. + const MachineOperand &IntrIDOp = MI->getOperand(MI->getNumExplicitDefs()); + if (!IntrIDOp.isIntrinsicID()) { + report("G_INTRINSIC first src operand must be an intrinsic ID", MI); + break; + } + + bool NoSideEffects = MI->getOpcode() == TargetOpcode::G_INTRINSIC; + unsigned IntrID = IntrIDOp.getIntrinsicID(); + if (IntrID != 0 && IntrID < Intrinsic::num_intrinsics) { + AttributeList Attrs + = Intrinsic::getAttributes(MF->getFunction().getContext(), + static_cast(IntrID)); + bool DeclHasSideEffects = !Attrs.hasFnAttribute(Attribute::ReadNone); + if (NoSideEffects && DeclHasSideEffects) { + report("G_INTRINSIC used with intrinsic that accesses memory", MI); + break; + } + if (!NoSideEffects && !DeclHasSideEffects) { + report("G_INTRINSIC_W_SIDE_EFFECTS used with readnone intrinsic", MI); + break; + } + } + + break; + } + default: + break; + } +} + +void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) { + const MCInstrDesc &MCID = MI->getDesc(); + if (MI->getNumOperands() < MCID.getNumOperands()) { + report("Too few operands", MI); + errs() << MCID.getNumOperands() << " operands expected, but " + << MI->getNumOperands() << " given.\n"; + } + + if (MI->isPHI()) { + if (MF->getProperties().hasProperty( + MachineFunctionProperties::Property::NoPHIs)) + report("Found PHI instruction with NoPHIs property set", MI); + + if (FirstNonPHI) + report("Found PHI instruction after non-PHI", MI); + } else if (FirstNonPHI == nullptr) + FirstNonPHI = MI; + + // Check the tied operands. + if (MI->isInlineAsm()) + verifyInlineAsm(MI); + + // Check the MachineMemOperands for basic consistency. + for (MachineInstr::mmo_iterator I = MI->memoperands_begin(), + E = MI->memoperands_end(); + I != E; ++I) { + if ((*I)->isLoad() && !MI->mayLoad()) + report("Missing mayLoad flag", MI); + if ((*I)->isStore() && !MI->mayStore()) + report("Missing mayStore flag", MI); + } + + // Debug values must not have a slot index. + // Other instructions must have one, unless they are inside a bundle. + if (LiveInts) { + bool mapped = !LiveInts->isNotInMIMap(*MI); + if (MI->isDebugInstr()) { + if (mapped) + report("Debug instruction has a slot index", MI); + } else if (MI->isInsideBundle()) { + if (mapped) + report("Instruction inside bundle has a slot index", MI); + } else { + if (!mapped) + report("Missing slot index", MI); + } + } + + if (isPreISelGenericOpcode(MCID.getOpcode())) { + verifyPreISelGenericInstruction(MI); + return; + } + + StringRef ErrorInfo; + if (!TII->verifyInstruction(*MI, ErrorInfo)) + report(ErrorInfo.data(), MI); + + // Verify properties of various specific instruction types + switch (MI->getOpcode()) { case TargetOpcode::COPY: { if (foundErrors) break; @@ -1193,7 +1484,8 @@ void MachineVerifier::visitMachineInstrBefore(const MachineInstr *MI) { VerifyStackMapConstant(VarStart + StatepointOpers::NumDeoptOperandsOffset); // TODO: verify we have properly encoded deopt arguments - }; + break; + } } void @@ -1356,7 +1648,7 @@ MachineVerifier::visitMachineOperand(const MachineOperand *MO, unsigned MONum) { return; } if (SubIdx) { - report("Generic virtual register does not subregister index", MO, + report("Generic virtual register does not allow subregister index", MO, MONum); return; } @@ -1911,6 +2203,10 @@ void MachineVerifier::visitMachineFunctionAfter() { verifyLiveVariables(); if (LiveInts) verifyLiveIntervals(); + + for (auto CSInfo : MF->getCallSitesInfo()) + if (!CSInfo.first->isCall()) + report("Call site info referencing instruction that is not call", MF); } void MachineVerifier::verifyLiveVariables() { diff --git a/lib/CodeGen/MacroFusion.cpp b/lib/CodeGen/MacroFusion.cpp index 82b6d642c73b..2db1e86905a4 100644 --- a/lib/CodeGen/MacroFusion.cpp +++ b/lib/CodeGen/MacroFusion.cpp @@ -1,9 +1,8 @@ //===- MacroFusion.cpp - Macro Fusion -------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -37,7 +36,7 @@ static bool isHazard(const SDep &Dep) { return Dep.getKind() == SDep::Anti || Dep.getKind() == SDep::Output; } -static bool fuseInstructionPair(ScheduleDAGMI &DAG, SUnit &FirstSU, +static bool fuseInstructionPair(ScheduleDAGInstrs &DAG, SUnit &FirstSU, SUnit &SecondSU) { // Check that neither instr is already paired with another along the edge // between them. @@ -49,7 +48,7 @@ static bool fuseInstructionPair(ScheduleDAGMI &DAG, SUnit &FirstSU, if (SI.isCluster()) return false; // Though the reachability checks above could be made more generic, - // perhaps as part of ScheduleDAGMI::addEdge(), since such edges are valid, + // perhaps as part of ScheduleDAGInstrs::addEdge(), since such edges are valid, // the extra computation cost makes it less interesting in general cases. // Create a single weak edge between the adjacent instrs. The only effect is @@ -118,7 +117,7 @@ namespace { class MacroFusion : public ScheduleDAGMutation { ShouldSchedulePredTy shouldScheduleAdjacent; bool FuseBlock; - bool scheduleAdjacentImpl(ScheduleDAGMI &DAG, SUnit &AnchorSU); + bool scheduleAdjacentImpl(ScheduleDAGInstrs &DAG, SUnit &AnchorSU); public: MacroFusion(ShouldSchedulePredTy shouldScheduleAdjacent, bool FuseBlock) @@ -129,9 +128,7 @@ public: } // end anonymous namespace -void MacroFusion::apply(ScheduleDAGInstrs *DAGInstrs) { - ScheduleDAGMI *DAG = static_cast(DAGInstrs); - +void MacroFusion::apply(ScheduleDAGInstrs *DAG) { if (FuseBlock) // For each of the SUnits in the scheduling block, try to fuse the instr in // it with one in its predecessors. @@ -145,7 +142,7 @@ void MacroFusion::apply(ScheduleDAGInstrs *DAGInstrs) { /// Implement the fusion of instr pairs in the scheduling DAG, /// anchored at the instr in AnchorSU.. -bool MacroFusion::scheduleAdjacentImpl(ScheduleDAGMI &DAG, SUnit &AnchorSU) { +bool MacroFusion::scheduleAdjacentImpl(ScheduleDAGInstrs &DAG, SUnit &AnchorSU) { const MachineInstr &AnchorMI = *AnchorSU.getInstr(); const TargetInstrInfo &TII = *DAG.TII; const TargetSubtargetInfo &ST = DAG.MF.getSubtarget(); diff --git a/lib/CodeGen/OptimizePHIs.cpp b/lib/CodeGen/OptimizePHIs.cpp index 770f6c5b0403..c70b62252139 100644 --- a/lib/CodeGen/OptimizePHIs.cpp +++ b/lib/CodeGen/OptimizePHIs.cpp @@ -1,9 +1,8 @@ //===- OptimizePHIs.cpp - Optimize machine instruction PHIs ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -182,11 +181,12 @@ bool OptimizePHIs::OptimizeBB(MachineBasicBlock &MBB) { if (!MRI->constrainRegClass(SingleValReg, MRI->getRegClass(OldReg))) continue; - // for the case SingleValReg taken from copy instr - MRI->clearKillFlags(SingleValReg); - MRI->replaceRegWith(OldReg, SingleValReg); MI->eraseFromParent(); + + // The kill flags on OldReg and SingleValReg may no longer be correct. + MRI->clearKillFlags(SingleValReg); + ++NumPHICycles; Changed = true; continue; diff --git a/lib/CodeGen/PHIElimination.cpp b/lib/CodeGen/PHIElimination.cpp index b9801c6fd97b..948a5835438c 100644 --- a/lib/CodeGen/PHIElimination.cpp +++ b/lib/CodeGen/PHIElimination.cpp @@ -1,9 +1,8 @@ //===- PhiElimination.cpp - Eliminate PHI nodes by inserting copies -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/PHIEliminationUtils.cpp b/lib/CodeGen/PHIEliminationUtils.cpp index 4e67ff2e5088..3a2cdaf3bd3c 100644 --- a/lib/CodeGen/PHIEliminationUtils.cpp +++ b/lib/CodeGen/PHIEliminationUtils.cpp @@ -1,9 +1,8 @@ //===-- PHIEliminationUtils.cpp - Helper functions for PHI elimination ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/PHIEliminationUtils.h b/lib/CodeGen/PHIEliminationUtils.h index b997d7ac5f4f..0ff3a41f47d3 100644 --- a/lib/CodeGen/PHIEliminationUtils.h +++ b/lib/CodeGen/PHIEliminationUtils.h @@ -1,9 +1,8 @@ //=- PHIEliminationUtils.h - Helper functions for PHI elimination -*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/ParallelCG.cpp b/lib/CodeGen/ParallelCG.cpp index bc3f2a6e9b5a..e4c73658cb4f 100644 --- a/lib/CodeGen/ParallelCG.cpp +++ b/lib/CodeGen/ParallelCG.cpp @@ -1,9 +1,8 @@ //===-- ParallelCG.cpp ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/PatchableFunction.cpp b/lib/CodeGen/PatchableFunction.cpp index afb4b0a7e174..a3fa1b0ad8ed 100644 --- a/lib/CodeGen/PatchableFunction.cpp +++ b/lib/CodeGen/PatchableFunction.cpp @@ -1,9 +1,8 @@ //===-- PatchableFunction.cpp - Patchable prologues for LLVM -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/PeepholeOptimizer.cpp b/lib/CodeGen/PeepholeOptimizer.cpp index 1d058ccfb633..b918396aa8c5 100644 --- a/lib/CodeGen/PeepholeOptimizer.cpp +++ b/lib/CodeGen/PeepholeOptimizer.cpp @@ -1,9 +1,8 @@ //===- PeepholeOptimizer.cpp - Peephole Optimizations ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -1307,7 +1306,7 @@ bool PeepholeOptimizer::optimizeUncoalescableCopy( /// Check whether MI is a candidate for folding into a later instruction. /// We only fold loads to virtual registers and the virtual register defined -/// has a single use. +/// has a single user. bool PeepholeOptimizer::isLoadFoldable( MachineInstr &MI, SmallSet &FoldAsLoadDefCandidates) { if (!MI.canFoldAsLoad() || !MI.mayLoad()) @@ -1317,12 +1316,12 @@ bool PeepholeOptimizer::isLoadFoldable( return false; unsigned Reg = MI.getOperand(0).getReg(); - // To reduce compilation time, we check MRI->hasOneNonDBGUse when inserting + // To reduce compilation time, we check MRI->hasOneNonDBGUser when inserting // loads. It should be checked when processing uses of the load, since // uses can be removed during peephole. if (!MI.getOperand(0).getSubReg() && TargetRegisterInfo::isVirtualRegister(Reg) && - MRI->hasOneNonDBGUse(Reg)) { + MRI->hasOneNonDBGUser(Reg)) { FoldAsLoadDefCandidates.insert(Reg); return true; } @@ -1778,6 +1777,8 @@ bool PeepholeOptimizer::runOnMachineFunction(MachineFunction &MF) { LocalMIs.erase(MI); LocalMIs.erase(DefMI); LocalMIs.insert(FoldMI); + if (MI->isCall()) + MI->getMF()->updateCallSiteInfo(MI, FoldMI); MI->eraseFromParent(); DefMI->eraseFromParent(); MRI->markUsesInDebugValueAsUndef(FoldedReg); @@ -1826,7 +1827,7 @@ ValueTrackerResult ValueTracker::getNextSourceFromBitcast() { assert(Def->isBitcast() && "Invalid definition"); // Bail if there are effects that a plain copy will not expose. - if (Def->hasUnmodeledSideEffects()) + if (Def->mayRaiseFPException() || Def->hasUnmodeledSideEffects()) return ValueTrackerResult(); // Bitcasts with more than one def are not supported. @@ -1901,13 +1902,8 @@ ValueTrackerResult ValueTracker::getNextSourceFromRegSequence() { // Def = REG_SEQUENCE v0, sub0, v1, sub1, ... // Check if one of the operand defines the subreg we are interested in. for (const RegSubRegPairAndIdx &RegSeqInput : RegSeqInputRegs) { - if (RegSeqInput.SubIdx == DefSubReg) { - if (RegSeqInput.SubReg) - // Bail if we have to compose sub registers. - return ValueTrackerResult(); - + if (RegSeqInput.SubIdx == DefSubReg) return ValueTrackerResult(RegSeqInput.Reg, RegSeqInput.SubReg); - } } // If the subreg we are tracking is super-defined by another subreg, diff --git a/lib/CodeGen/PostRAHazardRecognizer.cpp b/lib/CodeGen/PostRAHazardRecognizer.cpp index f9d4a9746e41..0a3838617bc5 100644 --- a/lib/CodeGen/PostRAHazardRecognizer.cpp +++ b/lib/CodeGen/PostRAHazardRecognizer.cpp @@ -1,9 +1,8 @@ //===----- PostRAHazardRecognizer.cpp - hazard recognizer -----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/PostRASchedulerList.cpp b/lib/CodeGen/PostRASchedulerList.cpp index dd0a5fe1b39d..5bea9f2893c9 100644 --- a/lib/CodeGen/PostRASchedulerList.cpp +++ b/lib/CodeGen/PostRASchedulerList.cpp @@ -1,9 +1,8 @@ //===----- SchedulePostRAList.cpp - list scheduler ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/PreISelIntrinsicLowering.cpp b/lib/CodeGen/PreISelIntrinsicLowering.cpp index b0e9ac03612d..2752e186875c 100644 --- a/lib/CodeGen/PreISelIntrinsicLowering.cpp +++ b/lib/CodeGen/PreISelIntrinsicLowering.cpp @@ -1,9 +1,8 @@ //===- PreISelIntrinsicLowering.cpp - Pre-ISel intrinsic lowering pass ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -45,7 +44,7 @@ static bool lowerLoadRelative(Function &F) { Value *OffsetPtr = B.CreateGEP(Int8Ty, CI->getArgOperand(0), CI->getArgOperand(1)); Value *OffsetPtrI32 = B.CreateBitCast(OffsetPtr, Int32PtrTy); - Value *OffsetI32 = B.CreateAlignedLoad(OffsetPtrI32, 4); + Value *OffsetI32 = B.CreateAlignedLoad(Int32Ty, OffsetPtrI32, 4); Value *ResultPtr = B.CreateGEP(Int8Ty, CI->getArgOperand(0), OffsetI32); @@ -65,9 +64,9 @@ static bool lowerObjCCall(Function &F, const char *NewFn, // If we haven't already looked up this function, check to see if the // program already contains a function with this name. Module *M = F.getParent(); - Constant* FCache = M->getOrInsertFunction(NewFn, F.getFunctionType()); + FunctionCallee FCache = M->getOrInsertFunction(NewFn, F.getFunctionType()); - if (Function* Fn = dyn_cast(FCache)) { + if (Function *Fn = dyn_cast(FCache.getCallee())) { Fn->setLinkage(F.getLinkage()); if (setNonLazyBind && !Fn->isWeakForLinker()) { // If we have Native ARC, set nonlazybind attribute for these APIs for diff --git a/lib/CodeGen/ProcessImplicitDefs.cpp b/lib/CodeGen/ProcessImplicitDefs.cpp index 7e9b4af12ee9..b38987ad1c90 100644 --- a/lib/CodeGen/ProcessImplicitDefs.cpp +++ b/lib/CodeGen/ProcessImplicitDefs.cpp @@ -1,9 +1,8 @@ //===---------------------- ProcessImplicitDefs.cpp -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/PrologEpilogInserter.cpp b/lib/CodeGen/PrologEpilogInserter.cpp index 23754e487a18..d463bee67595 100644 --- a/lib/CodeGen/PrologEpilogInserter.cpp +++ b/lib/CodeGen/PrologEpilogInserter.cpp @@ -1,9 +1,8 @@ //===- PrologEpilogInserter.cpp - Insert Prolog/Epilog code in function ---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -32,6 +31,7 @@ #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineLoopInfo.h" #include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineOperand.h" @@ -169,6 +169,46 @@ void PEI::getAnalysisUsage(AnalysisUsage &AU) const { /// StackObjSet - A set of stack object indexes using StackObjSet = SmallSetVector; +using SavedDbgValuesMap = + SmallDenseMap, 4>; + +/// Stash DBG_VALUEs that describe parameters and which are placed at the start +/// of the block. Later on, after the prologue code has been emitted, the +/// stashed DBG_VALUEs will be reinserted at the start of the block. +static void stashEntryDbgValues(MachineBasicBlock &MBB, + SavedDbgValuesMap &EntryDbgValues) { + SmallVector FrameIndexValues; + + for (auto &MI : MBB) { + if (!MI.isDebugInstr()) + break; + if (!MI.isDebugValue() || !MI.getDebugVariable()->isParameter()) + continue; + if (MI.getOperand(0).isFI()) { + // We can only emit valid locations for frame indices after the frame + // setup, so do not stash away them. + FrameIndexValues.push_back(&MI); + continue; + } + const DILocalVariable *Var = MI.getDebugVariable(); + const DIExpression *Expr = MI.getDebugExpression(); + auto Overlaps = [Var, Expr](const MachineInstr *DV) { + return Var == DV->getDebugVariable() && + Expr->fragmentsOverlap(DV->getDebugExpression()); + }; + // See if the debug value overlaps with any preceding debug value that will + // not be stashed. If that is the case, then we can't stash this value, as + // we would then reorder the values at reinsertion. + if (llvm::none_of(FrameIndexValues, Overlaps)) + EntryDbgValues[&MBB].push_back(&MI); + } + + // Remove stashed debug values from the block. + if (EntryDbgValues.count(&MBB)) + for (auto *MI : EntryDbgValues[&MBB]) + MI->removeFromParent(); +} + /// runOnMachineFunction - Insert prolog/epilog code and replace abstract /// frame indexes with appropriate references. bool PEI::runOnMachineFunction(MachineFunction &MF) { @@ -179,8 +219,6 @@ bool PEI::runOnMachineFunction(MachineFunction &MF) { RS = TRI->requiresRegisterScavenging(MF) ? new RegScavenger() : nullptr; FrameIndexVirtualScavenging = TRI->requiresFrameIndexScavenging(MF); - FrameIndexEliminationScavenging = (RS && !FrameIndexVirtualScavenging) || - TRI->requiresFrameIndexReplacementScavenging(MF); ORE = &getAnalysis().getORE(); // Calculate the MaxCallFrameSize and AdjustsStack variables for the @@ -192,6 +230,11 @@ bool PEI::runOnMachineFunction(MachineFunction &MF) { // place all spills in the entry block, all restores in return blocks. calculateSaveRestoreBlocks(MF); + // Stash away DBG_VALUEs that should not be moved by insertion of prolog code. + SavedDbgValuesMap EntryDbgValues; + for (MachineBasicBlock *SaveBlock : SaveBlocks) + stashEntryDbgValues(*SaveBlock, EntryDbgValues); + // Handle CSR spilling and restoring, for targets that need it. if (MF.getTarget().usesPhysRegsForPEI()) spillCalleeSavedRegs(MF); @@ -211,6 +254,10 @@ bool PEI::runOnMachineFunction(MachineFunction &MF) { if (!F.hasFnAttribute(Attribute::Naked)) insertPrologEpilogCode(MF); + // Reinsert stashed debug values at the start of the entry blocks. + for (auto &I : EntryDbgValues) + I.first->insert(I.first->begin(), I.second.begin(), I.second.end()); + // Replace all MO_FrameIndex operands with physical register references // and actual offsets. // @@ -495,9 +542,16 @@ static void insertCSRSaves(MachineBasicBlock &SaveBlock, for (const CalleeSavedInfo &CS : CSI) { // Insert the spill to the stack frame. unsigned Reg = CS.getReg(); - const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg); - TII.storeRegToStackSlot(SaveBlock, I, Reg, true, CS.getFrameIdx(), RC, - TRI); + + if (CS.isSpilledToReg()) { + BuildMI(SaveBlock, I, DebugLoc(), + TII.get(TargetOpcode::COPY), CS.getDstReg()) + .addReg(Reg, getKillRegState(true)); + } else { + const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg); + TII.storeRegToStackSlot(SaveBlock, I, Reg, true, CS.getFrameIdx(), RC, + TRI); + } } } } @@ -517,12 +571,17 @@ static void insertCSRRestores(MachineBasicBlock &RestoreBlock, if (!TFI->restoreCalleeSavedRegisters(RestoreBlock, I, CSI, TRI)) { for (const CalleeSavedInfo &CI : reverse(CSI)) { unsigned Reg = CI.getReg(); - const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg); - TII.loadRegFromStackSlot(RestoreBlock, I, Reg, CI.getFrameIdx(), RC, TRI); - assert(I != RestoreBlock.begin() && - "loadRegFromStackSlot didn't insert any code!"); - // Insert in reverse order. loadRegFromStackSlot can insert - // multiple instructions. + if (CI.isSpilledToReg()) { + BuildMI(RestoreBlock, I, DebugLoc(), TII.get(TargetOpcode::COPY), Reg) + .addReg(CI.getDstReg(), getKillRegState(true)); + } else { + const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg); + TII.loadRegFromStackSlot(RestoreBlock, I, Reg, CI.getFrameIdx(), RC, TRI); + assert(I != RestoreBlock.begin() && + "loadRegFromStackSlot didn't insert any code!"); + // Insert in reverse order. loadRegFromStackSlot can insert + // multiple instructions. + } } } } @@ -615,10 +674,13 @@ computeFreeStackSlots(MachineFrameInfo &MFI, bool StackGrowsDown, SmallVector AllocatedFrameSlots; // Add fixed objects. for (int i = MFI.getObjectIndexBegin(); i != 0; ++i) - AllocatedFrameSlots.push_back(i); + // StackSlot scavenging is only implemented for the default stack. + if (MFI.getStackID(i) == TargetStackID::Default) + AllocatedFrameSlots.push_back(i); // Add callee-save objects. for (int i = MinCSFrameIndex; i <= (int)MaxCSFrameIndex; ++i) - AllocatedFrameSlots.push_back(i); + if (MFI.getStackID(i) == TargetStackID::Default) + AllocatedFrameSlots.push_back(i); for (int i : AllocatedFrameSlots) { // These are converted from int64_t, but they should always fit in int @@ -740,11 +802,23 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &MF) { // Skew to be applied to alignment. unsigned Skew = TFI.getStackAlignmentSkew(MF); +#ifdef EXPENSIVE_CHECKS + for (unsigned i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) + if (!MFI.isDeadObjectIndex(i) && + MFI.getStackID(i) == TargetStackID::Default) + assert(MFI.getObjectAlignment(i) <= MFI.getMaxAlignment() && + "MaxAlignment is invalid"); +#endif + // If there are fixed sized objects that are preallocated in the local area, // non-fixed objects can't be allocated right at the start of local area. // Adjust 'Offset' to point to the end of last fixed sized preallocated // object. for (int i = MFI.getObjectIndexBegin(); i != 0; ++i) { + if (MFI.getStackID(i) != + TargetStackID::Default) // Only allocate objects on the default stack. + continue; + int64_t FixedOff; if (StackGrowsDown) { // The maximum distance from the stack pointer is at lower address of @@ -763,6 +837,10 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &MF) { // callee saved registers. if (StackGrowsDown) { for (unsigned i = MinCSFrameIndex; i <= MaxCSFrameIndex; ++i) { + if (MFI.getStackID(i) != + TargetStackID::Default) // Only allocate objects on the default stack. + continue; + // If the stack grows down, we need to add the size to find the lowest // address of the object. Offset += MFI.getObjectSize(i); @@ -777,6 +855,10 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &MF) { } else if (MaxCSFrameIndex >= MinCSFrameIndex) { // Be careful about underflow in comparisons agains MinCSFrameIndex. for (unsigned i = MaxCSFrameIndex; i != MinCSFrameIndex - 1; --i) { + if (MFI.getStackID(i) != + TargetStackID::Default) // Only allocate objects on the default stack. + continue; + if (MFI.isDeadObjectIndex(i)) continue; @@ -845,18 +927,26 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &MF) { // Make sure that the stack protector comes before the local variables on the // stack. SmallSet ProtectedObjs; - if (MFI.getStackProtectorIndex() >= 0) { + if (MFI.hasStackProtectorIndex()) { + int StackProtectorFI = MFI.getStackProtectorIndex(); StackObjSet LargeArrayObjs; StackObjSet SmallArrayObjs; StackObjSet AddrOfObjs; - AdjustStackOffset(MFI, MFI.getStackProtectorIndex(), StackGrowsDown, - Offset, MaxAlign, Skew); + // If we need a stack protector, we need to make sure that + // LocalStackSlotPass didn't already allocate a slot for it. + // If we are told to use the LocalStackAllocationBlock, the stack protector + // is expected to be already pre-allocated. + if (!MFI.getUseLocalStackAllocationBlock()) + AdjustStackOffset(MFI, StackProtectorFI, StackGrowsDown, Offset, MaxAlign, + Skew); + else if (!MFI.isObjectPreAllocated(MFI.getStackProtectorIndex())) + llvm_unreachable( + "Stack protector not pre-allocated by LocalStackSlotPass."); // Assign large stack objects first. for (unsigned i = 0, e = MFI.getObjectIndexEnd(); i != e; ++i) { - if (MFI.isObjectPreAllocated(i) && - MFI.getUseLocalStackAllocationBlock()) + if (MFI.isObjectPreAllocated(i) && MFI.getUseLocalStackAllocationBlock()) continue; if (i >= MinCSFrameIndex && i <= MaxCSFrameIndex) continue; @@ -864,8 +954,10 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &MF) { continue; if (MFI.isDeadObjectIndex(i)) continue; - if (MFI.getStackProtectorIndex() == (int)i || - EHRegNodeFrameIndex == (int)i) + if (StackProtectorFI == (int)i || EHRegNodeFrameIndex == (int)i) + continue; + if (MFI.getStackID(i) != + TargetStackID::Default) // Only allocate objects on the default stack. continue; switch (MFI.getObjectSSPLayout(i)) { @@ -884,6 +976,15 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &MF) { llvm_unreachable("Unexpected SSPLayoutKind."); } + // We expect **all** the protected stack objects to be pre-allocated by + // LocalStackSlotPass. If it turns out that PEI still has to allocate some + // of them, we may end up messing up the expected order of the objects. + if (MFI.getUseLocalStackAllocationBlock() && + !(LargeArrayObjs.empty() && SmallArrayObjs.empty() && + AddrOfObjs.empty())) + llvm_unreachable("Found protected stack objects not pre-allocated by " + "LocalStackSlotPass."); + AssignProtectedObjSet(LargeArrayObjs, ProtectedObjs, MFI, StackGrowsDown, Offset, MaxAlign, Skew); AssignProtectedObjSet(SmallArrayObjs, ProtectedObjs, MFI, StackGrowsDown, @@ -905,11 +1006,13 @@ void PEI::calculateFrameObjectOffsets(MachineFunction &MF) { continue; if (MFI.isDeadObjectIndex(i)) continue; - if (MFI.getStackProtectorIndex() == (int)i || - EHRegNodeFrameIndex == (int)i) + if (MFI.getStackProtectorIndex() == (int)i || EHRegNodeFrameIndex == (int)i) continue; if (ProtectedObjs.count(i)) continue; + if (MFI.getStackID(i) != + TargetStackID::Default) // Only allocate objects on the default stack. + continue; // Add the objects that we need to allocate to our working set. ObjectsToAllocate.push_back(i); @@ -1026,8 +1129,16 @@ void PEI::insertPrologEpilogCode(MachineFunction &MF) { /// replaceFrameIndices - Replace all MO_FrameIndex operands with physical /// register references and actual offsets. void PEI::replaceFrameIndices(MachineFunction &MF) { - const TargetFrameLowering &TFI = *MF.getSubtarget().getFrameLowering(); - if (!TFI.needsFrameIndexResolution(MF)) return; + const auto &ST = MF.getSubtarget(); + const TargetFrameLowering &TFI = *ST.getFrameLowering(); + if (!TFI.needsFrameIndexResolution(MF)) + return; + + const TargetRegisterInfo *TRI = ST.getRegisterInfo(); + + // Allow the target to determine this after knowing the frame size. + FrameIndexEliminationScavenging = (RS && !FrameIndexVirtualScavenging) || + TRI->requiresFrameIndexReplacementScavenging(MF); // Store SPAdj at exit of a basic block. SmallVector SPState; @@ -1095,12 +1206,37 @@ void PEI::replaceFrameIndices(MachineBasicBlock *BB, MachineFunction &MF, assert(i == 0 && "Frame indices can only appear as the first " "operand of a DBG_VALUE machine instruction"); unsigned Reg; + unsigned FrameIdx = MI.getOperand(0).getIndex(); + unsigned Size = MF.getFrameInfo().getObjectSize(FrameIdx); + int64_t Offset = - TFI->getFrameIndexReference(MF, MI.getOperand(0).getIndex(), Reg); + TFI->getFrameIndexReference(MF, FrameIdx, Reg); MI.getOperand(0).ChangeToRegister(Reg, false /*isDef*/); MI.getOperand(0).setIsDebug(); - auto *DIExpr = DIExpression::prepend(MI.getDebugExpression(), - DIExpression::NoDeref, Offset); + + const DIExpression *DIExpr = MI.getDebugExpression(); + + // If we have a direct DBG_VALUE, and its location expression isn't + // currently complex, then adding an offset will morph it into a + // complex location that is interpreted as being a memory address. + // This changes a pointer-valued variable to dereference that pointer, + // which is incorrect. Fix by adding DW_OP_stack_value. + unsigned PrependFlags = DIExpression::ApplyOffset; + if (!MI.isIndirectDebugValue() && !DIExpr->isComplex()) + PrependFlags |= DIExpression::StackValue; + + // If we have DBG_VALUE that is indirect and has a Implicit location + // expression need to insert a deref before prepending a Memory + // location expression. Also after doing this we change the DBG_VALUE + // to be direct. + if (MI.isIndirectDebugValue() && DIExpr->isImplicit()) { + SmallVector Ops = {dwarf::DW_OP_deref_size, Size}; + bool WithStackValue = true; + DIExpr = DIExpression::prependOpcodes(DIExpr, Ops, WithStackValue); + // Make the DBG_VALUE direct. + MI.getOperand(1).ChangeToRegister(0, false); + } + DIExpr = DIExpression::prepend(DIExpr, PrependFlags, Offset); MI.getOperand(3).setMetadata(DIExpr); continue; } diff --git a/lib/CodeGen/PseudoSourceValue.cpp b/lib/CodeGen/PseudoSourceValue.cpp index 6ca8d86e3f8e..da3ef4b771f3 100644 --- a/lib/CodeGen/PseudoSourceValue.cpp +++ b/lib/CodeGen/PseudoSourceValue.cpp @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/PseudoSourceValue.cpp ----------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/ReachingDefAnalysis.cpp b/lib/CodeGen/ReachingDefAnalysis.cpp index a9f0a9387297..f05c97ad621e 100644 --- a/lib/CodeGen/ReachingDefAnalysis.cpp +++ b/lib/CodeGen/ReachingDefAnalysis.cpp @@ -1,9 +1,8 @@ //===---- ReachingDefAnalysis.cpp - Reaching Def Analysis ---*- C++ -*-----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/RegAllocBase.cpp b/lib/CodeGen/RegAllocBase.cpp index bc28a054c680..1cbe75c27d13 100644 --- a/lib/CodeGen/RegAllocBase.cpp +++ b/lib/CodeGen/RegAllocBase.cpp @@ -1,9 +1,8 @@ //===- RegAllocBase.cpp - Register Allocator Base Class -------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -20,6 +19,7 @@ #include "llvm/CodeGen/LiveIntervals.h" #include "llvm/CodeGen/LiveRegMatrix.h" #include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/CodeGen/VirtRegMap.h" @@ -119,16 +119,19 @@ void RegAllocBase::allocatePhysRegs() { for (MachineRegisterInfo::reg_instr_iterator I = MRI->reg_instr_begin(VirtReg->reg), E = MRI->reg_instr_end(); I != E; ) { - MachineInstr *TmpMI = &*(I++); - if (TmpMI->isInlineAsm()) { - MI = TmpMI; + MI = &*(I++); + if (MI->isInlineAsm()) break; - } } - if (MI) + if (MI && MI->isInlineAsm()) { MI->emitError("inline assembly requires more registers than available"); - else + } else if (MI) { + LLVMContext &Context = + MI->getParent()->getParent()->getMMI().getModule()->getContext(); + Context.emitError("ran out of registers during register allocation"); + } else { report_fatal_error("ran out of registers during register allocation"); + } // Keep going after reporting the error. VRM->assignVirt2Phys(VirtReg->reg, RegClassInfo.getOrder(MRI->getRegClass(VirtReg->reg)).front()); diff --git a/lib/CodeGen/RegAllocBase.h b/lib/CodeGen/RegAllocBase.h index 686ffc36e049..6a7cc5ba4308 100644 --- a/lib/CodeGen/RegAllocBase.h +++ b/lib/CodeGen/RegAllocBase.h @@ -1,9 +1,8 @@ //===- RegAllocBase.h - basic regalloc interface and driver -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/RegAllocBasic.cpp b/lib/CodeGen/RegAllocBasic.cpp index daeff3fc3963..46f6946f7003 100644 --- a/lib/CodeGen/RegAllocBasic.cpp +++ b/lib/CodeGen/RegAllocBasic.cpp @@ -1,9 +1,8 @@ //===-- RegAllocBasic.cpp - Basic Register Allocator ----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/RegAllocFast.cpp b/lib/CodeGen/RegAllocFast.cpp index eb3a4e481f5d..2ffa5e389f89 100644 --- a/lib/CodeGen/RegAllocFast.cpp +++ b/lib/CodeGen/RegAllocFast.cpp @@ -1,9 +1,8 @@ //===- RegAllocFast.cpp - A fast register allocator for debug code --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -102,6 +101,10 @@ namespace { DenseMap> LiveDbgValueMap; + /// Has a bit set for every virtual register for which it was determined + /// that it is alive across blocks. + BitVector MayLiveAcrossBlocks; + /// State of a physical register. enum RegState { /// A disabled register is not available for allocation, but an alias may @@ -152,6 +155,7 @@ namespace { enum : unsigned { spillClean = 50, spillDirty = 100, + spillPrefBonus = 20, spillImpossible = ~0u }; @@ -204,19 +208,26 @@ namespace { } void allocVirtReg(MachineInstr &MI, LiveReg &LR, unsigned Hint); + void allocVirtRegUndef(MachineOperand &MO); MCPhysReg defineVirtReg(MachineInstr &MI, unsigned OpNum, unsigned VirtReg, unsigned Hint); LiveReg &reloadVirtReg(MachineInstr &MI, unsigned OpNum, unsigned VirtReg, unsigned Hint); - void spillAll(MachineBasicBlock::iterator MI); + void spillAll(MachineBasicBlock::iterator MI, bool OnlyLiveOut); bool setPhysReg(MachineInstr &MI, MachineOperand &MO, MCPhysReg PhysReg); + unsigned traceCopies(unsigned VirtReg) const; + unsigned traceCopyChain(unsigned Reg) const; + int getStackSpaceFor(unsigned VirtReg); void spill(MachineBasicBlock::iterator Before, unsigned VirtReg, MCPhysReg AssignedReg, bool Kill); void reload(MachineBasicBlock::iterator Before, unsigned VirtReg, MCPhysReg PhysReg); + bool mayLiveOut(unsigned VirtReg); + bool mayLiveIn(unsigned VirtReg); + void dumpState(); }; @@ -251,6 +262,53 @@ int RegAllocFast::getStackSpaceFor(unsigned VirtReg) { return FrameIdx; } +/// Returns false if \p VirtReg is known to not live out of the current block. +bool RegAllocFast::mayLiveOut(unsigned VirtReg) { + if (MayLiveAcrossBlocks.test(TargetRegisterInfo::virtReg2Index(VirtReg))) { + // Cannot be live-out if there are no successors. + return !MBB->succ_empty(); + } + + // If this block loops back to itself, it would be necessary to check whether + // the use comes after the def. + if (MBB->isSuccessor(MBB)) { + MayLiveAcrossBlocks.set(TargetRegisterInfo::virtReg2Index(VirtReg)); + return true; + } + + // See if the first \p Limit uses of the register are all in the current + // block. + static const unsigned Limit = 8; + unsigned C = 0; + for (const MachineInstr &UseInst : MRI->reg_nodbg_instructions(VirtReg)) { + if (UseInst.getParent() != MBB || ++C >= Limit) { + MayLiveAcrossBlocks.set(TargetRegisterInfo::virtReg2Index(VirtReg)); + // Cannot be live-out if there are no successors. + return !MBB->succ_empty(); + } + } + + return false; +} + +/// Returns false if \p VirtReg is known to not be live into the current block. +bool RegAllocFast::mayLiveIn(unsigned VirtReg) { + if (MayLiveAcrossBlocks.test(TargetRegisterInfo::virtReg2Index(VirtReg))) + return !MBB->pred_empty(); + + // See if the first \p Limit def of the register are all in the current block. + static const unsigned Limit = 8; + unsigned C = 0; + for (const MachineInstr &DefInst : MRI->def_instructions(VirtReg)) { + if (DefInst.getParent() != MBB || ++C >= Limit) { + MayLiveAcrossBlocks.set(TargetRegisterInfo::virtReg2Index(VirtReg)); + return !MBB->pred_empty(); + } + } + + return false; +} + /// Insert spill instruction for \p AssignedReg before \p Before. Update /// DBG_VALUEs with \p VirtReg operands with the stack slot. void RegAllocFast::spill(MachineBasicBlock::iterator Before, unsigned VirtReg, @@ -374,7 +432,7 @@ void RegAllocFast::spillVirtReg(MachineBasicBlock::iterator MI, LiveReg &LR) { } /// Spill all dirty virtregs without killing them. -void RegAllocFast::spillAll(MachineBasicBlock::iterator MI) { +void RegAllocFast::spillAll(MachineBasicBlock::iterator MI, bool OnlyLiveOut) { if (LiveVirtRegs.empty()) return; // The LiveRegMap is keyed by an unsigned (the virtreg number), so the order @@ -382,6 +440,8 @@ void RegAllocFast::spillAll(MachineBasicBlock::iterator MI) { for (LiveReg &LR : LiveVirtRegs) { if (!LR.PhysReg) continue; + if (OnlyLiveOut && !mayLiveOut(LR.VirtReg)) + continue; spillVirtReg(MI, LR); } LiveVirtRegs.clear(); @@ -558,8 +618,48 @@ void RegAllocFast::assignVirtToPhysReg(LiveReg &LR, MCPhysReg PhysReg) { setPhysRegState(PhysReg, VirtReg); } +static bool isCoalescable(const MachineInstr &MI) { + return MI.isFullCopy(); +} + +unsigned RegAllocFast::traceCopyChain(unsigned Reg) const { + static const unsigned ChainLengthLimit = 3; + unsigned C = 0; + do { + if (TargetRegisterInfo::isPhysicalRegister(Reg)) + return Reg; + assert(TargetRegisterInfo::isVirtualRegister(Reg)); + + MachineInstr *VRegDef = MRI->getUniqueVRegDef(Reg); + if (!VRegDef || !isCoalescable(*VRegDef)) + return 0; + Reg = VRegDef->getOperand(1).getReg(); + } while (++C <= ChainLengthLimit); + return 0; +} + +/// Check if any of \p VirtReg's definitions is a copy. If it is follow the +/// chain of copies to check whether we reach a physical register we can +/// coalesce with. +unsigned RegAllocFast::traceCopies(unsigned VirtReg) const { + static const unsigned DefLimit = 3; + unsigned C = 0; + for (const MachineInstr &MI : MRI->def_instructions(VirtReg)) { + if (isCoalescable(MI)) { + unsigned Reg = MI.getOperand(1).getReg(); + Reg = traceCopyChain(Reg); + if (Reg != 0) + return Reg; + } + + if (++C >= DefLimit) + break; + } + return 0; +} + /// Allocates a physical register for VirtReg. -void RegAllocFast::allocVirtReg(MachineInstr &MI, LiveReg &LR, unsigned Hint) { +void RegAllocFast::allocVirtReg(MachineInstr &MI, LiveReg &LR, unsigned Hint0) { const unsigned VirtReg = LR.VirtReg; assert(TargetRegisterInfo::isVirtualRegister(VirtReg) && @@ -567,32 +667,54 @@ void RegAllocFast::allocVirtReg(MachineInstr &MI, LiveReg &LR, unsigned Hint) { const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg); LLVM_DEBUG(dbgs() << "Search register for " << printReg(VirtReg) - << " in class " << TRI->getRegClassName(&RC) << '\n'); + << " in class " << TRI->getRegClassName(&RC) + << " with hint " << printReg(Hint0, TRI) << '\n'); // Take hint when possible. - if (TargetRegisterInfo::isPhysicalRegister(Hint) && - MRI->isAllocatable(Hint) && RC.contains(Hint)) { + if (TargetRegisterInfo::isPhysicalRegister(Hint0) && + MRI->isAllocatable(Hint0) && RC.contains(Hint0)) { // Ignore the hint if we would have to spill a dirty register. - unsigned Cost = calcSpillCost(Hint); + unsigned Cost = calcSpillCost(Hint0); if (Cost < spillDirty) { + LLVM_DEBUG(dbgs() << "\tPreferred Register 1: " << printReg(Hint0, TRI) + << '\n'); if (Cost) - definePhysReg(MI, Hint, regFree); - assignVirtToPhysReg(LR, Hint); + definePhysReg(MI, Hint0, regFree); + assignVirtToPhysReg(LR, Hint0); return; + } else { + LLVM_DEBUG(dbgs() << "\tPreferred Register 1: " << printReg(Hint0, TRI) + << "occupied\n"); } + } else { + Hint0 = 0; } - // First try to find a completely free register. - ArrayRef AllocationOrder = RegClassInfo.getOrder(&RC); - for (MCPhysReg PhysReg : AllocationOrder) { - if (PhysRegState[PhysReg] == regFree && !isRegUsedInInstr(PhysReg)) { - assignVirtToPhysReg(LR, PhysReg); + // Try other hint. + unsigned Hint1 = traceCopies(VirtReg); + if (TargetRegisterInfo::isPhysicalRegister(Hint1) && + MRI->isAllocatable(Hint1) && RC.contains(Hint1) && + !isRegUsedInInstr(Hint1)) { + // Ignore the hint if we would have to spill a dirty register. + unsigned Cost = calcSpillCost(Hint1); + if (Cost < spillDirty) { + LLVM_DEBUG(dbgs() << "\tPreferred Register 0: " << printReg(Hint1, TRI) + << '\n'); + if (Cost) + definePhysReg(MI, Hint1, regFree); + assignVirtToPhysReg(LR, Hint1); return; + } else { + LLVM_DEBUG(dbgs() << "\tPreferred Register 0: " << printReg(Hint1, TRI) + << "occupied\n"); } + } else { + Hint1 = 0; } MCPhysReg BestReg = 0; unsigned BestCost = spillImpossible; + ArrayRef AllocationOrder = RegClassInfo.getOrder(&RC); for (MCPhysReg PhysReg : AllocationOrder) { LLVM_DEBUG(dbgs() << "\tRegister: " << printReg(PhysReg, TRI) << ' '); unsigned Cost = calcSpillCost(PhysReg); @@ -602,6 +724,10 @@ void RegAllocFast::allocVirtReg(MachineInstr &MI, LiveReg &LR, unsigned Hint) { assignVirtToPhysReg(LR, PhysReg); return; } + + if (PhysReg == Hint1 || PhysReg == Hint0) + Cost -= spillPrefBonus; + if (Cost < BestCost) { BestReg = PhysReg; BestCost = Cost; @@ -624,6 +750,31 @@ void RegAllocFast::allocVirtReg(MachineInstr &MI, LiveReg &LR, unsigned Hint) { assignVirtToPhysReg(LR, BestReg); } +void RegAllocFast::allocVirtRegUndef(MachineOperand &MO) { + assert(MO.isUndef() && "expected undef use"); + unsigned VirtReg = MO.getReg(); + assert(TargetRegisterInfo::isVirtualRegister(VirtReg) && "Expected virtreg"); + + LiveRegMap::const_iterator LRI = findLiveVirtReg(VirtReg); + MCPhysReg PhysReg; + if (LRI != LiveVirtRegs.end() && LRI->PhysReg) { + PhysReg = LRI->PhysReg; + } else { + const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg); + ArrayRef AllocationOrder = RegClassInfo.getOrder(&RC); + assert(!AllocationOrder.empty() && "Allocation order must not be empty"); + PhysReg = AllocationOrder[0]; + } + + unsigned SubRegIdx = MO.getSubReg(); + if (SubRegIdx != 0) { + PhysReg = TRI->getSubReg(PhysReg, SubRegIdx); + MO.setSubReg(0); + } + MO.setReg(PhysReg); + MO.setIsRenamable(true); +} + /// Allocates a register for VirtReg and mark it as dirty. MCPhysReg RegAllocFast::defineVirtReg(MachineInstr &MI, unsigned OpNum, unsigned VirtReg, unsigned Hint) { @@ -941,12 +1092,23 @@ void RegAllocFast::allocateInstruction(MachineInstr &MI) { // Second scan. // Allocate virtreg uses. + bool HasUndefUse = false; for (unsigned I = 0; I != VirtOpEnd; ++I) { MachineOperand &MO = MI.getOperand(I); if (!MO.isReg()) continue; unsigned Reg = MO.getReg(); if (!TargetRegisterInfo::isVirtualRegister(Reg)) continue; if (MO.isUse()) { + if (MO.isUndef()) { + HasUndefUse = true; + // There is no need to allocate a register for an undef use. + continue; + } + + // Populate MayLiveAcrossBlocks in case the use block is allocated before + // the def block (removing the vreg uses). + mayLiveIn(Reg); + LiveReg &LR = reloadVirtReg(MI, I, Reg, CopyDstReg); MCPhysReg PhysReg = LR.PhysReg; CopySrcReg = (CopySrcReg == Reg || CopySrcReg == PhysReg) ? PhysReg : 0; @@ -955,6 +1117,22 @@ void RegAllocFast::allocateInstruction(MachineInstr &MI) { } } + // Allocate undef operands. This is a separate step because in a situation + // like ` = OP undef %X, %X` both operands need the same register assign + // so we should perform the normal assignment first. + if (HasUndefUse) { + for (MachineOperand &MO : MI.uses()) { + if (!MO.isReg() || !MO.isUse()) + continue; + unsigned Reg = MO.getReg(); + if (!TargetRegisterInfo::isVirtualRegister(Reg)) + continue; + + assert(MO.isUndef() && "Should only have undef virtreg uses left"); + allocVirtRegUndef(MO); + } + } + // Track registers defined by instruction - early clobbers and tied uses at // this point. UsedInInstr.clear(); @@ -979,10 +1157,24 @@ void RegAllocFast::allocateInstruction(MachineInstr &MI) { // definitions may be used later on and we do not want to reuse // those for virtual registers in between. LLVM_DEBUG(dbgs() << " Spilling remaining registers before call.\n"); - spillAll(MI); + spillAll(MI, /*OnlyLiveOut*/ false); } // Third scan. + // Mark all physreg defs as used before allocating virtreg defs. + for (unsigned I = 0; I != DefOpEnd; ++I) { + const MachineOperand &MO = MI.getOperand(I); + if (!MO.isReg() || !MO.isDef() || !MO.getReg() || MO.isEarlyClobber()) + continue; + unsigned Reg = MO.getReg(); + + if (!Reg || !TargetRegisterInfo::isPhysicalRegister(Reg) || + !MRI->isAllocatable(Reg)) + continue; + definePhysReg(MI, Reg, MO.isDead() ? regFree : regReserved); + } + + // Fourth scan. // Allocate defs and collect dead defs. for (unsigned I = 0; I != DefOpEnd; ++I) { const MachineOperand &MO = MI.getOperand(I); @@ -990,11 +1182,9 @@ void RegAllocFast::allocateInstruction(MachineInstr &MI) { continue; unsigned Reg = MO.getReg(); - if (TargetRegisterInfo::isPhysicalRegister(Reg)) { - if (!MRI->isAllocatable(Reg)) continue; - definePhysReg(MI, Reg, MO.isDead() ? regFree : regReserved); + // We have already dealt with phys regs in the previous scan. + if (TargetRegisterInfo::isPhysicalRegister(Reg)) continue; - } MCPhysReg PhysReg = defineVirtReg(MI, I, Reg, CopySrcReg); if (setPhysReg(MI, MI.getOperand(I), PhysReg)) { VirtDead.push_back(Reg); @@ -1089,7 +1279,7 @@ void RegAllocFast::allocateBasicBlock(MachineBasicBlock &MBB) { // Spill all physical registers holding virtual registers now. LLVM_DEBUG(dbgs() << "Spilling live registers at end of block.\n"); - spillAll(MBB.getFirstTerminator()); + spillAll(MBB.getFirstTerminator(), /*OnlyLiveOut*/ true); // Erase all the coalesced copies. We are delaying it until now because // LiveVirtRegs might refer to the instrs. @@ -1118,6 +1308,8 @@ bool RegAllocFast::runOnMachineFunction(MachineFunction &MF) { unsigned NumVirtRegs = MRI->getNumVirtRegs(); StackSlotForVirtReg.resize(NumVirtRegs); LiveVirtRegs.setUniverse(NumVirtRegs); + MayLiveAcrossBlocks.clear(); + MayLiveAcrossBlocks.resize(NumVirtRegs); // Loop over all of the basic blocks, eliminating virtual register references for (MachineBasicBlock &MBB : MF) diff --git a/lib/CodeGen/RegAllocGreedy.cpp b/lib/CodeGen/RegAllocGreedy.cpp index 81b21b442437..771fc46415db 100644 --- a/lib/CodeGen/RegAllocGreedy.cpp +++ b/lib/CodeGen/RegAllocGreedy.cpp @@ -1,9 +1,8 @@ //===- RegAllocGreedy.cpp - greedy register allocator ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -138,7 +137,7 @@ CSRFirstTimeCost("regalloc-csr-first-time-cost", cl::init(0), cl::Hidden); static cl::opt ConsiderLocalIntervalCost( - "condsider-local-interval-cost", cl::Hidden, + "consider-local-interval-cost", cl::Hidden, cl::desc("Consider the cost of local intervals created by a split " "candidate when choosing the best split candidate."), cl::init(false)); @@ -465,7 +464,8 @@ private: void calcGapWeights(unsigned, SmallVectorImpl&); unsigned canReassign(LiveInterval &VirtReg, unsigned PrevReg); bool shouldEvict(LiveInterval &A, bool, LiveInterval &B, bool); - bool canEvictInterference(LiveInterval&, unsigned, bool, EvictionCost&); + bool canEvictInterference(LiveInterval&, unsigned, bool, EvictionCost&, + const SmallVirtRegSet&); bool canEvictInterferenceInRange(LiveInterval &VirtReg, unsigned PhysReg, SlotIndex Start, SlotIndex End, EvictionCost &MaxCost); @@ -479,9 +479,11 @@ private: const SmallVirtRegSet &FixedRegisters); unsigned tryAssign(LiveInterval&, AllocationOrder&, - SmallVectorImpl&); + SmallVectorImpl&, + const SmallVirtRegSet&); unsigned tryEvict(LiveInterval&, AllocationOrder&, - SmallVectorImpl&, unsigned = ~0u); + SmallVectorImpl&, unsigned, + const SmallVirtRegSet&); unsigned tryRegionSplit(LiveInterval&, AllocationOrder&, SmallVectorImpl&); unsigned isSplitBenefitWorthCost(LiveInterval &VirtReg); @@ -508,7 +510,8 @@ private: unsigned tryLocalSplit(LiveInterval&, AllocationOrder&, SmallVectorImpl&); unsigned trySplit(LiveInterval&, AllocationOrder&, - SmallVectorImpl&); + SmallVectorImpl&, + const SmallVirtRegSet&); unsigned tryLastChanceRecoloring(LiveInterval &, AllocationOrder &, SmallVectorImpl &, SmallVirtRegSet &, unsigned); @@ -758,7 +761,8 @@ LiveInterval *RAGreedy::dequeue(PQueue &CurQueue) { /// tryAssign - Try to assign VirtReg to an available register. unsigned RAGreedy::tryAssign(LiveInterval &VirtReg, AllocationOrder &Order, - SmallVectorImpl &NewVRegs) { + SmallVectorImpl &NewVRegs, + const SmallVirtRegSet &FixedRegisters) { Order.rewind(); unsigned PhysReg; while ((PhysReg = Order.next())) @@ -776,7 +780,7 @@ unsigned RAGreedy::tryAssign(LiveInterval &VirtReg, LLVM_DEBUG(dbgs() << "missed hint " << printReg(Hint, TRI) << '\n'); EvictionCost MaxCost; MaxCost.setBrokenHints(1); - if (canEvictInterference(VirtReg, Hint, true, MaxCost)) { + if (canEvictInterference(VirtReg, Hint, true, MaxCost, FixedRegisters)) { evictInterference(VirtReg, Hint, NewVRegs); return Hint; } @@ -794,7 +798,7 @@ unsigned RAGreedy::tryAssign(LiveInterval &VirtReg, LLVM_DEBUG(dbgs() << printReg(PhysReg, TRI) << " is available at cost " << Cost << '\n'); - unsigned CheapReg = tryEvict(VirtReg, Order, NewVRegs, Cost); + unsigned CheapReg = tryEvict(VirtReg, Order, NewVRegs, Cost, FixedRegisters); return CheapReg ? CheapReg : PhysReg; } @@ -866,7 +870,8 @@ bool RAGreedy::shouldEvict(LiveInterval &A, bool IsHint, /// when returning true. /// @returns True when interference can be evicted cheaper than MaxCost. bool RAGreedy::canEvictInterference(LiveInterval &VirtReg, unsigned PhysReg, - bool IsHint, EvictionCost &MaxCost) { + bool IsHint, EvictionCost &MaxCost, + const SmallVirtRegSet &FixedRegisters) { // It is only possible to evict virtual register interference. if (Matrix->checkInterference(VirtReg, PhysReg) > LiveRegMatrix::IK_VirtReg) return false; @@ -896,6 +901,13 @@ bool RAGreedy::canEvictInterference(LiveInterval &VirtReg, unsigned PhysReg, LiveInterval *Intf = Q.interferingVRegs()[i - 1]; assert(TargetRegisterInfo::isVirtualRegister(Intf->reg) && "Only expecting virtual register interference from query"); + + // Do not allow eviction of a virtual register if we are in the middle + // of last-chance recoloring and this virtual register is one that we + // have scavenged a physical register for. + if (FixedRegisters.count(Intf->reg)) + return false; + // Never evict spill products. They cannot split or spill. if (getStage(*Intf) == RS_Done) return false; @@ -1094,7 +1106,8 @@ bool RAGreedy::isUnusedCalleeSavedReg(unsigned PhysReg) const { unsigned RAGreedy::tryEvict(LiveInterval &VirtReg, AllocationOrder &Order, SmallVectorImpl &NewVRegs, - unsigned CostPerUseLimit) { + unsigned CostPerUseLimit, + const SmallVirtRegSet &FixedRegisters) { NamedRegionTimer T("evict", "Evict", TimerGroupName, TimerGroupDescription, TimePassesIsEnabled); @@ -1142,7 +1155,8 @@ unsigned RAGreedy::tryEvict(LiveInterval &VirtReg, continue; } - if (!canEvictInterference(VirtReg, PhysReg, false, BestCost)) + if (!canEvictInterference(VirtReg, PhysReg, false, BestCost, + FixedRegisters)) continue; // Best so far. @@ -2248,8 +2262,8 @@ unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order, ArrayRef RMS = LIS->getRegMaskSlotsInBlock(BI.MBB->getNumber()); LLVM_DEBUG(dbgs() << RMS.size() << " regmasks in block:"); // Constrain to VirtReg's live range. - unsigned ri = std::lower_bound(RMS.begin(), RMS.end(), - Uses.front().getRegSlot()) - RMS.begin(); + unsigned ri = + llvm::lower_bound(RMS, Uses.front().getRegSlot()) - RMS.begin(); unsigned re = RMS.size(); for (unsigned i = 0; i != NumGaps && ri != re; ++i) { // Look for Uses[i] <= RMS <= Uses[i+1]. @@ -2444,7 +2458,8 @@ unsigned RAGreedy::tryLocalSplit(LiveInterval &VirtReg, AllocationOrder &Order, /// assignable. /// @return Physreg when VirtReg may be assigned and/or new NewVRegs. unsigned RAGreedy::trySplit(LiveInterval &VirtReg, AllocationOrder &Order, - SmallVectorImpl&NewVRegs) { + SmallVectorImpl&NewVRegs, + const SmallVirtRegSet &FixedRegisters) { // Ranges must be Split2 or less. if (getStage(VirtReg) >= RS_Spill) return 0; @@ -2472,7 +2487,7 @@ unsigned RAGreedy::trySplit(LiveInterval &VirtReg, AllocationOrder &Order, if (SA->didRepairRange()) { // VirtReg has changed, so all cached queries are invalid. Matrix->invalidateVirtRegs(); - if (unsigned PhysReg = tryAssign(VirtReg, Order, NewVRegs)) + if (unsigned PhysReg = tryAssign(VirtReg, Order, NewVRegs, FixedRegisters)) return PhysReg; } @@ -2611,6 +2626,7 @@ unsigned RAGreedy::tryLastChanceRecoloring(LiveInterval &VirtReg, DenseMap VirtRegToPhysReg; // Mark VirtReg as fixed, i.e., it will not be recolored pass this point in // this recoloring "session". + assert(!FixedRegisters.count(VirtReg.reg)); FixedRegisters.insert(VirtReg.reg); SmallVector CurrentNewVRegs; @@ -2858,14 +2874,14 @@ void RAGreedy::collectHintInfo(unsigned Reg, HintsInfo &Out) { if (!Instr.isFullCopy()) continue; // Look for the other end of the copy. - unsigned OtherReg = Instr.getOperand(0).getReg(); + Register OtherReg = Instr.getOperand(0).getReg(); if (OtherReg == Reg) { OtherReg = Instr.getOperand(1).getReg(); if (OtherReg == Reg) continue; } // Get the current assignment. - unsigned OtherPhysReg = TargetRegisterInfo::isPhysicalRegister(OtherReg) + Register OtherPhysReg = TargetRegisterInfo::isPhysicalRegister(OtherReg) ? OtherReg : VRM->getPhys(OtherReg); // Push the collected information. @@ -3022,7 +3038,7 @@ unsigned RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg, unsigned CostPerUseLimit = ~0u; // First try assigning a free register. AllocationOrder Order(VirtReg.reg, *VRM, RegClassInfo, Matrix); - if (unsigned PhysReg = tryAssign(VirtReg, Order, NewVRegs)) { + if (unsigned PhysReg = tryAssign(VirtReg, Order, NewVRegs, FixedRegisters)) { // If VirtReg got an assignment, the eviction info is no longre relevant. LastEvicted.clearEvicteeInfo(VirtReg.reg); // When NewVRegs is not empty, we may have made decisions such as evicting @@ -3049,7 +3065,8 @@ unsigned RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg, // get a second chance until they have been split. if (Stage != RS_Split) if (unsigned PhysReg = - tryEvict(VirtReg, Order, NewVRegs, CostPerUseLimit)) { + tryEvict(VirtReg, Order, NewVRegs, CostPerUseLimit, + FixedRegisters)) { unsigned Hint = MRI->getSimpleHint(VirtReg.reg); // If VirtReg has a hint and that hint is broken record this // virtual register as a recoloring candidate for broken hint. @@ -3079,7 +3096,7 @@ unsigned RAGreedy::selectOrSplitImpl(LiveInterval &VirtReg, if (Stage < RS_Spill) { // Try splitting VirtReg or interferences. unsigned NewVRegSizeBefore = NewVRegs.size(); - unsigned PhysReg = trySplit(VirtReg, Order, NewVRegs); + unsigned PhysReg = trySplit(VirtReg, Order, NewVRegs, FixedRegisters); if (PhysReg || (NewVRegs.size() - NewVRegSizeBefore)) { // If VirtReg got split, the eviction info is no longre relevant. LastEvicted.clearEvicteeInfo(VirtReg.reg); diff --git a/lib/CodeGen/RegAllocPBQP.cpp b/lib/CodeGen/RegAllocPBQP.cpp index c19001c8403d..7a5a6c148ed4 100644 --- a/lib/CodeGen/RegAllocPBQP.cpp +++ b/lib/CodeGen/RegAllocPBQP.cpp @@ -1,9 +1,8 @@ //===- RegAllocPBQP.cpp ---- PBQP Register Allocator ----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/RegUsageInfoCollector.cpp b/lib/CodeGen/RegUsageInfoCollector.cpp index 66c7c5cd7dbf..b37dfada7101 100644 --- a/lib/CodeGen/RegUsageInfoCollector.cpp +++ b/lib/CodeGen/RegUsageInfoCollector.cpp @@ -1,9 +1,8 @@ //===-- RegUsageInfoCollector.cpp - Register Usage Information Collector --===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// @@ -78,14 +77,48 @@ FunctionPass *llvm::createRegUsageInfoCollector() { return new RegUsageInfoCollector(); } +// TODO: Move to hook somwehere? + +// Return true if it is useful to track the used registers for IPRA / no CSR +// optimizations. This is not useful for entry points, and computing the +// register usage information is expensive. +static bool isCallableFunction(const MachineFunction &MF) { + switch (MF.getFunction().getCallingConv()) { + case CallingConv::AMDGPU_VS: + case CallingConv::AMDGPU_GS: + case CallingConv::AMDGPU_PS: + case CallingConv::AMDGPU_CS: + case CallingConv::AMDGPU_HS: + case CallingConv::AMDGPU_ES: + case CallingConv::AMDGPU_LS: + case CallingConv::AMDGPU_KERNEL: + return false; + default: + return true; + } +} + bool RegUsageInfoCollector::runOnMachineFunction(MachineFunction &MF) { MachineRegisterInfo *MRI = &MF.getRegInfo(); const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo(); const LLVMTargetMachine &TM = MF.getTarget(); LLVM_DEBUG(dbgs() << " -------------------- " << getPassName() - << " -------------------- \n"); - LLVM_DEBUG(dbgs() << "Function Name : " << MF.getName() << "\n"); + << " -------------------- \nFunction Name : " + << MF.getName() << '\n'); + + // Analyzing the register usage may be expensive on some targets. + if (!isCallableFunction(MF)) { + LLVM_DEBUG(dbgs() << "Not analyzing non-callable function\n"); + return false; + } + + // If there are no callers, there's no point in computing more precise + // register usage here. + if (MF.getFunction().use_empty()) { + LLVM_DEBUG(dbgs() << "Not analyzing function with no callers\n"); + return false; + } std::vector RegMask; @@ -111,6 +144,7 @@ bool RegUsageInfoCollector::runOnMachineFunction(MachineFunction &MF) { }; // Scan all the physical registers. When a register is defined in the current // function set it and all the aliasing registers as defined in the regmask. + // FIXME: Rewrite to use regunits. for (unsigned PReg = 1, PRegE = TRI->getNumRegs(); PReg < PRegE; ++PReg) { // Don't count registers that are saved and restored. if (SavedRegs.test(PReg)) @@ -136,11 +170,14 @@ bool RegUsageInfoCollector::runOnMachineFunction(MachineFunction &MF) { << " function optimized for not having CSR.\n"); } - for (unsigned PReg = 1, PRegE = TRI->getNumRegs(); PReg < PRegE; ++PReg) - if (MachineOperand::clobbersPhysReg(&(RegMask[0]), PReg)) - LLVM_DEBUG(dbgs() << printReg(PReg, TRI) << " "); + LLVM_DEBUG( + for (unsigned PReg = 1, PRegE = TRI->getNumRegs(); PReg < PRegE; ++PReg) { + if (MachineOperand::clobbersPhysReg(&(RegMask[0]), PReg)) + dbgs() << printReg(PReg, TRI) << " "; + } - LLVM_DEBUG(dbgs() << " \n----------------------------------------\n"); + dbgs() << " \n----------------------------------------\n"; + ); PRUI.storeUpdateRegUsageInfo(F, RegMask); @@ -155,38 +192,17 @@ computeCalleeSavedRegs(BitVector &SavedRegs, MachineFunction &MF) { // Target will return the set of registers that it saves/restores as needed. SavedRegs.clear(); TFI.determineCalleeSaves(MF, SavedRegs); + if (SavedRegs.none()) + return; // Insert subregs. const MCPhysReg *CSRegs = TRI.getCalleeSavedRegs(&MF); for (unsigned i = 0; CSRegs[i]; ++i) { - unsigned Reg = CSRegs[i]; - if (SavedRegs.test(Reg)) - for (MCSubRegIterator SR(Reg, &TRI, false); SR.isValid(); ++SR) + MCPhysReg Reg = CSRegs[i]; + if (SavedRegs.test(Reg)) { + // Save subregisters + for (MCSubRegIterator SR(Reg, &TRI); SR.isValid(); ++SR) SavedRegs.set(*SR); - } - - // Insert any register fully saved via subregisters. - for (const TargetRegisterClass *RC : TRI.regclasses()) { - if (!RC->CoveredBySubRegs) - continue; - - for (unsigned PReg = 1, PRegE = TRI.getNumRegs(); PReg < PRegE; ++PReg) { - if (SavedRegs.test(PReg)) - continue; - - // Check if PReg is fully covered by its subregs. - if (!RC->contains(PReg)) - continue; - - // Add PReg to SavedRegs if all subregs are saved. - bool AllSubRegsSaved = true; - for (MCSubRegIterator SR(PReg, &TRI, false); SR.isValid(); ++SR) - if (!SavedRegs.test(*SR)) { - AllSubRegsSaved = false; - break; - } - if (AllSubRegsSaved) - SavedRegs.set(PReg); } } } diff --git a/lib/CodeGen/RegUsageInfoPropagate.cpp b/lib/CodeGen/RegUsageInfoPropagate.cpp index 256de295821d..fc4be82d215e 100644 --- a/lib/CodeGen/RegUsageInfoPropagate.cpp +++ b/lib/CodeGen/RegUsageInfoPropagate.cpp @@ -1,9 +1,8 @@ //=--- RegUsageInfoPropagate.cpp - Register Usage Informartion Propagation --=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/CodeGen/RegisterClassInfo.cpp b/lib/CodeGen/RegisterClassInfo.cpp index add8faec97d4..530e0cccf1d4 100644 --- a/lib/CodeGen/RegisterClassInfo.cpp +++ b/lib/CodeGen/RegisterClassInfo.cpp @@ -1,9 +1,8 @@ //===- RegisterClassInfo.cpp - Dynamic Register Class Info ----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -91,6 +90,7 @@ void RegisterClassInfo::runOnMachineFunction(const MachineFunction &mf) { void RegisterClassInfo::compute(const TargetRegisterClass *RC) const { assert(RC && "no register class given"); RCInfo &RCI = RegClass[RC->getID()]; + auto &STI = MF->getSubtarget(); // Raw register count, including all reserved regs. unsigned NumRegs = RC->getNumRegs(); @@ -115,7 +115,8 @@ void RegisterClassInfo::compute(const TargetRegisterClass *RC) const { unsigned Cost = TRI->getCostPerUse(PhysReg); MinCost = std::min(MinCost, Cost); - if (CalleeSavedAliases[PhysReg]) + if (CalleeSavedAliases[PhysReg] && + !STI.ignoreCSRForAllocationOrder(*MF, PhysReg)) // PhysReg aliases a CSR, save it for later. CSRAlias.push_back(PhysReg); else { diff --git a/lib/CodeGen/RegisterCoalescer.cpp b/lib/CodeGen/RegisterCoalescer.cpp index 2a06d5e95fbb..2db6ab454cea 100644 --- a/lib/CodeGen/RegisterCoalescer.cpp +++ b/lib/CodeGen/RegisterCoalescer.cpp @@ -1,9 +1,8 @@ //===- RegisterCoalescer.cpp - Generic Register Coalescing Interface ------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -105,6 +104,19 @@ static cl::opt LateRematUpdateThreshold( "repeated work. "), cl::init(100)); +static cl::opt LargeIntervalSizeThreshold( + "large-interval-size-threshold", cl::Hidden, + cl::desc("If the valnos size of an interval is larger than the threshold, " + "it is regarded as a large interval. "), + cl::init(100)); + +static cl::opt LargeIntervalFreqThreshold( + "large-interval-freq-threshold", cl::Hidden, + cl::desc("For a large interval, if it is coalesed with other live " + "intervals many times more than the threshold, stop its " + "coalescing to control the compile time. "), + cl::init(100)); + namespace { class RegisterCoalescer : public MachineFunctionPass, @@ -153,6 +165,10 @@ namespace { /// lateLiveIntervalUpdate is called. DenseSet ToBeUpdated; + /// Record how many times the large live interval with many valnos + /// has been tried to join with other live interval. + DenseMap LargeLIVisitCounter; + /// Recursively eliminate dead defs in DeadDefs. void eliminateDeadDefs(); @@ -195,6 +211,11 @@ namespace { /// Attempt joining two virtual registers. Return true on success. bool joinVirtRegs(CoalescerPair &CP); + /// If a live interval has many valnos and is coalesced with other + /// live intervals many times, we regard such live interval as having + /// high compile time cost. + bool isHighCostLiveInterval(LiveInterval &LI); + /// Attempt joining with a reserved physreg. bool joinReservedPhysReg(CoalescerPair &CP); @@ -337,9 +358,10 @@ INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) INITIALIZE_PASS_END(RegisterCoalescer, "simple-register-coalescing", "Simple Register Coalescing", false, false) -static bool isMoveInstr(const TargetRegisterInfo &tri, const MachineInstr *MI, - unsigned &Src, unsigned &Dst, - unsigned &SrcSub, unsigned &DstSub) { +LLVM_NODISCARD static bool isMoveInstr(const TargetRegisterInfo &tri, + const MachineInstr *MI, unsigned &Src, + unsigned &Dst, unsigned &SrcSub, + unsigned &DstSub) { if (MI->isCopy()) { Dst = MI->getOperand(0).getReg(); DstSub = MI->getOperand(0).getSubReg(); @@ -672,8 +694,7 @@ bool RegisterCoalescer::hasOtherReachingDefs(LiveInterval &IntA, for (LiveRange::Segment &ASeg : IntA.segments) { if (ASeg.valno != AValNo) continue; - LiveInterval::iterator BI = - std::upper_bound(IntB.begin(), IntB.end(), ASeg.start); + LiveInterval::iterator BI = llvm::upper_bound(IntB, ASeg.start); if (BI != IntB.begin()) --BI; for (; BI != IntB.end() && ASeg.end >= BI->start; ++BI) { @@ -903,23 +924,32 @@ RegisterCoalescer::removeCopyByCommutingDef(const CoalescerPair &CP, } SlotIndex AIdx = CopyIdx.getRegSlot(true); LaneBitmask MaskA; + const SlotIndexes &Indexes = *LIS->getSlotIndexes(); for (LiveInterval::SubRange &SA : IntA.subranges()) { VNInfo *ASubValNo = SA.getVNInfoAt(AIdx); - assert(ASubValNo != nullptr); + // Even if we are dealing with a full copy, some lanes can + // still be undefined. + // E.g., + // undef A.subLow = ... + // B = COPY A <== A.subHigh is undefined here and does + // not have a value number. + if (!ASubValNo) + continue; MaskA |= SA.LaneMask; - IntB.refineSubRanges(Allocator, SA.LaneMask, - [&Allocator,&SA,CopyIdx,ASubValNo,&ShrinkB] - (LiveInterval::SubRange &SR) { - VNInfo *BSubValNo = SR.empty() - ? SR.getNextValue(CopyIdx, Allocator) - : SR.getVNInfoAt(CopyIdx); - assert(BSubValNo != nullptr); - auto P = addSegmentsWithValNo(SR, BSubValNo, SA, ASubValNo); - ShrinkB |= P.second; - if (P.first) - BSubValNo->def = ASubValNo->def; - }); + IntB.refineSubRanges( + Allocator, SA.LaneMask, + [&Allocator, &SA, CopyIdx, ASubValNo, + &ShrinkB](LiveInterval::SubRange &SR) { + VNInfo *BSubValNo = SR.empty() ? SR.getNextValue(CopyIdx, Allocator) + : SR.getVNInfoAt(CopyIdx); + assert(BSubValNo != nullptr); + auto P = addSegmentsWithValNo(SR, BSubValNo, SA, ASubValNo); + ShrinkB |= P.second; + if (P.first) + BSubValNo->def = ASubValNo->def; + }, + Indexes, *TRI); } // Go over all subranges of IntB that have not been covered by IntA, // and delete the segments starting at CopyIdx. This can happen if @@ -947,7 +977,7 @@ RegisterCoalescer::removeCopyByCommutingDef(const CoalescerPair &CP, /// For copy B = A in BB2, if A is defined by A = B in BB0 which is a /// predecessor of BB2, and if B is not redefined on the way from A = B -/// in BB2 to B = A in BB2, B = A in BB2 is partially redundant if the +/// in BB0 to B = A in BB2, B = A in BB2 is partially redundant if the /// execution goes through the path from BB0 to BB2. We may move B = A /// to the predecessor without such reversed copy. /// So we will transform the program from: @@ -1494,7 +1524,8 @@ MachineInstr *RegisterCoalescer::eliminateUndefCopy(MachineInstr *CopyMI) { // CoalescerPair may have a new register class with adjusted subreg indices // at this point. unsigned SrcReg, DstReg, SrcSubIdx, DstSubIdx; - isMoveInstr(*TRI, CopyMI, SrcReg, DstReg, SrcSubIdx, DstSubIdx); + if(!isMoveInstr(*TRI, CopyMI, SrcReg, DstReg, SrcSubIdx, DstSubIdx)) + return nullptr; SlotIndex Idx = LIS->getInstructionIndex(*CopyMI); const LiveInterval &SrcLI = LIS->getInterval(SrcReg); @@ -1994,19 +2025,19 @@ bool RegisterCoalescer::joinReservedPhysReg(CoalescerPair &CP) { if (CP.isFlipped()) { // Physreg is copied into vreg // %y = COPY %physreg_x - // ... //< no other def of %x here + // ... //< no other def of %physreg_x here // use %y // => // ... - // use %x + // use %physreg_x CopyMI = MRI->getVRegDef(SrcReg); } else { // VReg is copied into physreg: // %y = def - // ... //< no other def or use of %y here - // %y = COPY %physreg_x + // ... //< no other def or use of %physreg_x here + // %physreg_x = COPY %y // => - // %y = def + // %physreg_x = def // ... if (!MRI->hasOneNonDBGUse(SrcReg)) { LLVM_DEBUG(dbgs() << "\t\tMultiple vreg uses!\n"); @@ -3010,7 +3041,9 @@ void JoinVals::pruneSubRegValues(LiveInterval &LI, LaneBitmask &ShrinkMask) { // If a subrange starts at the copy then an undefined value has been // copied and we must remove that subrange value as well. VNInfo *ValueOut = Q.valueOutOrDead(); - if (ValueOut != nullptr && Q.valueIn() == nullptr) { + if (ValueOut != nullptr && (Q.valueIn() == nullptr || + (V.Identical && V.Resolution == CR_Erase && + ValueOut->def == Def))) { LLVM_DEBUG(dbgs() << "\t\tPrune sublane " << PrintLaneMask(S.LaneMask) << " at " << Def << "\n"); SmallVector EndPoints; @@ -3019,7 +3052,7 @@ void JoinVals::pruneSubRegValues(LiveInterval &LI, LaneBitmask &ShrinkMask) { // Mark value number as unused. ValueOut->markUnused(); - if (V.Identical && S.Query(OtherDef).valueOut()) { + if (V.Identical && S.Query(OtherDef).valueOutOrDead()) { // If V is identical to V.OtherVNI (and S was live at OtherDef), // then we can't simply prune V from S. V needs to be replaced // with V.OtherVNI. @@ -3241,16 +3274,29 @@ void RegisterCoalescer::mergeSubRangeInto(LiveInterval &LI, LaneBitmask LaneMask, CoalescerPair &CP) { BumpPtrAllocator &Allocator = LIS->getVNInfoAllocator(); - LI.refineSubRanges(Allocator, LaneMask, - [this,&Allocator,&ToMerge,&CP](LiveInterval::SubRange &SR) { - if (SR.empty()) { - SR.assign(ToMerge, Allocator); - } else { - // joinSubRegRange() destroys the merged range, so we need a copy. - LiveRange RangeCopy(ToMerge, Allocator); - joinSubRegRanges(SR, RangeCopy, SR.LaneMask, CP); - } - }); + LI.refineSubRanges( + Allocator, LaneMask, + [this, &Allocator, &ToMerge, &CP](LiveInterval::SubRange &SR) { + if (SR.empty()) { + SR.assign(ToMerge, Allocator); + } else { + // joinSubRegRange() destroys the merged range, so we need a copy. + LiveRange RangeCopy(ToMerge, Allocator); + joinSubRegRanges(SR, RangeCopy, SR.LaneMask, CP); + } + }, + *LIS->getSlotIndexes(), *TRI); +} + +bool RegisterCoalescer::isHighCostLiveInterval(LiveInterval &LI) { + if (LI.valnos.size() < LargeIntervalSizeThreshold) + return false; + auto &Counter = LargeLIVisitCounter[LI.reg]; + if (Counter < LargeIntervalFreqThreshold) { + Counter++; + return false; + } + return true; } bool RegisterCoalescer::joinVirtRegs(CoalescerPair &CP) { @@ -3265,6 +3311,9 @@ bool RegisterCoalescer::joinVirtRegs(CoalescerPair &CP) { LLVM_DEBUG(dbgs() << "\t\tRHS = " << RHS << "\n\t\tLHS = " << LHS << '\n'); + if (isHighCostLiveInterval(LHS) || isHighCostLiveInterval(RHS)) + return false; + // First compute NewVNInfo and the simple value mappings. // Detect impossible conflicts early. if (!LHSVals.mapValues(RHSVals) || !RHSVals.mapValues(LHSVals)) @@ -3474,7 +3523,8 @@ bool RegisterCoalescer::applyTerminalRule(const MachineInstr &Copy) const { if (!UseTerminalRule) return false; unsigned DstReg, DstSubReg, SrcReg, SrcSubReg; - isMoveInstr(*TRI, &Copy, SrcReg, DstReg, SrcSubReg, DstSubReg); + if (!isMoveInstr(*TRI, &Copy, SrcReg, DstReg, SrcSubReg, DstSubReg)) + return false; // Check if the destination of this copy has any other affinity. if (TargetRegisterInfo::isPhysicalRegister(DstReg) || // If SrcReg is a physical register, the copy won't be coalesced. @@ -3498,8 +3548,9 @@ bool RegisterCoalescer::applyTerminalRule(const MachineInstr &Copy) const { if (&MI == &Copy || !MI.isCopyLike() || MI.getParent() != OrigBB) continue; unsigned OtherReg, OtherSubReg, OtherSrcReg, OtherSrcSubReg; - isMoveInstr(*TRI, &Copy, OtherSrcReg, OtherReg, OtherSrcSubReg, - OtherSubReg); + if (!isMoveInstr(*TRI, &Copy, OtherSrcReg, OtherReg, OtherSrcSubReg, + OtherSubReg)) + return false; if (OtherReg == SrcReg) OtherReg = OtherSrcReg; // Check if OtherReg is a non-terminal. @@ -3620,6 +3671,7 @@ void RegisterCoalescer::releaseMemory() { WorkList.clear(); DeadDefs.clear(); InflateRegs.clear(); + LargeLIVisitCounter.clear(); } bool RegisterCoalescer::runOnMachineFunction(MachineFunction &fn) { diff --git a/lib/CodeGen/RegisterCoalescer.h b/lib/CodeGen/RegisterCoalescer.h index 1a46f6d053e6..f505d46cd338 100644 --- a/lib/CodeGen/RegisterCoalescer.h +++ b/lib/CodeGen/RegisterCoalescer.h @@ -1,9 +1,8 @@ //===- RegisterCoalescer.h - Register Coalescing Interface ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/RegisterPressure.cpp b/lib/CodeGen/RegisterPressure.cpp index 1099e468e885..7d9b3aa9b2d7 100644 --- a/lib/CodeGen/RegisterPressure.cpp +++ b/lib/CodeGen/RegisterPressure.cpp @@ -1,9 +1,8 @@ //===- RegisterPressure.cpp - Dynamic Register Pressure -------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -846,7 +845,7 @@ void RegPressureTracker::recedeSkipDebugValues() { CurrPos = skipDebugInstructionsBackward(std::prev(CurrPos), MBB->begin()); SlotIndex SlotIdx; - if (RequireIntervals) + if (RequireIntervals && !CurrPos->isDebugInstr()) SlotIdx = LIS->getInstructionIndex(*CurrPos).getRegSlot(); // Open the top of the region using slot indexes. @@ -856,6 +855,12 @@ void RegPressureTracker::recedeSkipDebugValues() { void RegPressureTracker::recede(SmallVectorImpl *LiveUses) { recedeSkipDebugValues(); + if (CurrPos->isDebugValue()) { + // It's possible to only have debug_value instructions and hit the start of + // the block. + assert(CurrPos == MBB->begin()); + return; + } const MachineInstr &MI = *CurrPos; RegisterOperands RegOpers; diff --git a/lib/CodeGen/RegisterScavenging.cpp b/lib/CodeGen/RegisterScavenging.cpp index 3660586c1358..bb19110e6d70 100644 --- a/lib/CodeGen/RegisterScavenging.cpp +++ b/lib/CodeGen/RegisterScavenging.cpp @@ -1,9 +1,8 @@ //===- RegisterScavenging.cpp - Machine register scavenging ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -534,7 +533,7 @@ RegScavenger::spill(unsigned Reg, const TargetRegisterClass &RC, int SPAdj, unsigned RegScavenger::scavengeRegister(const TargetRegisterClass *RC, MachineBasicBlock::iterator I, - int SPAdj) { + int SPAdj, bool AllowSpill) { MachineInstr &MI = *I; const MachineFunction &MF = *MI.getMF(); // Consider all allocatable registers in the register class initially @@ -565,6 +564,9 @@ unsigned RegScavenger::scavengeRegister(const TargetRegisterClass *RC, return SReg; } + if (!AllowSpill) + return 0; + ScavengedInfo &Scavenged = spill(SReg, *RC, SPAdj, I, UseMI); Scavenged.Restore = &*std::prev(UseMI); @@ -576,7 +578,8 @@ unsigned RegScavenger::scavengeRegister(const TargetRegisterClass *RC, unsigned RegScavenger::scavengeRegisterBackwards(const TargetRegisterClass &RC, MachineBasicBlock::iterator To, - bool RestoreAfter, int SPAdj) { + bool RestoreAfter, int SPAdj, + bool AllowSpill) { const MachineBasicBlock &MBB = *To->getParent(); const MachineFunction &MF = *MBB.getParent(); @@ -590,21 +593,25 @@ unsigned RegScavenger::scavengeRegisterBackwards(const TargetRegisterClass &RC, MachineBasicBlock::iterator SpillBefore = P.second; assert(Reg != 0 && "No register left to scavenge!"); // Found an available register? - if (SpillBefore != MBB.end()) { - MachineBasicBlock::iterator ReloadAfter = - RestoreAfter ? std::next(MBBI) : MBBI; - MachineBasicBlock::iterator ReloadBefore = std::next(ReloadAfter); - if (ReloadBefore != MBB.end()) - LLVM_DEBUG(dbgs() << "Reload before: " << *ReloadBefore << '\n'); - ScavengedInfo &Scavenged = spill(Reg, RC, SPAdj, SpillBefore, ReloadBefore); - Scavenged.Restore = &*std::prev(SpillBefore); - LiveUnits.removeReg(Reg); - LLVM_DEBUG(dbgs() << "Scavenged register with spill: " << printReg(Reg, TRI) - << " until " << *SpillBefore); - } else { + if (SpillBefore == MBB.end()) { LLVM_DEBUG(dbgs() << "Scavenged free register: " << printReg(Reg, TRI) - << '\n'); + << '\n'); + return Reg; } + + if (!AllowSpill) + return 0; + + MachineBasicBlock::iterator ReloadAfter = + RestoreAfter ? std::next(MBBI) : MBBI; + MachineBasicBlock::iterator ReloadBefore = std::next(ReloadAfter); + if (ReloadBefore != MBB.end()) + LLVM_DEBUG(dbgs() << "Reload before: " << *ReloadBefore << '\n'); + ScavengedInfo &Scavenged = spill(Reg, RC, SPAdj, SpillBefore, ReloadBefore); + Scavenged.Restore = &*std::prev(SpillBefore); + LiveUnits.removeReg(Reg); + LLVM_DEBUG(dbgs() << "Scavenged register with spill: " << printReg(Reg, TRI) + << " until " << *SpillBefore); return Reg; } diff --git a/lib/CodeGen/RegisterUsageInfo.cpp b/lib/CodeGen/RegisterUsageInfo.cpp index 6b9880a8913f..6858d7233bc5 100644 --- a/lib/CodeGen/RegisterUsageInfo.cpp +++ b/lib/CodeGen/RegisterUsageInfo.cpp @@ -1,9 +1,8 @@ //===- RegisterUsageInfo.cpp - Register Usage Information Storage ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// diff --git a/lib/CodeGen/RenameIndependentSubregs.cpp b/lib/CodeGen/RenameIndependentSubregs.cpp index 156d1c81c238..22cff48c3051 100644 --- a/lib/CodeGen/RenameIndependentSubregs.cpp +++ b/lib/CodeGen/RenameIndependentSubregs.cpp @@ -1,9 +1,8 @@ //===-- RenameIndependentSubregs.cpp - Live Interval Analysis -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/ResetMachineFunctionPass.cpp b/lib/CodeGen/ResetMachineFunctionPass.cpp index a02302e6ff99..019de6554d2a 100644 --- a/lib/CodeGen/ResetMachineFunctionPass.cpp +++ b/lib/CodeGen/ResetMachineFunctionPass.cpp @@ -1,9 +1,8 @@ //===-- ResetMachineFunctionPass.cpp - Reset Machine Function ----*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// /// \file @@ -27,6 +26,7 @@ using namespace llvm; #define DEBUG_TYPE "reset-machine-function" STATISTIC(NumFunctionsReset, "Number of functions reset"); +STATISTIC(NumFunctionsVisited, "Number of functions visited"); namespace { class ResetMachineFunction : public MachineFunctionPass { @@ -51,6 +51,7 @@ namespace { } bool runOnMachineFunction(MachineFunction &MF) override { + ++NumFunctionsVisited; // No matter what happened, whether we successfully selected the function // or not, nothing is going to use the vreg types after us. Make sure they // disappear. diff --git a/lib/CodeGen/SafeStack.cpp b/lib/CodeGen/SafeStack.cpp index c356fb57ac6d..a6bc7330e2cc 100644 --- a/lib/CodeGen/SafeStack.cpp +++ b/lib/CodeGen/SafeStack.cpp @@ -1,9 +1,8 @@ //===- SafeStack.cpp - Safe Stack Insertion -------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -372,7 +371,7 @@ Value *SafeStack::getStackGuard(IRBuilder<> &IRB, Function &F) { if (!StackGuardVar) StackGuardVar = F.getParent()->getOrInsertGlobal("__stack_chk_guard", StackPtrTy); - return IRB.CreateLoad(StackGuardVar, "StackGuard"); + return IRB.CreateLoad(StackPtrTy, StackGuardVar, "StackGuard"); } void SafeStack::findInsts(Function &F, @@ -453,7 +452,8 @@ SafeStack::createStackRestorePoints(IRBuilder<> &IRB, Function &F, ++NumUnsafeStackRestorePoints; IRB.SetInsertPoint(I->getNextNode()); - Value *CurrentTop = DynamicTop ? IRB.CreateLoad(DynamicTop) : StaticTop; + Value *CurrentTop = + DynamicTop ? IRB.CreateLoad(StackPtrTy, DynamicTop) : StaticTop; IRB.CreateStore(CurrentTop, UnsafeStackPtr); } @@ -462,7 +462,7 @@ SafeStack::createStackRestorePoints(IRBuilder<> &IRB, Function &F, void SafeStack::checkStackGuard(IRBuilder<> &IRB, Function &F, ReturnInst &RI, AllocaInst *StackGuardSlot, Value *StackGuard) { - Value *V = IRB.CreateLoad(StackGuardSlot); + Value *V = IRB.CreateLoad(StackPtrTy, StackGuardSlot); Value *Cmp = IRB.CreateICmpNE(StackGuard, V); auto SuccessProb = BranchProbabilityInfo::getBranchProbStackProtector(true); @@ -475,8 +475,8 @@ void SafeStack::checkStackGuard(IRBuilder<> &IRB, Function &F, ReturnInst &RI, /* Unreachable */ true, Weights); IRBuilder<> IRBFail(CheckTerm); // FIXME: respect -fsanitize-trap / -ftrap-function here? - Constant *StackChkFail = F.getParent()->getOrInsertFunction( - "__stack_chk_fail", IRB.getVoidTy()); + FunctionCallee StackChkFail = + F.getParent()->getOrInsertFunction("__stack_chk_fail", IRB.getVoidTy()); IRBFail.CreateCall(StackChkFail, {}); } @@ -550,7 +550,7 @@ Value *SafeStack::moveStaticAllocasToUnsafeStack( if (StackGuardSlot) { unsigned Offset = SSL.getObjectOffset(StackGuardSlot); - Value *Off = IRB.CreateGEP(BasePointer, // BasePointer is i8* + Value *Off = IRB.CreateGEP(Int8Ty, BasePointer, // BasePointer is i8* ConstantInt::get(Int32Ty, -Offset)); Value *NewAI = IRB.CreateBitCast(Off, StackGuardSlot->getType(), "StackGuardSlot"); @@ -569,14 +569,14 @@ Value *SafeStack::moveStaticAllocasToUnsafeStack( if (Size == 0) Size = 1; // Don't create zero-sized stack objects. - Value *Off = IRB.CreateGEP(BasePointer, // BasePointer is i8* + Value *Off = IRB.CreateGEP(Int8Ty, BasePointer, // BasePointer is i8* ConstantInt::get(Int32Ty, -Offset)); Value *NewArg = IRB.CreateBitCast(Off, Arg->getType(), Arg->getName() + ".unsafe-byval"); // Replace alloc with the new location. replaceDbgDeclare(Arg, BasePointer, BasePointer->getNextNode(), DIB, - DIExpression::NoDeref, -Offset, DIExpression::NoDeref); + DIExpression::ApplyOffset, -Offset); Arg->replaceAllUsesWith(NewArg); IRB.SetInsertPoint(cast(NewArg)->getNextNode()); IRB.CreateMemCpy(Off, Align, Arg, Arg->getParamAlignment(), Size); @@ -587,12 +587,8 @@ Value *SafeStack::moveStaticAllocasToUnsafeStack( IRB.SetInsertPoint(AI); unsigned Offset = SSL.getObjectOffset(AI); - uint64_t Size = getStaticAllocaAllocationSize(AI); - if (Size == 0) - Size = 1; // Don't create zero-sized stack objects. - - replaceDbgDeclareForAlloca(AI, BasePointer, DIB, DIExpression::NoDeref, - -Offset, DIExpression::NoDeref); + replaceDbgDeclareForAlloca(AI, BasePointer, DIB, DIExpression::ApplyOffset, + -Offset); replaceDbgValueForAlloca(AI, BasePointer, DIB, -Offset); // Replace uses of the alloca with the new location. @@ -609,20 +605,16 @@ Value *SafeStack::moveStaticAllocasToUnsafeStack( InsertBefore = User; IRBuilder<> IRBUser(InsertBefore); - Value *Off = IRBUser.CreateGEP(BasePointer, // BasePointer is i8* + Value *Off = IRBUser.CreateGEP(Int8Ty, BasePointer, // BasePointer is i8* ConstantInt::get(Int32Ty, -Offset)); Value *Replacement = IRBUser.CreateBitCast(Off, AI->getType(), Name); - if (auto *PHI = dyn_cast(User)) { + if (auto *PHI = dyn_cast(User)) // PHI nodes may have multiple incoming edges from the same BB (why??), // all must be updated at once with the same incoming value. - auto *BB = PHI->getIncomingBlock(U); - for (unsigned I = 0; I < PHI->getNumIncomingValues(); ++I) - if (PHI->getIncomingBlock(I) == BB) - PHI->setIncomingValue(I, Replacement); - } else { + PHI->setIncomingValueForBlock(PHI->getIncomingBlock(U), Replacement); + else U.set(Replacement); - } } AI->eraseFromParent(); @@ -637,7 +629,7 @@ Value *SafeStack::moveStaticAllocasToUnsafeStack( IRB.SetInsertPoint(BasePointer->getNextNode()); Value *StaticTop = - IRB.CreateGEP(BasePointer, ConstantInt::get(Int32Ty, -FrameSize), + IRB.CreateGEP(Int8Ty, BasePointer, ConstantInt::get(Int32Ty, -FrameSize), "unsafe_stack_static_top"); IRB.CreateStore(StaticTop, UnsafeStackPtr); return StaticTop; @@ -660,7 +652,8 @@ void SafeStack::moveDynamicAllocasToUnsafeStack( uint64_t TySize = DL.getTypeAllocSize(Ty); Value *Size = IRB.CreateMul(ArraySize, ConstantInt::get(IntPtrTy, TySize)); - Value *SP = IRB.CreatePtrToInt(IRB.CreateLoad(UnsafeStackPtr), IntPtrTy); + Value *SP = IRB.CreatePtrToInt(IRB.CreateLoad(StackPtrTy, UnsafeStackPtr), + IntPtrTy); SP = IRB.CreateSub(SP, Size); // Align the SP value to satisfy the AllocaInst, type and stack alignments. @@ -682,8 +675,7 @@ void SafeStack::moveDynamicAllocasToUnsafeStack( if (AI->hasName() && isa(NewAI)) NewAI->takeName(AI); - replaceDbgDeclareForAlloca(AI, NewAI, DIB, DIExpression::NoDeref, 0, - DIExpression::NoDeref); + replaceDbgDeclareForAlloca(AI, NewAI, DIB, DIExpression::ApplyOffset, 0); AI->replaceAllUsesWith(NewAI); AI->eraseFromParent(); } @@ -698,7 +690,7 @@ void SafeStack::moveDynamicAllocasToUnsafeStack( if (II->getIntrinsicID() == Intrinsic::stacksave) { IRBuilder<> IRB(II); - Instruction *LI = IRB.CreateLoad(UnsafeStackPtr); + Instruction *LI = IRB.CreateLoad(StackPtrTy, UnsafeStackPtr); LI->takeName(II); II->replaceAllUsesWith(LI); II->eraseFromParent(); @@ -727,7 +719,7 @@ void SafeStack::TryInlinePointerAddress() { if (!isa(UnsafeStackPtr)) return; - if(F.hasFnAttribute(Attribute::OptimizeNone)) + if(F.hasOptNone()) return; CallSite CS(UnsafeStackPtr); @@ -783,7 +775,7 @@ bool SafeStack::run() { if (DISubprogram *SP = F.getSubprogram()) IRB.SetCurrentDebugLocation(DebugLoc::get(SP->getScopeLine(), 0, SP)); if (SafeStackUsePointerAddress) { - Value *Fn = F.getParent()->getOrInsertFunction( + FunctionCallee Fn = F.getParent()->getOrInsertFunction( "__safestack_pointer_address", StackPtrTy->getPointerTo(0)); UnsafeStackPtr = IRB.CreateCall(Fn); } else { @@ -793,7 +785,7 @@ bool SafeStack::run() { // Load the current stack pointer (we'll also use it as a base pointer). // FIXME: use a dedicated register for it ? Instruction *BasePointer = - IRB.CreateLoad(UnsafeStackPtr, false, "unsafe_stack_ptr"); + IRB.CreateLoad(StackPtrTy, UnsafeStackPtr, false, "unsafe_stack_ptr"); assert(BasePointer->getType() == StackPtrTy); AllocaInst *StackGuardSlot = nullptr; diff --git a/lib/CodeGen/SafeStackColoring.cpp b/lib/CodeGen/SafeStackColoring.cpp index 726c38002817..04a5c4b6d892 100644 --- a/lib/CodeGen/SafeStackColoring.cpp +++ b/lib/CodeGen/SafeStackColoring.cpp @@ -1,9 +1,8 @@ //===- SafeStackColoring.cpp - SafeStack frame coloring -------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/SafeStackColoring.h b/lib/CodeGen/SafeStackColoring.h index 902e63ebeb7e..b696b1b6baed 100644 --- a/lib/CodeGen/SafeStackColoring.h +++ b/lib/CodeGen/SafeStackColoring.h @@ -1,9 +1,8 @@ //===- SafeStackColoring.h - SafeStack frame coloring ----------*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/SafeStackLayout.cpp b/lib/CodeGen/SafeStackLayout.cpp index 07b6a5d1883b..09964866e4d3 100644 --- a/lib/CodeGen/SafeStackLayout.cpp +++ b/lib/CodeGen/SafeStackLayout.cpp @@ -1,9 +1,8 @@ //===- SafeStackLayout.cpp - SafeStack frame layout -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/SafeStackLayout.h b/lib/CodeGen/SafeStackLayout.h index ac531d800f6e..349d9a8b595c 100644 --- a/lib/CodeGen/SafeStackLayout.h +++ b/lib/CodeGen/SafeStackLayout.h @@ -1,9 +1,8 @@ //===- SafeStackLayout.h - SafeStack frame layout --------------*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp b/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp index 2684f92b3a93..7776dffb4e9c 100644 --- a/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp +++ b/lib/CodeGen/ScalarizeMaskedMemIntrin.cpp @@ -1,10 +1,9 @@ //===- ScalarizeMaskedMemIntrin.cpp - Scalarize unsupported masked mem ----===// // instrinsics // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -124,7 +123,7 @@ static bool isConstantIntVector(Value *Mask) { // %10 = extractelement <16 x i1> %mask, i32 2 // br i1 %10, label %cond.load4, label %else5 // -static void scalarizeMaskedLoad(CallInst *CI) { +static void scalarizeMaskedLoad(CallInst *CI, bool &ModifiedDT) { Value *Ptr = CI->getArgOperand(0); Value *Alignment = CI->getArgOperand(1); Value *Mask = CI->getArgOperand(2); @@ -144,7 +143,7 @@ static void scalarizeMaskedLoad(CallInst *CI) { // Short-cut if the mask is all-true. if (isa(Mask) && cast(Mask)->isAllOnesValue()) { - Value *NewI = Builder.CreateAlignedLoad(Ptr, AlignVal); + Value *NewI = Builder.CreateAlignedLoad(VecType, Ptr, AlignVal); CI->replaceAllUsesWith(NewI); CI->eraseFromParent(); return; @@ -152,9 +151,9 @@ static void scalarizeMaskedLoad(CallInst *CI) { // Adjust alignment for the scalar instruction. AlignVal = MinAlign(AlignVal, EltTy->getPrimitiveSizeInBits() / 8); - // Bitcast %addr fron i8* to EltTy* + // Bitcast %addr from i8* to EltTy* Type *NewPtrType = - EltTy->getPointerTo(cast(Ptr->getType())->getAddressSpace()); + EltTy->getPointerTo(Ptr->getType()->getPointerAddressSpace()); Value *FirstEltPtr = Builder.CreateBitCast(Ptr, NewPtrType); unsigned VectorWidth = VecType->getNumElements(); @@ -165,11 +164,9 @@ static void scalarizeMaskedLoad(CallInst *CI) { for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) { if (cast(Mask)->getAggregateElement(Idx)->isNullValue()) continue; - Value *Gep = - Builder.CreateInBoundsGEP(EltTy, FirstEltPtr, Builder.getInt32(Idx)); - LoadInst *Load = Builder.CreateAlignedLoad(Gep, AlignVal); - VResult = - Builder.CreateInsertElement(VResult, Load, Builder.getInt32(Idx)); + Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, FirstEltPtr, Idx); + LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Gep, AlignVal); + VResult = Builder.CreateInsertElement(VResult, Load, Idx); } CI->replaceAllUsesWith(VResult); CI->eraseFromParent(); @@ -184,8 +181,7 @@ static void scalarizeMaskedLoad(CallInst *CI) { // br i1 %mask_1, label %cond.load, label %else // - Value *Predicate = - Builder.CreateExtractElement(Mask, Builder.getInt32(Idx)); + Value *Predicate = Builder.CreateExtractElement(Mask, Idx); // Create "cond" block // @@ -197,11 +193,9 @@ static void scalarizeMaskedLoad(CallInst *CI) { "cond.load"); Builder.SetInsertPoint(InsertPt); - Value *Gep = - Builder.CreateInBoundsGEP(EltTy, FirstEltPtr, Builder.getInt32(Idx)); - LoadInst *Load = Builder.CreateAlignedLoad(Gep, AlignVal); - Value *NewVResult = Builder.CreateInsertElement(VResult, Load, - Builder.getInt32(Idx)); + Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, FirstEltPtr, Idx); + LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Gep, AlignVal); + Value *NewVResult = Builder.CreateInsertElement(VResult, Load, Idx); // Create "else" block, fill it in the next iteration BasicBlock *NewIfBlock = @@ -222,6 +216,8 @@ static void scalarizeMaskedLoad(CallInst *CI) { CI->replaceAllUsesWith(VResult); CI->eraseFromParent(); + + ModifiedDT = true; } // Translate a masked store intrinsic, like @@ -250,7 +246,7 @@ static void scalarizeMaskedLoad(CallInst *CI) { // store i32 %6, i32* %7 // br label %else2 // . . . -static void scalarizeMaskedStore(CallInst *CI) { +static void scalarizeMaskedStore(CallInst *CI, bool &ModifiedDT) { Value *Src = CI->getArgOperand(0); Value *Ptr = CI->getArgOperand(1); Value *Alignment = CI->getArgOperand(2); @@ -276,9 +272,9 @@ static void scalarizeMaskedStore(CallInst *CI) { // Adjust alignment for the scalar instruction. AlignVal = MinAlign(AlignVal, EltTy->getPrimitiveSizeInBits() / 8); - // Bitcast %addr fron i8* to EltTy* + // Bitcast %addr from i8* to EltTy* Type *NewPtrType = - EltTy->getPointerTo(cast(Ptr->getType())->getAddressSpace()); + EltTy->getPointerTo(Ptr->getType()->getPointerAddressSpace()); Value *FirstEltPtr = Builder.CreateBitCast(Ptr, NewPtrType); unsigned VectorWidth = VecType->getNumElements(); @@ -286,9 +282,8 @@ static void scalarizeMaskedStore(CallInst *CI) { for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) { if (cast(Mask)->getAggregateElement(Idx)->isNullValue()) continue; - Value *OneElt = Builder.CreateExtractElement(Src, Builder.getInt32(Idx)); - Value *Gep = - Builder.CreateInBoundsGEP(EltTy, FirstEltPtr, Builder.getInt32(Idx)); + Value *OneElt = Builder.CreateExtractElement(Src, Idx); + Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, FirstEltPtr, Idx); Builder.CreateAlignedStore(OneElt, Gep, AlignVal); } CI->eraseFromParent(); @@ -301,8 +296,7 @@ static void scalarizeMaskedStore(CallInst *CI) { // %mask_1 = extractelement <16 x i1> %mask, i32 Idx // br i1 %mask_1, label %cond.store, label %else // - Value *Predicate = - Builder.CreateExtractElement(Mask, Builder.getInt32(Idx)); + Value *Predicate = Builder.CreateExtractElement(Mask, Idx); // Create "cond" block // @@ -314,9 +308,8 @@ static void scalarizeMaskedStore(CallInst *CI) { IfBlock->splitBasicBlock(InsertPt->getIterator(), "cond.store"); Builder.SetInsertPoint(InsertPt); - Value *OneElt = Builder.CreateExtractElement(Src, Builder.getInt32(Idx)); - Value *Gep = - Builder.CreateInBoundsGEP(EltTy, FirstEltPtr, Builder.getInt32(Idx)); + Value *OneElt = Builder.CreateExtractElement(Src, Idx); + Value *Gep = Builder.CreateConstInBoundsGEP1_32(EltTy, FirstEltPtr, Idx); Builder.CreateAlignedStore(OneElt, Gep, AlignVal); // Create "else" block, fill it in the next iteration @@ -329,6 +322,8 @@ static void scalarizeMaskedStore(CallInst *CI) { IfBlock = NewIfBlock; } CI->eraseFromParent(); + + ModifiedDT = true; } // Translate a masked gather intrinsic like @@ -360,13 +355,14 @@ static void scalarizeMaskedStore(CallInst *CI) { // . . . // %Result = select <16 x i1> %Mask, <16 x i32> %res.phi.select, <16 x i32> %Src // ret <16 x i32> %Result -static void scalarizeMaskedGather(CallInst *CI) { +static void scalarizeMaskedGather(CallInst *CI, bool &ModifiedDT) { Value *Ptrs = CI->getArgOperand(0); Value *Alignment = CI->getArgOperand(1); Value *Mask = CI->getArgOperand(2); Value *Src0 = CI->getArgOperand(3); VectorType *VecType = cast(CI->getType()); + Type *EltTy = VecType->getElementType(); IRBuilder<> Builder(CI->getContext()); Instruction *InsertPt = CI; @@ -385,12 +381,11 @@ static void scalarizeMaskedGather(CallInst *CI) { for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) { if (cast(Mask)->getAggregateElement(Idx)->isNullValue()) continue; - Value *Ptr = Builder.CreateExtractElement(Ptrs, Builder.getInt32(Idx), - "Ptr" + Twine(Idx)); + Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx)); LoadInst *Load = - Builder.CreateAlignedLoad(Ptr, AlignVal, "Load" + Twine(Idx)); - VResult = Builder.CreateInsertElement( - VResult, Load, Builder.getInt32(Idx), "Res" + Twine(Idx)); + Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx)); + VResult = + Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx)); } CI->replaceAllUsesWith(VResult); CI->eraseFromParent(); @@ -404,8 +399,8 @@ static void scalarizeMaskedGather(CallInst *CI) { // br i1 %Mask1, label %cond.load, label %else // - Value *Predicate = Builder.CreateExtractElement(Mask, Builder.getInt32(Idx), - "Mask" + Twine(Idx)); + Value *Predicate = + Builder.CreateExtractElement(Mask, Idx, "Mask" + Twine(Idx)); // Create "cond" block // @@ -416,13 +411,11 @@ static void scalarizeMaskedGather(CallInst *CI) { BasicBlock *CondBlock = IfBlock->splitBasicBlock(InsertPt, "cond.load"); Builder.SetInsertPoint(InsertPt); - Value *Ptr = Builder.CreateExtractElement(Ptrs, Builder.getInt32(Idx), - "Ptr" + Twine(Idx)); + Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx)); LoadInst *Load = - Builder.CreateAlignedLoad(Ptr, AlignVal, "Load" + Twine(Idx)); - Value *NewVResult = Builder.CreateInsertElement(VResult, Load, - Builder.getInt32(Idx), - "Res" + Twine(Idx)); + Builder.CreateAlignedLoad(EltTy, Ptr, AlignVal, "Load" + Twine(Idx)); + Value *NewVResult = + Builder.CreateInsertElement(VResult, Load, Idx, "Res" + Twine(Idx)); // Create "else" block, fill it in the next iteration BasicBlock *NewIfBlock = CondBlock->splitBasicBlock(InsertPt, "else"); @@ -441,6 +434,8 @@ static void scalarizeMaskedGather(CallInst *CI) { CI->replaceAllUsesWith(VResult); CI->eraseFromParent(); + + ModifiedDT = true; } // Translate a masked scatter intrinsic, like @@ -469,7 +464,7 @@ static void scalarizeMaskedGather(CallInst *CI) { // store i32 %Elt1, i32* %Ptr1, align 4 // br label %else2 // . . . -static void scalarizeMaskedScatter(CallInst *CI) { +static void scalarizeMaskedScatter(CallInst *CI, bool &ModifiedDT) { Value *Src = CI->getArgOperand(0); Value *Ptrs = CI->getArgOperand(1); Value *Alignment = CI->getArgOperand(2); @@ -493,12 +488,11 @@ static void scalarizeMaskedScatter(CallInst *CI) { // Shorten the way if the mask is a vector of constants. if (isConstantIntVector(Mask)) { for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) { - if (cast(Mask)->getAggregateElement(Idx)->isNullValue()) + if (cast(Mask)->getAggregateElement(Idx)->isNullValue()) continue; - Value *OneElt = Builder.CreateExtractElement(Src, Builder.getInt32(Idx), - "Elt" + Twine(Idx)); - Value *Ptr = Builder.CreateExtractElement(Ptrs, Builder.getInt32(Idx), - "Ptr" + Twine(Idx)); + Value *OneElt = + Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx)); + Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx)); Builder.CreateAlignedStore(OneElt, Ptr, AlignVal); } CI->eraseFromParent(); @@ -511,8 +505,8 @@ static void scalarizeMaskedScatter(CallInst *CI) { // %Mask1 = extractelement <16 x i1> %Mask, i32 Idx // br i1 %Mask1, label %cond.store, label %else // - Value *Predicate = Builder.CreateExtractElement(Mask, Builder.getInt32(Idx), - "Mask" + Twine(Idx)); + Value *Predicate = + Builder.CreateExtractElement(Mask, Idx, "Mask" + Twine(Idx)); // Create "cond" block // @@ -523,10 +517,8 @@ static void scalarizeMaskedScatter(CallInst *CI) { BasicBlock *CondBlock = IfBlock->splitBasicBlock(InsertPt, "cond.store"); Builder.SetInsertPoint(InsertPt); - Value *OneElt = Builder.CreateExtractElement(Src, Builder.getInt32(Idx), - "Elt" + Twine(Idx)); - Value *Ptr = Builder.CreateExtractElement(Ptrs, Builder.getInt32(Idx), - "Ptr" + Twine(Idx)); + Value *OneElt = Builder.CreateExtractElement(Src, Idx, "Elt" + Twine(Idx)); + Value *Ptr = Builder.CreateExtractElement(Ptrs, Idx, "Ptr" + Twine(Idx)); Builder.CreateAlignedStore(OneElt, Ptr, AlignVal); // Create "else" block, fill it in the next iteration @@ -538,6 +530,156 @@ static void scalarizeMaskedScatter(CallInst *CI) { IfBlock = NewIfBlock; } CI->eraseFromParent(); + + ModifiedDT = true; +} + +static void scalarizeMaskedExpandLoad(CallInst *CI, bool &ModifiedDT) { + Value *Ptr = CI->getArgOperand(0); + Value *Mask = CI->getArgOperand(1); + Value *PassThru = CI->getArgOperand(2); + + VectorType *VecType = cast(CI->getType()); + + Type *EltTy = VecType->getElementType(); + + IRBuilder<> Builder(CI->getContext()); + Instruction *InsertPt = CI; + BasicBlock *IfBlock = CI->getParent(); + + Builder.SetInsertPoint(InsertPt); + Builder.SetCurrentDebugLocation(CI->getDebugLoc()); + + unsigned VectorWidth = VecType->getNumElements(); + + // The result vector + Value *VResult = PassThru; + + for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) { + // Fill the "else" block, created in the previous iteration + // + // %res.phi.else3 = phi <16 x i32> [ %11, %cond.load1 ], [ %res.phi.else, %else ] + // %mask_1 = extractelement <16 x i1> %mask, i32 Idx + // br i1 %mask_1, label %cond.load, label %else + // + + Value *Predicate = + Builder.CreateExtractElement(Mask, Idx); + + // Create "cond" block + // + // %EltAddr = getelementptr i32* %1, i32 0 + // %Elt = load i32* %EltAddr + // VResult = insertelement <16 x i32> VResult, i32 %Elt, i32 Idx + // + BasicBlock *CondBlock = IfBlock->splitBasicBlock(InsertPt->getIterator(), + "cond.load"); + Builder.SetInsertPoint(InsertPt); + + LoadInst *Load = Builder.CreateAlignedLoad(EltTy, Ptr, 1); + Value *NewVResult = Builder.CreateInsertElement(VResult, Load, Idx); + + // Move the pointer if there are more blocks to come. + Value *NewPtr; + if ((Idx + 1) != VectorWidth) + NewPtr = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, 1); + + // Create "else" block, fill it in the next iteration + BasicBlock *NewIfBlock = + CondBlock->splitBasicBlock(InsertPt->getIterator(), "else"); + Builder.SetInsertPoint(InsertPt); + Instruction *OldBr = IfBlock->getTerminator(); + BranchInst::Create(CondBlock, NewIfBlock, Predicate, OldBr); + OldBr->eraseFromParent(); + BasicBlock *PrevIfBlock = IfBlock; + IfBlock = NewIfBlock; + + // Create the phi to join the new and previous value. + PHINode *ResultPhi = Builder.CreatePHI(VecType, 2, "res.phi.else"); + ResultPhi->addIncoming(NewVResult, CondBlock); + ResultPhi->addIncoming(VResult, PrevIfBlock); + VResult = ResultPhi; + + // Add a PHI for the pointer if this isn't the last iteration. + if ((Idx + 1) != VectorWidth) { + PHINode *PtrPhi = Builder.CreatePHI(Ptr->getType(), 2, "ptr.phi.else"); + PtrPhi->addIncoming(NewPtr, CondBlock); + PtrPhi->addIncoming(Ptr, PrevIfBlock); + Ptr = PtrPhi; + } + } + + CI->replaceAllUsesWith(VResult); + CI->eraseFromParent(); + + ModifiedDT = true; +} + +static void scalarizeMaskedCompressStore(CallInst *CI, bool &ModifiedDT) { + Value *Src = CI->getArgOperand(0); + Value *Ptr = CI->getArgOperand(1); + Value *Mask = CI->getArgOperand(2); + + VectorType *VecType = cast(Src->getType()); + + IRBuilder<> Builder(CI->getContext()); + Instruction *InsertPt = CI; + BasicBlock *IfBlock = CI->getParent(); + + Builder.SetInsertPoint(InsertPt); + Builder.SetCurrentDebugLocation(CI->getDebugLoc()); + + Type *EltTy = VecType->getVectorElementType(); + + unsigned VectorWidth = VecType->getNumElements(); + + for (unsigned Idx = 0; Idx < VectorWidth; ++Idx) { + // Fill the "else" block, created in the previous iteration + // + // %mask_1 = extractelement <16 x i1> %mask, i32 Idx + // br i1 %mask_1, label %cond.store, label %else + // + Value *Predicate = Builder.CreateExtractElement(Mask, Idx); + + // Create "cond" block + // + // %OneElt = extractelement <16 x i32> %Src, i32 Idx + // %EltAddr = getelementptr i32* %1, i32 0 + // %store i32 %OneElt, i32* %EltAddr + // + BasicBlock *CondBlock = + IfBlock->splitBasicBlock(InsertPt->getIterator(), "cond.store"); + Builder.SetInsertPoint(InsertPt); + + Value *OneElt = Builder.CreateExtractElement(Src, Idx); + Builder.CreateAlignedStore(OneElt, Ptr, 1); + + // Move the pointer if there are more blocks to come. + Value *NewPtr; + if ((Idx + 1) != VectorWidth) + NewPtr = Builder.CreateConstInBoundsGEP1_32(EltTy, Ptr, 1); + + // Create "else" block, fill it in the next iteration + BasicBlock *NewIfBlock = + CondBlock->splitBasicBlock(InsertPt->getIterator(), "else"); + Builder.SetInsertPoint(InsertPt); + Instruction *OldBr = IfBlock->getTerminator(); + BranchInst::Create(CondBlock, NewIfBlock, Predicate, OldBr); + OldBr->eraseFromParent(); + BasicBlock *PrevIfBlock = IfBlock; + IfBlock = NewIfBlock; + + // Add a PHI for the pointer if this isn't the last iteration. + if ((Idx + 1) != VectorWidth) { + PHINode *PtrPhi = Builder.CreatePHI(Ptr->getType(), 2, "ptr.phi.else"); + PtrPhi->addIncoming(NewPtr, CondBlock); + PtrPhi->addIncoming(Ptr, PrevIfBlock); + Ptr = PtrPhi; + } + } + CI->eraseFromParent(); + + ModifiedDT = true; } bool ScalarizeMaskedMemIntrin::runOnFunction(Function &F) { @@ -587,33 +729,35 @@ bool ScalarizeMaskedMemIntrin::optimizeCallInst(CallInst *CI, break; case Intrinsic::masked_load: // Scalarize unsupported vector masked load - if (!TTI->isLegalMaskedLoad(CI->getType())) { - scalarizeMaskedLoad(CI); - ModifiedDT = true; - return true; - } - return false; + if (TTI->isLegalMaskedLoad(CI->getType())) + return false; + scalarizeMaskedLoad(CI, ModifiedDT); + return true; case Intrinsic::masked_store: - if (!TTI->isLegalMaskedStore(CI->getArgOperand(0)->getType())) { - scalarizeMaskedStore(CI); - ModifiedDT = true; - return true; - } - return false; + if (TTI->isLegalMaskedStore(CI->getArgOperand(0)->getType())) + return false; + scalarizeMaskedStore(CI, ModifiedDT); + return true; case Intrinsic::masked_gather: - if (!TTI->isLegalMaskedGather(CI->getType())) { - scalarizeMaskedGather(CI); - ModifiedDT = true; - return true; - } - return false; + if (TTI->isLegalMaskedGather(CI->getType())) + return false; + scalarizeMaskedGather(CI, ModifiedDT); + return true; case Intrinsic::masked_scatter: - if (!TTI->isLegalMaskedScatter(CI->getArgOperand(0)->getType())) { - scalarizeMaskedScatter(CI); - ModifiedDT = true; - return true; - } - return false; + if (TTI->isLegalMaskedScatter(CI->getArgOperand(0)->getType())) + return false; + scalarizeMaskedScatter(CI, ModifiedDT); + return true; + case Intrinsic::masked_expandload: + if (TTI->isLegalMaskedExpandLoad(CI->getType())) + return false; + scalarizeMaskedExpandLoad(CI, ModifiedDT); + return true; + case Intrinsic::masked_compressstore: + if (TTI->isLegalMaskedCompressStore(CI->getArgOperand(0)->getType())) + return false; + scalarizeMaskedCompressStore(CI, ModifiedDT); + return true; } } diff --git a/lib/CodeGen/ScheduleDAG.cpp b/lib/CodeGen/ScheduleDAG.cpp index 6c135b3d69d6..dc3a11670a16 100644 --- a/lib/CodeGen/ScheduleDAG.cpp +++ b/lib/CodeGen/ScheduleDAG.cpp @@ -1,9 +1,8 @@ //===- ScheduleDAG.cpp - Implement the ScheduleDAG class ------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -15,6 +14,7 @@ #include "llvm/CodeGen/ScheduleDAG.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" #include "llvm/ADT/iterator_range.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/ScheduleHazardRecognizer.h" @@ -38,6 +38,10 @@ using namespace llvm; #define DEBUG_TYPE "pre-RA-sched" +STATISTIC(NumNewPredsAdded, "Number of times a single predecessor was added"); +STATISTIC(NumTopoInits, + "Number of times the topological order has been recomputed"); + #ifndef NDEBUG static cl::opt StressSchedOpt( "stress-sched", cl::Hidden, cl::init(false), @@ -458,6 +462,11 @@ void ScheduleDAGTopologicalSort::InitDAGTopologicalSorting() { // On insertion of the edge X->Y, the algorithm first marks by calling DFS // the nodes reachable from Y, and then shifts them using Shift to lie // immediately after X in Index2Node. + + // Cancel pending updates, mark as valid. + Dirty = false; + Updates.clear(); + unsigned DAGSize = SUnits.size(); std::vector WorkList; WorkList.reserve(DAGSize); @@ -498,6 +507,7 @@ void ScheduleDAGTopologicalSort::InitDAGTopologicalSorting() { } Visited.resize(DAGSize); + NumTopoInits++; #ifndef NDEBUG // Check correctness of the ordering @@ -510,6 +520,31 @@ void ScheduleDAGTopologicalSort::InitDAGTopologicalSorting() { #endif } +void ScheduleDAGTopologicalSort::FixOrder() { + // Recompute from scratch after new nodes have been added. + if (Dirty) { + InitDAGTopologicalSorting(); + return; + } + + // Otherwise apply updates one-by-one. + for (auto &U : Updates) + AddPred(U.first, U.second); + Updates.clear(); +} + +void ScheduleDAGTopologicalSort::AddPredQueued(SUnit *Y, SUnit *X) { + // Recomputing the order from scratch is likely more efficient than applying + // updates one-by-one for too many updates. The current cut-off is arbitrarily + // chosen. + Dirty = Dirty || Updates.size() > 10; + + if (Dirty) + return; + + Updates.emplace_back(Y, X); +} + void ScheduleDAGTopologicalSort::AddPred(SUnit *Y, SUnit *X) { int UpperBound, LowerBound; LowerBound = Node2Index[Y->NodeNum]; @@ -524,6 +559,8 @@ void ScheduleDAGTopologicalSort::AddPred(SUnit *Y, SUnit *X) { // Recompute topological indexes. Shift(Visited, LowerBound, UpperBound); } + + NumNewPredsAdded++; } void ScheduleDAGTopologicalSort::RemovePred(SUnit *M, SUnit *N) { @@ -665,6 +702,7 @@ void ScheduleDAGTopologicalSort::Shift(BitVector& Visited, int LowerBound, } bool ScheduleDAGTopologicalSort::WillCreateCycle(SUnit *TargetSU, SUnit *SU) { + FixOrder(); // Is SU reachable from TargetSU via successor edges? if (IsReachable(SU, TargetSU)) return true; @@ -677,6 +715,7 @@ bool ScheduleDAGTopologicalSort::WillCreateCycle(SUnit *TargetSU, SUnit *SU) { bool ScheduleDAGTopologicalSort::IsReachable(const SUnit *SU, const SUnit *TargetSU) { + FixOrder(); // If insertion of the edge SU->TargetSU would create a cycle // then there is a path from TargetSU to SU. int UpperBound, LowerBound; diff --git a/lib/CodeGen/ScheduleDAGInstrs.cpp b/lib/CodeGen/ScheduleDAGInstrs.cpp index 99406ed1496a..d5ad7e92299d 100644 --- a/lib/CodeGen/ScheduleDAGInstrs.cpp +++ b/lib/CodeGen/ScheduleDAGInstrs.cpp @@ -1,9 +1,8 @@ //===---- ScheduleDAGInstrs.cpp - MachineInstr Rescheduling ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -115,7 +114,7 @@ ScheduleDAGInstrs::ScheduleDAGInstrs(MachineFunction &mf, : ScheduleDAG(mf), MLI(mli), MFI(mf.getFrameInfo()), RemoveKillFlags(RemoveKillFlags), UnknownValue(UndefValue::get( - Type::getVoidTy(mf.getFunction().getContext()))) { + Type::getVoidTy(mf.getFunction().getContext()))), Topo(SUnits, &ExitSU) { DbgValues.clear(); const TargetSubtargetInfo &ST = mf.getSubtarget(); @@ -132,7 +131,8 @@ static bool getUnderlyingObjectsForInstr(const MachineInstr *MI, const DataLayout &DL) { auto allMMOsOkay = [&]() { for (const MachineMemOperand *MMO : MI->memoperands()) { - if (MMO->isVolatile()) + // TODO: Figure out whether isAtomic is really necessary (see D57601). + if (MMO->isVolatile() || MMO->isAtomic()) return false; if (const PseudoSourceValue *PSV = MMO->getPseudoValue()) { @@ -743,6 +743,14 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, // done. Value2SUsMap NonAliasStores, NonAliasLoads(1 /*TrueMemOrderLatency*/); + // Track all instructions that may raise floating-point exceptions. + // These do not depend on one other (or normal loads or stores), but + // must not be rescheduled across global barriers. Note that we don't + // really need a "map" here since we don't track those MIs by value; + // using the same Value2SUsMap data type here is simply a matter of + // convenience. + Value2SUsMap FPExceptions; + // Remove any stale debug info; sometimes BuildSchedGraph is called again // without emitting the info from the previous call. DbgValues.clear(); @@ -870,10 +878,26 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, addBarrierChain(Loads); addBarrierChain(NonAliasStores); addBarrierChain(NonAliasLoads); + addBarrierChain(FPExceptions); continue; } + // Instructions that may raise FP exceptions may not be moved + // across any global barriers. + if (MI.mayRaiseFPException()) { + if (BarrierChain) + BarrierChain->addPredBarrier(SU); + + FPExceptions.insert(SU, UnknownValue); + + if (FPExceptions.size() >= HugeRegion) { + LLVM_DEBUG(dbgs() << "Reducing FPExceptions map.\n";); + Value2SUsMap empty; + reduceHugeMemNodeMaps(FPExceptions, empty, getReductionSize()); + } + } + // If it's not a store or a variant load, we're done. if (!MI.mayStore() && !(MI.mayLoad() && !MI.isDereferenceableInvariantLoad(AA))) @@ -968,6 +992,8 @@ void ScheduleDAGInstrs::buildSchedGraph(AliasAnalysis *AA, Uses.clear(); CurrentVRegDefs.clear(); CurrentVRegUses.clear(); + + Topo.MarkDirty(); } raw_ostream &llvm::operator<<(raw_ostream &OS, const PseudoSourceValue* PSV) { @@ -1089,22 +1115,21 @@ void ScheduleDAGInstrs::fixupKills(MachineBasicBlock &MBB) { if (!MI.isBundled()) { toggleKills(MRI, LiveRegs, MI, true); } else { - MachineBasicBlock::instr_iterator First = MI.getIterator(); - if (MI.isBundle()) { + MachineBasicBlock::instr_iterator Bundle = MI.getIterator(); + if (MI.isBundle()) toggleKills(MRI, LiveRegs, MI, false); - ++First; - } + // Some targets make the (questionable) assumtion that the instructions // inside the bundle are ordered and consequently only the last use of // a register inside the bundle can kill it. - MachineBasicBlock::instr_iterator I = std::next(First); + MachineBasicBlock::instr_iterator I = std::next(Bundle); while (I->isBundledWithSucc()) ++I; do { if (!I->isDebugInstr()) toggleKills(MRI, LiveRegs, *I, true); --I; - } while(I != First); + } while (I != Bundle); } } } @@ -1146,6 +1171,23 @@ std::string ScheduleDAGInstrs::getDAGName() const { return "dag." + BB->getFullName(); } +bool ScheduleDAGInstrs::canAddEdge(SUnit *SuccSU, SUnit *PredSU) { + return SuccSU == &ExitSU || !Topo.IsReachable(PredSU, SuccSU); +} + +bool ScheduleDAGInstrs::addEdge(SUnit *SuccSU, const SDep &PredDep) { + if (SuccSU != &ExitSU) { + // Do not use WillCreateCycle, it assumes SD scheduling. + // If Pred is reachable from Succ, then the edge creates a cycle. + if (Topo.IsReachable(PredDep.getSUnit(), SuccSU)) + return false; + Topo.AddPredQueued(SuccSU, PredDep.getSUnit()); + } + SuccSU->addPred(PredDep, /*Required=*/!PredDep.isArtificial()); + // Return true regardless of whether a new edge needed to be inserted. + return true; +} + //===----------------------------------------------------------------------===// // SchedDFSResult Implementation //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/ScheduleDAGPrinter.cpp b/lib/CodeGen/ScheduleDAGPrinter.cpp index ff2085aae865..8d04711f07c6 100644 --- a/lib/CodeGen/ScheduleDAGPrinter.cpp +++ b/lib/CodeGen/ScheduleDAGPrinter.cpp @@ -1,9 +1,8 @@ //===-- ScheduleDAGPrinter.cpp - Implement ScheduleDAG::viewGraph() -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/ScoreboardHazardRecognizer.cpp b/lib/CodeGen/ScoreboardHazardRecognizer.cpp index 4301372179b8..a9fda56f2dac 100644 --- a/lib/CodeGen/ScoreboardHazardRecognizer.cpp +++ b/lib/CodeGen/ScoreboardHazardRecognizer.cpp @@ -1,9 +1,8 @@ //===- ScoreboardHazardRecognizer.cpp - Scheduler Support -----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index ff5505c97721..49c922f560fa 100644 --- a/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -1,9 +1,8 @@ //===- DAGCombiner.cpp - Implement a DAG node combiner --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -112,6 +111,10 @@ static cl::opt MaySplitLoadIndex("combiner-split-load-index", cl::Hidden, cl::init(true), cl::desc("DAG combiner may split indexing from loads")); +static cl::opt TokenFactorInlineLimit( + "combiner-tokenfactor-inline-limit", cl::Hidden, cl::init(2048), + cl::desc("Limit the number of operands to inline for Token Factors")); + namespace { class DAGCombiner { @@ -138,6 +141,10 @@ namespace { /// them) when they are deleted from the underlying DAG. It relies on /// stable indices of nodes within the worklist. DenseMap WorklistMap; + /// This records all nodes attempted to add to the worklist since we + /// considered a new worklist entry. As we keep do not add duplicate nodes + /// in the worklist, this is different from the tail of the worklist. + SmallSetVector PruningList; /// Set of nodes which have been combined (at least once). /// @@ -155,6 +162,37 @@ namespace { AddToWorklist(Node); } + // Prune potentially dangling nodes. This is called after + // any visit to a node, but should also be called during a visit after any + // failed combine which may have created a DAG node. + void clearAddedDanglingWorklistEntries() { + // Check any nodes added to the worklist to see if they are prunable. + while (!PruningList.empty()) { + auto *N = PruningList.pop_back_val(); + if (N->use_empty()) + recursivelyDeleteUnusedNodes(N); + } + } + + SDNode *getNextWorklistEntry() { + // Before we do any work, remove nodes that are not in use. + clearAddedDanglingWorklistEntries(); + SDNode *N = nullptr; + // The Worklist holds the SDNodes in order, but it may contain null + // entries. + while (!N && !Worklist.empty()) { + N = Worklist.pop_back_val(); + } + + if (N) { + bool GoodWorklistEntry = WorklistMap.erase(N); + (void)GoodWorklistEntry; + assert(GoodWorklistEntry && + "Found a worklist entry without a corresponding map entry!"); + } + return N; + } + /// Call the node-specific routine that folds each particular type of node. SDValue visit(SDNode *N); @@ -162,7 +200,7 @@ namespace { DAGCombiner(SelectionDAG &D, AliasAnalysis *AA, CodeGenOpt::Level OL) : DAG(D), TLI(D.getTargetLoweringInfo()), Level(BeforeLegalizeTypes), OptLevel(OL), AA(AA) { - ForCodeSize = DAG.getMachineFunction().getFunction().optForSize(); + ForCodeSize = DAG.getMachineFunction().getFunction().hasOptSize(); MaximumLegalStoreInBits = 0; for (MVT VT : MVT::all_valuetypes()) @@ -172,6 +210,11 @@ namespace { MaximumLegalStoreInBits = VT.getSizeInBits(); } + void ConsiderForPruning(SDNode *N) { + // Mark this for potential pruning. + PruningList.insert(N); + } + /// Add to the worklist making sure its instance is at the back (next to be /// processed.) void AddToWorklist(SDNode *N) { @@ -183,6 +226,8 @@ namespace { if (N->getOpcode() == ISD::HANDLENODE) return; + ConsiderForPruning(N); + if (WorklistMap.insert(std::make_pair(N, Worklist.size())).second) Worklist.push_back(N); } @@ -190,6 +235,7 @@ namespace { /// Remove all instances of N from the worklist. void removeFromWorklist(SDNode *N) { CombinedNodes.erase(N); + PruningList.remove(N); auto It = WorklistMap.find(N); if (It == WorklistMap.end()) @@ -229,8 +275,15 @@ namespace { /// If so, return true. bool SimplifyDemandedBits(SDValue Op) { unsigned BitWidth = Op.getScalarValueSizeInBits(); - APInt Demanded = APInt::getAllOnesValue(BitWidth); - return SimplifyDemandedBits(Op, Demanded); + APInt DemandedBits = APInt::getAllOnesValue(BitWidth); + return SimplifyDemandedBits(Op, DemandedBits); + } + + bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits) { + EVT VT = Op.getValueType(); + unsigned NumElts = VT.isVector() ? VT.getVectorNumElements() : 1; + APInt DemandedElts = APInt::getAllOnesValue(NumElts); + return SimplifyDemandedBits(Op, DemandedBits, DemandedElts); } /// Check the specified vector node value to see if it can be simplified or @@ -238,12 +291,13 @@ namespace { /// elements. If so, return true. bool SimplifyDemandedVectorElts(SDValue Op) { unsigned NumElts = Op.getValueType().getVectorNumElements(); - APInt Demanded = APInt::getAllOnesValue(NumElts); - return SimplifyDemandedVectorElts(Op, Demanded); + APInt DemandedElts = APInt::getAllOnesValue(NumElts); + return SimplifyDemandedVectorElts(Op, DemandedElts); } - bool SimplifyDemandedBits(SDValue Op, const APInt &Demanded); - bool SimplifyDemandedVectorElts(SDValue Op, const APInt &Demanded, + bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, + const APInt &DemandedElts); + bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedElts, bool AssumeSingleUse = false); bool CombineToPreIndexedLoadStore(SDNode *N); @@ -291,15 +345,16 @@ namespace { SDValue visitTokenFactor(SDNode *N); SDValue visitMERGE_VALUES(SDNode *N); SDValue visitADD(SDNode *N); - SDValue visitADDLike(SDValue N0, SDValue N1, SDNode *LocReference); + SDValue visitADDLike(SDNode *N); + SDValue visitADDLikeCommutative(SDValue N0, SDValue N1, SDNode *LocReference); SDValue visitSUB(SDNode *N); SDValue visitADDSAT(SDNode *N); SDValue visitSUBSAT(SDNode *N); SDValue visitADDC(SDNode *N); - SDValue visitUADDO(SDNode *N); + SDValue visitADDO(SDNode *N); SDValue visitUADDOLike(SDValue N0, SDValue N1, SDNode *N); SDValue visitSUBC(SDNode *N); - SDValue visitUSUBO(SDNode *N); + SDValue visitSUBO(SDNode *N); SDValue visitADDE(SDNode *N); SDValue visitADDCARRY(SDNode *N); SDValue visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn, SDNode *N); @@ -316,8 +371,7 @@ namespace { SDValue visitMULHS(SDNode *N); SDValue visitSMUL_LOHI(SDNode *N); SDValue visitUMUL_LOHI(SDNode *N); - SDValue visitSMULO(SDNode *N); - SDValue visitUMULO(SDNode *N); + SDValue visitMULO(SDNode *N); SDValue visitIMINMAX(SDNode *N); SDValue visitAND(SDNode *N); SDValue visitANDLike(SDValue N0, SDValue N1, SDNode *N); @@ -386,6 +440,7 @@ namespace { SDValue replaceStoreOfFPConstant(StoreSDNode *ST); SDValue visitSTORE(SDNode *N); + SDValue visitLIFETIME_END(SDNode *N); SDValue visitINSERT_VECTOR_ELT(SDNode *N); SDValue visitEXTRACT_VECTOR_ELT(SDNode *N); SDValue visitBUILD_VECTOR(SDNode *N); @@ -400,13 +455,19 @@ namespace { SDValue visitMSCATTER(SDNode *N); SDValue visitFP_TO_FP16(SDNode *N); SDValue visitFP16_TO_FP(SDNode *N); + SDValue visitVECREDUCE(SDNode *N); SDValue visitFADDForFMACombine(SDNode *N); SDValue visitFSUBForFMACombine(SDNode *N); SDValue visitFMULForFMADistributiveCombine(SDNode *N); SDValue XformToShuffleWithZero(SDNode *N); - SDValue ReassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0, + bool reassociationCanBreakAddressingModePattern(unsigned Opc, + const SDLoc &DL, SDValue N0, + SDValue N1); + SDValue reassociateOpsCommutative(unsigned Opc, const SDLoc &DL, SDValue N0, + SDValue N1); + SDValue reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0, SDValue N1, SDNodeFlags Flags); SDValue visitShiftByConstant(SDNode *N, ConstantSDNode *Amt); @@ -466,6 +527,7 @@ namespace { const SDLoc &DL); SDNode *MatchRotate(SDValue LHS, SDValue RHS, const SDLoc &DL); SDValue MatchLoadCombine(SDNode *N); + SDValue MatchStoreCombine(StoreSDNode *N); SDValue ReduceLoadWidth(SDNode *N); SDValue ReduceLoadOpStoreWidth(SDNode *N); SDValue splitMergedValStore(StoreSDNode *ST); @@ -475,7 +537,8 @@ namespace { SDValue reduceBuildVecToShuffle(SDNode *N); SDValue createBuildVecShuffle(const SDLoc &DL, SDNode *N, ArrayRef VectorMask, SDValue VecIn1, - SDValue VecIn2, unsigned LeftIdx); + SDValue VecIn2, unsigned LeftIdx, + bool DidSplitVec); SDValue matchVSelectOpSizesWithSetCC(SDNode *Cast); /// Walk up chain skipping non-aliasing memory nodes, @@ -484,7 +547,7 @@ namespace { SmallVectorImpl &Aliases); /// Return true if there is any possibility that the two addresses overlap. - bool isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) const; + bool isAlias(SDNode *Op0, SDNode *Op1) const; /// Walk up chain skipping non-aliasing memory nodes, looking for a better /// chain (aliasing node.) @@ -642,6 +705,18 @@ public: } }; +class WorklistInserter : public SelectionDAG::DAGUpdateListener { + DAGCombiner &DC; + +public: + explicit WorklistInserter(DAGCombiner &dc) + : SelectionDAG::DAGUpdateListener(dc.getDAG()), DC(dc) {} + + // FIXME: Ideally we could add N to the worklist, but this causes exponential + // compile time costs in large DAGs, e.g. Halide. + void NodeInserted(SDNode *N) override { DC.ConsiderForPruning(N); } +}; + } // end anonymous namespace //===----------------------------------------------------------------------===// @@ -697,20 +772,23 @@ void DAGCombiner::deleteAndRecombine(SDNode *N) { static char isNegatibleForFree(SDValue Op, bool LegalOperations, const TargetLowering &TLI, const TargetOptions *Options, + bool ForCodeSize, unsigned Depth = 0) { // fneg is removable even if it has multiple uses. - if (Op.getOpcode() == ISD::FNEG) return 2; + if (Op.getOpcode() == ISD::FNEG) + return 2; // Don't allow anything with multiple uses unless we know it is free. EVT VT = Op.getValueType(); const SDNodeFlags Flags = Op->getFlags(); - if (!Op.hasOneUse()) - if (!(Op.getOpcode() == ISD::FP_EXTEND && - TLI.isFPExtFree(VT, Op.getOperand(0).getValueType()))) - return 0; + if (!Op.hasOneUse() && + !(Op.getOpcode() == ISD::FP_EXTEND && + TLI.isFPExtFree(VT, Op.getOperand(0).getValueType()))) + return 0; // Don't recurse exponentially. - if (Depth > 6) return 0; + if (Depth > 6) + return 0; switch (Op.getOpcode()) { default: return false; @@ -721,7 +799,25 @@ static char isNegatibleForFree(SDValue Op, bool LegalOperations, // Don't invert constant FP values after legalization unless the target says // the negated constant is legal. return TLI.isOperationLegal(ISD::ConstantFP, VT) || - TLI.isFPImmLegal(neg(cast(Op)->getValueAPF()), VT); + TLI.isFPImmLegal(neg(cast(Op)->getValueAPF()), VT, + ForCodeSize); + } + case ISD::BUILD_VECTOR: { + // Only permit BUILD_VECTOR of constants. + if (llvm::any_of(Op->op_values(), [&](SDValue N) { + return !N.isUndef() && !isa(N); + })) + return 0; + if (!LegalOperations) + return 1; + if (TLI.isOperationLegal(ISD::ConstantFP, VT) && + TLI.isOperationLegal(ISD::BUILD_VECTOR, VT)) + return 1; + return llvm::all_of(Op->op_values(), [&](SDValue N) { + return N.isUndef() || + TLI.isFPImmLegal(neg(cast(N)->getValueAPF()), VT, + ForCodeSize); + }); } case ISD::FADD: if (!Options->UnsafeFPMath && !Flags.hasNoSignedZeros()) @@ -733,15 +829,14 @@ static char isNegatibleForFree(SDValue Op, bool LegalOperations, // fold (fneg (fadd A, B)) -> (fsub (fneg A), B) if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, - Options, Depth + 1)) + Options, ForCodeSize, Depth + 1)) return V; // fold (fneg (fadd A, B)) -> (fsub (fneg B), A) return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options, - Depth + 1); + ForCodeSize, Depth + 1); case ISD::FSUB: // We can't turn -(A-B) into B-A when we honor signed zeros. - if (!Options->NoSignedZerosFPMath && - !Flags.hasNoSignedZeros()) + if (!Options->NoSignedZerosFPMath && !Flags.hasNoSignedZeros()) return 0; // fold (fneg (fsub A, B)) -> (fsub B, A) @@ -751,30 +846,31 @@ static char isNegatibleForFree(SDValue Op, bool LegalOperations, case ISD::FDIV: // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) or (fmul X, (fneg Y)) if (char V = isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, - Options, Depth + 1)) + Options, ForCodeSize, Depth + 1)) return V; return isNegatibleForFree(Op.getOperand(1), LegalOperations, TLI, Options, - Depth + 1); + ForCodeSize, Depth + 1); case ISD::FP_EXTEND: case ISD::FP_ROUND: case ISD::FSIN: return isNegatibleForFree(Op.getOperand(0), LegalOperations, TLI, Options, - Depth + 1); + ForCodeSize, Depth + 1); } } /// If isNegatibleForFree returns true, return the newly negated expression. static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG, - bool LegalOperations, unsigned Depth = 0) { - const TargetOptions &Options = DAG.getTarget().Options; + bool LegalOperations, bool ForCodeSize, + unsigned Depth = 0) { // fneg is removable even if it has multiple uses. - if (Op.getOpcode() == ISD::FNEG) return Op.getOperand(0); + if (Op.getOpcode() == ISD::FNEG) + return Op.getOperand(0); assert(Depth <= 6 && "GetNegatedExpression doesn't match isNegatibleForFree"); - - const SDNodeFlags Flags = Op.getNode()->getFlags(); + const TargetOptions &Options = DAG.getTarget().Options; + const SDNodeFlags Flags = Op->getFlags(); switch (Op.getOpcode()) { default: llvm_unreachable("Unknown code"); @@ -783,24 +879,41 @@ static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG, V.changeSign(); return DAG.getConstantFP(V, SDLoc(Op), Op.getValueType()); } + case ISD::BUILD_VECTOR: { + SmallVector Ops; + for (SDValue C : Op->op_values()) { + if (C.isUndef()) { + Ops.push_back(C); + continue; + } + APFloat V = cast(C)->getValueAPF(); + V.changeSign(); + Ops.push_back(DAG.getConstantFP(V, SDLoc(Op), C.getValueType())); + } + return DAG.getBuildVector(Op.getValueType(), SDLoc(Op), Ops); + } case ISD::FADD: assert(Options.UnsafeFPMath || Flags.hasNoSignedZeros()); // fold (fneg (fadd A, B)) -> (fsub (fneg A), B) if (isNegatibleForFree(Op.getOperand(0), LegalOperations, - DAG.getTargetLoweringInfo(), &Options, Depth+1)) + DAG.getTargetLoweringInfo(), &Options, ForCodeSize, + Depth + 1)) return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(), GetNegatedExpression(Op.getOperand(0), DAG, - LegalOperations, Depth+1), + LegalOperations, ForCodeSize, + Depth + 1), Op.getOperand(1), Flags); // fold (fneg (fadd A, B)) -> (fsub (fneg B), A) return DAG.getNode(ISD::FSUB, SDLoc(Op), Op.getValueType(), GetNegatedExpression(Op.getOperand(1), DAG, - LegalOperations, Depth+1), + LegalOperations, ForCodeSize, + Depth + 1), Op.getOperand(0), Flags); case ISD::FSUB: // fold (fneg (fsub 0, B)) -> B - if (ConstantFPSDNode *N0CFP = dyn_cast(Op.getOperand(0))) + if (ConstantFPSDNode *N0CFP = + isConstOrConstSplatFP(Op.getOperand(0), /*AllowUndefs*/ true)) if (N0CFP->isZero()) return Op.getOperand(1); @@ -812,28 +925,33 @@ static SDValue GetNegatedExpression(SDValue Op, SelectionDAG &DAG, case ISD::FDIV: // fold (fneg (fmul X, Y)) -> (fmul (fneg X), Y) if (isNegatibleForFree(Op.getOperand(0), LegalOperations, - DAG.getTargetLoweringInfo(), &Options, Depth+1)) + DAG.getTargetLoweringInfo(), &Options, ForCodeSize, + Depth + 1)) return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(), GetNegatedExpression(Op.getOperand(0), DAG, - LegalOperations, Depth+1), + LegalOperations, ForCodeSize, + Depth + 1), Op.getOperand(1), Flags); // fold (fneg (fmul X, Y)) -> (fmul X, (fneg Y)) return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(), Op.getOperand(0), GetNegatedExpression(Op.getOperand(1), DAG, - LegalOperations, Depth+1), Flags); + LegalOperations, ForCodeSize, + Depth + 1), Flags); case ISD::FP_EXTEND: case ISD::FSIN: return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(), GetNegatedExpression(Op.getOperand(0), DAG, - LegalOperations, Depth+1)); + LegalOperations, ForCodeSize, + Depth + 1)); case ISD::FP_ROUND: - return DAG.getNode(ISD::FP_ROUND, SDLoc(Op), Op.getValueType(), - GetNegatedExpression(Op.getOperand(0), DAG, - LegalOperations, Depth+1), - Op.getOperand(1)); + return DAG.getNode(ISD::FP_ROUND, SDLoc(Op), Op.getValueType(), + GetNegatedExpression(Op.getOperand(0), DAG, + LegalOperations, ForCodeSize, + Depth + 1), + Op.getOperand(1)); } } @@ -924,53 +1042,113 @@ static bool isAnyConstantBuildVector(SDValue V, bool NoOpaques = false) { ISD::isBuildVectorOfConstantFPSDNodes(V.getNode()); } -SDValue DAGCombiner::ReassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0, - SDValue N1, SDNodeFlags Flags) { - // Don't reassociate reductions. - if (Flags.hasVectorReduction()) - return SDValue(); +bool DAGCombiner::reassociationCanBreakAddressingModePattern(unsigned Opc, + const SDLoc &DL, + SDValue N0, + SDValue N1) { + // Currently this only tries to ensure we don't undo the GEP splits done by + // CodeGenPrepare when shouldConsiderGEPOffsetSplit is true. To ensure this, + // we check if the following transformation would be problematic: + // (load/store (add, (add, x, offset1), offset2)) -> + // (load/store (add, x, offset1+offset2)). - EVT VT = N0.getValueType(); - if (N0.getOpcode() == Opc && !N0->getFlags().hasVectorReduction()) { - if (SDNode *L = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1))) { - if (SDNode *R = DAG.isConstantIntBuildVectorOrConstantInt(N1)) { - // reassoc. (op (op x, c1), c2) -> (op x, (op c1, c2)) - if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, L, R)) - return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode); - return SDValue(); - } - if (N0.hasOneUse()) { - // reassoc. (op (op x, c1), y) -> (op (op x, y), c1) iff x+c1 has one - // use - SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0.getOperand(0), N1); - if (!OpNode.getNode()) - return SDValue(); - AddToWorklist(OpNode.getNode()); - return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1)); - } + if (Opc != ISD::ADD || N0.getOpcode() != ISD::ADD) + return false; + + if (N0.hasOneUse()) + return false; + + auto *C1 = dyn_cast(N0.getOperand(1)); + auto *C2 = dyn_cast(N1); + if (!C1 || !C2) + return false; + + const APInt &C1APIntVal = C1->getAPIntValue(); + const APInt &C2APIntVal = C2->getAPIntValue(); + if (C1APIntVal.getBitWidth() > 64 || C2APIntVal.getBitWidth() > 64) + return false; + + const APInt CombinedValueIntVal = C1APIntVal + C2APIntVal; + if (CombinedValueIntVal.getBitWidth() > 64) + return false; + const int64_t CombinedValue = CombinedValueIntVal.getSExtValue(); + + for (SDNode *Node : N0->uses()) { + auto LoadStore = dyn_cast(Node); + if (LoadStore) { + // Is x[offset2] already not a legal addressing mode? If so then + // reassociating the constants breaks nothing (we test offset2 because + // that's the one we hope to fold into the load or store). + TargetLoweringBase::AddrMode AM; + AM.HasBaseReg = true; + AM.BaseOffs = C2APIntVal.getSExtValue(); + EVT VT = LoadStore->getMemoryVT(); + unsigned AS = LoadStore->getAddressSpace(); + Type *AccessTy = VT.getTypeForEVT(*DAG.getContext()); + if (!TLI.isLegalAddressingMode(DAG.getDataLayout(), AM, AccessTy, AS)) + continue; + + // Would x[offset1+offset2] still be a legal addressing mode? + AM.BaseOffs = CombinedValue; + if (!TLI.isLegalAddressingMode(DAG.getDataLayout(), AM, AccessTy, AS)) + return true; } } - if (N1.getOpcode() == Opc && !N1->getFlags().hasVectorReduction()) { - if (SDNode *R = DAG.isConstantIntBuildVectorOrConstantInt(N1.getOperand(1))) { - if (SDNode *L = DAG.isConstantIntBuildVectorOrConstantInt(N0)) { - // reassoc. (op c2, (op x, c1)) -> (op x, (op c1, c2)) - if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, R, L)) - return DAG.getNode(Opc, DL, VT, N1.getOperand(0), OpNode); + return false; +} + +// Helper for DAGCombiner::reassociateOps. Try to reassociate an expression +// such as (Opc N0, N1), if \p N0 is the same kind of operation as \p Opc. +SDValue DAGCombiner::reassociateOpsCommutative(unsigned Opc, const SDLoc &DL, + SDValue N0, SDValue N1) { + EVT VT = N0.getValueType(); + + if (N0.getOpcode() != Opc) + return SDValue(); + + // Don't reassociate reductions. + if (N0->getFlags().hasVectorReduction()) + return SDValue(); + + if (SDNode *C1 = DAG.isConstantIntBuildVectorOrConstantInt(N0.getOperand(1))) { + if (SDNode *C2 = DAG.isConstantIntBuildVectorOrConstantInt(N1)) { + // Reassociate: (op (op x, c1), c2) -> (op x, (op c1, c2)) + if (SDValue OpNode = DAG.FoldConstantArithmetic(Opc, DL, VT, C1, C2)) + return DAG.getNode(Opc, DL, VT, N0.getOperand(0), OpNode); + return SDValue(); + } + if (N0.hasOneUse()) { + // Reassociate: (op (op x, c1), y) -> (op (op x, y), c1) + // iff (op x, c1) has one use + SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0.getOperand(0), N1); + if (!OpNode.getNode()) return SDValue(); - } - if (N1.hasOneUse()) { - // reassoc. (op x, (op y, c1)) -> (op (op x, y), c1) iff x+c1 has one - // use - SDValue OpNode = DAG.getNode(Opc, SDLoc(N0), VT, N0, N1.getOperand(0)); - if (!OpNode.getNode()) - return SDValue(); - AddToWorklist(OpNode.getNode()); - return DAG.getNode(Opc, DL, VT, OpNode, N1.getOperand(1)); - } + AddToWorklist(OpNode.getNode()); + return DAG.getNode(Opc, DL, VT, OpNode, N0.getOperand(1)); } } + return SDValue(); +} + +// Try to reassociate commutative binops. +SDValue DAGCombiner::reassociateOps(unsigned Opc, const SDLoc &DL, SDValue N0, + SDValue N1, SDNodeFlags Flags) { + assert(TLI.isCommutativeBinOp(Opc) && "Operation not commutative."); + // Don't reassociate reductions. + if (Flags.hasVectorReduction()) + return SDValue(); + // Floating-point reassociation is not allowed without loose FP math. + if (N0.getValueType().isFloatingPoint() || + N1.getValueType().isFloatingPoint()) + if (!Flags.hasAllowReassociation() || !Flags.hasNoSignedZeros()) + return SDValue(); + + if (SDValue Combined = reassociateOpsCommutative(Opc, DL, N0, N1)) + return Combined; + if (SDValue Combined = reassociateOpsCommutative(Opc, DL, N1, N0)) + return Combined; return SDValue(); } @@ -1026,10 +1204,11 @@ CommitTargetLoweringOpt(const TargetLowering::TargetLoweringOpt &TLO) { /// Check the specified integer node value to see if it can be simplified or if /// things it uses can be simplified by bit propagation. If so, return true. -bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &Demanded) { +bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, + const APInt &DemandedElts) { TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations); KnownBits Known; - if (!TLI.SimplifyDemandedBits(Op, Demanded, Known, TLO)) + if (!TLI.SimplifyDemandedBits(Op, DemandedBits, DemandedElts, Known, TLO)) return false; // Revisit the node. @@ -1048,12 +1227,13 @@ bool DAGCombiner::SimplifyDemandedBits(SDValue Op, const APInt &Demanded) { /// Check the specified vector node value to see if it can be simplified or /// if things it uses can be simplified as it only uses some of the elements. /// If so, return true. -bool DAGCombiner::SimplifyDemandedVectorElts(SDValue Op, const APInt &Demanded, +bool DAGCombiner::SimplifyDemandedVectorElts(SDValue Op, + const APInt &DemandedElts, bool AssumeSingleUse) { TargetLowering::TargetLoweringOpt TLO(DAG, LegalTypes, LegalOperations); APInt KnownUndef, KnownZero; - if (!TLI.SimplifyDemandedVectorElts(Op, Demanded, KnownUndef, KnownZero, TLO, - 0, AssumeSingleUse)) + if (!TLI.SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, + TLO, 0, AssumeSingleUse)) return false; // Revisit the node. @@ -1383,6 +1563,8 @@ void DAGCombiner::Run(CombineLevel AtLevel) { LegalOperations = Level >= AfterLegalizeVectorOps; LegalTypes = Level >= AfterLegalizeTypes; + WorklistInserter AddNodes(*this); + // Add all the dag nodes to the worklist. for (SDNode &Node : DAG.allnodes()) AddToWorklist(&Node); @@ -1392,19 +1574,8 @@ void DAGCombiner::Run(CombineLevel AtLevel) { // changes of the root. HandleSDNode Dummy(DAG.getRoot()); - // While the worklist isn't empty, find a node and try to combine it. - while (!WorklistMap.empty()) { - SDNode *N; - // The Worklist holds the SDNodes in order, but it may contain null entries. - do { - N = Worklist.pop_back_val(); - } while (!N); - - bool GoodWorklistEntry = WorklistMap.erase(N); - (void)GoodWorklistEntry; - assert(GoodWorklistEntry && - "Found a worklist entry without a corresponding map entry!"); - + // While we have a valid worklist entry node, try to combine it. + while (SDNode *N = getNextWorklistEntry()) { // If N has no uses, it is dead. Make sure to revisit all N's operands once // N is deleted from the DAG, since they too may now be dead or may have a // reduced number of uses, allowing other xforms. @@ -1493,9 +1664,11 @@ SDValue DAGCombiner::visit(SDNode *N) { case ISD::SSUBSAT: case ISD::USUBSAT: return visitSUBSAT(N); case ISD::ADDC: return visitADDC(N); - case ISD::UADDO: return visitUADDO(N); + case ISD::SADDO: + case ISD::UADDO: return visitADDO(N); case ISD::SUBC: return visitSUBC(N); - case ISD::USUBO: return visitUSUBO(N); + case ISD::SSUBO: + case ISD::USUBO: return visitSUBO(N); case ISD::ADDE: return visitADDE(N); case ISD::ADDCARRY: return visitADDCARRY(N); case ISD::SUBE: return visitSUBE(N); @@ -1509,8 +1682,8 @@ SDValue DAGCombiner::visit(SDNode *N) { case ISD::MULHS: return visitMULHS(N); case ISD::SMUL_LOHI: return visitSMUL_LOHI(N); case ISD::UMUL_LOHI: return visitUMUL_LOHI(N); - case ISD::SMULO: return visitSMULO(N); - case ISD::UMULO: return visitUMULO(N); + case ISD::SMULO: + case ISD::UMULO: return visitMULO(N); case ISD::SMIN: case ISD::SMAX: case ISD::UMIN: @@ -1590,8 +1763,22 @@ SDValue DAGCombiner::visit(SDNode *N) { case ISD::MLOAD: return visitMLOAD(N); case ISD::MSCATTER: return visitMSCATTER(N); case ISD::MSTORE: return visitMSTORE(N); + case ISD::LIFETIME_END: return visitLIFETIME_END(N); case ISD::FP_TO_FP16: return visitFP_TO_FP16(N); case ISD::FP16_TO_FP: return visitFP16_TO_FP(N); + case ISD::VECREDUCE_FADD: + case ISD::VECREDUCE_FMUL: + case ISD::VECREDUCE_ADD: + case ISD::VECREDUCE_MUL: + case ISD::VECREDUCE_AND: + case ISD::VECREDUCE_OR: + case ISD::VECREDUCE_XOR: + case ISD::VECREDUCE_SMAX: + case ISD::VECREDUCE_SMIN: + case ISD::VECREDUCE_UMAX: + case ISD::VECREDUCE_UMIN: + case ISD::VECREDUCE_FMAX: + case ISD::VECREDUCE_FMIN: return visitVECREDUCE(N); } return SDValue(); } @@ -1644,7 +1831,7 @@ SDValue DAGCombiner::combine(SDNode *N) { } } - // If N is a commutative binary node, try eliminate it if the commuted + // If N is a commutative binary node, try to eliminate it if the commuted // version is already present in the DAG. if (!RV.getNode() && TLI.isCommutativeBinOp(N->getOpcode()) && N->getNumValues() == 1) { @@ -1693,6 +1880,12 @@ SDValue DAGCombiner::visitTokenFactor(SDNode *N) { if (OptLevel == CodeGenOpt::None) return SDValue(); + // If the sole user is a token factor, we should make sure we have a + // chance to merge them together. This prevents TF chains from inhibiting + // optimizations. + if (N->hasOneUse() && N->use_begin()->getOpcode() == ISD::TokenFactor) + AddToWorklist(*(N->use_begin())); + SmallVector TFs; // List of token factors to visit. SmallVector Ops; // Ops for replacing token factor. SmallPtrSet SeenOps; @@ -1704,8 +1897,19 @@ SDValue DAGCombiner::visitTokenFactor(SDNode *N) { // Iterate through token factors. The TFs grows when new token factors are // encountered. for (unsigned i = 0; i < TFs.size(); ++i) { - SDNode *TF = TFs[i]; + // Limit number of nodes to inline, to avoid quadratic compile times. + // We have to add the outstanding Token Factors to Ops, otherwise we might + // drop Ops from the resulting Token Factors. + if (Ops.size() > TokenFactorInlineLimit) { + for (unsigned j = i; j < TFs.size(); j++) + Ops.emplace_back(TFs[j], 0); + // Drop unprocessed Token Factors from TFs, so we do not add them to the + // combiner worklist later. + TFs.resize(i); + break; + } + SDNode *TF = TFs[i]; // Check each of the operands. for (const SDValue &Op : TF->op_values()) { switch (Op.getOpcode()) { @@ -1719,8 +1923,6 @@ SDValue DAGCombiner::visitTokenFactor(SDNode *N) { if (Op.hasOneUse() && !is_contained(TFs, Op.getNode())) { // Queue up for processing. TFs.push_back(Op.getNode()); - // Clean up in case the token factor is removed. - AddToWorklist(Op.getNode()); Changed = true; break; } @@ -1737,6 +1939,11 @@ SDValue DAGCombiner::visitTokenFactor(SDNode *N) { } } + // Re-visit inlined Token Factors, to clean them up in case they have been + // removed. Skip the first Token Factor, as this is the current node. + for (unsigned i = 1, e = TFs.size(); i < e; i++) + AddToWorklist(TFs[i]); + // Remove Nodes that are chained to another node in the list. Do so // by walking up chains breath-first stopping when we've seen // another operand. In general we must climb to the EntryNode, but we can exit @@ -1803,6 +2010,8 @@ SDValue DAGCombiner::visitTokenFactor(SDNode *N) { for (const SDValue &Op : CurNode->op_values()) AddToWorklist(i, Op.getNode(), CurOpNumber); break; + case ISD::LIFETIME_START: + case ISD::LIFETIME_END: case ISD::CopyFromReg: case ISD::CopyToReg: AddToWorklist(i, CurNode->getOperand(0).getNode(), CurOpNumber); @@ -1831,9 +2040,9 @@ SDValue DAGCombiner::visitTokenFactor(SDNode *N) { if (SeenChains.count(Op.getNode()) == 0) PrunedOps.push_back(Op); } - Result = DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, PrunedOps); + Result = DAG.getTokenFactor(SDLoc(N), PrunedOps); } else { - Result = DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, Ops); + Result = DAG.getTokenFactor(SDLoc(N), Ops); } } return Result; @@ -1869,7 +2078,8 @@ static ConstantSDNode *getAsNonOpaqueConstant(SDValue N) { } SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) { - assert(ISD::isBinaryOp(BO) && "Unexpected binary operator"); + assert(TLI.isBinOp(BO->getOpcode()) && BO->getNumValues() == 1 && + "Unexpected binary operator"); // Don't do this unless the old select is going away. We want to eliminate the // binary operator, not replace a binop with a select. @@ -1940,7 +2150,9 @@ SDValue DAGCombiner::foldBinOpIntoSelect(SDNode *BO) { !isConstantFPBuildVectorOrConstantFP(NewCF)) return SDValue(); - return DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF); + SDValue SelectOp = DAG.getSelect(DL, VT, Sel.getOperand(0), NewCT, NewCF); + SelectOp->setFlags(BO->getFlags()); + return SelectOp; } static SDValue foldAddSubBoolOfMaskedVal(SDNode *N, SelectionDAG &DAG) { @@ -1990,6 +2202,7 @@ static SDValue foldAddSubOfSignBit(SDNode *N, SelectionDAG &DAG) { // We need a constant operand for the add/sub, and the other operand is a // logical shift right: add (srl), C or sub C, (srl). + // TODO - support non-uniform vector amounts. bool IsAdd = N->getOpcode() == ISD::ADD; SDValue ConstantOp = IsAdd ? N->getOperand(1) : N->getOperand(0); SDValue ShiftOp = IsAdd ? N->getOperand(0) : N->getOperand(1); @@ -2006,7 +2219,7 @@ static SDValue foldAddSubOfSignBit(SDNode *N, SelectionDAG &DAG) { EVT VT = ShiftOp.getValueType(); SDValue ShAmt = ShiftOp.getOperand(1); ConstantSDNode *ShAmtC = isConstOrConstSplat(ShAmt); - if (!ShAmtC || ShAmtC->getZExtValue() != VT.getScalarSizeInBits() - 1) + if (!ShAmtC || ShAmtC->getAPIntValue() != (VT.getScalarSizeInBits() - 1)) return SDValue(); // Eliminate the 'not' by adjusting the shift and add/sub constant: @@ -2019,7 +2232,10 @@ static SDValue foldAddSubOfSignBit(SDNode *N, SelectionDAG &DAG) { return DAG.getNode(ISD::ADD, DL, VT, NewShift, DAG.getConstant(NewC, DL, VT)); } -SDValue DAGCombiner::visitADD(SDNode *N) { +/// Try to fold a node that behaves like an ADD (note that N isn't necessarily +/// an ISD::ADD here, it could for example be an ISD::OR if we know that there +/// are no common bits set in the operands). +SDValue DAGCombiner::visitADDLike(SDNode *N) { SDValue N0 = N->getOperand(0); SDValue N1 = N->getOperand(1); EVT VT = N0.getValueType(); @@ -2058,13 +2274,22 @@ SDValue DAGCombiner::visitADD(SDNode *N) { return N0; if (isConstantOrConstantVector(N1, /* NoOpaque */ true)) { + // fold ((A-c1)+c2) -> (A+(c2-c1)) + if (N0.getOpcode() == ISD::SUB && + isConstantOrConstantVector(N0.getOperand(1), /* NoOpaque */ true)) { + SDValue Sub = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, N1.getNode(), + N0.getOperand(1).getNode()); + assert(Sub && "Constant folding failed"); + return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), Sub); + } + // fold ((c1-A)+c2) -> (c1+c2)-A if (N0.getOpcode() == ISD::SUB && isConstantOrConstantVector(N0.getOperand(0), /* NoOpaque */ true)) { - // FIXME: Adding 2 constants should be handled by FoldConstantArithmetic. - return DAG.getNode(ISD::SUB, DL, VT, - DAG.getNode(ISD::ADD, DL, VT, N1, N0.getOperand(0)), - N0.getOperand(1)); + SDValue Add = DAG.FoldConstantArithmetic(ISD::ADD, DL, VT, N1.getNode(), + N0.getOperand(0).getNode()); + assert(Add && "Constant folding failed"); + return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1)); } // add (sext i1 X), 1 -> zext (not i1 X) @@ -2097,9 +2322,10 @@ SDValue DAGCombiner::visitADD(SDNode *N) { return NewSel; // reassociate add - if (SDValue RADD = ReassociateOps(ISD::ADD, DL, N0, N1, N->getFlags())) - return RADD; - + if (!reassociationCanBreakAddressingModePattern(ISD::ADD, DL, N0, N1)) { + if (SDValue RADD = reassociateOps(ISD::ADD, DL, N0, N1, N->getFlags())) + return RADD; + } // fold ((0-A) + B) -> B-A if (N0.getOpcode() == ISD::SUB && isNullOrNullSplat(N0.getOperand(0))) return DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1)); @@ -2116,6 +2342,18 @@ SDValue DAGCombiner::visitADD(SDNode *N) { if (N0.getOpcode() == ISD::SUB && N1 == N0.getOperand(1)) return N0.getOperand(0); + // fold ((A-B)+(C-A)) -> (C-B) + if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB && + N0.getOperand(0) == N1.getOperand(1)) + return DAG.getNode(ISD::SUB, DL, VT, N1.getOperand(0), + N0.getOperand(1)); + + // fold ((A-B)+(B-C)) -> (A-C) + if (N0.getOpcode() == ISD::SUB && N1.getOpcode() == ISD::SUB && + N0.getOperand(1) == N1.getOperand(0)) + return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), + N1.getOperand(1)); + // fold (A+(B-(A+C))) to (B-C) if (N1.getOpcode() == ISD::SUB && N1.getOperand(1).getOpcode() == ISD::ADD && N0 == N1.getOperand(1).getOperand(0)) @@ -2148,31 +2386,93 @@ SDValue DAGCombiner::visitADD(SDNode *N) { DAG.getNode(ISD::ADD, SDLoc(N1), VT, N01, N11)); } + // fold (add (umax X, C), -C) --> (usubsat X, C) + if (N0.getOpcode() == ISD::UMAX && hasOperation(ISD::USUBSAT, VT)) { + auto MatchUSUBSAT = [](ConstantSDNode *Max, ConstantSDNode *Op) { + return (!Max && !Op) || + (Max && Op && Max->getAPIntValue() == (-Op->getAPIntValue())); + }; + if (ISD::matchBinaryPredicate(N0.getOperand(1), N1, MatchUSUBSAT, + /*AllowUndefs*/ true)) + return DAG.getNode(ISD::USUBSAT, DL, VT, N0.getOperand(0), + N0.getOperand(1)); + } + + if (SimplifyDemandedBits(SDValue(N, 0))) + return SDValue(N, 0); + + if (isOneOrOneSplat(N1)) { + // fold (add (xor a, -1), 1) -> (sub 0, a) + if (isBitwiseNot(N0)) + return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), + N0.getOperand(0)); + + // fold (add (add (xor a, -1), b), 1) -> (sub b, a) + if (N0.getOpcode() == ISD::ADD || + N0.getOpcode() == ISD::UADDO || + N0.getOpcode() == ISD::SADDO) { + SDValue A, Xor; + + if (isBitwiseNot(N0.getOperand(0))) { + A = N0.getOperand(1); + Xor = N0.getOperand(0); + } else if (isBitwiseNot(N0.getOperand(1))) { + A = N0.getOperand(0); + Xor = N0.getOperand(1); + } + + if (Xor) + return DAG.getNode(ISD::SUB, DL, VT, A, Xor.getOperand(0)); + } + + // Look for: + // add (add x, y), 1 + // And if the target does not like this form then turn into: + // sub y, (xor x, -1) + if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.hasOneUse() && + N0.getOpcode() == ISD::ADD) { + SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0), + DAG.getAllOnesConstant(DL, VT)); + return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(1), Not); + } + } + + // (x - y) + -1 -> add (xor y, -1), x + if (N0.hasOneUse() && N0.getOpcode() == ISD::SUB && + isAllOnesOrAllOnesSplat(N1)) { + SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1), N1); + return DAG.getNode(ISD::ADD, DL, VT, Xor, N0.getOperand(0)); + } + + if (SDValue Combined = visitADDLikeCommutative(N0, N1, N)) + return Combined; + + if (SDValue Combined = visitADDLikeCommutative(N1, N0, N)) + return Combined; + + return SDValue(); +} + +SDValue DAGCombiner::visitADD(SDNode *N) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + EVT VT = N0.getValueType(); + SDLoc DL(N); + + if (SDValue Combined = visitADDLike(N)) + return Combined; + if (SDValue V = foldAddSubBoolOfMaskedVal(N, DAG)) return V; if (SDValue V = foldAddSubOfSignBit(N, DAG)) return V; - if (SimplifyDemandedBits(SDValue(N, 0))) - return SDValue(N, 0); - // fold (a+b) -> (a|b) iff a and b share no bits. if ((!LegalOperations || TLI.isOperationLegal(ISD::OR, VT)) && DAG.haveNoCommonBitsSet(N0, N1)) return DAG.getNode(ISD::OR, DL, VT, N0, N1); - // fold (add (xor a, -1), 1) -> (sub 0, a) - if (isBitwiseNot(N0) && isOneOrOneSplat(N1)) - return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), - N0.getOperand(0)); - - if (SDValue Combined = visitADDLike(N0, N1, N)) - return Combined; - - if (SDValue Combined = visitADDLike(N1, N0, N)) - return Combined; - return SDValue(); } @@ -2246,6 +2546,10 @@ static SDValue getAsCarry(const TargetLowering &TLI, SDValue V) { V.getOpcode() != ISD::UADDO && V.getOpcode() != ISD::USUBO) return SDValue(); + EVT VT = V.getNode()->getValueType(0); + if (!TLI.isOperationLegalOrCustom(V.getOpcode(), VT)) + return SDValue(); + // If the result is masked, then no matter what kind of bool it is we can // return. If it isn't, then we need to make sure the bool type is either 0 or // 1 and not other values. @@ -2257,7 +2561,26 @@ static SDValue getAsCarry(const TargetLowering &TLI, SDValue V) { return SDValue(); } -SDValue DAGCombiner::visitADDLike(SDValue N0, SDValue N1, SDNode *LocReference) { +/// Given the operands of an add/sub operation, see if the 2nd operand is a +/// masked 0/1 whose source operand is actually known to be 0/-1. If so, invert +/// the opcode and bypass the mask operation. +static SDValue foldAddSubMasked1(bool IsAdd, SDValue N0, SDValue N1, + SelectionDAG &DAG, const SDLoc &DL) { + if (N1.getOpcode() != ISD::AND || !isOneOrOneSplat(N1->getOperand(1))) + return SDValue(); + + EVT VT = N0.getValueType(); + if (DAG.ComputeNumSignBits(N1.getOperand(0)) != VT.getScalarSizeInBits()) + return SDValue(); + + // add N0, (and (AssertSext X, i1), 1) --> sub N0, X + // sub N0, (and (AssertSext X, i1), 1) --> add N0, X + return DAG.getNode(IsAdd ? ISD::SUB : ISD::ADD, DL, VT, N0, N1.getOperand(0)); +} + +/// Helper for doing combines based on N0 and N1 being added to each other. +SDValue DAGCombiner::visitADDLikeCommutative(SDValue N0, SDValue N1, + SDNode *LocReference) { EVT VT = N0.getValueType(); SDLoc DL(LocReference); @@ -2269,21 +2592,42 @@ SDValue DAGCombiner::visitADDLike(SDValue N0, SDValue N1, SDNode *LocReference) N1.getOperand(0).getOperand(1), N1.getOperand(1))); - if (N1.getOpcode() == ISD::AND) { - SDValue AndOp0 = N1.getOperand(0); - unsigned NumSignBits = DAG.ComputeNumSignBits(AndOp0); - unsigned DestBits = VT.getScalarSizeInBits(); - - // (add z, (and (sbbl x, x), 1)) -> (sub z, (sbbl x, x)) - // and similar xforms where the inner op is either ~0 or 0. - if (NumSignBits == DestBits && isOneOrOneSplat(N1->getOperand(1))) - return DAG.getNode(ISD::SUB, DL, VT, N0, AndOp0); - } + if (SDValue V = foldAddSubMasked1(true, N0, N1, DAG, DL)) + return V; - // add (sext i1), X -> sub X, (zext i1) + // Look for: + // add (add x, 1), y + // And if the target does not like this form then turn into: + // sub y, (xor x, -1) + if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.hasOneUse() && + N0.getOpcode() == ISD::ADD && isOneOrOneSplat(N0.getOperand(1))) { + SDValue Not = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(0), + DAG.getAllOnesConstant(DL, VT)); + return DAG.getNode(ISD::SUB, DL, VT, N1, Not); + } + + // Hoist one-use subtraction by non-opaque constant: + // (x - C) + y -> (x + y) - C + // This is necessary because SUB(X,C) -> ADD(X,-C) doesn't work for vectors. + if (N0.hasOneUse() && N0.getOpcode() == ISD::SUB && + isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true)) { + SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), N1); + return DAG.getNode(ISD::SUB, DL, VT, Add, N0.getOperand(1)); + } + // Hoist one-use subtraction from non-opaque constant: + // (C - x) + y -> (y - x) + C + if (N0.hasOneUse() && N0.getOpcode() == ISD::SUB && + isConstantOrConstantVector(N0.getOperand(0), /*NoOpaques=*/true)) { + SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N1, N0.getOperand(1)); + return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(0)); + } + + // If the target's bool is represented as 0/1, prefer to make this 'sub 0/1' + // rather than 'add 0/-1' (the zext should get folded). + // add (sext i1 Y), X --> sub X, (zext i1 Y) if (N0.getOpcode() == ISD::SIGN_EXTEND && - N0.getOperand(0).getValueType() == MVT::i1 && - !TLI.isOperationLegal(ISD::SIGN_EXTEND, MVT::i1)) { + N0.getOperand(0).getScalarValueSizeInBits() == 1 && + TLI.getBooleanContents(VT) == TargetLowering::ZeroOrOneBooleanContent) { SDValue ZExt = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0)); return DAG.getNode(ISD::SUB, DL, VT, N1, ZExt); } @@ -2344,8 +2688,10 @@ SDValue DAGCombiner::visitADDC(SDNode *N) { return SDValue(); } -static SDValue flipBoolean(SDValue V, const SDLoc &DL, EVT VT, +static SDValue flipBoolean(SDValue V, const SDLoc &DL, SelectionDAG &DAG, const TargetLowering &TLI) { + EVT VT = V.getValueType(); + SDValue Cst; switch (TLI.getBooleanContents(VT)) { case TargetLowering::ZeroOrOneBooleanContent: @@ -2353,35 +2699,60 @@ static SDValue flipBoolean(SDValue V, const SDLoc &DL, EVT VT, Cst = DAG.getConstant(1, DL, VT); break; case TargetLowering::ZeroOrNegativeOneBooleanContent: - Cst = DAG.getConstant(-1, DL, VT); + Cst = DAG.getAllOnesConstant(DL, VT); break; } return DAG.getNode(ISD::XOR, DL, VT, V, Cst); } -static bool isBooleanFlip(SDValue V, EVT VT, const TargetLowering &TLI) { - if (V.getOpcode() != ISD::XOR) return false; - ConstantSDNode *Const = dyn_cast(V.getOperand(1)); - if (!Const) return false; +/** + * Flips a boolean if it is cheaper to compute. If the Force parameters is set, + * then the flip also occurs if computing the inverse is the same cost. + * This function returns an empty SDValue in case it cannot flip the boolean + * without increasing the cost of the computation. If you want to flip a boolean + * no matter what, use flipBoolean. + */ +static SDValue extractBooleanFlip(SDValue V, SelectionDAG &DAG, + const TargetLowering &TLI, + bool Force) { + if (Force && isa(V)) + return flipBoolean(V, SDLoc(V), DAG, TLI); + + if (V.getOpcode() != ISD::XOR) + return SDValue(); + + ConstantSDNode *Const = isConstOrConstSplat(V.getOperand(1), false); + if (!Const) + return SDValue(); + EVT VT = V.getValueType(); + + bool IsFlip = false; switch(TLI.getBooleanContents(VT)) { case TargetLowering::ZeroOrOneBooleanContent: - return Const->isOne(); + IsFlip = Const->isOne(); + break; case TargetLowering::ZeroOrNegativeOneBooleanContent: - return Const->isAllOnesValue(); + IsFlip = Const->isAllOnesValue(); + break; case TargetLowering::UndefinedBooleanContent: - return (Const->getAPIntValue() & 0x01) == 1; + IsFlip = (Const->getAPIntValue() & 0x01) == 1; + break; } - llvm_unreachable("Unsupported boolean content"); + + if (IsFlip) + return V.getOperand(0); + if (Force) + return flipBoolean(V, SDLoc(V), DAG, TLI); + return SDValue(); } -SDValue DAGCombiner::visitUADDO(SDNode *N) { +SDValue DAGCombiner::visitADDO(SDNode *N) { SDValue N0 = N->getOperand(0); SDValue N1 = N->getOperand(1); EVT VT = N0.getValueType(); - if (VT.isVector()) - return SDValue(); + bool IsSigned = (ISD::SADDO == N->getOpcode()); EVT CarryVT = N->getValueType(1); SDLoc DL(N); @@ -2392,40 +2763,42 @@ SDValue DAGCombiner::visitUADDO(SDNode *N) { DAG.getUNDEF(CarryVT)); // canonicalize constant to RHS. - ConstantSDNode *N0C = dyn_cast(N0); - ConstantSDNode *N1C = dyn_cast(N1); - if (N0C && !N1C) - return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N1, N0); + if (DAG.isConstantIntBuildVectorOrConstantInt(N0) && + !DAG.isConstantIntBuildVectorOrConstantInt(N1)) + return DAG.getNode(N->getOpcode(), DL, N->getVTList(), N1, N0); - // fold (uaddo x, 0) -> x + no carry out - if (isNullConstant(N1)) + // fold (addo x, 0) -> x + no carry out + if (isNullOrNullSplat(N1)) return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT)); - // If it cannot overflow, transform into an add. - if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never) - return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), - DAG.getConstant(0, DL, CarryVT)); + if (!IsSigned) { + // If it cannot overflow, transform into an add. + if (DAG.computeOverflowKind(N0, N1) == SelectionDAG::OFK_Never) + return CombineTo(N, DAG.getNode(ISD::ADD, DL, VT, N0, N1), + DAG.getConstant(0, DL, CarryVT)); - // fold (uaddo (xor a, -1), 1) -> (usub 0, a) and flip carry. - if (isBitwiseNot(N0) && isOneOrOneSplat(N1)) { - SDValue Sub = DAG.getNode(ISD::USUBO, DL, N->getVTList(), - DAG.getConstant(0, DL, VT), - N0.getOperand(0)); - return CombineTo(N, Sub, - flipBoolean(Sub.getValue(1), DL, CarryVT, DAG, TLI)); - } + // fold (uaddo (xor a, -1), 1) -> (usub 0, a) and flip carry. + if (isBitwiseNot(N0) && isOneOrOneSplat(N1)) { + SDValue Sub = DAG.getNode(ISD::USUBO, DL, N->getVTList(), + DAG.getConstant(0, DL, VT), N0.getOperand(0)); + return CombineTo(N, Sub, + flipBoolean(Sub.getValue(1), DL, DAG, TLI)); + } - if (SDValue Combined = visitUADDOLike(N0, N1, N)) - return Combined; + if (SDValue Combined = visitUADDOLike(N0, N1, N)) + return Combined; - if (SDValue Combined = visitUADDOLike(N1, N0, N)) - return Combined; + if (SDValue Combined = visitUADDOLike(N1, N0, N)) + return Combined; + } return SDValue(); } SDValue DAGCombiner::visitUADDOLike(SDValue N0, SDValue N1, SDNode *N) { - auto VT = N0.getValueType(); + EVT VT = N0.getValueType(); + if (VT.isVector()) + return SDValue(); // (uaddo X, (addcarry Y, 0, Carry)) -> (addcarry X, Y, Carry) // If Y + 1 cannot overflow. @@ -2484,11 +2857,10 @@ SDValue DAGCombiner::visitADDCARRY(SDNode *N) { return DAG.getNode(ISD::UADDO, DL, N->getVTList(), N0, N1); } - EVT CarryVT = CarryIn.getValueType(); - // fold (addcarry 0, 0, X) -> (and (ext/trunc X), 1) and no carry. if (isNullConstant(N0) && isNullConstant(N1)) { EVT VT = N0.getValueType(); + EVT CarryVT = CarryIn.getValueType(); SDValue CarryExt = DAG.getBoolExtOrTrunc(CarryIn, DL, VT, CarryVT); AddToWorklist(CarryExt.getNode()); return CombineTo(N, DAG.getNode(ISD::AND, DL, VT, CarryExt, @@ -2496,16 +2868,6 @@ SDValue DAGCombiner::visitADDCARRY(SDNode *N) { DAG.getConstant(0, DL, CarryVT)); } - // fold (addcarry (xor a, -1), 0, !b) -> (subcarry 0, a, b) and flip carry. - if (isBitwiseNot(N0) && isNullConstant(N1) && - isBooleanFlip(CarryIn, CarryVT, TLI)) { - SDValue Sub = DAG.getNode(ISD::SUBCARRY, DL, N->getVTList(), - DAG.getConstant(0, DL, N0.getValueType()), - N0.getOperand(0), CarryIn.getOperand(0)); - return CombineTo(N, Sub, - flipBoolean(Sub.getValue(1), DL, CarryVT, DAG, TLI)); - } - if (SDValue Combined = visitADDCARRYLike(N0, N1, CarryIn, N)) return Combined; @@ -2515,12 +2877,112 @@ SDValue DAGCombiner::visitADDCARRY(SDNode *N) { return SDValue(); } +/** + * If we are facing some sort of diamond carry propapagtion pattern try to + * break it up to generate something like: + * (addcarry X, 0, (addcarry A, B, Z):Carry) + * + * The end result is usually an increase in operation required, but because the + * carry is now linearized, other tranforms can kick in and optimize the DAG. + * + * Patterns typically look something like + * (uaddo A, B) + * / \ + * Carry Sum + * | \ + * | (addcarry *, 0, Z) + * | / + * \ Carry + * | / + * (addcarry X, *, *) + * + * But numerous variation exist. Our goal is to identify A, B, X and Z and + * produce a combine with a single path for carry propagation. + */ +static SDValue combineADDCARRYDiamond(DAGCombiner &Combiner, SelectionDAG &DAG, + SDValue X, SDValue Carry0, SDValue Carry1, + SDNode *N) { + if (Carry1.getResNo() != 1 || Carry0.getResNo() != 1) + return SDValue(); + if (Carry1.getOpcode() != ISD::UADDO) + return SDValue(); + + SDValue Z; + + /** + * First look for a suitable Z. It will present itself in the form of + * (addcarry Y, 0, Z) or its equivalent (uaddo Y, 1) for Z=true + */ + if (Carry0.getOpcode() == ISD::ADDCARRY && + isNullConstant(Carry0.getOperand(1))) { + Z = Carry0.getOperand(2); + } else if (Carry0.getOpcode() == ISD::UADDO && + isOneConstant(Carry0.getOperand(1))) { + EVT VT = Combiner.getSetCCResultType(Carry0.getValueType()); + Z = DAG.getConstant(1, SDLoc(Carry0.getOperand(1)), VT); + } else { + // We couldn't find a suitable Z. + return SDValue(); + } + + + auto cancelDiamond = [&](SDValue A,SDValue B) { + SDLoc DL(N); + SDValue NewY = DAG.getNode(ISD::ADDCARRY, DL, Carry0->getVTList(), A, B, Z); + Combiner.AddToWorklist(NewY.getNode()); + return DAG.getNode(ISD::ADDCARRY, DL, N->getVTList(), X, + DAG.getConstant(0, DL, X.getValueType()), + NewY.getValue(1)); + }; + + /** + * (uaddo A, B) + * | + * Sum + * | + * (addcarry *, 0, Z) + */ + if (Carry0.getOperand(0) == Carry1.getValue(0)) { + return cancelDiamond(Carry1.getOperand(0), Carry1.getOperand(1)); + } + + /** + * (addcarry A, 0, Z) + * | + * Sum + * | + * (uaddo *, B) + */ + if (Carry1.getOperand(0) == Carry0.getValue(0)) { + return cancelDiamond(Carry0.getOperand(0), Carry1.getOperand(1)); + } + + if (Carry1.getOperand(1) == Carry0.getValue(0)) { + return cancelDiamond(Carry1.getOperand(0), Carry0.getOperand(0)); + } + + return SDValue(); +} + SDValue DAGCombiner::visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn, SDNode *N) { + // fold (addcarry (xor a, -1), b, c) -> (subcarry b, a, !c) and flip carry. + if (isBitwiseNot(N0)) + if (SDValue NotC = extractBooleanFlip(CarryIn, DAG, TLI, true)) { + SDLoc DL(N); + SDValue Sub = DAG.getNode(ISD::SUBCARRY, DL, N->getVTList(), N1, + N0.getOperand(0), NotC); + return CombineTo(N, Sub, + flipBoolean(Sub.getValue(1), DL, DAG, TLI)); + } + // Iff the flag result is dead: // (addcarry (add|uaddo X, Y), 0, Carry) -> (addcarry X, Y, Carry) + // Don't do this if the Carry comes from the uaddo. It won't remove the uaddo + // or the dependency between the instructions. if ((N0.getOpcode() == ISD::ADD || - (N0.getOpcode() == ISD::UADDO && N0.getResNo() == 0)) && + (N0.getOpcode() == ISD::UADDO && N0.getResNo() == 0 && + N0.getValue(1) != CarryIn)) && isNullConstant(N1) && !N->hasAnyUseOfValue(1)) return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0.getOperand(0), N0.getOperand(1), CarryIn); @@ -2529,35 +2991,13 @@ SDValue DAGCombiner::visitADDCARRYLike(SDValue N0, SDValue N1, SDValue CarryIn, * When one of the addcarry argument is itself a carry, we may be facing * a diamond carry propagation. In which case we try to transform the DAG * to ensure linear carry propagation if that is possible. - * - * We are trying to get: - * (addcarry X, 0, (addcarry A, B, Z):Carry) */ if (auto Y = getAsCarry(TLI, N1)) { - /** - * (uaddo A, B) - * / \ - * Carry Sum - * | \ - * | (addcarry *, 0, Z) - * | / - * \ Carry - * | / - * (addcarry X, *, *) - */ - if (Y.getOpcode() == ISD::UADDO && - CarryIn.getResNo() == 1 && - CarryIn.getOpcode() == ISD::ADDCARRY && - isNullConstant(CarryIn.getOperand(1)) && - CarryIn.getOperand(0) == Y.getValue(0)) { - auto NewY = DAG.getNode(ISD::ADDCARRY, SDLoc(N), Y->getVTList(), - Y.getOperand(0), Y.getOperand(1), - CarryIn.getOperand(2)); - AddToWorklist(NewY.getNode()); - return DAG.getNode(ISD::ADDCARRY, SDLoc(N), N->getVTList(), N0, - DAG.getConstant(0, SDLoc(N), N0.getValueType()), - NewY.getValue(1)); - } + // Because both are carries, Y and Z can be swapped. + if (auto R = combineADDCARRYDiamond(*this, DAG, N0, Y, CarryIn, N)) + return R; + if (auto R = combineADDCARRYDiamond(*this, DAG, N0, CarryIn, Y, N)) + return R; } return SDValue(); @@ -2620,7 +3060,7 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { // -(X >>s 31) -> (X >>u 31) if (N1->getOpcode() == ISD::SRA || N1->getOpcode() == ISD::SRL) { ConstantSDNode *ShiftAmt = isConstOrConstSplat(N1.getOperand(1)); - if (ShiftAmt && ShiftAmt->getZExtValue() == BitWidth - 1) { + if (ShiftAmt && ShiftAmt->getAPIntValue() == (BitWidth - 1)) { auto NewSh = N1->getOpcode() == ISD::SRA ? ISD::SRL : ISD::SRA; if (!LegalOperations || TLI.isOperationLegal(NewSh, VT)) return DAG.getNode(NewSh, DL, VT, N1.getOperand(0), N1.getOperand(1)); @@ -2662,16 +3102,48 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { if (N0.getOpcode() == ISD::ADD && N0.getOperand(1) == N1) return N0.getOperand(0); + // fold (A+C1)-C2 -> A+(C1-C2) + if (N0.getOpcode() == ISD::ADD && + isConstantOrConstantVector(N1, /* NoOpaques */ true) && + isConstantOrConstantVector(N0.getOperand(1), /* NoOpaques */ true)) { + SDValue NewC = DAG.FoldConstantArithmetic( + ISD::SUB, DL, VT, N0.getOperand(1).getNode(), N1.getNode()); + assert(NewC && "Constant folding failed"); + return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(0), NewC); + } + // fold C2-(A+C1) -> (C2-C1)-A if (N1.getOpcode() == ISD::ADD) { SDValue N11 = N1.getOperand(1); if (isConstantOrConstantVector(N0, /* NoOpaques */ true) && isConstantOrConstantVector(N11, /* NoOpaques */ true)) { - SDValue NewC = DAG.getNode(ISD::SUB, DL, VT, N0, N11); + SDValue NewC = DAG.FoldConstantArithmetic(ISD::SUB, DL, VT, N0.getNode(), + N11.getNode()); + assert(NewC && "Constant folding failed"); return DAG.getNode(ISD::SUB, DL, VT, NewC, N1.getOperand(0)); } } + // fold (A-C1)-C2 -> A-(C1+C2) + if (N0.getOpcode() == ISD::SUB && + isConstantOrConstantVector(N1, /* NoOpaques */ true) && + isConstantOrConstantVector(N0.getOperand(1), /* NoOpaques */ true)) { + SDValue NewC = DAG.FoldConstantArithmetic( + ISD::ADD, DL, VT, N0.getOperand(1).getNode(), N1.getNode()); + assert(NewC && "Constant folding failed"); + return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), NewC); + } + + // fold (c1-A)-c2 -> (c1-c2)-A + if (N0.getOpcode() == ISD::SUB && + isConstantOrConstantVector(N1, /* NoOpaques */ true) && + isConstantOrConstantVector(N0.getOperand(0), /* NoOpaques */ true)) { + SDValue NewC = DAG.FoldConstantArithmetic( + ISD::SUB, DL, VT, N0.getOperand(0).getNode(), N1.getNode()); + assert(NewC && "Constant folding failed"); + return DAG.getNode(ISD::SUB, DL, VT, NewC, N0.getOperand(1)); + } + // fold ((A+(B+or-C))-B) -> A+or-C if (N0.getOpcode() == ISD::ADD && (N0.getOperand(1).getOpcode() == ISD::SUB || @@ -2728,6 +3200,63 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { if (SDValue V = foldAddSubOfSignBit(N, DAG)) return V; + if (SDValue V = foldAddSubMasked1(false, N0, N1, DAG, SDLoc(N))) + return V; + + // (x - y) - 1 -> add (xor y, -1), x + if (N0.hasOneUse() && N0.getOpcode() == ISD::SUB && isOneOrOneSplat(N1)) { + SDValue Xor = DAG.getNode(ISD::XOR, DL, VT, N0.getOperand(1), + DAG.getAllOnesConstant(DL, VT)); + return DAG.getNode(ISD::ADD, DL, VT, Xor, N0.getOperand(0)); + } + + // Look for: + // sub y, (xor x, -1) + // And if the target does not like this form then turn into: + // add (add x, y), 1 + if (TLI.preferIncOfAddToSubOfNot(VT) && N1.hasOneUse() && isBitwiseNot(N1)) { + SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, N1.getOperand(0)); + return DAG.getNode(ISD::ADD, DL, VT, Add, DAG.getConstant(1, DL, VT)); + } + + // Hoist one-use addition by non-opaque constant: + // (x + C) - y -> (x - y) + C + if (N0.hasOneUse() && N0.getOpcode() == ISD::ADD && + isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true)) { + SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1); + return DAG.getNode(ISD::ADD, DL, VT, Sub, N0.getOperand(1)); + } + // y - (x + C) -> (y - x) - C + if (N1.hasOneUse() && N1.getOpcode() == ISD::ADD && + isConstantOrConstantVector(N1.getOperand(1), /*NoOpaques=*/true)) { + SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0, N1.getOperand(0)); + return DAG.getNode(ISD::SUB, DL, VT, Sub, N1.getOperand(1)); + } + // (x - C) - y -> (x - y) - C + // This is necessary because SUB(X,C) -> ADD(X,-C) doesn't work for vectors. + if (N0.hasOneUse() && N0.getOpcode() == ISD::SUB && + isConstantOrConstantVector(N0.getOperand(1), /*NoOpaques=*/true)) { + SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), N1); + return DAG.getNode(ISD::SUB, DL, VT, Sub, N0.getOperand(1)); + } + // (C - x) - y -> C - (x + y) + if (N0.hasOneUse() && N0.getOpcode() == ISD::SUB && + isConstantOrConstantVector(N0.getOperand(0), /*NoOpaques=*/true)) { + SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), N1); + return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), Add); + } + + // If the target's bool is represented as 0/-1, prefer to make this 'add 0/-1' + // rather than 'sub 0/1' (the sext should get folded). + // sub X, (zext i1 Y) --> add X, (sext i1 Y) + if (N1.getOpcode() == ISD::ZERO_EXTEND && + N1.getOperand(0).getScalarValueSizeInBits() == 1 && + TLI.getBooleanContents(VT) == + TargetLowering::ZeroOrNegativeOneBooleanContent) { + SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, N1.getOperand(0)); + return DAG.getNode(ISD::ADD, DL, VT, N0, SExt); + } + // fold Y = sra (X, size(X)-1); sub (xor (X, Y), Y) -> (abs X) if (TLI.isOperationLegalOrCustom(ISD::ABS, VT)) { if (N0.getOpcode() == ISD::XOR && N1.getOpcode() == ISD::SRA) { @@ -2772,7 +3301,8 @@ SDValue DAGCombiner::visitSUB(SDNode *N) { if (!LegalOperations && N1.getOpcode() == ISD::SRL && N1.hasOneUse()) { SDValue ShAmt = N1.getOperand(1); ConstantSDNode *ShAmtC = isConstOrConstSplat(ShAmt); - if (ShAmtC && ShAmtC->getZExtValue() == N1.getScalarValueSizeInBits() - 1) { + if (ShAmtC && + ShAmtC->getAPIntValue() == (N1.getScalarValueSizeInBits() - 1)) { SDValue SRA = DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0), ShAmt); return DAG.getNode(ISD::ADD, DL, VT, N0, SRA); } @@ -2846,12 +3376,11 @@ SDValue DAGCombiner::visitSUBC(SDNode *N) { return SDValue(); } -SDValue DAGCombiner::visitUSUBO(SDNode *N) { +SDValue DAGCombiner::visitSUBO(SDNode *N) { SDValue N0 = N->getOperand(0); SDValue N1 = N->getOperand(1); EVT VT = N0.getValueType(); - if (VT.isVector()) - return SDValue(); + bool IsSigned = (ISD::SSUBO == N->getOpcode()); EVT CarryVT = N->getValueType(1); SDLoc DL(N); @@ -2861,17 +3390,25 @@ SDValue DAGCombiner::visitUSUBO(SDNode *N) { return CombineTo(N, DAG.getNode(ISD::SUB, DL, VT, N0, N1), DAG.getUNDEF(CarryVT)); - // fold (usubo x, x) -> 0 + no borrow + // fold (subo x, x) -> 0 + no borrow if (N0 == N1) return CombineTo(N, DAG.getConstant(0, DL, VT), DAG.getConstant(0, DL, CarryVT)); - // fold (usubo x, 0) -> x + no borrow - if (isNullConstant(N1)) + ConstantSDNode *N1C = getAsNonOpaqueConstant(N1); + + // fold (subox, c) -> (addo x, -c) + if (IsSigned && N1C && !N1C->getAPIntValue().isMinSignedValue()) { + return DAG.getNode(ISD::SADDO, DL, N->getVTList(), N0, + DAG.getConstant(-N1C->getAPIntValue(), DL, VT)); + } + + // fold (subo x, 0) -> x + no borrow + if (isNullOrNullSplat(N1)) return CombineTo(N, N0, DAG.getConstant(0, DL, CarryVT)); // Canonicalize (usubo -1, x) -> ~x, i.e. (xor x, -1) + no borrow - if (isAllOnesConstant(N0)) + if (!IsSigned && isAllOnesOrAllOnesSplat(N0)) return CombineTo(N, DAG.getNode(ISD::XOR, DL, VT, N1, N0), DAG.getConstant(0, DL, CarryVT)); @@ -3012,13 +3549,13 @@ SDValue DAGCombiner::visitMUL(SDNode *N) { MathOp = ISD::SUB; if (MathOp != ISD::DELETED_NODE) { - unsigned ShAmt = MathOp == ISD::ADD ? (MulC - 1).logBase2() - : (MulC + 1).logBase2(); - assert(ShAmt > 0 && ShAmt < VT.getScalarSizeInBits() && - "Not expecting multiply-by-constant that could have simplified"); + unsigned ShAmt = + MathOp == ISD::ADD ? (MulC - 1).logBase2() : (MulC + 1).logBase2(); + assert(ShAmt < VT.getScalarSizeInBits() && + "multiply-by-constant generated out of bounds shift"); SDLoc DL(N); - SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, N0, - DAG.getConstant(ShAmt, DL, VT)); + SDValue Shl = + DAG.getNode(ISD::SHL, DL, VT, N0, DAG.getConstant(ShAmt, DL, VT)); SDValue R = DAG.getNode(MathOp, DL, VT, Shl, N0); if (ConstValue1.isNegative()) R = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), R); @@ -3069,7 +3606,7 @@ SDValue DAGCombiner::visitMUL(SDNode *N) { N0.getOperand(1), N1)); // reassociate mul - if (SDValue RMUL = ReassociateOps(ISD::MUL, SDLoc(N), N0, N1, N->getFlags())) + if (SDValue RMUL = reassociateOps(ISD::MUL, SDLoc(N), N0, N1, N->getFlags())) return RMUL; return SDValue(); @@ -3612,7 +4149,6 @@ SDValue DAGCombiner::visitMULHU(SDNode *N) { // fold (mulhu x, (1 << c)) -> x >> (bitwidth - c) if (isConstantOrConstantVector(N1, /*NoOpaques*/ true) && DAG.isKnownToBeAPowerOfTwo(N1) && hasOperation(ISD::SRL, VT)) { - SDLoc DL(N); unsigned NumEltBits = VT.getScalarSizeInBits(); SDValue LogBase2 = BuildLogBase2(N1, DL); SDValue SRLAmt = DAG.getNode( @@ -3753,22 +4289,14 @@ SDValue DAGCombiner::visitUMUL_LOHI(SDNode *N) { return SDValue(); } -SDValue DAGCombiner::visitSMULO(SDNode *N) { - // (smulo x, 2) -> (saddo x, x) - if (ConstantSDNode *C2 = dyn_cast(N->getOperand(1))) - if (C2->getAPIntValue() == 2) - return DAG.getNode(ISD::SADDO, SDLoc(N), N->getVTList(), - N->getOperand(0), N->getOperand(0)); +SDValue DAGCombiner::visitMULO(SDNode *N) { + bool IsSigned = (ISD::SMULO == N->getOpcode()); - return SDValue(); -} - -SDValue DAGCombiner::visitUMULO(SDNode *N) { - // (umulo x, 2) -> (uaddo x, x) - if (ConstantSDNode *C2 = dyn_cast(N->getOperand(1))) + // (mulo x, 2) -> (addo x, x) + if (ConstantSDNode *C2 = isConstOrConstSplat(N->getOperand(1))) if (C2->getAPIntValue() == 2) - return DAG.getNode(ISD::UADDO, SDLoc(N), N->getVTList(), - N->getOperand(0), N->getOperand(0)); + return DAG.getNode(IsSigned ? ISD::SADDO : ISD::UADDO, SDLoc(N), + N->getVTList(), N->getOperand(0), N->getOperand(0)); return SDValue(); } @@ -4075,6 +4603,33 @@ SDValue DAGCombiner::foldLogicOfSetCCs(bool IsAnd, SDValue N0, SDValue N1, SDValue Zero = DAG.getConstant(0, DL, OpVT); return DAG.getSetCC(DL, VT, Or, Zero, CC1); } + + // Turn compare of constants whose difference is 1 bit into add+and+setcc. + // TODO - support non-uniform vector amounts. + if ((IsAnd && CC1 == ISD::SETNE) || (!IsAnd && CC1 == ISD::SETEQ)) { + // Match a shared variable operand and 2 non-opaque constant operands. + ConstantSDNode *C0 = isConstOrConstSplat(LR); + ConstantSDNode *C1 = isConstOrConstSplat(RR); + if (LL == RL && C0 && C1 && !C0->isOpaque() && !C1->isOpaque()) { + // Canonicalize larger constant as C0. + if (C1->getAPIntValue().ugt(C0->getAPIntValue())) + std::swap(C0, C1); + + // The difference of the constants must be a single bit. + const APInt &C0Val = C0->getAPIntValue(); + const APInt &C1Val = C1->getAPIntValue(); + if ((C0Val - C1Val).isPowerOf2()) { + // and/or (setcc X, C0, ne), (setcc X, C1, ne/eq) --> + // setcc ((add X, -C1), ~(C0 - C1)), 0, ne/eq + SDValue OffsetC = DAG.getConstant(-C1Val, DL, OpVT); + SDValue Add = DAG.getNode(ISD::ADD, DL, OpVT, LL, OffsetC); + SDValue MaskC = DAG.getConstant(~(C0Val - C1Val), DL, OpVT); + SDValue And = DAG.getNode(ISD::AND, DL, OpVT, Add, MaskC); + SDValue Zero = DAG.getConstant(0, DL, OpVT); + return DAG.getSetCC(DL, VT, And, Zero, CC0); + } + } + } } // Canonicalize equivalent operands to LL == RL. @@ -4259,7 +4814,8 @@ bool DAGCombiner::isLegalNarrowLdSt(LSBaseSDNode *LDST, // Ensure that this isn't going to produce an unsupported unaligned access. if (ShAmt && !TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), MemVT, - LDST->getAddressSpace(), ShAmt / 8)) + LDST->getAddressSpace(), ShAmt / 8, + LDST->getMemOperand()->getFlags())) return false; // It's not possible to generate a constant of extended or untyped type. @@ -4316,9 +4872,7 @@ bool DAGCombiner::SearchForAndLoads(SDNode *N, SDNode *&NodeToMask) { // Recursively search for the operands, looking for loads which can be // narrowed. - for (unsigned i = 0, e = N->getNumOperands(); i < e; ++i) { - SDValue Op = N->getOperand(i); - + for (SDValue Op : N->op_values()) { if (Op.getValueType().isVector()) return false; @@ -4480,7 +5034,7 @@ SDValue DAGCombiner::unfoldExtremeBitClearingToShifts(SDNode *N) { SDValue N1 = N->getOperand(1); // Do we actually prefer shifts over mask? - if (!TLI.preferShiftsToClearExtremeBits(N0)) + if (!TLI.shouldFoldMaskToVariableShiftPair(N0)) return SDValue(); // Try to match (-1 '[outer] logical shift' y) @@ -4575,7 +5129,7 @@ SDValue DAGCombiner::visitAND(SDNode *N) { return NewSel; // reassociate and - if (SDValue RAND = ReassociateOps(ISD::AND, SDLoc(N), N0, N1, N->getFlags())) + if (SDValue RAND = reassociateOps(ISD::AND, SDLoc(N), N0, N1, N->getFlags())) return RAND; // Try to convert a constant mask AND into a shuffle clear mask. @@ -4644,24 +5198,22 @@ SDValue DAGCombiner::visitAND(SDNode *N) { // the first vector value and FF for the rest, repeating. We need a mask // that will apply equally to all members of the vector, so AND all the // lanes of the constant together. - EVT VT = Vector->getValueType(0); - unsigned BitWidth = VT.getScalarSizeInBits(); + unsigned EltBitWidth = Vector->getValueType(0).getScalarSizeInBits(); // If the splat value has been compressed to a bitlength lower // than the size of the vector lane, we need to re-expand it to // the lane size. - if (BitWidth > SplatBitSize) - for (SplatValue = SplatValue.zextOrTrunc(BitWidth); - SplatBitSize < BitWidth; - SplatBitSize = SplatBitSize * 2) + if (EltBitWidth > SplatBitSize) + for (SplatValue = SplatValue.zextOrTrunc(EltBitWidth); + SplatBitSize < EltBitWidth; SplatBitSize = SplatBitSize * 2) SplatValue |= SplatValue.shl(SplatBitSize); // Make sure that variable 'Constant' is only set if 'SplatBitSize' is a // multiple of 'BitWidth'. Otherwise, we could propagate a wrong value. - if (SplatBitSize % BitWidth == 0) { - Constant = APInt::getAllOnesValue(BitWidth); - for (unsigned i = 0, n = SplatBitSize/BitWidth; i < n; ++i) - Constant &= SplatValue.lshr(i*BitWidth).zextOrTrunc(BitWidth); + if ((SplatBitSize % EltBitWidth) == 0) { + Constant = APInt::getAllOnesValue(EltBitWidth); + for (unsigned i = 0, n = (SplatBitSize / EltBitWidth); i < n; ++i) + Constant &= SplatValue.extractBits(EltBitWidth, i * EltBitWidth); } } } @@ -4763,54 +5315,39 @@ SDValue DAGCombiner::visitAND(SDNode *N) { return SubRHS; if (SubRHS.getOpcode() == ISD::SIGN_EXTEND && SubRHS.getOperand(0).getScalarValueSizeInBits() == 1) - return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, SubRHS.getOperand(0)); - } - } - - // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1) - // fold (and (sra)) -> (and (srl)) when possible. - if (SimplifyDemandedBits(SDValue(N, 0))) - return SDValue(N, 0); - - // fold (zext_inreg (extload x)) -> (zextload x) - if (ISD::isEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode())) { - LoadSDNode *LN0 = cast(N0); - EVT MemVT = LN0->getMemoryVT(); - // If we zero all the possible extended bits, then we can turn this into - // a zextload if we are running before legalize or the operation is legal. - unsigned BitWidth = N1.getScalarValueSizeInBits(); - if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth, - BitWidth - MemVT.getScalarSizeInBits())) && - ((!LegalOperations && !LN0->isVolatile()) || - TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) { - SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, - LN0->getChain(), LN0->getBasePtr(), - MemVT, LN0->getMemOperand()); - AddToWorklist(N); - CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); - return SDValue(N, 0); // Return N so it doesn't get rechecked! + return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), VT, SubRHS.getOperand(0)); } } + + // fold (and (sign_extend_inreg x, i16 to i32), 1) -> (and x, 1) + // fold (and (sra)) -> (and (srl)) when possible. + if (SimplifyDemandedBits(SDValue(N, 0))) + return SDValue(N, 0); + + // fold (zext_inreg (extload x)) -> (zextload x) // fold (zext_inreg (sextload x)) -> (zextload x) iff load has one use - if (ISD::isSEXTLoad(N0.getNode()) && ISD::isUNINDEXEDLoad(N0.getNode()) && - N0.hasOneUse()) { + if (ISD::isUNINDEXEDLoad(N0.getNode()) && + (ISD::isEXTLoad(N0.getNode()) || + (ISD::isSEXTLoad(N0.getNode()) && N0.hasOneUse()))) { LoadSDNode *LN0 = cast(N0); EVT MemVT = LN0->getMemoryVT(); // If we zero all the possible extended bits, then we can turn this into // a zextload if we are running before legalize or the operation is legal. - unsigned BitWidth = N1.getScalarValueSizeInBits(); - if (DAG.MaskedValueIsZero(N1, APInt::getHighBitsSet(BitWidth, - BitWidth - MemVT.getScalarSizeInBits())) && + unsigned ExtBitSize = N1.getScalarValueSizeInBits(); + unsigned MemBitSize = MemVT.getScalarSizeInBits(); + APInt ExtBits = APInt::getHighBitsSet(ExtBitSize, ExtBitSize - MemBitSize); + if (DAG.MaskedValueIsZero(N1, ExtBits) && ((!LegalOperations && !LN0->isVolatile()) || TLI.isLoadExtLegal(ISD::ZEXTLOAD, VT, MemVT))) { - SDValue ExtLoad = DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, - LN0->getChain(), LN0->getBasePtr(), - MemVT, LN0->getMemOperand()); + SDValue ExtLoad = + DAG.getExtLoad(ISD::ZEXTLOAD, SDLoc(N0), VT, LN0->getChain(), + LN0->getBasePtr(), MemVT, LN0->getMemOperand()); AddToWorklist(N); CombineTo(N0.getNode(), ExtLoad, ExtLoad.getValue(1)); - return SDValue(N, 0); // Return N so it doesn't get rechecked! + return SDValue(N, 0); // Return N so it doesn't get rechecked! } } + // fold (and (or (srl N, 8), (shl N, 8)), 0xffff) -> (srl (bswap N), const) if (N1C && N1C->getAPIntValue() == 0xffff && N0.getOpcode() == ISD::OR) { if (SDValue BSwap = MatchBSwapHWordLow(N0.getNode(), N0.getOperand(0), @@ -5155,6 +5692,23 @@ SDValue DAGCombiner::visitORLike(SDValue N0, SDValue N1, SDNode *N) { return SDValue(); } +/// OR combines for which the commuted variant will be tried as well. +static SDValue visitORCommutative( + SelectionDAG &DAG, SDValue N0, SDValue N1, SDNode *N) { + EVT VT = N0.getValueType(); + if (N0.getOpcode() == ISD::AND) { + // fold (or (and X, (xor Y, -1)), Y) -> (or X, Y) + if (isBitwiseNot(N0.getOperand(1)) && N0.getOperand(1).getOperand(0) == N1) + return DAG.getNode(ISD::OR, SDLoc(N), VT, N0.getOperand(0), N1); + + // fold (or (and (xor Y, -1), X), Y) -> (or X, Y) + if (isBitwiseNot(N0.getOperand(0)) && N0.getOperand(0).getOperand(0) == N1) + return DAG.getNode(ISD::OR, SDLoc(N), VT, N0.getOperand(1), N1); + } + + return SDValue(); +} + SDValue DAGCombiner::visitOR(SDNode *N) { SDValue N0 = N->getOperand(0); SDValue N1 = N->getOperand(1); @@ -5284,7 +5838,7 @@ SDValue DAGCombiner::visitOR(SDNode *N) { return BSwap; // reassociate or - if (SDValue ROR = ReassociateOps(ISD::OR, SDLoc(N), N0, N1, N->getFlags())) + if (SDValue ROR = reassociateOps(ISD::OR, SDLoc(N), N0, N1, N->getFlags())) return ROR; // Canonicalize (or (and X, c1), c2) -> (and (or X, c2), c1|c2) @@ -5302,6 +5856,11 @@ SDValue DAGCombiner::visitOR(SDNode *N) { } } + if (SDValue Combined = visitORCommutative(DAG, N0, N1, N)) + return Combined; + if (SDValue Combined = visitORCommutative(DAG, N1, N0, N)) + return Combined; + // Simplify: (or (op x...), (op y...)) -> (op (or x, y)) if (N0.getOpcode() == N1.getOpcode()) if (SDValue V = hoistLogicOpWithSameOpcodeHands(N)) @@ -5318,6 +5877,12 @@ SDValue DAGCombiner::visitOR(SDNode *N) { if (SimplifyDemandedBits(SDValue(N, 0))) return SDValue(N, 0); + // If OR can be rewritten into ADD, try combines based on ADD. + if ((!LegalOperations || TLI.isOperationLegal(ISD::ADD, VT)) && + DAG.haveNoCommonBitsSet(N0, N1)) + if (SDValue Combined = visitADDLike(N)) + return Combined; + return SDValue(); } @@ -5869,6 +6434,213 @@ calculateByteProvider(SDValue Op, unsigned Index, unsigned Depth, return None; } +static unsigned LittleEndianByteAt(unsigned BW, unsigned i) { + return i; +} + +static unsigned BigEndianByteAt(unsigned BW, unsigned i) { + return BW - i - 1; +} + +// Check if the bytes offsets we are looking at match with either big or +// little endian value loaded. Return true for big endian, false for little +// endian, and None if match failed. +static Optional isBigEndian(const SmallVector &ByteOffsets, + int64_t FirstOffset) { + // The endian can be decided only when it is 2 bytes at least. + unsigned Width = ByteOffsets.size(); + if (Width < 2) + return None; + + bool BigEndian = true, LittleEndian = true; + for (unsigned i = 0; i < Width; i++) { + int64_t CurrentByteOffset = ByteOffsets[i] - FirstOffset; + LittleEndian &= CurrentByteOffset == LittleEndianByteAt(Width, i); + BigEndian &= CurrentByteOffset == BigEndianByteAt(Width, i); + if (!BigEndian && !LittleEndian) + return None; + } + + assert((BigEndian != LittleEndian) && "It should be either big endian or" + "little endian"); + return BigEndian; +} + +static SDValue stripTruncAndExt(SDValue Value) { + switch (Value.getOpcode()) { + case ISD::TRUNCATE: + case ISD::ZERO_EXTEND: + case ISD::SIGN_EXTEND: + case ISD::ANY_EXTEND: + return stripTruncAndExt(Value.getOperand(0)); + } + return Value; +} + +/// Match a pattern where a wide type scalar value is stored by several narrow +/// stores. Fold it into a single store or a BSWAP and a store if the targets +/// supports it. +/// +/// Assuming little endian target: +/// i8 *p = ... +/// i32 val = ... +/// p[0] = (val >> 0) & 0xFF; +/// p[1] = (val >> 8) & 0xFF; +/// p[2] = (val >> 16) & 0xFF; +/// p[3] = (val >> 24) & 0xFF; +/// => +/// *((i32)p) = val; +/// +/// i8 *p = ... +/// i32 val = ... +/// p[0] = (val >> 24) & 0xFF; +/// p[1] = (val >> 16) & 0xFF; +/// p[2] = (val >> 8) & 0xFF; +/// p[3] = (val >> 0) & 0xFF; +/// => +/// *((i32)p) = BSWAP(val); +SDValue DAGCombiner::MatchStoreCombine(StoreSDNode *N) { + // Collect all the stores in the chain. + SDValue Chain; + SmallVector Stores; + for (StoreSDNode *Store = N; Store; Store = dyn_cast(Chain)) { + if (Store->getMemoryVT() != MVT::i8 || + Store->isVolatile() || Store->isIndexed()) + return SDValue(); + Stores.push_back(Store); + Chain = Store->getChain(); + } + // Handle the simple type only. + unsigned Width = Stores.size(); + EVT VT = EVT::getIntegerVT( + *DAG.getContext(), Width * N->getMemoryVT().getSizeInBits()); + if (VT != MVT::i16 && VT != MVT::i32 && VT != MVT::i64) + return SDValue(); + + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + if (LegalOperations && !TLI.isOperationLegal(ISD::STORE, VT)) + return SDValue(); + + // Check if all the bytes of the combined value we are looking at are stored + // to the same base address. Collect bytes offsets from Base address into + // ByteOffsets. + SDValue CombinedValue; + SmallVector ByteOffsets(Width, INT64_MAX); + int64_t FirstOffset = INT64_MAX; + StoreSDNode *FirstStore = nullptr; + Optional Base; + for (auto Store : Stores) { + // All the stores store different byte of the CombinedValue. A truncate is + // required to get that byte value. + SDValue Trunc = Store->getValue(); + if (Trunc.getOpcode() != ISD::TRUNCATE) + return SDValue(); + // A shift operation is required to get the right byte offset, except the + // first byte. + int64_t Offset = 0; + SDValue Value = Trunc.getOperand(0); + if (Value.getOpcode() == ISD::SRL || + Value.getOpcode() == ISD::SRA) { + ConstantSDNode *ShiftOffset = + dyn_cast(Value.getOperand(1)); + // Trying to match the following pattern. The shift offset must be + // a constant and a multiple of 8. It is the byte offset in "y". + // + // x = srl y, offset + // i8 z = trunc x + // store z, ... + if (!ShiftOffset || (ShiftOffset->getSExtValue() % 8)) + return SDValue(); + + Offset = ShiftOffset->getSExtValue()/8; + Value = Value.getOperand(0); + } + + // Stores must share the same combined value with different offsets. + if (!CombinedValue) + CombinedValue = Value; + else if (stripTruncAndExt(CombinedValue) != stripTruncAndExt(Value)) + return SDValue(); + + // The trunc and all the extend operation should be stripped to get the + // real value we are stored. + else if (CombinedValue.getValueType() != VT) { + if (Value.getValueType() == VT || + Value.getValueSizeInBits() > CombinedValue.getValueSizeInBits()) + CombinedValue = Value; + // Give up if the combined value type is smaller than the store size. + if (CombinedValue.getValueSizeInBits() < VT.getSizeInBits()) + return SDValue(); + } + + // Stores must share the same base address + BaseIndexOffset Ptr = BaseIndexOffset::match(Store, DAG); + int64_t ByteOffsetFromBase = 0; + if (!Base) + Base = Ptr; + else if (!Base->equalBaseIndex(Ptr, DAG, ByteOffsetFromBase)) + return SDValue(); + + // Remember the first byte store + if (ByteOffsetFromBase < FirstOffset) { + FirstStore = Store; + FirstOffset = ByteOffsetFromBase; + } + // Map the offset in the store and the offset in the combined value, and + // early return if it has been set before. + if (Offset < 0 || Offset >= Width || ByteOffsets[Offset] != INT64_MAX) + return SDValue(); + ByteOffsets[Offset] = ByteOffsetFromBase; + } + + assert(FirstOffset != INT64_MAX && "First byte offset must be set"); + assert(FirstStore && "First store must be set"); + + // Check if the bytes of the combined value we are looking at match with + // either big or little endian value store. + Optional IsBigEndian = isBigEndian(ByteOffsets, FirstOffset); + if (!IsBigEndian.hasValue()) + return SDValue(); + + // The node we are looking at matches with the pattern, check if we can + // replace it with a single bswap if needed and store. + + // If the store needs byte swap check if the target supports it + bool NeedsBswap = DAG.getDataLayout().isBigEndian() != *IsBigEndian; + + // Before legalize we can introduce illegal bswaps which will be later + // converted to an explicit bswap sequence. This way we end up with a single + // store and byte shuffling instead of several stores and byte shuffling. + if (NeedsBswap && LegalOperations && !TLI.isOperationLegal(ISD::BSWAP, VT)) + return SDValue(); + + // Check that a store of the wide type is both allowed and fast on the target + bool Fast = false; + bool Allowed = + TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT, + *FirstStore->getMemOperand(), &Fast); + if (!Allowed || !Fast) + return SDValue(); + + if (VT != CombinedValue.getValueType()) { + assert(CombinedValue.getValueType().getSizeInBits() > VT.getSizeInBits() && + "Get unexpected store value to combine"); + CombinedValue = DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, + CombinedValue); + } + + if (NeedsBswap) + CombinedValue = DAG.getNode(ISD::BSWAP, SDLoc(N), VT, CombinedValue); + + SDValue NewStore = + DAG.getStore(Chain, SDLoc(N), CombinedValue, FirstStore->getBasePtr(), + FirstStore->getPointerInfo(), FirstStore->getAlignment()); + + // Rely on other DAG combine rules to remove the other individual stores. + DAG.ReplaceAllUsesWith(N, NewStore.getNode()); + return NewStore; +} + /// Match a pattern where a wide type scalar value is loaded by several narrow /// loads and combined by shifts and ors. Fold it into a single load or a load /// and a BSWAP if the targets supports it. @@ -5916,11 +6688,6 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) { if (LegalOperations && !TLI.isOperationLegal(ISD::LOAD, VT)) return SDValue(); - std::function LittleEndianByteAt = []( - unsigned BW, unsigned i) { return i; }; - std::function BigEndianByteAt = []( - unsigned BW, unsigned i) { return BW - i - 1; }; - bool IsBigEndianTarget = DAG.getDataLayout().isBigEndian(); auto MemoryByteOffset = [&] (ByteProvider P) { assert(P.isMemory() && "Must be a memory byte provider"); @@ -5987,15 +6754,10 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) { // Check if the bytes of the OR we are looking at match with either big or // little endian value load - bool BigEndian = true, LittleEndian = true; - for (unsigned i = 0; i < ByteWidth; i++) { - int64_t CurrentByteOffset = ByteOffsets[i] - FirstOffset; - LittleEndian &= CurrentByteOffset == LittleEndianByteAt(ByteWidth, i); - BigEndian &= CurrentByteOffset == BigEndianByteAt(ByteWidth, i); - if (!BigEndian && !LittleEndian) - return SDValue(); - } - assert((BigEndian != LittleEndian) && "should be either or"); + Optional IsBigEndian = isBigEndian(ByteOffsets, FirstOffset); + if (!IsBigEndian.hasValue()) + return SDValue(); + assert(FirstByteProvider && "must be set"); // Ensure that the first byte is loaded from zero offset of the first load. @@ -6008,7 +6770,7 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) { // replace it with a single load and bswap if needed. // If the load needs byte swap check if the target supports it - bool NeedsBswap = IsBigEndianTarget != BigEndian; + bool NeedsBswap = IsBigEndianTarget != *IsBigEndian; // Before legalize we can introduce illegal bswaps which will be later // converted to an explicit bswap sequence. This way we end up with a single @@ -6019,8 +6781,7 @@ SDValue DAGCombiner::MatchLoadCombine(SDNode *N) { // Check that a load of the wide type is both allowed and fast on the target bool Fast = false; bool Allowed = TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), - VT, FirstLoad->getAddressSpace(), - FirstLoad->getAlignment(), &Fast); + VT, *FirstLoad->getMemOperand(), &Fast); if (!Allowed || !Fast) return SDValue(); @@ -6160,7 +6921,7 @@ SDValue DAGCombiner::visitXOR(SDNode *N) { return NewSel; // reassociate xor - if (SDValue RXOR = ReassociateOps(ISD::XOR, DL, N0, N1, N->getFlags())) + if (SDValue RXOR = reassociateOps(ISD::XOR, DL, N0, N1, N->getFlags())) return RXOR; // fold !(x cc y) -> (x !cc y) @@ -6218,6 +6979,16 @@ SDValue DAGCombiner::visitXOR(SDNode *N) { return DAG.getNode(NewOpcode, DL, VT, LHS, RHS); } } + + // fold (not (neg x)) -> (add X, -1) + // FIXME: This can be generalized to (not (sub Y, X)) -> (add X, ~Y) if + // Y is a constant or the subtract has a single use. + if (isAllOnesConstant(N1) && N0.getOpcode() == ISD::SUB && + isNullConstant(N0.getOperand(0))) { + return DAG.getNode(ISD::ADD, DL, VT, N0.getOperand(1), + DAG.getAllOnesConstant(DL, VT)); + } + // fold (xor (and x, y), y) -> (and (not x), y) if (N0Opcode == ISD::AND && N0.hasOneUse() && N0->getOperand(1) == N1) { SDValue X = N0.getOperand(0); @@ -6310,11 +7081,16 @@ SDValue DAGCombiner::visitXOR(SDNode *N) { /// Handle transforms common to the three shifts, when the shift amount is a /// constant. +/// We are looking for: (shift being one of shl/sra/srl) +/// shift (binop X, C0), C1 +/// And want to transform into: +/// binop (shift X, C1), (shift C0, C1) SDValue DAGCombiner::visitShiftByConstant(SDNode *N, ConstantSDNode *Amt) { // Do not turn a 'not' into a regular xor. if (isBitwiseNot(N->getOperand(0))) return SDValue(); + // The inner binop must be one-use, since we want to replace it. SDNode *LHS = N->getOperand(0).getNode(); if (!LHS->hasOneUse()) return SDValue(); @@ -6322,56 +7098,43 @@ SDValue DAGCombiner::visitShiftByConstant(SDNode *N, ConstantSDNode *Amt) { // instead of (shift (and)), likewise for add, or, xor, etc. This sort of // thing happens with address calculations, so it's important to canonicalize // it. - bool HighBitSet = false; // Can we transform this if the high bit is set? - switch (LHS->getOpcode()) { - default: return SDValue(); + default: + return SDValue(); case ISD::OR: case ISD::XOR: - HighBitSet = false; // We can only transform sra if the high bit is clear. - break; case ISD::AND: - HighBitSet = true; // We can only transform sra if the high bit is set. break; case ISD::ADD: if (N->getOpcode() != ISD::SHL) return SDValue(); // only shl(add) not sr[al](add). - HighBitSet = false; // We can only transform sra if the high bit is clear. break; } // We require the RHS of the binop to be a constant and not opaque as well. ConstantSDNode *BinOpCst = getAsNonOpaqueConstant(LHS->getOperand(1)); - if (!BinOpCst) return SDValue(); + if (!BinOpCst) + return SDValue(); // FIXME: disable this unless the input to the binop is a shift by a constant - // or is copy/select.Enable this in other cases when figure out it's exactly profitable. - SDNode *BinOpLHSVal = LHS->getOperand(0).getNode(); - bool isShift = BinOpLHSVal->getOpcode() == ISD::SHL || - BinOpLHSVal->getOpcode() == ISD::SRA || - BinOpLHSVal->getOpcode() == ISD::SRL; - bool isCopyOrSelect = BinOpLHSVal->getOpcode() == ISD::CopyFromReg || - BinOpLHSVal->getOpcode() == ISD::SELECT; - - if ((!isShift || !isa(BinOpLHSVal->getOperand(1))) && - !isCopyOrSelect) + // or is copy/select. Enable this in other cases when figure out it's exactly + // profitable. + SDValue BinOpLHSVal = LHS->getOperand(0); + bool IsShiftByConstant = (BinOpLHSVal.getOpcode() == ISD::SHL || + BinOpLHSVal.getOpcode() == ISD::SRA || + BinOpLHSVal.getOpcode() == ISD::SRL) && + isa(BinOpLHSVal.getOperand(1)); + bool IsCopyOrSelect = BinOpLHSVal.getOpcode() == ISD::CopyFromReg || + BinOpLHSVal.getOpcode() == ISD::SELECT; + + if (!IsShiftByConstant && !IsCopyOrSelect) return SDValue(); - if (isCopyOrSelect && N->hasOneUse()) + if (IsCopyOrSelect && N->hasOneUse()) return SDValue(); EVT VT = N->getValueType(0); - // If this is a signed shift right, and the high bit is modified by the - // logical operation, do not perform the transformation. The highBitSet - // boolean indicates the value of the high bit of the constant which would - // cause it to be modified for this operation. - if (N->getOpcode() == ISD::SRA) { - bool BinOpRHSSignSet = BinOpCst->getAPIntValue().isNegative(); - if (BinOpRHSSignSet != HighBitSet) - return SDValue(); - } - if (!TLI.isDesirableToCommuteWithShift(N, Level)) return SDValue(); @@ -6395,11 +7158,12 @@ SDValue DAGCombiner::distributeTruncateThroughAnd(SDNode *N) { assert(N->getOperand(0).getOpcode() == ISD::AND); // (truncate:TruncVT (and N00, N01C)) -> (and (truncate:TruncVT N00), TruncC) - if (N->hasOneUse() && N->getOperand(0).hasOneUse()) { + EVT TruncVT = N->getValueType(0); + if (N->hasOneUse() && N->getOperand(0).hasOneUse() && + TLI.isTypeDesirableForOp(ISD::AND, TruncVT)) { SDValue N01 = N->getOperand(0).getOperand(1); if (isConstantOrConstantVector(N01, /* NoOpaques */ true)) { SDLoc DL(N); - EVT TruncVT = N->getValueType(0); SDValue N00 = N->getOperand(0).getOperand(0); SDValue Trunc00 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N00); SDValue Trunc01 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N01); @@ -6431,6 +7195,7 @@ SDValue DAGCombiner::visitRotate(SDNode *N) { } // fold (rot x, c) -> (rot x, c % BitSize) + // TODO - support non-uniform vector amounts. if (ConstantSDNode *Cst = isConstOrConstSplat(N1)) { if (Cst->getAPIntValue().uge(Bitsize)) { uint64_t RotAmt = Cst->getAPIntValue().urem(Bitsize); @@ -6476,6 +7241,7 @@ SDValue DAGCombiner::visitSHL(SDNode *N) { return V; EVT VT = N0.getValueType(); + EVT ShiftVT = N1.getValueType(); unsigned OpSizeInBits = VT.getScalarSizeInBits(); // fold vector ops @@ -6506,6 +7272,7 @@ SDValue DAGCombiner::visitSHL(SDNode *N) { ConstantSDNode *N1C = isConstOrConstSplat(N1); // fold (shl c1, c2) -> c1<isOpaque()) return DAG.FoldConstantArithmetic(ISD::SHL, SDLoc(N), VT, N0C, N1C); @@ -6517,6 +7284,7 @@ SDValue DAGCombiner::visitSHL(SDNode *N) { if (DAG.MaskedValueIsZero(SDValue(N, 0), APInt::getAllOnesValue(OpSizeInBits))) return DAG.getConstant(0, SDLoc(N), VT); + // fold (shl x, (trunc (and y, c))) -> (shl x, (and (trunc y), (trunc c))). if (N1.getOpcode() == ISD::TRUNCATE && N1.getOperand(0).getOpcode() == ISD::AND) { @@ -6524,6 +7292,7 @@ SDValue DAGCombiner::visitSHL(SDNode *N) { return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, NewOp1); } + // TODO - support non-uniform vector shift amounts. if (N1C && SimplifyDemandedBits(SDValue(N, 0))) return SDValue(N, 0); @@ -6548,69 +7317,86 @@ SDValue DAGCombiner::visitSHL(SDNode *N) { }; if (ISD::matchBinaryPredicate(N1, N0.getOperand(1), MatchInRange)) { SDLoc DL(N); - EVT ShiftVT = N1.getValueType(); SDValue Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, N1, N0.getOperand(1)); return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), Sum); } } - // fold (shl (ext (shl x, c1)), c2) -> (ext (shl x, (add c1, c2))) + // fold (shl (ext (shl x, c1)), c2) -> (shl (ext x), (add c1, c2)) // For this to be valid, the second form must not preserve any of the bits // that are shifted out by the inner shift in the first form. This means // the outer shift size must be >= the number of bits added by the ext. // As a corollary, we don't care what kind of ext it is. - if (N1C && (N0.getOpcode() == ISD::ZERO_EXTEND || - N0.getOpcode() == ISD::ANY_EXTEND || - N0.getOpcode() == ISD::SIGN_EXTEND) && + if ((N0.getOpcode() == ISD::ZERO_EXTEND || + N0.getOpcode() == ISD::ANY_EXTEND || + N0.getOpcode() == ISD::SIGN_EXTEND) && N0.getOperand(0).getOpcode() == ISD::SHL) { SDValue N0Op0 = N0.getOperand(0); - if (ConstantSDNode *N0Op0C1 = isConstOrConstSplat(N0Op0.getOperand(1))) { - APInt c1 = N0Op0C1->getAPIntValue(); - APInt c2 = N1C->getAPIntValue(); - zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */); + SDValue InnerShiftAmt = N0Op0.getOperand(1); + EVT InnerVT = N0Op0.getValueType(); + uint64_t InnerBitwidth = InnerVT.getScalarSizeInBits(); - EVT InnerShiftVT = N0Op0.getValueType(); - uint64_t InnerShiftSize = InnerShiftVT.getScalarSizeInBits(); - if (c2.uge(OpSizeInBits - InnerShiftSize)) { - SDLoc DL(N0); - APInt Sum = c1 + c2; - if (Sum.uge(OpSizeInBits)) - return DAG.getConstant(0, DL, VT); + auto MatchOutOfRange = [OpSizeInBits, InnerBitwidth](ConstantSDNode *LHS, + ConstantSDNode *RHS) { + APInt c1 = LHS->getAPIntValue(); + APInt c2 = RHS->getAPIntValue(); + zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */); + return c2.uge(OpSizeInBits - InnerBitwidth) && + (c1 + c2).uge(OpSizeInBits); + }; + if (ISD::matchBinaryPredicate(InnerShiftAmt, N1, MatchOutOfRange, + /*AllowUndefs*/ false, + /*AllowTypeMismatch*/ true)) + return DAG.getConstant(0, SDLoc(N), VT); - return DAG.getNode( - ISD::SHL, DL, VT, - DAG.getNode(N0.getOpcode(), DL, VT, N0Op0->getOperand(0)), - DAG.getConstant(Sum.getZExtValue(), DL, N1.getValueType())); - } + auto MatchInRange = [OpSizeInBits, InnerBitwidth](ConstantSDNode *LHS, + ConstantSDNode *RHS) { + APInt c1 = LHS->getAPIntValue(); + APInt c2 = RHS->getAPIntValue(); + zeroExtendToMatch(c1, c2, 1 /* Overflow Bit */); + return c2.uge(OpSizeInBits - InnerBitwidth) && + (c1 + c2).ult(OpSizeInBits); + }; + if (ISD::matchBinaryPredicate(InnerShiftAmt, N1, MatchInRange, + /*AllowUndefs*/ false, + /*AllowTypeMismatch*/ true)) { + SDLoc DL(N); + SDValue Ext = DAG.getNode(N0.getOpcode(), DL, VT, N0Op0.getOperand(0)); + SDValue Sum = DAG.getZExtOrTrunc(InnerShiftAmt, DL, ShiftVT); + Sum = DAG.getNode(ISD::ADD, DL, ShiftVT, Sum, N1); + return DAG.getNode(ISD::SHL, DL, VT, Ext, Sum); } } // fold (shl (zext (srl x, C)), C) -> (zext (shl (srl x, C), C)) // Only fold this if the inner zext has no other uses to avoid increasing // the total number of instructions. - if (N1C && N0.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse() && + if (N0.getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse() && N0.getOperand(0).getOpcode() == ISD::SRL) { SDValue N0Op0 = N0.getOperand(0); - if (ConstantSDNode *N0Op0C1 = isConstOrConstSplat(N0Op0.getOperand(1))) { - if (N0Op0C1->getAPIntValue().ult(VT.getScalarSizeInBits())) { - uint64_t c1 = N0Op0C1->getZExtValue(); - uint64_t c2 = N1C->getZExtValue(); - if (c1 == c2) { - SDValue NewOp0 = N0.getOperand(0); - EVT CountVT = NewOp0.getOperand(1).getValueType(); - SDLoc DL(N); - SDValue NewSHL = DAG.getNode(ISD::SHL, DL, NewOp0.getValueType(), - NewOp0, - DAG.getConstant(c2, DL, CountVT)); - AddToWorklist(NewSHL.getNode()); - return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL); - } - } + SDValue InnerShiftAmt = N0Op0.getOperand(1); + + auto MatchEqual = [VT](ConstantSDNode *LHS, ConstantSDNode *RHS) { + APInt c1 = LHS->getAPIntValue(); + APInt c2 = RHS->getAPIntValue(); + zeroExtendToMatch(c1, c2); + return c1.ult(VT.getScalarSizeInBits()) && (c1 == c2); + }; + if (ISD::matchBinaryPredicate(InnerShiftAmt, N1, MatchEqual, + /*AllowUndefs*/ false, + /*AllowTypeMismatch*/ true)) { + SDLoc DL(N); + EVT InnerShiftAmtVT = N0Op0.getOperand(1).getValueType(); + SDValue NewSHL = DAG.getZExtOrTrunc(N1, DL, InnerShiftAmtVT); + NewSHL = DAG.getNode(ISD::SHL, DL, N0Op0.getValueType(), N0Op0, NewSHL); + AddToWorklist(NewSHL.getNode()); + return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N0), VT, NewSHL); } } // fold (shl (sr[la] exact X, C1), C2) -> (shl X, (C2-C1)) if C1 <= C2 // fold (shl (sr[la] exact X, C1), C2) -> (sr[la] X, (C2-C1)) if C1 > C2 + // TODO - support non-uniform vector shift amounts. if (N1C && (N0.getOpcode() == ISD::SRL || N0.getOpcode() == ISD::SRA) && N0->getFlags().hasExact()) { if (ConstantSDNode *N0C1 = isConstOrConstSplat(N0.getOperand(1))) { @@ -6619,9 +7405,9 @@ SDValue DAGCombiner::visitSHL(SDNode *N) { SDLoc DL(N); if (C1 <= C2) return DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), - DAG.getConstant(C2 - C1, DL, N1.getValueType())); + DAG.getConstant(C2 - C1, DL, ShiftVT)); return DAG.getNode(N0.getOpcode(), DL, VT, N0.getOperand(0), - DAG.getConstant(C1 - C2, DL, N1.getValueType())); + DAG.getConstant(C1 - C2, DL, ShiftVT)); } } @@ -6629,11 +7415,13 @@ SDValue DAGCombiner::visitSHL(SDNode *N) { // (and (srl x, (sub c1, c2), MASK) // Only fold this if the inner shift has no other uses -- if it does, folding // this will increase the total number of instructions. + // TODO - drop hasOneUse requirement if c1 == c2? + // TODO - support non-uniform vector shift amounts. if (N1C && N0.getOpcode() == ISD::SRL && N0.hasOneUse() && - TLI.shouldFoldShiftPairToMask(N, Level)) { + TLI.shouldFoldConstantShiftPairToMask(N, Level)) { if (ConstantSDNode *N0C1 = isConstOrConstSplat(N0.getOperand(1))) { - uint64_t c1 = N0C1->getZExtValue(); - if (c1 < OpSizeInBits) { + if (N0C1->getAPIntValue().ult(OpSizeInBits)) { + uint64_t c1 = N0C1->getZExtValue(); uint64_t c2 = N1C->getZExtValue(); APInt Mask = APInt::getHighBitsSet(OpSizeInBits, OpSizeInBits - c1); SDValue Shift; @@ -6641,12 +7429,12 @@ SDValue DAGCombiner::visitSHL(SDNode *N) { Mask <<= c2 - c1; SDLoc DL(N); Shift = DAG.getNode(ISD::SHL, DL, VT, N0.getOperand(0), - DAG.getConstant(c2 - c1, DL, N1.getValueType())); + DAG.getConstant(c2 - c1, DL, ShiftVT)); } else { Mask.lshrInPlace(c1 - c2); SDLoc DL(N); Shift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), - DAG.getConstant(c1 - c2, DL, N1.getValueType())); + DAG.getConstant(c1 - c2, DL, ShiftVT)); } SDLoc DL(N0); return DAG.getNode(ISD::AND, DL, VT, Shift, @@ -6719,6 +7507,7 @@ SDValue DAGCombiner::visitSRA(SDNode *N) { ConstantSDNode *N1C = isConstOrConstSplat(N1); // fold (sra c1, c2) -> (sra c1, c2) + // TODO - support non-uniform vector shift amounts. ConstantSDNode *N0C = getAsNonOpaqueConstant(N0); if (N0C && N1C && !N1C->isOpaque()) return DAG.FoldConstantArithmetic(ISD::SRA, SDLoc(N), VT, N0C, N1C); @@ -6815,32 +7604,32 @@ SDValue DAGCombiner::visitSRA(SDNode *N) { return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0, NewOp1); } + // fold (sra (trunc (sra x, c1)), c2) -> (trunc (sra x, c1 + c2)) // fold (sra (trunc (srl x, c1)), c2) -> (trunc (sra x, c1 + c2)) // if c1 is equal to the number of bits the trunc removes + // TODO - support non-uniform vector shift amounts. if (N0.getOpcode() == ISD::TRUNCATE && (N0.getOperand(0).getOpcode() == ISD::SRL || N0.getOperand(0).getOpcode() == ISD::SRA) && N0.getOperand(0).hasOneUse() && - N0.getOperand(0).getOperand(1).hasOneUse() && - N1C) { + N0.getOperand(0).getOperand(1).hasOneUse() && N1C) { SDValue N0Op0 = N0.getOperand(0); if (ConstantSDNode *LargeShift = isConstOrConstSplat(N0Op0.getOperand(1))) { - unsigned LargeShiftVal = LargeShift->getZExtValue(); EVT LargeVT = N0Op0.getValueType(); - - if (LargeVT.getScalarSizeInBits() - OpSizeInBits == LargeShiftVal) { + unsigned TruncBits = LargeVT.getScalarSizeInBits() - OpSizeInBits; + if (LargeShift->getAPIntValue() == TruncBits) { SDLoc DL(N); - SDValue Amt = - DAG.getConstant(LargeShiftVal + N1C->getZExtValue(), DL, - getShiftAmountTy(N0Op0.getOperand(0).getValueType())); - SDValue SRA = DAG.getNode(ISD::SRA, DL, LargeVT, - N0Op0.getOperand(0), Amt); + SDValue Amt = DAG.getConstant(N1C->getZExtValue() + TruncBits, DL, + getShiftAmountTy(LargeVT)); + SDValue SRA = + DAG.getNode(ISD::SRA, DL, LargeVT, N0Op0.getOperand(0), Amt); return DAG.getNode(ISD::TRUNCATE, DL, VT, SRA); } } } // Simplify, based on bits shifted out of the LHS. + // TODO - support non-uniform vector shift amounts. if (N1C && SimplifyDemandedBits(SDValue(N, 0))) return SDValue(N, 0); @@ -6872,6 +7661,7 @@ SDValue DAGCombiner::visitSRL(SDNode *N) { ConstantSDNode *N1C = isConstOrConstSplat(N1); // fold (srl c1, c2) -> c1 >>u c2 + // TODO - support non-uniform vector shift amounts. ConstantSDNode *N0C = getAsNonOpaqueConstant(N0); if (N0C && N1C && !N1C->isOpaque()) return DAG.FoldConstantArithmetic(ISD::SRL, SDLoc(N), VT, N0C, N1C); @@ -6912,6 +7702,7 @@ SDValue DAGCombiner::visitSRL(SDNode *N) { } // fold (srl (trunc (srl x, c1)), c2) -> 0 or (trunc (srl x, (add c1, c2))) + // TODO - support non-uniform vector shift amounts. if (N1C && N0.getOpcode() == ISD::TRUNCATE && N0.getOperand(0).getOpcode() == ISD::SRL) { if (auto N001C = isConstOrConstSplat(N0.getOperand(0).getOperand(1))) { @@ -6935,6 +7726,7 @@ SDValue DAGCombiner::visitSRL(SDNode *N) { } // fold (srl (shl x, c), c) -> (and x, cst2) + // TODO - (srl (shl x, c1), c2). if (N0.getOpcode() == ISD::SHL && N0.getOperand(1) == N1 && isConstantOrConstantVector(N1, /* NoOpaques */ true)) { SDLoc DL(N); @@ -6945,11 +7737,12 @@ SDValue DAGCombiner::visitSRL(SDNode *N) { } // fold (srl (anyextend x), c) -> (and (anyextend (srl x, c)), mask) + // TODO - support non-uniform vector shift amounts. if (N1C && N0.getOpcode() == ISD::ANY_EXTEND) { // Shifting in all undef bits? EVT SmallVT = N0.getOperand(0).getValueType(); unsigned BitSize = SmallVT.getScalarSizeInBits(); - if (N1C->getZExtValue() >= BitSize) + if (N1C->getAPIntValue().uge(BitSize)) return DAG.getUNDEF(VT); if (!LegalTypes || TLI.isTypeDesirableForOp(ISD::SRL, SmallVT)) { @@ -6970,7 +7763,7 @@ SDValue DAGCombiner::visitSRL(SDNode *N) { // fold (srl (sra X, Y), 31) -> (srl X, 31). This srl only looks at the sign // bit, which is unmodified by sra. - if (N1C && N1C->getZExtValue() + 1 == OpSizeInBits) { + if (N1C && N1C->getAPIntValue() == (OpSizeInBits - 1)) { if (N0.getOpcode() == ISD::SRA) return DAG.getNode(ISD::SRL, SDLoc(N), VT, N0.getOperand(0), N1); } @@ -7021,6 +7814,7 @@ SDValue DAGCombiner::visitSRL(SDNode *N) { // fold operands of srl based on knowledge that the low bits are not // demanded. + // TODO - support non-uniform vector shift amounts. if (N1C && SimplifyDemandedBits(SDValue(N, 0))) return SDValue(N, 0); @@ -7079,13 +7873,49 @@ SDValue DAGCombiner::visitFunnelShift(SDNode *N) { N2, APInt(N2.getScalarValueSizeInBits(), BitWidth - 1))) return IsFSHL ? N0 : N1; - // fold (fsh* N0, N1, c) -> (fsh* N0, N1, c % BitWidth) + auto IsUndefOrZero = [](SDValue V) { + return V.isUndef() || isNullOrNullSplat(V, /*AllowUndefs*/ true); + }; + + // TODO - support non-uniform vector shift amounts. if (ConstantSDNode *Cst = isConstOrConstSplat(N2)) { + EVT ShAmtTy = N2.getValueType(); + + // fold (fsh* N0, N1, c) -> (fsh* N0, N1, c % BitWidth) if (Cst->getAPIntValue().uge(BitWidth)) { uint64_t RotAmt = Cst->getAPIntValue().urem(BitWidth); return DAG.getNode(N->getOpcode(), SDLoc(N), VT, N0, N1, - DAG.getConstant(RotAmt, SDLoc(N), N2.getValueType())); + DAG.getConstant(RotAmt, SDLoc(N), ShAmtTy)); } + + unsigned ShAmt = Cst->getZExtValue(); + if (ShAmt == 0) + return IsFSHL ? N0 : N1; + + // fold fshl(undef_or_zero, N1, C) -> lshr(N1, BW-C) + // fold fshr(undef_or_zero, N1, C) -> lshr(N1, C) + // fold fshl(N0, undef_or_zero, C) -> shl(N0, C) + // fold fshr(N0, undef_or_zero, C) -> shl(N0, BW-C) + if (IsUndefOrZero(N0)) + return DAG.getNode(ISD::SRL, SDLoc(N), VT, N1, + DAG.getConstant(IsFSHL ? BitWidth - ShAmt : ShAmt, + SDLoc(N), ShAmtTy)); + if (IsUndefOrZero(N1)) + return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, + DAG.getConstant(IsFSHL ? ShAmt : BitWidth - ShAmt, + SDLoc(N), ShAmtTy)); + } + + // fold fshr(undef_or_zero, N1, N2) -> lshr(N1, N2) + // fold fshl(N0, undef_or_zero, N2) -> shl(N0, N2) + // iff We know the shift amount is in range. + // TODO: when is it worth doing SUB(BW, N2) as well? + if (isPowerOf2_32(BitWidth)) { + APInt ModuloBits(N2.getScalarValueSizeInBits(), BitWidth - 1); + if (IsUndefOrZero(N0) && !IsFSHL && DAG.MaskedValueIsZero(N2, ~ModuloBits)) + return DAG.getNode(ISD::SRL, SDLoc(N), VT, N1, N2); + if (IsUndefOrZero(N1) && IsFSHL && DAG.MaskedValueIsZero(N2, ~ModuloBits)) + return DAG.getNode(ISD::SHL, SDLoc(N), VT, N0, N2); } // fold (fshl N0, N0, N2) -> (rotl N0, N2) @@ -7096,6 +7926,10 @@ SDValue DAGCombiner::visitFunnelShift(SDNode *N) { if (N0 == N1 && hasOperation(RotOpc, VT)) return DAG.getNode(RotOpc, SDLoc(N), VT, N0, N2); + // Simplify, based on bits shifted out of N0/N1. + if (SimplifyDemandedBits(SDValue(N, 0))) + return SDValue(N, 0); + return SDValue(); } @@ -7207,11 +8041,14 @@ SDValue DAGCombiner::visitCTPOP(SDNode *N) { // FIXME: This should be checking for no signed zeros on individual operands, as // well as no nans. -static bool isLegalToCombineMinNumMaxNum(SelectionDAG &DAG, SDValue LHS, SDValue RHS) { +static bool isLegalToCombineMinNumMaxNum(SelectionDAG &DAG, SDValue LHS, + SDValue RHS, + const TargetLowering &TLI) { const TargetOptions &Options = DAG.getTarget().Options; EVT VT = LHS.getValueType(); return Options.NoSignedZerosFPMath && VT.isFloatingPoint() && + TLI.isProfitableToCombineMinNumMaxNum(VT) && DAG.isKnownNeverNaN(LHS) && DAG.isKnownNeverNaN(RHS); } @@ -7364,6 +8201,7 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) { EVT VT = N->getValueType(0); EVT VT0 = N0.getValueType(); SDLoc DL(N); + SDNodeFlags Flags = N->getFlags(); if (SDValue V = DAG.simplifySelect(N0, N1, N2)) return V; @@ -7414,20 +8252,26 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) { SDValue Cond0 = N0->getOperand(0); SDValue Cond1 = N0->getOperand(1); SDValue InnerSelect = - DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond1, N1, N2); + DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond1, N1, N2, Flags); if (normalizeToSequence || !InnerSelect.use_empty()) return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0, - InnerSelect, N2); + InnerSelect, N2, Flags); + // Cleanup on failure. + if (InnerSelect.use_empty()) + recursivelyDeleteUnusedNodes(InnerSelect.getNode()); } // select (or Cond0, Cond1), X, Y -> select Cond0, X, (select Cond1, X, Y) if (N0->getOpcode() == ISD::OR && N0->hasOneUse()) { SDValue Cond0 = N0->getOperand(0); SDValue Cond1 = N0->getOperand(1); - SDValue InnerSelect = - DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond1, N1, N2); + SDValue InnerSelect = DAG.getNode(ISD::SELECT, DL, N1.getValueType(), + Cond1, N1, N2, Flags); if (normalizeToSequence || !InnerSelect.use_empty()) return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Cond0, N1, - InnerSelect); + InnerSelect, Flags); + // Cleanup on failure. + if (InnerSelect.use_empty()) + recursivelyDeleteUnusedNodes(InnerSelect.getNode()); } // select Cond0, (select Cond1, X, Y), Y -> select (and Cond0, Cond1), X, Y @@ -7439,12 +8283,14 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) { // Create the actual and node if we can generate good code for it. if (!normalizeToSequence) { SDValue And = DAG.getNode(ISD::AND, DL, N0.getValueType(), N0, N1_0); - return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), And, N1_1, N2); + return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), And, N1_1, + N2, Flags); } // Otherwise see if we can optimize the "and" to a better pattern. - if (SDValue Combined = visitANDLike(N0, N1_0, N)) + if (SDValue Combined = visitANDLike(N0, N1_0, N)) { return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1_1, - N2); + N2, Flags); + } } } // select Cond0, X, (select Cond1, X, Y) -> select (or Cond0, Cond1), X, Y @@ -7456,20 +8302,22 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) { // Create the actual or node if we can generate good code for it. if (!normalizeToSequence) { SDValue Or = DAG.getNode(ISD::OR, DL, N0.getValueType(), N0, N2_0); - return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Or, N1, N2_2); + return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Or, N1, + N2_2, Flags); } // Otherwise see if we can optimize to a better pattern. if (SDValue Combined = visitORLike(N0, N2_0, N)) return DAG.getNode(ISD::SELECT, DL, N1.getValueType(), Combined, N1, - N2_2); + N2_2, Flags); } } } - if (VT0 == MVT::i1) { - // select (not Cond), N1, N2 -> select Cond, N2, N1 - if (isBitwiseNot(N0)) - return DAG.getNode(ISD::SELECT, DL, VT, N0->getOperand(0), N2, N1); + // select (not Cond), N1, N2 -> select Cond, N2, N1 + if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false)) { + SDValue SelectOp = DAG.getSelect(DL, VT, F, N2, N1); + SelectOp->setFlags(Flags); + return SelectOp; } // Fold selects based on a setcc into other things, such as min/max/abs. @@ -7481,7 +8329,7 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) { // select (fcmp gt x, y), x, y -> fmaxnum x, y // // This is OK if we don't care what happens if either operand is a NaN. - if (N0.hasOneUse() && isLegalToCombineMinNumMaxNum(DAG, N1, N2)) + if (N0.hasOneUse() && isLegalToCombineMinNumMaxNum(DAG, N1, N2, TLI)) if (SDValue FMinMax = combineMinNumMaxNum(DL, VT, Cond0, Cond1, N1, N2, CC, TLI, DAG)) return FMinMax; @@ -7516,9 +8364,16 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) { } if (TLI.isOperationLegal(ISD::SELECT_CC, VT) || - (!LegalOperations && TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT))) - return DAG.getNode(ISD::SELECT_CC, DL, VT, Cond0, Cond1, N1, N2, - N0.getOperand(2)); + (!LegalOperations && + TLI.isOperationLegalOrCustom(ISD::SELECT_CC, VT))) { + // Any flags available in a select/setcc fold will be on the setcc as they + // migrated from fcmp + Flags = N0.getNode()->getFlags(); + SDValue SelectNode = DAG.getNode(ISD::SELECT_CC, DL, VT, Cond0, Cond1, N1, + N2, N0.getOperand(2)); + SelectNode->setFlags(Flags); + return SelectNode; + } return SimplifySelect(DL, N0, N1, N2); } @@ -7599,14 +8454,19 @@ static SDValue ConvertSelectToConcatVector(SDNode *N, SelectionDAG &DAG) { } SDValue DAGCombiner::visitMSCATTER(SDNode *N) { - if (Level >= AfterLegalizeTypes) - return SDValue(); - MaskedScatterSDNode *MSC = cast(N); SDValue Mask = MSC->getMask(); - SDValue Data = MSC->getValue(); + SDValue Data = MSC->getValue(); + SDValue Chain = MSC->getChain(); SDLoc DL(N); + // Zap scatters with a zero mask. + if (ISD::isBuildVectorAllZeros(Mask.getNode())) + return Chain; + + if (Level >= AfterLegalizeTypes) + return SDValue(); + // If the MSCATTER data type requires splitting and the mask is provided by a // SETCC, then split both nodes and its operands before legalization. This // prevents the type legalizer from unrolling SETCC into scalar comparisons @@ -7624,8 +8484,6 @@ SDValue DAGCombiner::visitMSCATTER(SDNode *N) { EVT LoVT, HiVT; std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(MSC->getValueType(0)); - SDValue Chain = MSC->getChain(); - EVT MemoryVT = MSC->getMemoryVT(); unsigned Alignment = MSC->getOriginalAlignment(); @@ -7658,15 +8516,20 @@ SDValue DAGCombiner::visitMSCATTER(SDNode *N) { } SDValue DAGCombiner::visitMSTORE(SDNode *N) { - if (Level >= AfterLegalizeTypes) - return SDValue(); - - MaskedStoreSDNode *MST = dyn_cast(N); + MaskedStoreSDNode *MST = cast(N); SDValue Mask = MST->getMask(); - SDValue Data = MST->getValue(); + SDValue Data = MST->getValue(); + SDValue Chain = MST->getChain(); EVT VT = Data.getValueType(); SDLoc DL(N); + // Zap masked stores with a zero mask. + if (ISD::isBuildVectorAllZeros(Mask.getNode())) + return Chain; + + if (Level >= AfterLegalizeTypes) + return SDValue(); + // If the MSTORE data type requires splitting and the mask is provided by a // SETCC, then split both nodes and its operands before legalization. This // prevents the type legalizer from unrolling SETCC into scalar comparisons @@ -7680,17 +8543,11 @@ SDValue DAGCombiner::visitMSTORE(SDNode *N) { SDValue MaskLo, MaskHi, Lo, Hi; std::tie(MaskLo, MaskHi) = SplitVSETCC(Mask.getNode(), DAG); - SDValue Chain = MST->getChain(); SDValue Ptr = MST->getBasePtr(); EVT MemoryVT = MST->getMemoryVT(); unsigned Alignment = MST->getOriginalAlignment(); - // if Alignment is equal to the vector size, - // take the half of it for the second part - unsigned SecondHalfAlignment = - (Alignment == VT.getSizeInBits() / 8) ? Alignment / 2 : Alignment; - EVT LoMemVT, HiMemVT; std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); @@ -7712,7 +8569,7 @@ SDValue DAGCombiner::visitMSTORE(SDNode *N) { MMO = DAG.getMachineFunction().getMachineMemOperand( MST->getPointerInfo().getWithOffset(HiOffset), - MachineMemOperand::MOStore, HiMemVT.getStoreSize(), SecondHalfAlignment, + MachineMemOperand::MOStore, HiMemVT.getStoreSize(), Alignment, MST->getAAInfo(), MST->getRanges()); Hi = DAG.getMaskedStore(Chain, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO, @@ -7728,13 +8585,17 @@ SDValue DAGCombiner::visitMSTORE(SDNode *N) { } SDValue DAGCombiner::visitMGATHER(SDNode *N) { - if (Level >= AfterLegalizeTypes) - return SDValue(); - MaskedGatherSDNode *MGT = cast(N); SDValue Mask = MGT->getMask(); SDLoc DL(N); + // Zap gathers with a zero mask. + if (ISD::isBuildVectorAllZeros(Mask.getNode())) + return CombineTo(N, MGT->getPassThru(), MGT->getChain()); + + if (Level >= AfterLegalizeTypes) + return SDValue(); + // If the MGATHER result requires splitting and the mask is provided by a // SETCC, then split both nodes and its operands before legalization. This // prevents the type legalizer from unrolling SETCC into scalar comparisons @@ -7805,13 +8666,17 @@ SDValue DAGCombiner::visitMGATHER(SDNode *N) { } SDValue DAGCombiner::visitMLOAD(SDNode *N) { - if (Level >= AfterLegalizeTypes) - return SDValue(); - - MaskedLoadSDNode *MLD = dyn_cast(N); + MaskedLoadSDNode *MLD = cast(N); SDValue Mask = MLD->getMask(); SDLoc DL(N); + // Zap masked loads with a zero mask. + if (ISD::isBuildVectorAllZeros(Mask.getNode())) + return CombineTo(N, MLD->getPassThru(), MLD->getChain()); + + if (Level >= AfterLegalizeTypes) + return SDValue(); + // If the MLOAD result requires splitting and the mask is provided by a // SETCC, then split both nodes and its operands before legalization. This // prevents the type legalizer from unrolling SETCC into scalar comparisons @@ -7839,12 +8704,6 @@ SDValue DAGCombiner::visitMLOAD(SDNode *N) { EVT MemoryVT = MLD->getMemoryVT(); unsigned Alignment = MLD->getOriginalAlignment(); - // if Alignment is equal to the vector size, - // take the half of it for the second part - unsigned SecondHalfAlignment = - (Alignment == MLD->getValueType(0).getSizeInBits()/8) ? - Alignment/2 : Alignment; - EVT LoMemVT, HiMemVT; std::tie(LoMemVT, HiMemVT) = DAG.GetSplitDestVTs(MemoryVT); @@ -7862,7 +8721,7 @@ SDValue DAGCombiner::visitMLOAD(SDNode *N) { MMO = DAG.getMachineFunction().getMachineMemOperand( MLD->getPointerInfo().getWithOffset(HiOffset), - MachineMemOperand::MOLoad, HiMemVT.getStoreSize(), SecondHalfAlignment, + MachineMemOperand::MOLoad, HiMemVT.getStoreSize(), Alignment, MLD->getAAInfo(), MLD->getRanges()); Hi = DAG.getMaskedLoad(HiVT, DL, Chain, Ptr, MaskHi, PassThruHi, HiMemVT, @@ -7943,11 +8802,16 @@ SDValue DAGCombiner::visitVSELECT(SDNode *N) { SDValue N0 = N->getOperand(0); SDValue N1 = N->getOperand(1); SDValue N2 = N->getOperand(2); + EVT VT = N->getValueType(0); SDLoc DL(N); if (SDValue V = DAG.simplifySelect(N0, N1, N2)) return V; + // vselect (not Cond), N1, N2 -> vselect Cond, N2, N1 + if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false)) + return DAG.getSelect(DL, VT, F, N2, N1); + // Canonicalize integer abs. // vselect (setg[te] X, 0), X, -X -> // vselect (setgt X, -1), X, -X -> @@ -7987,11 +8851,10 @@ SDValue DAGCombiner::visitVSELECT(SDNode *N) { // This is OK if we don't care about what happens if either operand is a // NaN. // - EVT VT = N->getValueType(0); - if (N0.hasOneUse() && isLegalToCombineMinNumMaxNum(DAG, N0.getOperand(0), N0.getOperand(1))) { - ISD::CondCode CC = cast(N0.getOperand(2))->get(); + if (N0.hasOneUse() && isLegalToCombineMinNumMaxNum(DAG, N0.getOperand(0), + N0.getOperand(1), TLI)) { if (SDValue FMinMax = combineMinNumMaxNum( - DL, VT, N0.getOperand(0), N0.getOperand(1), N1, N2, CC, TLI, DAG)) + DL, VT, N0.getOperand(0), N0.getOperand(1), N1, N2, CC, TLI, DAG)) return FMinMax; } @@ -8080,9 +8943,11 @@ SDValue DAGCombiner::visitSELECT_CC(SDNode *N) { return N2; } else if (SCC.getOpcode() == ISD::SETCC) { // Fold to a simpler select_cc - return DAG.getNode(ISD::SELECT_CC, SDLoc(N), N2.getValueType(), - SCC.getOperand(0), SCC.getOperand(1), N2, N3, - SCC.getOperand(2)); + SDValue SelectOp = DAG.getNode( + ISD::SELECT_CC, SDLoc(N), N2.getValueType(), SCC.getOperand(0), + SCC.getOperand(1), N2, N3, SCC.getOperand(2)); + SelectOp->setFlags(SCC->getFlags()); + return SelectOp; } } @@ -8148,6 +9013,7 @@ static SDValue tryToFoldExtendOfConstant(SDNode *N, const TargetLowering &TLI, unsigned Opcode = N->getOpcode(); SDValue N0 = N->getOperand(0); EVT VT = N->getValueType(0); + SDLoc DL(N); assert((Opcode == ISD::SIGN_EXTEND || Opcode == ISD::ZERO_EXTEND || Opcode == ISD::ANY_EXTEND || Opcode == ISD::SIGN_EXTEND_VECTOR_INREG || @@ -8158,7 +9024,33 @@ static SDValue tryToFoldExtendOfConstant(SDNode *N, const TargetLowering &TLI, // fold (zext c1) -> c1 // fold (aext c1) -> c1 if (isa(N0)) - return DAG.getNode(Opcode, SDLoc(N), VT, N0); + return DAG.getNode(Opcode, DL, VT, N0); + + // fold (sext (select cond, c1, c2)) -> (select cond, sext c1, sext c2) + // fold (zext (select cond, c1, c2)) -> (select cond, zext c1, zext c2) + // fold (aext (select cond, c1, c2)) -> (select cond, sext c1, sext c2) + if (N0->getOpcode() == ISD::SELECT) { + SDValue Op1 = N0->getOperand(1); + SDValue Op2 = N0->getOperand(2); + if (isa(Op1) && isa(Op2) && + (Opcode != ISD::ZERO_EXTEND || !TLI.isZExtFree(N0.getValueType(), VT))) { + // For any_extend, choose sign extension of the constants to allow a + // possible further transform to sign_extend_inreg.i.e. + // + // t1: i8 = select t0, Constant:i8<-1>, Constant:i8<0> + // t2: i64 = any_extend t1 + // --> + // t3: i64 = select t0, Constant:i64<-1>, Constant:i64<0> + // --> + // t4: i64 = sign_extend_inreg t3 + unsigned FoldOpc = Opcode; + if (FoldOpc == ISD::ANY_EXTEND) + FoldOpc = ISD::SIGN_EXTEND; + return DAG.getSelect(DL, VT, N0->getOperand(0), + DAG.getNode(FoldOpc, DL, VT, Op1), + DAG.getNode(FoldOpc, DL, VT, Op2)); + } + } // fold (sext (build_vector AllConstants) -> (build_vector AllConstants) // fold (zext (build_vector AllConstants) -> (build_vector AllConstants) @@ -8173,7 +9065,6 @@ static SDValue tryToFoldExtendOfConstant(SDNode *N, const TargetLowering &TLI, unsigned EVTBits = N0->getValueType(0).getScalarSizeInBits(); SmallVector Elts; unsigned NumElts = VT.getVectorNumElements(); - SDLoc DL(N); // For zero-extensions, UNDEF elements still guarantee to have the upper // bits set to zero. @@ -8387,6 +9278,9 @@ SDValue DAGCombiner::CombineExtLoad(SDNode *N) { SDValue DAGCombiner::CombineZExtLogicopShiftLoad(SDNode *N) { assert(N->getOpcode() == ISD::ZERO_EXTEND); EVT VT = N->getValueType(0); + EVT OrigVT = N->getOperand(0).getValueType(); + if (TLI.isZExtFree(OrigVT, VT)) + return SDValue(); // and/or/xor SDValue N0 = N->getOperand(0); @@ -8450,6 +9344,10 @@ SDValue DAGCombiner::CombineZExtLogicopShiftLoad(SDNode *N) { Load->getValueType(0), ExtLoad); CombineTo(Load, Trunc, ExtLoad.getValue(1)); } + + // N0 is dead at this point. + recursivelyDeleteUnusedNodes(N0.getNode()); + return SDValue(N,0); // Return N so it doesn't get rechecked! } @@ -8509,19 +9407,21 @@ static SDValue tryToFoldExtOfExtload(SelectionDAG &DAG, DAGCombiner &Combiner, : ISD::isZEXTLoad(N0Node); if ((!isAExtLoad && !ISD::isEXTLoad(N0Node)) || !ISD::isUNINDEXEDLoad(N0Node) || !N0.hasOneUse()) - return {}; + return SDValue(); LoadSDNode *LN0 = cast(N0); EVT MemVT = LN0->getMemoryVT(); if ((LegalOperations || LN0->isVolatile() || VT.isVector()) && !TLI.isLoadExtLegal(ExtLoadType, VT, MemVT)) - return {}; + return SDValue(); SDValue ExtLoad = DAG.getExtLoad(ExtLoadType, SDLoc(LN0), VT, LN0->getChain(), LN0->getBasePtr(), MemVT, LN0->getMemOperand()); Combiner.CombineTo(N, ExtLoad); DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1)); + if (LN0->use_empty()) + Combiner.recursivelyDeleteUnusedNodes(LN0); return SDValue(N, 0); // Return N so it doesn't get rechecked! } @@ -8559,6 +9459,7 @@ static SDValue tryToFoldExtOfLoad(SelectionDAG &DAG, DAGCombiner &Combiner, Combiner.CombineTo(N, ExtLoad); if (NoReplaceTrunc) { DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1)); + Combiner.recursivelyDeleteUnusedNodes(LN0); } else { SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), ExtLoad); @@ -8804,6 +9705,25 @@ SDValue DAGCombiner::visitSIGN_EXTEND(SDNode *N) { if (SDValue NewVSel = matchVSelectOpSizesWithSetCC(N)) return NewVSel; + // Eliminate this sign extend by doing a negation in the destination type: + // sext i32 (0 - (zext i8 X to i32)) to i64 --> 0 - (zext i8 X to i64) + if (N0.getOpcode() == ISD::SUB && N0.hasOneUse() && + isNullOrNullSplat(N0.getOperand(0)) && + N0.getOperand(1).getOpcode() == ISD::ZERO_EXTEND && + TLI.isOperationLegalOrCustom(ISD::SUB, VT)) { + SDValue Zext = DAG.getZExtOrTrunc(N0.getOperand(1).getOperand(0), DL, VT); + return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Zext); + } + // Eliminate this sign extend by doing a decrement in the destination type: + // sext i32 ((zext i8 X to i32) + (-1)) to i64 --> (zext i8 X to i64) + (-1) + if (N0.getOpcode() == ISD::ADD && N0.hasOneUse() && + isAllOnesOrAllOnesSplat(N0.getOperand(1)) && + N0.getOperand(0).getOpcode() == ISD::ZERO_EXTEND && + TLI.isOperationLegalOrCustom(ISD::ADD, VT)) { + SDValue Zext = DAG.getZExtOrTrunc(N0.getOperand(0).getOperand(0), DL, VT); + return DAG.getNode(ISD::ADD, DL, VT, Zext, DAG.getAllOnesConstant(DL, VT)); + } + return SDValue(); } @@ -9061,14 +9981,13 @@ SDValue DAGCombiner::visitZERO_EXTEND(SDNode *N) { N0.getOperand(0).getOpcode() == ISD::ZERO_EXTEND && N0.hasOneUse()) { SDValue ShAmt = N0.getOperand(1); - unsigned ShAmtVal = cast(ShAmt)->getZExtValue(); if (N0.getOpcode() == ISD::SHL) { SDValue InnerZExt = N0.getOperand(0); // If the original shl may be shifting out bits, do not perform this // transformation. unsigned KnownZeroBits = InnerZExt.getValueSizeInBits() - InnerZExt.getOperand(0).getValueSizeInBits(); - if (ShAmtVal > KnownZeroBits) + if (cast(ShAmt)->getAPIntValue().ugt(KnownZeroBits)) return SDValue(); } @@ -9162,6 +10081,7 @@ SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) { CombineTo(N, ExtLoad); if (NoReplaceTrunc) { DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1)); + recursivelyDeleteUnusedNodes(LN0); } else { SDValue Trunc = DAG.getNode(ISD::TRUNCATE, SDLoc(N0), N0.getValueType(), ExtLoad); @@ -9185,6 +10105,7 @@ SDValue DAGCombiner::visitANY_EXTEND(SDNode *N) { MemVT, LN0->getMemOperand()); CombineTo(N, ExtLoad); DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), ExtLoad.getValue(1)); + recursivelyDeleteUnusedNodes(LN0); return SDValue(N, 0); // Return N so it doesn't get rechecked! } } @@ -9574,14 +10495,14 @@ SDValue DAGCombiner::visitSIGN_EXTEND_INREG(SDNode *N) { // fold (sext_in_reg (srl X, 23), i8) -> (sra X, 23) iff possible. // We already fold "(sext_in_reg (srl X, 25), i8) -> srl X, 25" above. if (N0.getOpcode() == ISD::SRL) { - if (ConstantSDNode *ShAmt = dyn_cast(N0.getOperand(1))) - if (ShAmt->getZExtValue()+EVTBits <= VTBits) { + if (auto *ShAmt = dyn_cast(N0.getOperand(1))) + if (ShAmt->getAPIntValue().ule(VTBits - EVTBits)) { // We can turn this into an SRA iff the input to the SRL is already sign // extended enough. unsigned InSignBits = DAG.ComputeNumSignBits(N0.getOperand(0)); - if (VTBits-(ShAmt->getZExtValue()+EVTBits) < InSignBits) - return DAG.getNode(ISD::SRA, SDLoc(N), VT, - N0.getOperand(0), N0.getOperand(1)); + if (((VTBits - EVTBits) - ShAmt->getZExtValue()) < InSignBits) + return DAG.getNode(ISD::SRA, SDLoc(N), VT, N0.getOperand(0), + N0.getOperand(1)); } } @@ -9667,10 +10588,11 @@ SDValue DAGCombiner::visitZERO_EXTEND_VECTOR_INREG(SDNode *N) { SDValue DAGCombiner::visitTRUNCATE(SDNode *N) { SDValue N0 = N->getOperand(0); EVT VT = N->getValueType(0); + EVT SrcVT = N0.getValueType(); bool isLE = DAG.getDataLayout().isLittleEndian(); // noop truncate - if (N0.getValueType() == N->getValueType(0)) + if (SrcVT == VT) return N0; // fold (truncate (truncate x)) -> (truncate x) @@ -9740,7 +10662,6 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) { // trunc (select c, a, b) -> select c, (trunc a), (trunc b) if (N0.getOpcode() == ISD::SELECT && N0.hasOneUse()) { - EVT SrcVT = N0.getValueType(); if ((!LegalOperations || TLI.isOperationLegal(ISD::SELECT, SrcVT)) && TLI.isTruncateFree(SrcVT, VT)) { SDLoc SL(N0); @@ -9753,7 +10674,7 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) { // trunc (shl x, K) -> shl (trunc x), K => K < VT.getScalarSizeInBits() if (N0.getOpcode() == ISD::SHL && N0.hasOneUse() && - (!LegalOperations || TLI.isOperationLegalOrCustom(ISD::SHL, VT)) && + (!LegalOperations || TLI.isOperationLegal(ISD::SHL, VT)) && TLI.isTypeDesirableForOp(ISD::SHL, VT)) { SDValue Amt = N0.getOperand(1); KnownBits Known = DAG.computeKnownBits(Amt); @@ -9771,6 +10692,19 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) { } } + // Attempt to pre-truncate BUILD_VECTOR sources. + if (N0.getOpcode() == ISD::BUILD_VECTOR && !LegalOperations && + TLI.isTruncateFree(SrcVT.getScalarType(), VT.getScalarType())) { + SDLoc DL(N); + EVT SVT = VT.getScalarType(); + SmallVector TruncOps; + for (const SDValue &Op : N0->op_values()) { + SDValue TruncOp = DAG.getNode(ISD::TRUNCATE, DL, SVT, Op); + TruncOps.push_back(TruncOp); + } + return DAG.getBuildVector(VT, DL, TruncOps); + } + // Fold a series of buildvector, bitcast, and truncate if possible. // For example fold // (2xi32 trunc (bitcast ((4xi32)buildvector x, x, y, y) 2xi64)) to @@ -9906,7 +10840,9 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) { // When the adde's carry is not used. if ((N0.getOpcode() == ISD::ADDE || N0.getOpcode() == ISD::ADDCARRY) && N0.hasOneUse() && !N0.getNode()->hasAnyUseOfValue(1) && - (!LegalOperations || TLI.isOperationLegal(N0.getOpcode(), VT))) { + // We only do for addcarry before legalize operation + ((!LegalOperations && N0.getOpcode() == ISD::ADDCARRY) || + TLI.isOperationLegal(N0.getOpcode(), VT))) { SDLoc SL(N); auto X = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(0)); auto Y = DAG.getNode(ISD::TRUNCATE, SL, VT, N0.getOperand(1)); @@ -10070,14 +11006,17 @@ SDValue DAGCombiner::visitBITCAST(SDNode *N) { return DAG.getUNDEF(VT); // If the input is a BUILD_VECTOR with all constant elements, fold this now. - // Only do this before legalize types, since we might create an illegal - // scalar type. Even if we knew we wouldn't create an illegal scalar type - // we can only do this before legalize ops, since the target maybe - // depending on the bitcast. + // Only do this before legalize types, unless both types are integer and the + // scalar type is legal. Only do this before legalize ops, since the target + // maybe depending on the bitcast. // First check to see if this is all constant. - if (!LegalTypes && + // TODO: Support FP bitcasts after legalize types. + if (VT.isVector() && + (!LegalTypes || + (!LegalOperations && VT.isInteger() && N0.getValueType().isInteger() && + TLI.isTypeLegal(VT.getVectorElementType()))) && N0.getOpcode() == ISD::BUILD_VECTOR && N0.getNode()->hasOneUse() && - VT.isVector() && cast(N0)->isConstant()) + cast(N0)->isConstant()) return ConstantFoldBITCASTofBUILD_VECTOR(N0.getNode(), VT.getVectorElementType()); @@ -10113,18 +11052,14 @@ SDValue DAGCombiner::visitBITCAST(SDNode *N) { // as we assume software couldn't rely on the number of accesses of an // illegal type. ((!LegalOperations && !cast(N0)->isVolatile()) || - TLI.isOperationLegal(ISD::LOAD, VT)) && - TLI.isLoadBitCastBeneficial(N0.getValueType(), VT)) { + TLI.isOperationLegal(ISD::LOAD, VT))) { LoadSDNode *LN0 = cast(N0); - unsigned OrigAlign = LN0->getAlignment(); - bool Fast = false; - if (TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT, - LN0->getAddressSpace(), OrigAlign, &Fast) && - Fast) { + if (TLI.isLoadBitCastBeneficial(N0.getValueType(), VT, DAG, + *LN0->getMemOperand())) { SDValue Load = DAG.getLoad(VT, SDLoc(N), LN0->getChain(), LN0->getBasePtr(), - LN0->getPointerInfo(), OrigAlign, + LN0->getPointerInfo(), LN0->getAlignment(), LN0->getMemOperand()->getFlags(), LN0->getAAInfo()); DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), Load.getValue(1)); return Load; @@ -11071,15 +12006,17 @@ SDValue DAGCombiner::visitFADD(SDNode *N) { // fold (fadd A, (fneg B)) -> (fsub A, B) if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) && - isNegatibleForFree(N1, LegalOperations, TLI, &Options) == 2) + isNegatibleForFree(N1, LegalOperations, TLI, &Options, ForCodeSize) == 2) return DAG.getNode(ISD::FSUB, DL, VT, N0, - GetNegatedExpression(N1, DAG, LegalOperations), Flags); + GetNegatedExpression(N1, DAG, LegalOperations, + ForCodeSize), Flags); // fold (fadd (fneg A), B) -> (fsub B, A) if ((!LegalOperations || TLI.isOperationLegalOrCustom(ISD::FSUB, VT)) && - isNegatibleForFree(N0, LegalOperations, TLI, &Options) == 2) + isNegatibleForFree(N0, LegalOperations, TLI, &Options, ForCodeSize) == 2) return DAG.getNode(ISD::FSUB, DL, VT, N1, - GetNegatedExpression(N0, DAG, LegalOperations), Flags); + GetNegatedExpression(N0, DAG, LegalOperations, + ForCodeSize), Flags); auto isFMulNegTwo = [](SDValue FMul) { if (!FMul.hasOneUse() || FMul.getOpcode() != ISD::FMUL) @@ -11105,8 +12042,8 @@ SDValue DAGCombiner::visitFADD(SDNode *N) { // Selection pass has a hard time dealing with FP constants. bool AllowNewConst = (Level < AfterLegalizeDAG); - // If 'unsafe math' or nnan is enabled, fold lots of things. - if ((Options.UnsafeFPMath || Flags.hasNoNaNs()) && AllowNewConst) { + // If nnan is enabled, fold lots of things. + if ((Options.NoNaNsFPMath || Flags.hasNoNaNs()) && AllowNewConst) { // If allowed, fold (fadd (fneg x), x) -> 0.0 if (N0.getOpcode() == ISD::FNEG && N0.getOperand(0) == N1) return DAG.getConstantFP(0.0, DL, VT); @@ -11246,16 +12183,20 @@ SDValue DAGCombiner::visitFSUB(SDNode *N) { if (N0 == N1) { // (fsub x, x) -> 0.0 - if (Options.UnsafeFPMath || Flags.hasNoNaNs()) + if (Options.NoNaNsFPMath || Flags.hasNoNaNs()) return DAG.getConstantFP(0.0f, DL, VT); } // (fsub -0.0, N1) -> -N1 + // NOTE: It is safe to transform an FSUB(-0.0,X) into an FNEG(X), since the + // FSUB does not specify the sign bit of a NaN. Also note that for + // the same reason, the inverse transform is not safe, unless fast math + // flags are in play. if (N0CFP && N0CFP->isZero()) { if (N0CFP->isNegative() || (Options.NoSignedZerosFPMath || Flags.hasNoSignedZeros())) { - if (isNegatibleForFree(N1, LegalOperations, TLI, &Options)) - return GetNegatedExpression(N1, DAG, LegalOperations); + if (isNegatibleForFree(N1, LegalOperations, TLI, &Options, ForCodeSize)) + return GetNegatedExpression(N1, DAG, LegalOperations, ForCodeSize); if (!LegalOperations || TLI.isOperationLegal(ISD::FNEG, VT)) return DAG.getNode(ISD::FNEG, DL, VT, N1, Flags); } @@ -11273,9 +12214,10 @@ SDValue DAGCombiner::visitFSUB(SDNode *N) { } // fold (fsub A, (fneg B)) -> (fadd A, B) - if (isNegatibleForFree(N1, LegalOperations, TLI, &Options)) + if (isNegatibleForFree(N1, LegalOperations, TLI, &Options, ForCodeSize)) return DAG.getNode(ISD::FADD, DL, VT, N0, - GetNegatedExpression(N1, DAG, LegalOperations), Flags); + GetNegatedExpression(N1, DAG, LegalOperations, + ForCodeSize), Flags); // FSUB -> FMA combines: if (SDValue Fused = visitFSUBForFMACombine(N)) { @@ -11319,7 +12261,7 @@ SDValue DAGCombiner::visitFMUL(SDNode *N) { if (SDValue NewSel = foldBinOpIntoSelect(N)) return NewSel; - if (Options.UnsafeFPMath || + if ((Options.NoNaNsFPMath && Options.NoSignedZerosFPMath) || (Flags.hasNoNaNs() && Flags.hasNoSignedZeros())) { // fold (fmul A, 0) -> 0 if (N1CFP && N1CFP->isZero()) @@ -11361,14 +12303,18 @@ SDValue DAGCombiner::visitFMUL(SDNode *N) { return DAG.getNode(ISD::FNEG, DL, VT, N0); // fold (fmul (fneg X), (fneg Y)) -> (fmul X, Y) - if (char LHSNeg = isNegatibleForFree(N0, LegalOperations, TLI, &Options)) { - if (char RHSNeg = isNegatibleForFree(N1, LegalOperations, TLI, &Options)) { + if (char LHSNeg = isNegatibleForFree(N0, LegalOperations, TLI, &Options, + ForCodeSize)) { + if (char RHSNeg = isNegatibleForFree(N1, LegalOperations, TLI, &Options, + ForCodeSize)) { // Both can be negated for free, check to see if at least one is cheaper // negated. if (LHSNeg == 2 || RHSNeg == 2) return DAG.getNode(ISD::FMUL, DL, VT, - GetNegatedExpression(N0, DAG, LegalOperations), - GetNegatedExpression(N1, DAG, LegalOperations), + GetNegatedExpression(N0, DAG, LegalOperations, + ForCodeSize), + GetNegatedExpression(N1, DAG, LegalOperations, + ForCodeSize), Flags); } } @@ -11506,7 +12452,8 @@ SDValue DAGCombiner::visitFMA(SDNode *N) { // fma (fneg x), K, y -> fma x -K, y if (N0.getOpcode() == ISD::FNEG && (TLI.isOperationLegal(ISD::ConstantFP, VT) || - (N1.hasOneUse() && !TLI.isFPImmLegal(N1CFP->getValueAPF(), VT)))) { + (N1.hasOneUse() && !TLI.isFPImmLegal(N1CFP->getValueAPF(), VT, + ForCodeSize)))) { return DAG.getNode(ISD::FMA, DL, VT, N0.getOperand(0), DAG.getNode(ISD::FNEG, DL, VT, N1, Flags), N2); } @@ -11541,22 +12488,33 @@ SDValue DAGCombiner::visitFMA(SDNode *N) { // FDIVs may be lower than the cost of one FDIV and two FMULs. Another reason // is the critical path is increased from "one FDIV" to "one FDIV + one FMUL". SDValue DAGCombiner::combineRepeatedFPDivisors(SDNode *N) { + // TODO: Limit this transform based on optsize/minsize - it always creates at + // least 1 extra instruction. But the perf win may be substantial enough + // that only minsize should restrict this. bool UnsafeMath = DAG.getTarget().Options.UnsafeFPMath; const SDNodeFlags Flags = N->getFlags(); if (!UnsafeMath && !Flags.hasAllowReciprocal()) return SDValue(); - // Skip if current node is a reciprocal. + // Skip if current node is a reciprocal/fneg-reciprocal. SDValue N0 = N->getOperand(0); - ConstantFPSDNode *N0CFP = dyn_cast(N0); - if (N0CFP && N0CFP->isExactlyValue(1.0)) + ConstantFPSDNode *N0CFP = isConstOrConstSplatFP(N0, /* AllowUndefs */ true); + if (N0CFP && (N0CFP->isExactlyValue(1.0) || N0CFP->isExactlyValue(-1.0))) return SDValue(); // Exit early if the target does not want this transform or if there can't // possibly be enough uses of the divisor to make the transform worthwhile. SDValue N1 = N->getOperand(1); unsigned MinUses = TLI.combineRepeatedFPDivisors(); - if (!MinUses || N1->use_size() < MinUses) + + // For splat vectors, scale the number of uses by the splat factor. If we can + // convert the division into a scalar op, that will likely be much faster. + unsigned NumElts = 1; + EVT VT = N->getValueType(0); + if (VT.isVector() && DAG.isSplatValue(N1)) + NumElts = VT.getVectorNumElements(); + + if (!MinUses || (N1->use_size() * NumElts) < MinUses) return SDValue(); // Find all FDIV users of the same divisor. @@ -11573,10 +12531,9 @@ SDValue DAGCombiner::combineRepeatedFPDivisors(SDNode *N) { // Now that we have the actual number of divisor uses, make sure it meets // the minimum threshold specified by the target. - if (Users.size() < MinUses) + if ((Users.size() * NumElts) < MinUses) return SDValue(); - EVT VT = N->getValueType(0); SDLoc DL(N); SDValue FPOne = DAG.getConstantFP(1.0, DL, VT); SDValue Reciprocal = DAG.getNode(ISD::FDIV, DL, VT, FPOne, N1, Flags); @@ -11619,6 +12576,9 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) { if (SDValue NewSel = foldBinOpIntoSelect(N)) return NewSel; + if (SDValue V = combineRepeatedFPDivisors(N)) + return V; + if (Options.UnsafeFPMath || Flags.hasAllowReciprocal()) { // fold (fdiv X, c2) -> fmul X, 1/c2 if losing precision is acceptable. if (N1CFP) { @@ -11634,7 +12594,7 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) { // backend)... we should handle this gracefully after Legalize. // TLI.isOperationLegalOrCustom(ISD::ConstantFP, VT) || TLI.isOperationLegal(ISD::ConstantFP, VT) || - TLI.isFPImmLegal(Recip, VT))) + TLI.isFPImmLegal(Recip, VT, ForCodeSize))) return DAG.getNode(ISD::FMUL, DL, VT, N0, DAG.getConstantFP(Recip, DL, VT), Flags); } @@ -11692,21 +12652,22 @@ SDValue DAGCombiner::visitFDIV(SDNode *N) { } // (fdiv (fneg X), (fneg Y)) -> (fdiv X, Y) - if (char LHSNeg = isNegatibleForFree(N0, LegalOperations, TLI, &Options)) { - if (char RHSNeg = isNegatibleForFree(N1, LegalOperations, TLI, &Options)) { + if (char LHSNeg = isNegatibleForFree(N0, LegalOperations, TLI, &Options, + ForCodeSize)) { + if (char RHSNeg = isNegatibleForFree(N1, LegalOperations, TLI, &Options, + ForCodeSize)) { // Both can be negated for free, check to see if at least one is cheaper // negated. if (LHSNeg == 2 || RHSNeg == 2) return DAG.getNode(ISD::FDIV, SDLoc(N), VT, - GetNegatedExpression(N0, DAG, LegalOperations), - GetNegatedExpression(N1, DAG, LegalOperations), + GetNegatedExpression(N0, DAG, LegalOperations, + ForCodeSize), + GetNegatedExpression(N1, DAG, LegalOperations, + ForCodeSize), Flags); } } - if (SDValue CombineRepeatedDivisors = combineRepeatedFPDivisors(N)) - return CombineRepeatedDivisors; - return SDValue(); } @@ -11838,18 +12799,24 @@ SDValue DAGCombiner::visitFPOW(SDNode *N) { return DAG.getNode(ISD::FCBRT, SDLoc(N), VT, N->getOperand(0), Flags); } - // Try to convert x ** (1/4) into square roots. + // Try to convert x ** (1/4) and x ** (3/4) into square roots. // x ** (1/2) is canonicalized to sqrt, so we do not bother with that case. // TODO: This could be extended (using a target hook) to handle smaller // power-of-2 fractional exponents. - if (ExponentC->getValueAPF().isExactlyValue(0.25)) { + bool ExponentIs025 = ExponentC->getValueAPF().isExactlyValue(0.25); + bool ExponentIs075 = ExponentC->getValueAPF().isExactlyValue(0.75); + if (ExponentIs025 || ExponentIs075) { // pow(-0.0, 0.25) = +0.0; sqrt(sqrt(-0.0)) = -0.0. // pow(-inf, 0.25) = +inf; sqrt(sqrt(-inf)) = NaN. + // pow(-0.0, 0.75) = +0.0; sqrt(-0.0) * sqrt(sqrt(-0.0)) = +0.0. + // pow(-inf, 0.75) = +inf; sqrt(-inf) * sqrt(sqrt(-inf)) = NaN. // For regular numbers, rounding may cause the results to differ. // Therefore, we require { nsz ninf afn } for this transform. // TODO: We could select out the special cases if we don't have nsz/ninf. SDNodeFlags Flags = N->getFlags(); - if (!Flags.hasNoSignedZeros() || !Flags.hasNoInfs() || + + // We only need no signed zeros for the 0.25 case. + if ((!Flags.hasNoSignedZeros() && ExponentIs025) || !Flags.hasNoInfs() || !Flags.hasApproximateFuncs()) return SDValue(); @@ -11859,13 +12826,17 @@ SDValue DAGCombiner::visitFPOW(SDNode *N) { // Assume that libcalls are the smallest code. // TODO: This restriction should probably be lifted for vectors. - if (DAG.getMachineFunction().getFunction().optForSize()) + if (DAG.getMachineFunction().getFunction().hasOptSize()) return SDValue(); // pow(X, 0.25) --> sqrt(sqrt(X)) SDLoc DL(N); SDValue Sqrt = DAG.getNode(ISD::FSQRT, DL, VT, N->getOperand(0), Flags); - return DAG.getNode(ISD::FSQRT, DL, VT, Sqrt, Flags); + SDValue SqrtSqrt = DAG.getNode(ISD::FSQRT, DL, VT, Sqrt, Flags); + if (ExponentIs025) + return SqrtSqrt; + // pow(X, 0.75) --> sqrt(X) * sqrt(sqrt(X)) + return DAG.getNode(ISD::FMUL, DL, VT, Sqrt, SqrtSqrt, Flags); } return SDValue(); @@ -11911,6 +12882,10 @@ SDValue DAGCombiner::visitSINT_TO_FP(SDNode *N) { EVT VT = N->getValueType(0); EVT OpVT = N0.getValueType(); + // [us]itofp(undef) = 0, because the result value is bounded. + if (N0.isUndef()) + return DAG.getConstantFP(0.0, SDLoc(N), VT); + // fold (sint_to_fp c1) -> c1fp if (DAG.isConstantIntBuildVectorOrConstantInt(N0) && // ...but only if the target supports immediate floating-point values @@ -11968,6 +12943,10 @@ SDValue DAGCombiner::visitUINT_TO_FP(SDNode *N) { EVT VT = N->getValueType(0); EVT OpVT = N0.getValueType(); + // [us]itofp(undef) = 0, because the result value is bounded. + if (N0.isUndef()) + return DAG.getConstantFP(0.0, SDLoc(N), VT); + // fold (uint_to_fp c1) -> c1fp if (DAG.isConstantIntBuildVectorOrConstantInt(N0) && // ...but only if the target supports immediate floating-point values @@ -12051,6 +13030,10 @@ SDValue DAGCombiner::visitFP_TO_SINT(SDNode *N) { SDValue N0 = N->getOperand(0); EVT VT = N->getValueType(0); + // fold (fp_to_sint undef) -> undef + if (N0.isUndef()) + return DAG.getUNDEF(VT); + // fold (fp_to_sint c1fp) -> c1 if (isConstantFPBuildVectorOrConstantFP(N0)) return DAG.getNode(ISD::FP_TO_SINT, SDLoc(N), VT, N0); @@ -12062,6 +13045,10 @@ SDValue DAGCombiner::visitFP_TO_UINT(SDNode *N) { SDValue N0 = N->getOperand(0); EVT VT = N->getValueType(0); + // fold (fp_to_uint undef) -> undef + if (N0.isUndef()) + return DAG.getUNDEF(VT); + // fold (fp_to_uint c1fp) -> c1 if (isConstantFPBuildVectorOrConstantFP(N0)) return DAG.getNode(ISD::FP_TO_UINT, SDLoc(N), VT, N0); @@ -12250,8 +13237,8 @@ SDValue DAGCombiner::visitFNEG(SDNode *N) { return DAG.getNode(ISD::FNEG, SDLoc(N), VT, N0); if (isNegatibleForFree(N0, LegalOperations, DAG.getTargetLoweringInfo(), - &DAG.getTarget().Options)) - return GetNegatedExpression(N0, DAG, LegalOperations); + &DAG.getTarget().Options, ForCodeSize)) + return GetNegatedExpression(N0, DAG, LegalOperations, ForCodeSize); // Transform fneg(bitconvert(x)) -> bitconvert(x ^ sign) to avoid loading // constant pool values. @@ -12287,7 +13274,7 @@ SDValue DAGCombiner::visitFNEG(SDNode *N) { APFloat CVal = CFP1->getValueAPF(); CVal.changeSign(); if (Level >= AfterLegalizeDAG && - (TLI.isFPImmLegal(CVal, VT) || + (TLI.isFPImmLegal(CVal, VT, ForCodeSize) || TLI.isOperationLegal(ISD::ConstantFP, VT))) return DAG.getNode( ISD::FMUL, SDLoc(N), VT, N0.getOperand(0), @@ -12556,6 +13543,7 @@ static bool canFoldInAddressingMode(SDNode *N, SDNode *Use, TargetLowering::AddrMode AM; if (N->getOpcode() == ISD::ADD) { + AM.HasBaseReg = true; ConstantSDNode *Offset = dyn_cast(N->getOperand(1)); if (Offset) // [reg +/- imm] @@ -12564,6 +13552,7 @@ static bool canFoldInAddressingMode(SDNode *N, SDNode *Use, // [reg +/- reg] AM.Scale = 1; } else if (N->getOpcode() == ISD::SUB) { + AM.HasBaseReg = true; ConstantSDNode *Offset = dyn_cast(N->getOperand(1)); if (Offset) // [reg +/- imm] @@ -12653,7 +13642,13 @@ bool DAGCombiner::CombineToPreIndexedLoadStore(SDNode *N) { // Check #2. if (!isLoad) { SDValue Val = cast(N)->getValue(); - if (Val == BasePtr || BasePtr.getNode()->isPredecessorOf(Val.getNode())) + + // Would require a copy. + if (Val == BasePtr) + return false; + + // Would create a cycle. + if (Val == Ptr || Ptr->isPredecessorOf(Val.getNode())) return false; } @@ -13190,7 +14185,7 @@ SDValue DAGCombiner::visitLOAD(SDNode *N) { if (LD->isUnindexed()) { // Walk up chain skipping non-aliasing memory nodes. - SDValue BetterChain = FindBetterChain(N, Chain); + SDValue BetterChain = FindBetterChain(LD, Chain); // If there is a better chain. if (Chain != BetterChain) { @@ -13378,7 +14373,7 @@ struct LoadedSlice { /// Get the alignment of the load used for this slice. unsigned getAlignment() const { unsigned Alignment = Origin->getAlignment(); - unsigned Offset = getOffsetFromBase(); + uint64_t Offset = getOffsetFromBase(); if (Offset != 0) Alignment = MinAlign(Alignment, Alignment + Offset); return Alignment; @@ -13500,9 +14495,11 @@ struct LoadedSlice { assert(DAG && "Missing context"); const TargetLowering &TLI = DAG->getTargetLoweringInfo(); EVT ResVT = Use->getValueType(0); - const TargetRegisterClass *ResRC = TLI.getRegClassFor(ResVT.getSimpleVT()); + const TargetRegisterClass *ResRC = + TLI.getRegClassFor(ResVT.getSimpleVT(), Use->isDivergent()); const TargetRegisterClass *ArgRC = - TLI.getRegClassFor(Use->getOperand(0).getValueType().getSimpleVT()); + TLI.getRegClassFor(Use->getOperand(0).getValueType().getSimpleVT(), + Use->getOperand(0)->isDivergent()); if (ArgRC == ResRC || !TLI.isOperationLegal(ISD::LOAD, ResVT)) return false; @@ -13826,7 +14823,7 @@ CheckForMaskedLoad(SDValue V, SDValue Ptr, SDValue Chain) { if (NotMaskTZ && NotMaskTZ/8 % MaskedBytes) return Result; // For narrowing to be valid, it must be the case that the load the - // immediately preceeding memory operation before the store. + // immediately preceding memory operation before the store. if (LD == Chain.getNode()) ; // ok. else if (Chain->getOpcode() == ISD::TokenFactor && @@ -14039,11 +15036,9 @@ SDValue DAGCombiner::ReduceLoadOpStoreWidth(SDNode *N) { /// load / store operations if the target deems the transformation profitable. SDValue DAGCombiner::TransformFPLoadStorePair(SDNode *N) { StoreSDNode *ST = cast(N); - SDValue Chain = ST->getChain(); SDValue Value = ST->getValue(); if (ISD::isNormalStore(ST) && ISD::isNormalLoad(Value.getNode()) && - Value.hasOneUse() && - Chain == SDValue(Value.getNode(), 1)) { + Value.hasOneUse()) { LoadSDNode *LD = cast(Value); EVT VT = LD->getMemoryVT(); if (!VT.isFloatingPoint() || @@ -14073,7 +15068,7 @@ SDValue DAGCombiner::TransformFPLoadStorePair(SDNode *N) { LD->getPointerInfo(), LDAlign); SDValue NewST = - DAG.getStore(NewLD.getValue(1), SDLoc(N), NewLD, ST->getBasePtr(), + DAG.getStore(ST->getChain(), SDLoc(N), NewLD, ST->getBasePtr(), ST->getPointerInfo(), STAlign); AddToWorklist(NewLD.getNode()); @@ -14171,14 +15166,14 @@ SDValue DAGCombiner::getMergeStoreChains(SmallVectorImpl &StoreNodes, Visited.insert(StoreNodes[i].MemNode); } - // don't include nodes that are children + // don't include nodes that are children or repeated nodes. for (unsigned i = 0; i < NumStores; ++i) { - if (Visited.count(StoreNodes[i].MemNode->getChain().getNode()) == 0) + if (Visited.insert(StoreNodes[i].MemNode->getChain().getNode()).second) Chains.push_back(StoreNodes[i].MemNode->getChain()); } assert(Chains.size() > 0 && "Chain should have generated a chain"); - return DAG.getNode(ISD::TokenFactor, StoreDL, MVT::Other, Chains); + return DAG.getTokenFactor(StoreDL, Chains); } bool DAGCombiner::MergeStoresOfConstantsOrVecElts( @@ -14372,15 +15367,19 @@ void DAGCombiner::getStoreMergeCandidates( // Loads must only have one use. if (!Ld->hasNUsesOfValue(1, 0)) return; - // The memory operands must not be volatile. + // The memory operands must not be volatile/indexed. if (Ld->isVolatile() || Ld->isIndexed()) return; } auto CandidateMatch = [&](StoreSDNode *Other, BaseIndexOffset &Ptr, int64_t &Offset) -> bool { + // The memory operands must not be volatile/indexed. if (Other->isVolatile() || Other->isIndexed()) return false; - SDValue Val = peekThroughBitcasts(Other->getValue()); + // Don't mix temporal stores with non-temporal stores. + if (St->isNonTemporal() != Other->isNonTemporal()) + return false; + SDValue OtherBC = peekThroughBitcasts(Other->getValue()); // Allow merging constants of different types as integers. bool NoTypeMatch = (MemVT.isInteger()) ? !MemVT.bitsEq(Other->getMemoryVT()) : Other->getMemoryVT() != MemVT; @@ -14388,16 +15387,19 @@ void DAGCombiner::getStoreMergeCandidates( if (NoTypeMatch) return false; // The Load's Base Ptr must also match - if (LoadSDNode *OtherLd = dyn_cast(Val)) { - auto LPtr = BaseIndexOffset::match(OtherLd, DAG); + if (LoadSDNode *OtherLd = dyn_cast(OtherBC)) { + BaseIndexOffset LPtr = BaseIndexOffset::match(OtherLd, DAG); if (LoadVT != OtherLd->getMemoryVT()) return false; // Loads must only have one use. if (!OtherLd->hasNUsesOfValue(1, 0)) return false; - // The memory operands must not be volatile. + // The memory operands must not be volatile/indexed. if (OtherLd->isVolatile() || OtherLd->isIndexed()) return false; + // Don't mix temporal loads with non-temporal loads. + if (cast(Val)->isNonTemporal() != OtherLd->isNonTemporal()) + return false; if (!(LBasePtr.equalBaseIndex(LPtr, DAG))) return false; } else @@ -14406,17 +15408,17 @@ void DAGCombiner::getStoreMergeCandidates( if (IsConstantSrc) { if (NoTypeMatch) return false; - if (!(isa(Val) || isa(Val))) + if (!(isa(OtherBC) || isa(OtherBC))) return false; } if (IsExtractVecSrc) { // Do not merge truncated stores here. if (Other->isTruncatingStore()) return false; - if (!MemVT.bitsEq(Val.getValueType())) + if (!MemVT.bitsEq(OtherBC.getValueType())) return false; - if (Val.getOpcode() != ISD::EXTRACT_VECTOR_ELT && - Val.getOpcode() != ISD::EXTRACT_SUBVECTOR) + if (OtherBC.getOpcode() != ISD::EXTRACT_VECTOR_ELT && + OtherBC.getOpcode() != ISD::EXTRACT_SUBVECTOR) return false; } Ptr = BaseIndexOffset::match(Other, DAG); @@ -14441,9 +15443,11 @@ void DAGCombiner::getStoreMergeCandidates( RootNode = St->getChain().getNode(); + unsigned NumNodesExplored = 0; if (LoadSDNode *Ldn = dyn_cast(RootNode)) { RootNode = Ldn->getChain().getNode(); - for (auto I = RootNode->use_begin(), E = RootNode->use_end(); I != E; ++I) + for (auto I = RootNode->use_begin(), E = RootNode->use_end(); + I != E && NumNodesExplored < 1024; ++I, ++NumNodesExplored) if (I.getOperandNo() == 0 && isa(*I)) // walk down chain for (auto I2 = (*I)->use_begin(), E2 = (*I)->use_end(); I2 != E2; ++I2) if (I2.getOperandNo() == 0) @@ -14454,7 +15458,8 @@ void DAGCombiner::getStoreMergeCandidates( StoreNodes.push_back(MemOpLink(OtherST, PtrDiff)); } } else - for (auto I = RootNode->use_begin(), E = RootNode->use_end(); I != E; ++I) + for (auto I = RootNode->use_begin(), E = RootNode->use_end(); + I != E && NumNodesExplored < 1024; ++I, ++NumNodesExplored) if (I.getOperandNo() == 0) if (StoreSDNode *OtherST = dyn_cast(*I)) { BaseIndexOffset Ptr; @@ -14551,6 +15556,9 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) { isa(StoredVal); bool IsExtractVecSrc = (StoredVal.getOpcode() == ISD::EXTRACT_VECTOR_ELT || StoredVal.getOpcode() == ISD::EXTRACT_SUBVECTOR); + bool IsNonTemporalStore = St->isNonTemporal(); + bool IsNonTemporalLoad = + IsLoadSrc && cast(StoredVal)->isNonTemporal(); if (!IsConstantSrc && !IsLoadSrc && !IsExtractVecSrc) return false; @@ -14652,8 +15660,8 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) { if (TLI.isTypeLegal(StoreTy) && TLI.canMergeStoresTo(FirstStoreAS, StoreTy, DAG) && - TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstStoreAS, - FirstStoreAlign, &IsFast) && + TLI.allowsMemoryAccess(Context, DL, StoreTy, + *FirstInChain->getMemOperand(), &IsFast) && IsFast) { LastIntegerTrunc = false; LastLegalType = i + 1; @@ -14664,8 +15672,9 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) { TLI.getTypeToTransformTo(Context, StoredVal.getValueType()); if (TLI.isTruncStoreLegal(LegalizedStoredValTy, StoreTy) && TLI.canMergeStoresTo(FirstStoreAS, LegalizedStoredValTy, DAG) && - TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstStoreAS, - FirstStoreAlign, &IsFast) && + TLI.allowsMemoryAccess(Context, DL, StoreTy, + *FirstInChain->getMemOperand(), + &IsFast) && IsFast) { LastIntegerTrunc = true; LastLegalType = i + 1; @@ -14683,8 +15692,8 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) { EVT Ty = EVT::getVectorVT(Context, MemVT.getScalarType(), Elts); if (TLI.isTypeLegal(Ty) && TLI.isTypeLegal(MemVT) && TLI.canMergeStoresTo(FirstStoreAS, Ty, DAG) && - TLI.allowsMemoryAccess(Context, DL, Ty, FirstStoreAS, - FirstStoreAlign, &IsFast) && + TLI.allowsMemoryAccess( + Context, DL, Ty, *FirstInChain->getMemOperand(), &IsFast) && IsFast) LastLegalVectorType = i + 1; } @@ -14755,8 +15764,8 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) { if (TLI.isTypeLegal(Ty) && TLI.canMergeStoresTo(FirstStoreAS, Ty, DAG) && - TLI.allowsMemoryAccess(Context, DL, Ty, FirstStoreAS, - FirstStoreAlign, &IsFast) && + TLI.allowsMemoryAccess(Context, DL, Ty, + *FirstInChain->getMemOperand(), &IsFast) && IsFast) NumStoresToMerge = i + 1; } @@ -14847,7 +15856,6 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) { unsigned FirstStoreAS = FirstInChain->getAddressSpace(); unsigned FirstStoreAlign = FirstInChain->getAlignment(); LoadSDNode *FirstLoad = cast(LoadNodes[0].MemNode); - unsigned FirstLoadAS = FirstLoad->getAddressSpace(); unsigned FirstLoadAlign = FirstLoad->getAlignment(); // Scan the memory operations on the chain and find the first @@ -14887,11 +15895,11 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) { bool IsFastSt, IsFastLd; if (TLI.isTypeLegal(StoreTy) && TLI.canMergeStoresTo(FirstStoreAS, StoreTy, DAG) && - TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstStoreAS, - FirstStoreAlign, &IsFastSt) && + TLI.allowsMemoryAccess(Context, DL, StoreTy, + *FirstInChain->getMemOperand(), &IsFastSt) && IsFastSt && - TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstLoadAS, - FirstLoadAlign, &IsFastLd) && + TLI.allowsMemoryAccess(Context, DL, StoreTy, + *FirstLoad->getMemOperand(), &IsFastLd) && IsFastLd) { LastLegalVectorType = i + 1; } @@ -14901,11 +15909,11 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) { StoreTy = EVT::getIntegerVT(Context, SizeInBits); if (TLI.isTypeLegal(StoreTy) && TLI.canMergeStoresTo(FirstStoreAS, StoreTy, DAG) && - TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstStoreAS, - FirstStoreAlign, &IsFastSt) && + TLI.allowsMemoryAccess(Context, DL, StoreTy, + *FirstInChain->getMemOperand(), &IsFastSt) && IsFastSt && - TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstLoadAS, - FirstLoadAlign, &IsFastLd) && + TLI.allowsMemoryAccess(Context, DL, StoreTy, + *FirstLoad->getMemOperand(), &IsFastLd) && IsFastLd) { LastLegalIntegerType = i + 1; DoIntegerTruncate = false; @@ -14920,11 +15928,12 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) { TLI.isLoadExtLegal(ISD::SEXTLOAD, LegalizedStoredValTy, StoreTy) && TLI.isLoadExtLegal(ISD::EXTLOAD, LegalizedStoredValTy, StoreTy) && - TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstStoreAS, - FirstStoreAlign, &IsFastSt) && + TLI.allowsMemoryAccess(Context, DL, StoreTy, + *FirstInChain->getMemOperand(), + &IsFastSt) && IsFastSt && - TLI.allowsMemoryAccess(Context, DL, StoreTy, FirstLoadAS, - FirstLoadAlign, &IsFastLd) && + TLI.allowsMemoryAccess(Context, DL, StoreTy, + *FirstLoad->getMemOperand(), &IsFastLd) && IsFastLd) { LastLegalIntegerType = i + 1; DoIntegerTruncate = true; @@ -14994,26 +16003,32 @@ bool DAGCombiner::MergeConsecutiveStores(StoreSDNode *St) { SDValue NewStoreChain = getMergeStoreChains(StoreNodes, NumElem); AddToWorklist(NewStoreChain.getNode()); - MachineMemOperand::Flags MMOFlags = + MachineMemOperand::Flags LdMMOFlags = isDereferenceable ? MachineMemOperand::MODereferenceable : MachineMemOperand::MONone; + if (IsNonTemporalLoad) + LdMMOFlags |= MachineMemOperand::MONonTemporal; + + MachineMemOperand::Flags StMMOFlags = + IsNonTemporalStore ? MachineMemOperand::MONonTemporal + : MachineMemOperand::MONone; SDValue NewLoad, NewStore; if (UseVectorTy || !DoIntegerTruncate) { NewLoad = DAG.getLoad(JointMemOpVT, LoadDL, FirstLoad->getChain(), FirstLoad->getBasePtr(), FirstLoad->getPointerInfo(), - FirstLoadAlign, MMOFlags); + FirstLoadAlign, LdMMOFlags); NewStore = DAG.getStore( NewStoreChain, StoreDL, NewLoad, FirstInChain->getBasePtr(), - FirstInChain->getPointerInfo(), FirstStoreAlign); + FirstInChain->getPointerInfo(), FirstStoreAlign, StMMOFlags); } else { // This must be the truncstore/extload case EVT ExtendedTy = TLI.getTypeToTransformTo(*DAG.getContext(), JointMemOpVT); NewLoad = DAG.getExtLoad(ISD::EXTLOAD, LoadDL, ExtendedTy, FirstLoad->getChain(), FirstLoad->getBasePtr(), FirstLoad->getPointerInfo(), JointMemOpVT, - FirstLoadAlign, MMOFlags); + FirstLoadAlign, LdMMOFlags); NewStore = DAG.getTruncStore(NewStoreChain, StoreDL, NewLoad, FirstInChain->getBasePtr(), FirstInChain->getPointerInfo(), @@ -15168,16 +16183,11 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) { // illegal type. if (((!LegalOperations && !ST->isVolatile()) || TLI.isOperationLegal(ISD::STORE, SVT)) && - TLI.isStoreBitCastBeneficial(Value.getValueType(), SVT)) { - unsigned OrigAlign = ST->getAlignment(); - bool Fast = false; - if (TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), SVT, - ST->getAddressSpace(), OrigAlign, &Fast) && - Fast) { - return DAG.getStore(Chain, SDLoc(N), Value.getOperand(0), Ptr, - ST->getPointerInfo(), OrigAlign, - ST->getMemOperand()->getFlags(), ST->getAAInfo()); - } + TLI.isStoreBitCastBeneficial(Value.getValueType(), SVT, + DAG, *ST->getMemOperand())) { + return DAG.getStore(Chain, SDLoc(N), Value.getOperand(0), Ptr, + ST->getPointerInfo(), ST->getAlignment(), + ST->getMemOperand()->getFlags(), ST->getAAInfo()); } } @@ -15205,6 +16215,10 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) { if (SDValue NewST = TransformFPLoadStorePair(N)) return NewST; + // Try transforming several stores into STORE (BSWAP). + if (SDValue Store = MatchStoreCombine(ST)) + return Store; + if (ST->isUnindexed()) { // Walk up chain skipping non-aliasing memory nodes, on this store and any // adjacent stores. @@ -15221,23 +16235,22 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) { Value.getValueType().isInteger() && (!isa(Value) || !cast(Value)->isOpaque())) { + APInt TruncDemandedBits = + APInt::getLowBitsSet(Value.getScalarValueSizeInBits(), + ST->getMemoryVT().getScalarSizeInBits()); + // See if we can simplify the input to this truncstore with knowledge that // only the low bits are being used. For example: // "truncstore (or (shl x, 8), y), i8" -> "truncstore y, i8" - SDValue Shorter = DAG.GetDemandedBits( - Value, APInt::getLowBitsSet(Value.getScalarValueSizeInBits(), - ST->getMemoryVT().getScalarSizeInBits())); + SDValue Shorter = DAG.GetDemandedBits(Value, TruncDemandedBits); AddToWorklist(Value.getNode()); - if (Shorter.getNode()) - return DAG.getTruncStore(Chain, SDLoc(N), Shorter, - Ptr, ST->getMemoryVT(), ST->getMemOperand()); + if (Shorter) + return DAG.getTruncStore(Chain, SDLoc(N), Shorter, Ptr, ST->getMemoryVT(), + ST->getMemOperand()); // Otherwise, see if we can simplify the operation with // SimplifyDemandedBits, which only works if the value has a single use. - if (SimplifyDemandedBits( - Value, - APInt::getLowBitsSet(Value.getScalarValueSizeInBits(), - ST->getMemoryVT().getScalarSizeInBits()))) { + if (SimplifyDemandedBits(Value, TruncDemandedBits)) { // Re-visit the store if anything changed and the store hasn't been merged // with another node (N is deleted) SimplifyDemandedBits will add Value's // node back to the worklist if necessary, but we also need to re-visit @@ -15263,25 +16276,55 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) { if (StoreSDNode *ST1 = dyn_cast(Chain)) { if (ST->isUnindexed() && !ST->isVolatile() && ST1->isUnindexed() && - !ST1->isVolatile() && ST1->getBasePtr() == Ptr && - ST->getMemoryVT() == ST1->getMemoryVT()) { - // If this is a store followed by a store with the same value to the same - // location, then the store is dead/noop. - if (ST1->getValue() == Value) { - // The store is dead, remove it. + !ST1->isVolatile()) { + if (ST1->getBasePtr() == Ptr && ST1->getValue() == Value && + ST->getMemoryVT() == ST1->getMemoryVT()) { + // If this is a store followed by a store with the same value to the + // same location, then the store is dead/noop. return Chain; } - // If this is a store who's preceeding store to the same location - // and no one other node is chained to that store we can effectively - // drop the store. Do not remove stores to undef as they may be used as - // data sinks. if (OptLevel != CodeGenOpt::None && ST1->hasOneUse() && !ST1->getBasePtr().isUndef()) { - // ST1 is fully overwritten and can be elided. Combine with it's chain - // value. - CombineTo(ST1, ST1->getChain()); - return SDValue(); + const BaseIndexOffset STBase = BaseIndexOffset::match(ST, DAG); + const BaseIndexOffset ChainBase = BaseIndexOffset::match(ST1, DAG); + unsigned STBitSize = ST->getMemoryVT().getSizeInBits(); + unsigned ChainBitSize = ST1->getMemoryVT().getSizeInBits(); + // If this is a store who's preceding store to a subset of the current + // location and no one other node is chained to that store we can + // effectively drop the store. Do not remove stores to undef as they may + // be used as data sinks. + if (STBase.contains(DAG, STBitSize, ChainBase, ChainBitSize)) { + CombineTo(ST1, ST1->getChain()); + return SDValue(); + } + + // If ST stores to a subset of preceding store's write set, we may be + // able to fold ST's value into the preceding stored value. As we know + // the other uses of ST1's chain are unconcerned with ST, this folding + // will not affect those nodes. + int64_t BitOffset; + if (ChainBase.contains(DAG, ChainBitSize, STBase, STBitSize, + BitOffset)) { + SDValue ChainValue = ST1->getValue(); + if (auto *C1 = dyn_cast(ChainValue)) { + if (auto *C = dyn_cast(Value)) { + APInt Val = C1->getAPIntValue(); + APInt InsertVal = C->getAPIntValue().zextOrTrunc(STBitSize); + // FIXME: Handle Big-endian mode. + if (!DAG.getDataLayout().isBigEndian()) { + Val.insertBits(InsertVal, BitOffset); + SDValue NewSDVal = + DAG.getConstant(Val, SDLoc(C), ChainValue.getValueType(), + C1->isTargetOpcode(), C1->isOpaque()); + SDNode *NewST1 = DAG.UpdateNodeOperands( + ST1, ST1->getChain(), NewSDVal, ST1->getOperand(2), + ST1->getOperand(3)); + return CombineTo(ST, SDValue(NewST1, 0)); + } + } + } + } // End ST subset of ST1 case. } } } @@ -15299,7 +16342,7 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) { // Always perform this optimization before types are legal. If the target // prefers, also try this after legalization to catch stores that were created // by intrinsics or other nodes. - if (!LegalTypes || (TLI.mergeStoresAfterLegalization())) { + if (!LegalTypes || (TLI.mergeStoresAfterLegalization(ST->getMemoryVT()))) { while (true) { // There can be multiple store sequences on the same chain. // Keep trying to merge store sequences until we are unable to do so @@ -15333,6 +16376,54 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) { return ReduceLoadOpStoreWidth(N); } +SDValue DAGCombiner::visitLIFETIME_END(SDNode *N) { + const auto *LifetimeEnd = cast(N); + if (!LifetimeEnd->hasOffset()) + return SDValue(); + + const BaseIndexOffset LifetimeEndBase(N->getOperand(1), SDValue(), + LifetimeEnd->getOffset(), false); + + // We walk up the chains to find stores. + SmallVector Chains = {N->getOperand(0)}; + while (!Chains.empty()) { + SDValue Chain = Chains.back(); + Chains.pop_back(); + if (!Chain.hasOneUse()) + continue; + switch (Chain.getOpcode()) { + case ISD::TokenFactor: + for (unsigned Nops = Chain.getNumOperands(); Nops;) + Chains.push_back(Chain.getOperand(--Nops)); + break; + case ISD::LIFETIME_START: + case ISD::LIFETIME_END: + // We can forward past any lifetime start/end that can be proven not to + // alias the node. + if (!isAlias(Chain.getNode(), N)) + Chains.push_back(Chain.getOperand(0)); + break; + case ISD::STORE: { + StoreSDNode *ST = dyn_cast(Chain); + if (ST->isVolatile() || ST->isIndexed()) + continue; + const BaseIndexOffset StoreBase = BaseIndexOffset::match(ST, DAG); + // If we store purely within object bounds just before its lifetime ends, + // we can remove the store. + if (LifetimeEndBase.contains(DAG, LifetimeEnd->getSize() * 8, StoreBase, + ST->getMemoryVT().getStoreSizeInBits())) { + LLVM_DEBUG(dbgs() << "\nRemoving store:"; StoreBase.dump(); + dbgs() << "\nwithin LIFETIME_END of : "; + LifetimeEndBase.dump(); dbgs() << "\n"); + CombineTo(ST, ST->getChain()); + return SDValue(N, 0); + } + } + } + } + return SDValue(); +} + /// For the instruction sequence of store below, F and I values /// are bundled together as an i64 value before being stored into memory. /// Sometimes it is more efficent to generate separate stores for F and I, @@ -15616,7 +16707,9 @@ SDValue DAGCombiner::scalarizeExtractedVectorLoad(SDNode *EVE, EVT InVecVT, Offset = DAG.getNode( ISD::MUL, DL, PtrType, Offset, DAG.getConstant(VecEltVT.getStoreSize(), DL, PtrType)); - MPI = OriginalLoad->getPointerInfo(); + // Discard the pointer info except the address space because the memory + // operand can't represent this new access since the offset is variable. + MPI = MachinePointerInfo(OriginalLoad->getPointerInfo().getAddrSpace()); } NewPtr = DAG.getNode(ISD::ADD, DL, PtrType, NewPtr, Offset); @@ -15668,14 +16761,15 @@ SDValue DAGCombiner::scalarizeExtractedVectorLoad(SDNode *EVE, EVT InVecVT, /// the math/logic after an extract element of a vector. static SDValue scalarizeExtractedBinop(SDNode *ExtElt, SelectionDAG &DAG, bool LegalOperations) { + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDValue Vec = ExtElt->getOperand(0); SDValue Index = ExtElt->getOperand(1); auto *IndexC = dyn_cast(Index); - if (!IndexC || !ISD::isBinaryOp(Vec.getNode()) || !Vec.hasOneUse()) + if (!IndexC || !TLI.isBinOp(Vec.getOpcode()) || !Vec.hasOneUse() || + Vec.getNode()->getNumValues() != 1) return SDValue(); // Targets may want to avoid this to prevent an expensive register transfer. - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); if (!TLI.shouldScalarizeBinop(Vec)) return SDValue(); @@ -16073,7 +17167,7 @@ SDValue DAGCombiner::reduceBuildVecExtToExtBuildVec(SDNode *N) { SDValue DAGCombiner::createBuildVecShuffle(const SDLoc &DL, SDNode *N, ArrayRef VectorMask, SDValue VecIn1, SDValue VecIn2, - unsigned LeftIdx) { + unsigned LeftIdx, bool DidSplitVec) { MVT IdxTy = TLI.getVectorIdxTy(DAG.getDataLayout()); SDValue ZeroIdx = DAG.getConstant(0, DL, IdxTy); @@ -16081,17 +17175,12 @@ SDValue DAGCombiner::createBuildVecShuffle(const SDLoc &DL, SDNode *N, EVT InVT1 = VecIn1.getValueType(); EVT InVT2 = VecIn2.getNode() ? VecIn2.getValueType() : InVT1; - unsigned Vec2Offset = 0; unsigned NumElems = VT.getVectorNumElements(); unsigned ShuffleNumElems = NumElems; - // In case both the input vectors are extracted from same base - // vector we do not need extra addend (Vec2Offset) while - // computing shuffle mask. - if (!VecIn2 || !(VecIn1.getOpcode() == ISD::EXTRACT_SUBVECTOR) || - !(VecIn2.getOpcode() == ISD::EXTRACT_SUBVECTOR) || - !(VecIn1.getOperand(0) == VecIn2.getOperand(0))) - Vec2Offset = InVT1.getVectorNumElements(); + // If we artificially split a vector in two already, then the offsets in the + // operands will all be based off of VecIn1, even those in VecIn2. + unsigned Vec2Offset = DidSplitVec ? 0 : InVT1.getVectorNumElements(); // We can't generate a shuffle node with mismatched input and output types. // Try to make the types match the type of the output. @@ -16214,23 +17303,29 @@ static SDValue reduceBuildVecToShuffleWithZero(SDNode *BV, SelectionDAG &DAG) { // The build vector contains some number of undef elements and exactly // one other element. That other element must be a zero-extended scalar // extracted from a vector at a constant index to turn this into a shuffle. + // Also, require that the build vector does not implicitly truncate/extend + // its elements. // TODO: This could be enhanced to allow ANY_EXTEND as well as ZERO_EXTEND. + EVT VT = BV->getValueType(0); SDValue Zext = BV->getOperand(ZextElt); if (Zext.getOpcode() != ISD::ZERO_EXTEND || !Zext.hasOneUse() || Zext.getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT || - !isa(Zext.getOperand(0).getOperand(1))) + !isa(Zext.getOperand(0).getOperand(1)) || + Zext.getValueSizeInBits() != VT.getScalarSizeInBits()) return SDValue(); - // The zero-extend must be a multiple of the source size. + // The zero-extend must be a multiple of the source size, and we must be + // building a vector of the same size as the source of the extract element. SDValue Extract = Zext.getOperand(0); unsigned DestSize = Zext.getValueSizeInBits(); unsigned SrcSize = Extract.getValueSizeInBits(); - if (DestSize % SrcSize != 0) + if (DestSize % SrcSize != 0 || + Extract.getOperand(0).getValueSizeInBits() != VT.getSizeInBits()) return SDValue(); // Create a shuffle mask that will combine the extracted element with zeros // and undefs. - int ZextRatio = DestSize / SrcSize; + int ZextRatio = DestSize / SrcSize; int NumMaskElts = NumBVOps * ZextRatio; SmallVector ShufMask(NumMaskElts, -1); for (int i = 0; i != NumMaskElts; ++i) { @@ -16260,7 +17355,7 @@ static SDValue reduceBuildVecToShuffleWithZero(SDNode *BV, SelectionDAG &DAG) { SDValue ZeroVec = DAG.getConstant(0, DL, VecVT); SDValue Shuf = DAG.getVectorShuffle(VecVT, DL, Extract.getOperand(0), ZeroVec, ShufMask); - return DAG.getBitcast(BV->getValueType(0), Shuf); + return DAG.getBitcast(VT, Shuf); } // Check to see if this is a BUILD_VECTOR of a bunch of EXTRACT_VECTOR_ELT @@ -16316,7 +17411,7 @@ SDValue DAGCombiner::reduceBuildVecToShuffle(SDNode *N) { return SDValue(); SDValue ExtractedFromVec = Op.getOperand(0); - APInt ExtractIdx = cast(Op.getOperand(1))->getAPIntValue(); + const APInt &ExtractIdx = Op.getConstantOperandAPInt(1); if (ExtractIdx.uge(ExtractedFromVec.getValueType().getVectorNumElements())) return SDValue(); @@ -16344,6 +17439,7 @@ SDValue DAGCombiner::reduceBuildVecToShuffle(SDNode *N) { // vector, then split the vector efficiently based on the maximum // vector access index and adjust the VectorMask and // VecIn accordingly. + bool DidSplitVec = false; if (VecIn.size() == 2) { unsigned MaxIndex = 0; unsigned NearestPow2 = 0; @@ -16374,6 +17470,7 @@ SDValue DAGCombiner::reduceBuildVecToShuffle(SDNode *N) { VecIn.pop_back(); VecIn.push_back(VecIn1); VecIn.push_back(VecIn2); + DidSplitVec = true; for (unsigned i = 0; i < NumElems; i++) { if (VectorMask[i] <= 0) @@ -16411,7 +17508,7 @@ SDValue DAGCombiner::reduceBuildVecToShuffle(SDNode *N) { (LeftIdx + 1) < VecIn.size() ? VecIn[LeftIdx + 1] : SDValue(); if (SDValue Shuffle = createBuildVecShuffle(DL, N, VectorMask, VecLeft, - VecRight, LeftIdx)) + VecRight, LeftIdx, DidSplitVec)) Shuffles.push_back(Shuffle); else return SDValue(); @@ -16477,18 +17574,20 @@ SDValue DAGCombiner::reduceBuildVecToShuffle(SDNode *N) { // Try to turn a build vector of zero extends of extract vector elts into a // a vector zero extend and possibly an extract subvector. -// TODO: Support sign extend or any extend? +// TODO: Support sign extend? // TODO: Allow undef elements? -// TODO: Don't require the extracts to start at element 0. SDValue DAGCombiner::convertBuildVecZextToZext(SDNode *N) { if (LegalOperations) return SDValue(); EVT VT = N->getValueType(0); + bool FoundZeroExtend = false; SDValue Op0 = N->getOperand(0); auto checkElem = [&](SDValue Op) -> int64_t { - if (Op.getOpcode() == ISD::ZERO_EXTEND && + unsigned Opc = Op.getOpcode(); + FoundZeroExtend |= (Opc == ISD::ZERO_EXTEND); + if ((Opc == ISD::ZERO_EXTEND || Opc == ISD::ANY_EXTEND) && Op.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT && Op0.getOperand(0).getOperand(0) == Op.getOperand(0).getOperand(0)) if (auto *C = dyn_cast(Op.getOperand(0).getOperand(1))) @@ -16520,7 +17619,8 @@ SDValue DAGCombiner::convertBuildVecZextToZext(SDNode *N) { SDLoc DL(N); In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, InVT, In, Op0.getOperand(0).getOperand(1)); - return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, In); + return DAG.getNode(FoundZeroExtend ? ISD::ZERO_EXTEND : ISD::ANY_EXTEND, DL, + VT, In); } SDValue DAGCombiner::visitBUILD_VECTOR(SDNode *N) { @@ -16885,14 +17985,14 @@ SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) { return SDValue(); } - unsigned IdentityIndex = i * PartNumElem; - ConstantSDNode *CS = dyn_cast(Op.getOperand(1)); + auto *CS = dyn_cast(Op.getOperand(1)); // The extract index must be constant. if (!CS) return SDValue(); // Check that we are reading from the identity index. - if (CS->getZExtValue() != IdentityIndex) + unsigned IdentityIndex = i * PartNumElem; + if (CS->getAPIntValue() != IdentityIndex) return SDValue(); } @@ -16902,12 +18002,59 @@ SDValue DAGCombiner::visitCONCAT_VECTORS(SDNode *N) { return SDValue(); } +static SDValue narrowInsertExtractVectorBinOp(SDNode *Extract, + SelectionDAG &DAG) { + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + SDValue BinOp = Extract->getOperand(0); + unsigned BinOpcode = BinOp.getOpcode(); + if (!TLI.isBinOp(BinOpcode) || BinOp.getNode()->getNumValues() != 1) + return SDValue(); + + SDValue Bop0 = BinOp.getOperand(0), Bop1 = BinOp.getOperand(1); + SDValue Index = Extract->getOperand(1); + EVT VT = Extract->getValueType(0); + + // Helper that peeks through INSERT_SUBVECTOR/CONCAT_VECTORS to find + // if the source subvector is the same type as the one being extracted. + auto GetSubVector = [VT, Index](SDValue V) -> SDValue { + if (V.getOpcode() == ISD::INSERT_SUBVECTOR && + V.getOperand(1).getValueType() == VT && V.getOperand(2) == Index) { + return V.getOperand(1); + } + auto *IndexC = dyn_cast(Index); + if (IndexC && V.getOpcode() == ISD::CONCAT_VECTORS && + V.getOperand(0).getValueType() == VT && + (IndexC->getZExtValue() % VT.getVectorNumElements()) == 0) { + uint64_t SubIdx = IndexC->getZExtValue() / VT.getVectorNumElements(); + return V.getOperand(SubIdx); + } + return SDValue(); + }; + SDValue Sub0 = GetSubVector(Bop0); + SDValue Sub1 = GetSubVector(Bop1); + + // TODO: We could handle the case where only 1 operand is being inserted by + // creating an extract of the other operand, but that requires checking + // number of uses and/or costs. + if (!Sub0 || !Sub1 || !TLI.isOperationLegalOrCustom(BinOpcode, VT)) + return SDValue(); + + // We are inserting both operands of the wide binop only to extract back + // to the narrow vector size. Eliminate all of the insert/extract: + // ext (binop (ins ?, X, Index), (ins ?, Y, Index)), Index --> binop X, Y + return DAG.getNode(BinOpcode, SDLoc(Extract), VT, Sub0, Sub1, + BinOp->getFlags()); +} + /// If we are extracting a subvector produced by a wide binary operator try /// to use a narrow binary operator and/or avoid concatenation and extraction. static SDValue narrowExtractedVectorBinOp(SDNode *Extract, SelectionDAG &DAG) { // TODO: Refactor with the caller (visitEXTRACT_SUBVECTOR), so we can share // some of these bailouts with other transforms. + if (SDValue V = narrowInsertExtractVectorBinOp(Extract, DAG)) + return V; + // The extract index must be a constant, so we can map it to a concat operand. auto *ExtractIndexC = dyn_cast(Extract->getOperand(1)); if (!ExtractIndexC) @@ -16915,8 +18062,10 @@ static SDValue narrowExtractedVectorBinOp(SDNode *Extract, SelectionDAG &DAG) { // We are looking for an optionally bitcasted wide vector binary operator // feeding an extract subvector. + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDValue BinOp = peekThroughBitcasts(Extract->getOperand(0)); - if (!ISD::isBinaryOp(BinOp.getNode())) + unsigned BOpcode = BinOp.getOpcode(); + if (!TLI.isBinOp(BOpcode) || BinOp.getNode()->getNumValues() != 1) return SDValue(); // The binop must be a vector type, so we can extract some fraction of it. @@ -16945,8 +18094,6 @@ static SDValue narrowExtractedVectorBinOp(SDNode *Extract, SelectionDAG &DAG) { // Bail out if the target does not support a narrower version of the binop. EVT NarrowBVT = EVT::getVectorVT(*DAG.getContext(), WideBVT.getScalarType(), WideNumElts / NarrowingRatio); - unsigned BOpcode = BinOp.getOpcode(); - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); if (!TLI.isOperationLegalOrCustomOrPromote(BOpcode, NarrowBVT)) return SDValue(); @@ -16986,35 +18133,35 @@ static SDValue narrowExtractedVectorBinOp(SDNode *Extract, SelectionDAG &DAG) { // We need at least one concatenation operation of a binop operand to make // this transform worthwhile. The concat must double the input vector sizes. - // TODO: Should we also handle INSERT_SUBVECTOR patterns? - SDValue LHS = peekThroughBitcasts(BinOp.getOperand(0)); - SDValue RHS = peekThroughBitcasts(BinOp.getOperand(1)); - bool ConcatL = - LHS.getOpcode() == ISD::CONCAT_VECTORS && LHS.getNumOperands() == 2; - bool ConcatR = - RHS.getOpcode() == ISD::CONCAT_VECTORS && RHS.getNumOperands() == 2; - if (!ConcatL && !ConcatR) + auto GetSubVector = [ConcatOpNum](SDValue V) -> SDValue { + if (V.getOpcode() == ISD::CONCAT_VECTORS && V.getNumOperands() == 2) + return V.getOperand(ConcatOpNum); return SDValue(); + }; + SDValue SubVecL = GetSubVector(peekThroughBitcasts(BinOp.getOperand(0))); + SDValue SubVecR = GetSubVector(peekThroughBitcasts(BinOp.getOperand(1))); + + if (SubVecL || SubVecR) { + // If a binop operand was not the result of a concat, we must extract a + // half-sized operand for our new narrow binop: + // extract (binop (concat X1, X2), (concat Y1, Y2)), N --> binop XN, YN + // extract (binop (concat X1, X2), Y), N --> binop XN, (extract Y, IndexC) + // extract (binop X, (concat Y1, Y2)), N --> binop (extract X, IndexC), YN + SDLoc DL(Extract); + SDValue IndexC = DAG.getConstant(ExtBOIdx, DL, ExtBOIdxVT); + SDValue X = SubVecL ? DAG.getBitcast(NarrowBVT, SubVecL) + : DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT, + BinOp.getOperand(0), IndexC); - // If one of the binop operands was not the result of a concat, we must - // extract a half-sized operand for our new narrow binop. - SDLoc DL(Extract); - - // extract (binop (concat X1, X2), (concat Y1, Y2)), N --> binop XN, YN - // extract (binop (concat X1, X2), Y), N --> binop XN, (extract Y, N) - // extract (binop X, (concat Y1, Y2)), N --> binop (extract X, N), YN - SDValue X = ConcatL ? DAG.getBitcast(NarrowBVT, LHS.getOperand(ConcatOpNum)) - : DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT, - BinOp.getOperand(0), - DAG.getConstant(ExtBOIdx, DL, ExtBOIdxVT)); + SDValue Y = SubVecR ? DAG.getBitcast(NarrowBVT, SubVecR) + : DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT, + BinOp.getOperand(1), IndexC); - SDValue Y = ConcatR ? DAG.getBitcast(NarrowBVT, RHS.getOperand(ConcatOpNum)) - : DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NarrowBVT, - BinOp.getOperand(1), - DAG.getConstant(ExtBOIdx, DL, ExtBOIdxVT)); + SDValue NarrowBinOp = DAG.getNode(BOpcode, DL, NarrowBVT, X, Y); + return DAG.getBitcast(VT, NarrowBinOp); + } - SDValue NarrowBinOp = DAG.getNode(BOpcode, DL, NarrowBVT, X, Y); - return DAG.getBitcast(VT, NarrowBinOp); + return SDValue(); } /// If we are extracting a subvector from a wide vector load, convert to a @@ -17052,7 +18199,7 @@ static SDValue narrowExtractedVectorLoad(SDNode *Extract, SelectionDAG &DAG) { return NewLd; } -SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode* N) { +SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode *N) { EVT NVT = N->getValueType(0); SDValue V = N->getOperand(0); @@ -17064,14 +18211,51 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode* N) { if (SDValue NarrowLoad = narrowExtractedVectorLoad(N, DAG)) return NarrowLoad; + // Combine an extract of an extract into a single extract_subvector. + // ext (ext X, C), 0 --> ext X, C + SDValue Index = N->getOperand(1); + if (isNullConstant(Index) && V.getOpcode() == ISD::EXTRACT_SUBVECTOR && + V.hasOneUse() && isa(V.getOperand(1))) { + if (TLI.isExtractSubvectorCheap(NVT, V.getOperand(0).getValueType(), + V.getConstantOperandVal(1)) && + TLI.isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, NVT)) { + return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), NVT, V.getOperand(0), + V.getOperand(1)); + } + } + + // Try to move vector bitcast after extract_subv by scaling extraction index: + // extract_subv (bitcast X), Index --> bitcast (extract_subv X, Index') + if (isa(Index) && V.getOpcode() == ISD::BITCAST && + V.getOperand(0).getValueType().isVector()) { + SDValue SrcOp = V.getOperand(0); + EVT SrcVT = SrcOp.getValueType(); + unsigned SrcNumElts = SrcVT.getVectorNumElements(); + unsigned DestNumElts = V.getValueType().getVectorNumElements(); + if ((SrcNumElts % DestNumElts) == 0) { + unsigned SrcDestRatio = SrcNumElts / DestNumElts; + unsigned NewExtNumElts = NVT.getVectorNumElements() * SrcDestRatio; + EVT NewExtVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(), + NewExtNumElts); + if (TLI.isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, NewExtVT)) { + unsigned IndexValScaled = N->getConstantOperandVal(1) * SrcDestRatio; + SDLoc DL(N); + SDValue NewIndex = DAG.getIntPtrConstant(IndexValScaled, DL); + SDValue NewExtract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, NewExtVT, + V.getOperand(0), NewIndex); + return DAG.getBitcast(NVT, NewExtract); + } + } + // TODO - handle (DestNumElts % SrcNumElts) == 0 + } + // Combine: // (extract_subvec (concat V1, V2, ...), i) // Into: // Vi if possible // Only operand 0 is checked as 'concat' assumes all inputs of the same // type. - if (V.getOpcode() == ISD::CONCAT_VECTORS && - isa(N->getOperand(1)) && + if (V.getOpcode() == ISD::CONCAT_VECTORS && isa(Index) && V.getOperand(0).getValueType() == NVT) { unsigned Idx = N->getConstantOperandVal(1); unsigned NumElems = NVT.getVectorNumElements(); @@ -17084,7 +18268,7 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode* N) { // If the input is a build vector. Try to make a smaller build vector. if (V.getOpcode() == ISD::BUILD_VECTOR) { - if (auto *Idx = dyn_cast(N->getOperand(1))) { + if (auto *IdxC = dyn_cast(Index)) { EVT InVT = V.getValueType(); unsigned ExtractSize = NVT.getSizeInBits(); unsigned EltSize = InVT.getScalarSizeInBits(); @@ -17092,26 +18276,27 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode* N) { if (ExtractSize % EltSize == 0) { unsigned NumElems = ExtractSize / EltSize; EVT EltVT = InVT.getVectorElementType(); - EVT ExtractVT = NumElems == 1 ? EltVT : - EVT::getVectorVT(*DAG.getContext(), EltVT, NumElems); + EVT ExtractVT = NumElems == 1 ? EltVT + : EVT::getVectorVT(*DAG.getContext(), + EltVT, NumElems); if ((Level < AfterLegalizeDAG || (NumElems == 1 || TLI.isOperationLegal(ISD::BUILD_VECTOR, ExtractVT))) && (!LegalTypes || TLI.isTypeLegal(ExtractVT))) { - unsigned IdxVal = (Idx->getZExtValue() * NVT.getScalarSizeInBits()) / - EltSize; + unsigned IdxVal = IdxC->getZExtValue(); + IdxVal *= NVT.getScalarSizeInBits(); + IdxVal /= EltSize; + if (NumElems == 1) { SDValue Src = V->getOperand(IdxVal); if (EltVT != Src.getValueType()) Src = DAG.getNode(ISD::TRUNCATE, SDLoc(N), InVT, Src); - return DAG.getBitcast(NVT, Src); } // Extract the pieces from the original build_vector. - SDValue BuildVec = DAG.getBuildVector(ExtractVT, SDLoc(N), - makeArrayRef(V->op_begin() + IdxVal, - NumElems)); + SDValue BuildVec = DAG.getBuildVector( + ExtractVT, SDLoc(N), V->ops().slice(IdxVal, NumElems)); return DAG.getBitcast(NVT, BuildVec); } } @@ -17126,9 +18311,8 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode* N) { return SDValue(); // Only handle cases where both indexes are constants. - auto *ExtIdx = dyn_cast(N->getOperand(1)); + auto *ExtIdx = dyn_cast(Index); auto *InsIdx = dyn_cast(V.getOperand(2)); - if (InsIdx && ExtIdx) { // Combine: // (extract_subvec (insert_subvec V1, V2, InsIdx), ExtIdx) @@ -17141,7 +18325,7 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode* N) { return DAG.getNode( ISD::EXTRACT_SUBVECTOR, SDLoc(N), NVT, DAG.getBitcast(N->getOperand(0).getValueType(), V.getOperand(0)), - N->getOperand(1)); + Index); } } @@ -17154,6 +18338,53 @@ SDValue DAGCombiner::visitEXTRACT_SUBVECTOR(SDNode* N) { return SDValue(); } +/// Try to convert a wide shuffle of concatenated vectors into 2 narrow shuffles +/// followed by concatenation. Narrow vector ops may have better performance +/// than wide ops, and this can unlock further narrowing of other vector ops. +/// Targets can invert this transform later if it is not profitable. +static SDValue foldShuffleOfConcatUndefs(ShuffleVectorSDNode *Shuf, + SelectionDAG &DAG) { + SDValue N0 = Shuf->getOperand(0), N1 = Shuf->getOperand(1); + if (N0.getOpcode() != ISD::CONCAT_VECTORS || N0.getNumOperands() != 2 || + N1.getOpcode() != ISD::CONCAT_VECTORS || N1.getNumOperands() != 2 || + !N0.getOperand(1).isUndef() || !N1.getOperand(1).isUndef()) + return SDValue(); + + // Split the wide shuffle mask into halves. Any mask element that is accessing + // operand 1 is offset down to account for narrowing of the vectors. + ArrayRef Mask = Shuf->getMask(); + EVT VT = Shuf->getValueType(0); + unsigned NumElts = VT.getVectorNumElements(); + unsigned HalfNumElts = NumElts / 2; + SmallVector Mask0(HalfNumElts, -1); + SmallVector Mask1(HalfNumElts, -1); + for (unsigned i = 0; i != NumElts; ++i) { + if (Mask[i] == -1) + continue; + int M = Mask[i] < (int)NumElts ? Mask[i] : Mask[i] - (int)HalfNumElts; + if (i < HalfNumElts) + Mask0[i] = M; + else + Mask1[i - HalfNumElts] = M; + } + + // Ask the target if this is a valid transform. + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), + HalfNumElts); + if (!TLI.isShuffleMaskLegal(Mask0, HalfVT) || + !TLI.isShuffleMaskLegal(Mask1, HalfVT)) + return SDValue(); + + // shuffle (concat X, undef), (concat Y, undef), Mask --> + // concat (shuffle X, Y, Mask0), (shuffle X, Y, Mask1) + SDValue X = N0.getOperand(0), Y = N1.getOperand(0); + SDLoc DL(Shuf); + SDValue Shuf0 = DAG.getVectorShuffle(HalfVT, DL, X, Y, Mask0); + SDValue Shuf1 = DAG.getVectorShuffle(HalfVT, DL, X, Y, Mask1); + return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Shuf0, Shuf1); +} + // Tries to turn a shuffle of two CONCAT_VECTORS into a single concat, // or turn a shuffle of a single concat into simpler shuffle then concat. static SDValue partitionShuffleOfConcats(SDNode *N, SelectionDAG &DAG) { @@ -17163,20 +18394,24 @@ static SDValue partitionShuffleOfConcats(SDNode *N, SelectionDAG &DAG) { SDValue N0 = N->getOperand(0); SDValue N1 = N->getOperand(1); ShuffleVectorSDNode *SVN = cast(N); + ArrayRef Mask = SVN->getMask(); SmallVector Ops; EVT ConcatVT = N0.getOperand(0).getValueType(); unsigned NumElemsPerConcat = ConcatVT.getVectorNumElements(); unsigned NumConcats = NumElts / NumElemsPerConcat; + auto IsUndefMaskElt = [](int i) { return i == -1; }; + // Special case: shuffle(concat(A,B)) can be more efficiently represented // as concat(shuffle(A,B),UNDEF) if the shuffle doesn't set any of the high // half vector elements. if (NumElemsPerConcat * 2 == NumElts && N1.isUndef() && - std::all_of(SVN->getMask().begin() + NumElemsPerConcat, - SVN->getMask().end(), [](int i) { return i == -1; })) { - N0 = DAG.getVectorShuffle(ConcatVT, SDLoc(N), N0.getOperand(0), N0.getOperand(1), - makeArrayRef(SVN->getMask().begin(), NumElemsPerConcat)); + llvm::all_of(Mask.slice(NumElemsPerConcat, NumElemsPerConcat), + IsUndefMaskElt)) { + N0 = DAG.getVectorShuffle(ConcatVT, SDLoc(N), N0.getOperand(0), + N0.getOperand(1), + Mask.slice(0, NumElemsPerConcat)); N1 = DAG.getUNDEF(ConcatVT); return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, N0, N1); } @@ -17184,35 +18419,32 @@ static SDValue partitionShuffleOfConcats(SDNode *N, SelectionDAG &DAG) { // Look at every vector that's inserted. We're looking for exact // subvector-sized copies from a concatenated vector for (unsigned I = 0; I != NumConcats; ++I) { - // Make sure we're dealing with a copy. unsigned Begin = I * NumElemsPerConcat; - bool AllUndef = true, NoUndef = true; - for (unsigned J = Begin; J != Begin + NumElemsPerConcat; ++J) { - if (SVN->getMaskElt(J) >= 0) - AllUndef = false; - else - NoUndef = false; + ArrayRef SubMask = Mask.slice(Begin, NumElemsPerConcat); + + // Make sure we're dealing with a copy. + if (llvm::all_of(SubMask, IsUndefMaskElt)) { + Ops.push_back(DAG.getUNDEF(ConcatVT)); + continue; } - if (NoUndef) { - if (SVN->getMaskElt(Begin) % NumElemsPerConcat != 0) + int OpIdx = -1; + for (int i = 0; i != (int)NumElemsPerConcat; ++i) { + if (IsUndefMaskElt(SubMask[i])) + continue; + if ((SubMask[i] % (int)NumElemsPerConcat) != i) return SDValue(); - - for (unsigned J = 1; J != NumElemsPerConcat; ++J) - if (SVN->getMaskElt(Begin + J - 1) + 1 != SVN->getMaskElt(Begin + J)) - return SDValue(); - - unsigned FirstElt = SVN->getMaskElt(Begin) / NumElemsPerConcat; - if (FirstElt < N0.getNumOperands()) - Ops.push_back(N0.getOperand(FirstElt)); - else - Ops.push_back(N1.getOperand(FirstElt - N0.getNumOperands())); - - } else if (AllUndef) { - Ops.push_back(DAG.getUNDEF(N0.getOperand(0).getValueType())); - } else { // Mixed with general masks and undefs, can't do optimization. - return SDValue(); + int EltOpIdx = SubMask[i] / NumElemsPerConcat; + if (0 <= OpIdx && EltOpIdx != OpIdx) + return SDValue(); + OpIdx = EltOpIdx; } + assert(0 <= OpIdx && "Unknown concat_vectors op"); + + if (OpIdx < (int)N0.getNumOperands()) + Ops.push_back(N0.getOperand(OpIdx)); + else + Ops.push_back(N1.getOperand(OpIdx - N0.getNumOperands())); } return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(N), VT, Ops); @@ -17278,8 +18510,8 @@ static SDValue combineShuffleOfScalars(ShuffleVectorSDNode *SVN, if (S.getOpcode() == ISD::BUILD_VECTOR) { Op = S.getOperand(Idx); } else if (S.getOpcode() == ISD::SCALAR_TO_VECTOR) { - assert(Idx == 0 && "Unexpected SCALAR_TO_VECTOR operand index."); - Op = S.getOperand(0); + SDValue Op0 = S.getOperand(0); + Op = Idx == 0 ? Op0 : DAG.getUNDEF(Op0.getValueType()); } else { // Operand can't be combined - bail out. return SDValue(); @@ -17433,11 +18665,17 @@ static SDValue combineTruncationShuffle(ShuffleVectorSDNode *SVN, // If splat-mask contains undef elements, we need to be careful about // introducing undef's in the folded mask which are not the result of composing // the masks of the shuffles. -static SDValue combineShuffleOfSplat(ArrayRef UserMask, - ShuffleVectorSDNode *Splat, - SelectionDAG &DAG) { +static SDValue combineShuffleOfSplatVal(ShuffleVectorSDNode *Shuf, + SelectionDAG &DAG) { + if (!Shuf->getOperand(1).isUndef()) + return SDValue(); + auto *Splat = dyn_cast(Shuf->getOperand(0)); + if (!Splat || !Splat->isSplat()) + return SDValue(); + + ArrayRef ShufMask = Shuf->getMask(); ArrayRef SplatMask = Splat->getMask(); - assert(UserMask.size() == SplatMask.size() && "Mask length mismatch"); + assert(ShufMask.size() == SplatMask.size() && "Mask length mismatch"); // Prefer simplifying to the splat-shuffle, if possible. This is legal if // every undef mask element in the splat-shuffle has a corresponding undef @@ -17463,13 +18701,13 @@ static SDValue combineShuffleOfSplat(ArrayRef UserMask, return false; return true; }; - if (CanSimplifyToExistingSplat(UserMask, SplatMask)) - return SDValue(Splat, 0); + if (CanSimplifyToExistingSplat(ShufMask, SplatMask)) + return Shuf->getOperand(0); // Create a new shuffle with a mask that is composed of the two shuffles' // masks. SmallVector NewMask; - for (int Idx : UserMask) + for (int Idx : ShufMask) NewMask.push_back(Idx == -1 ? -1 : SplatMask[Idx]); return DAG.getVectorShuffle(Splat->getValueType(0), SDLoc(Splat), @@ -17555,6 +18793,34 @@ static SDValue replaceShuffleOfInsert(ShuffleVectorSDNode *Shuf, Op1, Op0.getOperand(1), NewInsIndex); } +/// If we have a unary shuffle of a shuffle, see if it can be folded away +/// completely. This has the potential to lose undef knowledge because the first +/// shuffle may not have an undef mask element where the second one does. So +/// only call this after doing simplifications based on demanded elements. +static SDValue simplifyShuffleOfShuffle(ShuffleVectorSDNode *Shuf) { + // shuf (shuf0 X, Y, Mask0), undef, Mask + auto *Shuf0 = dyn_cast(Shuf->getOperand(0)); + if (!Shuf0 || !Shuf->getOperand(1).isUndef()) + return SDValue(); + + ArrayRef Mask = Shuf->getMask(); + ArrayRef Mask0 = Shuf0->getMask(); + for (int i = 0, e = (int)Mask.size(); i != e; ++i) { + // Ignore undef elements. + if (Mask[i] == -1) + continue; + assert(Mask[i] >= 0 && Mask[i] < e && "Unexpected shuffle mask value"); + + // Is the element of the shuffle operand chosen by this shuffle the same as + // the element chosen by the shuffle operand itself? + if (Mask0[Mask[i]] != Mask0[i]) + return SDValue(); + } + // Every element of this shuffle is identical to the result of the previous + // shuffle, so we can replace this value. + return Shuf->getOperand(0); +} + SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { EVT VT = N->getValueType(0); unsigned NumElts = VT.getVectorNumElements(); @@ -17604,19 +18870,35 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { if (SDValue InsElt = replaceShuffleOfInsert(SVN, DAG)) return InsElt; - // A shuffle of a single vector that is a splat can always be folded. - if (auto *N0Shuf = dyn_cast(N0)) - if (N1->isUndef() && N0Shuf->isSplat()) - return combineShuffleOfSplat(SVN->getMask(), N0Shuf, DAG); + // A shuffle of a single vector that is a splatted value can always be folded. + if (SDValue V = combineShuffleOfSplatVal(SVN, DAG)) + return V; // If it is a splat, check if the argument vector is another splat or a // build_vector. if (SVN->isSplat() && SVN->getSplatIndex() < (int)NumElts) { - SDNode *V = N0.getNode(); + int SplatIndex = SVN->getSplatIndex(); + if (TLI.isExtractVecEltCheap(VT, SplatIndex) && + TLI.isBinOp(N0.getOpcode()) && N0.getNode()->getNumValues() == 1) { + // splat (vector_bo L, R), Index --> + // splat (scalar_bo (extelt L, Index), (extelt R, Index)) + SDValue L = N0.getOperand(0), R = N0.getOperand(1); + SDLoc DL(N); + EVT EltVT = VT.getScalarType(); + SDValue Index = DAG.getIntPtrConstant(SplatIndex, DL); + SDValue ExtL = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, L, Index); + SDValue ExtR = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, R, Index); + SDValue NewBO = DAG.getNode(N0.getOpcode(), DL, EltVT, ExtL, ExtR, + N0.getNode()->getFlags()); + SDValue Insert = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, NewBO); + SmallVector ZeroMask(VT.getVectorNumElements(), 0); + return DAG.getVectorShuffle(VT, DL, Insert, DAG.getUNDEF(VT), ZeroMask); + } // If this is a bit convert that changes the element type of the vector but // not the number of vector elements, look through it. Be careful not to // look though conversions that change things like v4f32 to v2f64. + SDNode *V = N0.getNode(); if (V->getOpcode() == ISD::BITCAST) { SDValue ConvInput = V->getOperand(0); if (ConvInput.getValueType().isVector() && @@ -17649,7 +18931,7 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { return N0; // Canonicalize any other splat as a build_vector. - const SDValue &Splatted = V->getOperand(SVN->getSplatIndex()); + SDValue Splatted = V->getOperand(SplatIndex); SmallVector Ops(NumElts, Splatted); SDValue NewBV = DAG.getBuildVector(V->getValueType(0), SDLoc(N), Ops); @@ -17665,6 +18947,11 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { if (SimplifyDemandedVectorElts(SDValue(N, 0))) return SDValue(N, 0); + // This is intentionally placed after demanded elements simplification because + // it could eliminate knowledge of undef elements created by this shuffle. + if (SDValue ShufOp = simplifyShuffleOfShuffle(SVN)) + return ShufOp; + // Match shuffles that can be converted to any_vector_extend_in_reg. if (SDValue V = combineShuffleToVectorExtend(SVN, DAG, TLI, LegalOperations)) return V; @@ -17704,7 +18991,7 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { NewMask.push_back(M < 0 ? -1 : Scale * M + s); return NewMask; }; - + SDValue BC0 = peekThroughOneUseBitcasts(N0); if (BC0.getOpcode() == ISD::VECTOR_SHUFFLE && BC0.hasOneUse()) { EVT SVT = VT.getScalarType(); @@ -17884,6 +19171,9 @@ SDValue DAGCombiner::visitVECTOR_SHUFFLE(SDNode *N) { return DAG.getVectorShuffle(VT, SDLoc(N), SV0, SV1, Mask); } + if (SDValue V = foldShuffleOfConcatUndefs(SVN, DAG)) + return V; + return SDValue(); } @@ -18006,7 +19296,44 @@ SDValue DAGCombiner::visitINSERT_SUBVECTOR(SDNode *N) { if (!isa(N2)) return SDValue(); - unsigned InsIdx = cast(N2)->getZExtValue(); + uint64_t InsIdx = cast(N2)->getZExtValue(); + + // Push subvector bitcasts to the output, adjusting the index as we go. + // insert_subvector(bitcast(v), bitcast(s), c1) + // -> bitcast(insert_subvector(v, s, c2)) + if ((N0.isUndef() || N0.getOpcode() == ISD::BITCAST) && + N1.getOpcode() == ISD::BITCAST) { + SDValue N0Src = peekThroughBitcasts(N0); + SDValue N1Src = peekThroughBitcasts(N1); + EVT N0SrcSVT = N0Src.getValueType().getScalarType(); + EVT N1SrcSVT = N1Src.getValueType().getScalarType(); + if ((N0.isUndef() || N0SrcSVT == N1SrcSVT) && + N0Src.getValueType().isVector() && N1Src.getValueType().isVector()) { + EVT NewVT; + SDLoc DL(N); + SDValue NewIdx; + MVT IdxVT = TLI.getVectorIdxTy(DAG.getDataLayout()); + LLVMContext &Ctx = *DAG.getContext(); + unsigned NumElts = VT.getVectorNumElements(); + unsigned EltSizeInBits = VT.getScalarSizeInBits(); + if ((EltSizeInBits % N1SrcSVT.getSizeInBits()) == 0) { + unsigned Scale = EltSizeInBits / N1SrcSVT.getSizeInBits(); + NewVT = EVT::getVectorVT(Ctx, N1SrcSVT, NumElts * Scale); + NewIdx = DAG.getConstant(InsIdx * Scale, DL, IdxVT); + } else if ((N1SrcSVT.getSizeInBits() % EltSizeInBits) == 0) { + unsigned Scale = N1SrcSVT.getSizeInBits() / EltSizeInBits; + if ((NumElts % Scale) == 0 && (InsIdx % Scale) == 0) { + NewVT = EVT::getVectorVT(Ctx, N1SrcSVT, NumElts / Scale); + NewIdx = DAG.getConstant(InsIdx / Scale, DL, IdxVT); + } + } + if (NewIdx && hasOperation(ISD::INSERT_SUBVECTOR, NewVT)) { + SDValue Res = DAG.getBitcast(NewVT, N0Src); + Res = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, NewVT, Res, N1Src, NewIdx); + return DAG.getBitcast(VT, Res); + } + } + } // Canonicalize insert_subvector dag nodes. // Example: @@ -18070,6 +19397,36 @@ SDValue DAGCombiner::visitFP16_TO_FP(SDNode *N) { return SDValue(); } +SDValue DAGCombiner::visitVECREDUCE(SDNode *N) { + SDValue N0 = N->getOperand(0); + EVT VT = N0.getValueType(); + unsigned Opcode = N->getOpcode(); + + // VECREDUCE over 1-element vector is just an extract. + if (VT.getVectorNumElements() == 1) { + SDLoc dl(N); + SDValue Res = DAG.getNode( + ISD::EXTRACT_VECTOR_ELT, dl, VT.getVectorElementType(), N0, + DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); + if (Res.getValueType() != N->getValueType(0)) + Res = DAG.getNode(ISD::ANY_EXTEND, dl, N->getValueType(0), Res); + return Res; + } + + // On an boolean vector an and/or reduction is the same as a umin/umax + // reduction. Convert them if the latter is legal while the former isn't. + if (Opcode == ISD::VECREDUCE_AND || Opcode == ISD::VECREDUCE_OR) { + unsigned NewOpcode = Opcode == ISD::VECREDUCE_AND + ? ISD::VECREDUCE_UMIN : ISD::VECREDUCE_UMAX; + if (!TLI.isOperationLegalOrCustom(Opcode, VT) && + TLI.isOperationLegalOrCustom(NewOpcode, VT) && + DAG.ComputeNumSignBits(N0) == VT.getScalarSizeInBits()) + return DAG.getNode(NewOpcode, SDLoc(N), N->getValueType(0), N0); + } + + return SDValue(); +} + /// Returns a vector_shuffle if it able to transform an AND to a vector_shuffle /// with the destination vector and a zero vector. /// e.g. AND V, <0xffffffff, 0, 0xffffffff, 0>. ==> @@ -18161,6 +19518,53 @@ SDValue DAGCombiner::XformToShuffleWithZero(SDNode *N) { return SDValue(); } +/// If a vector binop is performed on splat values, it may be profitable to +/// extract, scalarize, and insert/splat. +static SDValue scalarizeBinOpOfSplats(SDNode *N, SelectionDAG &DAG) { + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + unsigned Opcode = N->getOpcode(); + EVT VT = N->getValueType(0); + EVT EltVT = VT.getVectorElementType(); + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + + // TODO: Remove/replace the extract cost check? If the elements are available + // as scalars, then there may be no extract cost. Should we ask if + // inserting a scalar back into a vector is cheap instead? + int Index0, Index1; + SDValue Src0 = DAG.getSplatSourceVector(N0, Index0); + SDValue Src1 = DAG.getSplatSourceVector(N1, Index1); + if (!Src0 || !Src1 || Index0 != Index1 || + Src0.getValueType().getVectorElementType() != EltVT || + Src1.getValueType().getVectorElementType() != EltVT || + !TLI.isExtractVecEltCheap(VT, Index0) || + !TLI.isOperationLegalOrCustom(Opcode, EltVT)) + return SDValue(); + + SDLoc DL(N); + SDValue IndexC = + DAG.getConstant(Index0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())); + SDValue X = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, N0, IndexC); + SDValue Y = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, N1, IndexC); + SDValue ScalarBO = DAG.getNode(Opcode, DL, EltVT, X, Y, N->getFlags()); + + // If all lanes but 1 are undefined, no need to splat the scalar result. + // TODO: Keep track of undefs and use that info in the general case. + if (N0.getOpcode() == ISD::BUILD_VECTOR && N0.getOpcode() == N1.getOpcode() && + count_if(N0->ops(), [](SDValue V) { return !V.isUndef(); }) == 1 && + count_if(N1->ops(), [](SDValue V) { return !V.isUndef(); }) == 1) { + // bo (build_vec ..undef, X, undef...), (build_vec ..undef, Y, undef...) --> + // build_vec ..undef, (bo X, Y), undef... + SmallVector Ops(VT.getVectorNumElements(), DAG.getUNDEF(EltVT)); + Ops[Index0] = ScalarBO; + return DAG.getBuildVector(VT, DL, Ops); + } + + // bo (splat X, Index), (splat Y, Index) --> splat (bo X, Y), Index + SmallVector Ops(VT.getVectorNumElements(), ScalarBO); + return DAG.getBuildVector(VT, DL, Ops); +} + /// Visit a binary vector operation, like ADD. SDValue DAGCombiner::SimplifyVBinOp(SDNode *N) { assert(N->getValueType(0).isVector() && @@ -18169,34 +19573,63 @@ SDValue DAGCombiner::SimplifyVBinOp(SDNode *N) { SDValue LHS = N->getOperand(0); SDValue RHS = N->getOperand(1); SDValue Ops[] = {LHS, RHS}; + EVT VT = N->getValueType(0); + unsigned Opcode = N->getOpcode(); // See if we can constant fold the vector operation. if (SDValue Fold = DAG.FoldConstantVectorArithmetic( - N->getOpcode(), SDLoc(LHS), LHS.getValueType(), Ops, N->getFlags())) + Opcode, SDLoc(LHS), LHS.getValueType(), Ops, N->getFlags())) return Fold; - // Type legalization might introduce new shuffles in the DAG. - // Fold (VBinOp (shuffle (A, Undef, Mask)), (shuffle (B, Undef, Mask))) - // -> (shuffle (VBinOp (A, B)), Undef, Mask). - if (LegalTypes && isa(LHS) && - isa(RHS) && LHS.hasOneUse() && RHS.hasOneUse() && - LHS.getOperand(1).isUndef() && - RHS.getOperand(1).isUndef()) { - ShuffleVectorSDNode *SVN0 = cast(LHS); - ShuffleVectorSDNode *SVN1 = cast(RHS); - - if (SVN0->getMask().equals(SVN1->getMask())) { - EVT VT = N->getValueType(0); - SDValue UndefVector = LHS.getOperand(1); - SDValue NewBinOp = DAG.getNode(N->getOpcode(), SDLoc(N), VT, - LHS.getOperand(0), RHS.getOperand(0), - N->getFlags()); - AddUsersToWorklist(N); - return DAG.getVectorShuffle(VT, SDLoc(N), NewBinOp, UndefVector, - SVN0->getMask()); + // Move unary shuffles with identical masks after a vector binop: + // VBinOp (shuffle A, Undef, Mask), (shuffle B, Undef, Mask)) + // --> shuffle (VBinOp A, B), Undef, Mask + // This does not require type legality checks because we are creating the + // same types of operations that are in the original sequence. We do have to + // restrict ops like integer div that have immediate UB (eg, div-by-zero) + // though. This code is adapted from the identical transform in instcombine. + if (Opcode != ISD::UDIV && Opcode != ISD::SDIV && + Opcode != ISD::UREM && Opcode != ISD::SREM && + Opcode != ISD::UDIVREM && Opcode != ISD::SDIVREM) { + auto *Shuf0 = dyn_cast(LHS); + auto *Shuf1 = dyn_cast(RHS); + if (Shuf0 && Shuf1 && Shuf0->getMask().equals(Shuf1->getMask()) && + LHS.getOperand(1).isUndef() && RHS.getOperand(1).isUndef() && + (LHS.hasOneUse() || RHS.hasOneUse() || LHS == RHS)) { + SDLoc DL(N); + SDValue NewBinOp = DAG.getNode(Opcode, DL, VT, LHS.getOperand(0), + RHS.getOperand(0), N->getFlags()); + SDValue UndefV = LHS.getOperand(1); + return DAG.getVectorShuffle(VT, DL, NewBinOp, UndefV, Shuf0->getMask()); + } + } + + // The following pattern is likely to emerge with vector reduction ops. Moving + // the binary operation ahead of insertion may allow using a narrower vector + // instruction that has better performance than the wide version of the op: + // VBinOp (ins undef, X, Z), (ins undef, Y, Z) --> ins VecC, (VBinOp X, Y), Z + if (LHS.getOpcode() == ISD::INSERT_SUBVECTOR && LHS.getOperand(0).isUndef() && + RHS.getOpcode() == ISD::INSERT_SUBVECTOR && RHS.getOperand(0).isUndef() && + LHS.getOperand(2) == RHS.getOperand(2) && + (LHS.hasOneUse() || RHS.hasOneUse())) { + SDValue X = LHS.getOperand(1); + SDValue Y = RHS.getOperand(1); + SDValue Z = LHS.getOperand(2); + EVT NarrowVT = X.getValueType(); + if (NarrowVT == Y.getValueType() && + TLI.isOperationLegalOrCustomOrPromote(Opcode, NarrowVT)) { + // (binop undef, undef) may not return undef, so compute that result. + SDLoc DL(N); + SDValue VecC = + DAG.getNode(Opcode, DL, VT, DAG.getUNDEF(VT), DAG.getUNDEF(VT)); + SDValue NarrowBO = DAG.getNode(Opcode, DL, NarrowVT, X, Y); + return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, VecC, NarrowBO, Z); } } + if (SDValue V = scalarizeBinOpOfSplats(N, DAG)) + return V; + return SDValue(); } @@ -18214,13 +19647,16 @@ SDValue DAGCombiner::SimplifySelect(const SDLoc &DL, SDValue N0, SDValue N1, // Check to see if we got a select_cc back (to turn into setcc/select). // Otherwise, just return whatever node we got back, like fabs. if (SCC.getOpcode() == ISD::SELECT_CC) { + const SDNodeFlags Flags = N0.getNode()->getFlags(); SDValue SETCC = DAG.getNode(ISD::SETCC, SDLoc(N0), N0.getValueType(), SCC.getOperand(0), SCC.getOperand(1), - SCC.getOperand(4)); + SCC.getOperand(4), Flags); AddToWorklist(SETCC.getNode()); - return DAG.getSelect(SDLoc(SCC), SCC.getValueType(), SETCC, - SCC.getOperand(2), SCC.getOperand(3)); + SDValue SelectNode = DAG.getSelect(SDLoc(SCC), SCC.getValueType(), SETCC, + SCC.getOperand(2), SCC.getOperand(3)); + SelectNode->setFlags(Flags); + return SelectNode; } return SCC; @@ -18305,6 +19741,10 @@ bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDValue LHS, // locations are not in the default address space. LLD->getPointerInfo().getAddrSpace() != 0 || RLD->getPointerInfo().getAddrSpace() != 0 || + // We can't produce a CMOV of a TargetFrameIndex since we won't + // generate the address generation required. + LLD->getBasePtr().getOpcode() == ISD::TargetFrameIndex || + RLD->getBasePtr().getOpcode() == ISD::TargetFrameIndex || !TLI.isOperationLegalOrCustom(TheSelect->getOpcode(), LLD->getBasePtr().getValueType())) return false; @@ -18501,8 +19941,8 @@ SDValue DAGCombiner::convertSelectOfFPConstantsToLoadOffset( // If a constant can be materialized without loads, this does not make sense. if (TLI.getOperationAction(ISD::ConstantFP, VT) == TargetLowering::Legal || - TLI.isFPImmLegal(TV->getValueAPF(), TV->getValueType(0)) || - TLI.isFPImmLegal(FV->getValueAPF(), FV->getValueType(0))) + TLI.isFPImmLegal(TV->getValueAPF(), TV->getValueType(0), ForCodeSize) || + TLI.isFPImmLegal(FV->getValueAPF(), FV->getValueType(0), ForCodeSize)) return SDValue(); // If both constants have multiple uses, then we won't need to do an extra @@ -18547,20 +19987,20 @@ SDValue DAGCombiner::SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1, if (N2 == N3) return N2; EVT CmpOpVT = N0.getValueType(); + EVT CmpResVT = getSetCCResultType(CmpOpVT); EVT VT = N2.getValueType(); auto *N1C = dyn_cast(N1.getNode()); auto *N2C = dyn_cast(N2.getNode()); auto *N3C = dyn_cast(N3.getNode()); // Determine if the condition we're dealing with is constant. - SDValue SCC = SimplifySetCC(getSetCCResultType(CmpOpVT), N0, N1, CC, DL, - false); - if (SCC.getNode()) AddToWorklist(SCC.getNode()); - - if (auto *SCCC = dyn_cast_or_null(SCC.getNode())) { - // fold select_cc true, x, y -> x - // fold select_cc false, x, y -> y - return !SCCC->isNullValue() ? N2 : N3; + if (SDValue SCC = DAG.FoldSetCC(CmpResVT, N0, N1, CC, DL)) { + AddToWorklist(SCC.getNode()); + if (auto *SCCC = dyn_cast(SCC)) { + // fold select_cc true, x, y -> x + // fold select_cc false, x, y -> y + return !(SCCC->isNullValue()) ? N2 : N3; + } } if (SDValue V = @@ -18621,7 +20061,7 @@ SDValue DAGCombiner::SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1, SDValue Temp, SCC; // zext (setcc n0, n1) if (LegalTypes) { - SCC = DAG.getSetCC(DL, getSetCCResultType(CmpOpVT), N0, N1, CC); + SCC = DAG.getSetCC(DL, CmpResVT, N0, N1, CC); if (VT.bitsLT(SCC.getValueType())) Temp = DAG.getZeroExtendInReg(SCC, SDLoc(N2), VT); else @@ -18644,36 +20084,6 @@ SDValue DAGCombiner::SimplifySelectCC(const SDLoc &DL, SDValue N0, SDValue N1, getShiftAmountTy(Temp.getValueType()))); } - // Check to see if this is an integer abs. - // select_cc setg[te] X, 0, X, -X -> - // select_cc setgt X, -1, X, -X -> - // select_cc setl[te] X, 0, -X, X -> - // select_cc setlt X, 1, -X, X -> - // Y = sra (X, size(X)-1); xor (add (X, Y), Y) - if (N1C) { - ConstantSDNode *SubC = nullptr; - if (((N1C->isNullValue() && (CC == ISD::SETGT || CC == ISD::SETGE)) || - (N1C->isAllOnesValue() && CC == ISD::SETGT)) && - N0 == N2 && N3.getOpcode() == ISD::SUB && N0 == N3.getOperand(1)) - SubC = dyn_cast(N3.getOperand(0)); - else if (((N1C->isNullValue() && (CC == ISD::SETLT || CC == ISD::SETLE)) || - (N1C->isOne() && CC == ISD::SETLT)) && - N0 == N3 && N2.getOpcode() == ISD::SUB && N0 == N2.getOperand(1)) - SubC = dyn_cast(N2.getOperand(0)); - - if (SubC && SubC->isNullValue() && CmpOpVT.isInteger()) { - SDLoc DL(N0); - SDValue Shift = DAG.getNode(ISD::SRA, DL, CmpOpVT, N0, - DAG.getConstant(CmpOpVT.getSizeInBits() - 1, - DL, - getShiftAmountTy(CmpOpVT))); - SDValue Add = DAG.getNode(ISD::ADD, DL, CmpOpVT, N0, Shift); - AddToWorklist(Shift.getNode()); - AddToWorklist(Add.getNode()); - return DAG.getNode(ISD::XOR, DL, CmpOpVT, Add, Shift); - } - } - // select_cc seteq X, 0, sizeof(X), ctlz(X) -> ctlz(X) // select_cc seteq X, 0, sizeof(X), ctlz_zero_undef(X) -> ctlz(X) // select_cc seteq X, 0, sizeof(X), cttz(X) -> cttz(X) @@ -18728,7 +20138,7 @@ SDValue DAGCombiner::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, SDValue DAGCombiner::BuildSDIV(SDNode *N) { // when optimising for minimum size, we don't want to expand a div to a mul // and a shift. - if (DAG.getMachineFunction().getFunction().optForMinSize()) + if (DAG.getMachineFunction().getFunction().hasMinSize()) return SDValue(); SmallVector Built; @@ -18769,7 +20179,7 @@ SDValue DAGCombiner::BuildSDIVPow2(SDNode *N) { SDValue DAGCombiner::BuildUDIV(SDNode *N) { // when optimising for minimum size, we don't want to expand a div to a mul // and a shift. - if (DAG.getMachineFunction().getFunction().optForMinSize()) + if (DAG.getMachineFunction().getFunction().hasMinSize()) return SDValue(); SmallVector Built; @@ -18821,7 +20231,6 @@ SDValue DAGCombiner::BuildReciprocalEstimate(SDValue Op, SDNodeFlags Flags) { AddToWorklist(Est.getNode()); if (Iterations) { - EVT VT = Op.getValueType(); SDLoc DL(Op); SDValue FPOne = DAG.getConstantFP(1.0, DL, VT); @@ -18977,7 +20386,6 @@ SDValue DAGCombiner::buildSqrtEstimateImpl(SDValue Op, SDNodeFlags Flags, if (!Reciprocal) { // The estimate is now completely wrong if the input was exactly 0.0 or // possibly a denormal. Force the answer to 0.0 for those cases. - EVT VT = Op.getValueType(); SDLoc DL(Op); EVT CCVT = getSetCCResultType(VT); ISD::NodeType SelOpcode = VT.isVector() ? ISD::VSELECT : ISD::SELECT; @@ -19020,79 +20428,95 @@ SDValue DAGCombiner::buildSqrtEstimate(SDValue Op, SDNodeFlags Flags) { } /// Return true if there is any possibility that the two addresses overlap. -bool DAGCombiner::isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) const { - // If they are the same then they must be aliases. - if (Op0->getBasePtr() == Op1->getBasePtr()) return true; +bool DAGCombiner::isAlias(SDNode *Op0, SDNode *Op1) const { - // If they are both volatile then they cannot be reordered. - if (Op0->isVolatile() && Op1->isVolatile()) return true; + struct MemUseCharacteristics { + bool IsVolatile; + SDValue BasePtr; + int64_t Offset; + Optional NumBytes; + MachineMemOperand *MMO; + }; - // If one operation reads from invariant memory, and the other may store, they - // cannot alias. These should really be checking the equivalent of mayWrite, - // but it only matters for memory nodes other than load /store. - if (Op0->isInvariant() && Op1->writeMem()) - return false; + auto getCharacteristics = [](SDNode *N) -> MemUseCharacteristics { + if (const auto *LSN = dyn_cast(N)) { + int64_t Offset = 0; + if (auto *C = dyn_cast(LSN->getOffset())) + Offset = (LSN->getAddressingMode() == ISD::PRE_INC) + ? C->getSExtValue() + : (LSN->getAddressingMode() == ISD::PRE_DEC) + ? -1 * C->getSExtValue() + : 0; + return {LSN->isVolatile(), LSN->getBasePtr(), Offset /*base offset*/, + Optional(LSN->getMemoryVT().getStoreSize()), + LSN->getMemOperand()}; + } + if (const auto *LN = cast(N)) + return {false /*isVolatile*/, LN->getOperand(1), + (LN->hasOffset()) ? LN->getOffset() : 0, + (LN->hasOffset()) ? Optional(LN->getSize()) + : Optional(), + (MachineMemOperand *)nullptr}; + // Default. + return {false /*isvolatile*/, SDValue(), (int64_t)0 /*offset*/, + Optional() /*size*/, (MachineMemOperand *)nullptr}; + }; - if (Op1->isInvariant() && Op0->writeMem()) - return false; + MemUseCharacteristics MUC0 = getCharacteristics(Op0), + MUC1 = getCharacteristics(Op1); - unsigned NumBytes0 = Op0->getMemoryVT().getStoreSize(); - unsigned NumBytes1 = Op1->getMemoryVT().getStoreSize(); - - // Check for BaseIndexOffset matching. - BaseIndexOffset BasePtr0 = BaseIndexOffset::match(Op0, DAG); - BaseIndexOffset BasePtr1 = BaseIndexOffset::match(Op1, DAG); - int64_t PtrDiff; - if (BasePtr0.getBase().getNode() && BasePtr1.getBase().getNode()) { - if (BasePtr0.equalBaseIndex(BasePtr1, DAG, PtrDiff)) - return !((NumBytes0 <= PtrDiff) || (PtrDiff + NumBytes1 <= 0)); - - // If both BasePtr0 and BasePtr1 are FrameIndexes, we will not be - // able to calculate their relative offset if at least one arises - // from an alloca. However, these allocas cannot overlap and we - // can infer there is no alias. - if (auto *A = dyn_cast(BasePtr0.getBase())) - if (auto *B = dyn_cast(BasePtr1.getBase())) { - MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); - // If the base are the same frame index but the we couldn't find a - // constant offset, (indices are different) be conservative. - if (A != B && (!MFI.isFixedObjectIndex(A->getIndex()) || - !MFI.isFixedObjectIndex(B->getIndex()))) - return false; - } + // If they are to the same address, then they must be aliases. + if (MUC0.BasePtr.getNode() && MUC0.BasePtr == MUC1.BasePtr && + MUC0.Offset == MUC1.Offset) + return true; + + // If they are both volatile then they cannot be reordered. + if (MUC0.IsVolatile && MUC1.IsVolatile) + return true; - bool IsFI0 = isa(BasePtr0.getBase()); - bool IsFI1 = isa(BasePtr1.getBase()); - bool IsGV0 = isa(BasePtr0.getBase()); - bool IsGV1 = isa(BasePtr1.getBase()); - bool IsCV0 = isa(BasePtr0.getBase()); - bool IsCV1 = isa(BasePtr1.getBase()); - - // If of mismatched base types or checkable indices we can check - // they do not alias. - if ((BasePtr0.getIndex() == BasePtr1.getIndex() || (IsFI0 != IsFI1) || - (IsGV0 != IsGV1) || (IsCV0 != IsCV1)) && - (IsFI0 || IsGV0 || IsCV0) && (IsFI1 || IsGV1 || IsCV1)) + if (MUC0.MMO && MUC1.MMO) { + if ((MUC0.MMO->isInvariant() && MUC1.MMO->isStore()) || + (MUC1.MMO->isInvariant() && MUC0.MMO->isStore())) return false; } + // Try to prove that there is aliasing, or that there is no aliasing. Either + // way, we can return now. If nothing can be proved, proceed with more tests. + bool IsAlias; + if (BaseIndexOffset::computeAliasing(Op0, MUC0.NumBytes, Op1, MUC1.NumBytes, + DAG, IsAlias)) + return IsAlias; + + // The following all rely on MMO0 and MMO1 being valid. Fail conservatively if + // either are not known. + if (!MUC0.MMO || !MUC1.MMO) + return true; + + // If one operation reads from invariant memory, and the other may store, they + // cannot alias. These should really be checking the equivalent of mayWrite, + // but it only matters for memory nodes other than load /store. + if ((MUC0.MMO->isInvariant() && MUC1.MMO->isStore()) || + (MUC1.MMO->isInvariant() && MUC0.MMO->isStore())) + return false; + // If we know required SrcValue1 and SrcValue2 have relatively large // alignment compared to the size and offset of the access, we may be able // to prove they do not alias. This check is conservative for now to catch // cases created by splitting vector types. - int64_t SrcValOffset0 = Op0->getSrcValueOffset(); - int64_t SrcValOffset1 = Op1->getSrcValueOffset(); - unsigned OrigAlignment0 = Op0->getOriginalAlignment(); - unsigned OrigAlignment1 = Op1->getOriginalAlignment(); + int64_t SrcValOffset0 = MUC0.MMO->getOffset(); + int64_t SrcValOffset1 = MUC1.MMO->getOffset(); + unsigned OrigAlignment0 = MUC0.MMO->getBaseAlignment(); + unsigned OrigAlignment1 = MUC1.MMO->getBaseAlignment(); if (OrigAlignment0 == OrigAlignment1 && SrcValOffset0 != SrcValOffset1 && - NumBytes0 == NumBytes1 && OrigAlignment0 > NumBytes0) { + MUC0.NumBytes.hasValue() && MUC1.NumBytes.hasValue() && + *MUC0.NumBytes == *MUC1.NumBytes && OrigAlignment0 > *MUC0.NumBytes) { int64_t OffAlign0 = SrcValOffset0 % OrigAlignment0; int64_t OffAlign1 = SrcValOffset1 % OrigAlignment1; // There is no overlap between these relatively aligned accesses of // similar size. Return no alias. - if ((OffAlign0 + NumBytes0) <= OffAlign1 || - (OffAlign1 + NumBytes1) <= OffAlign0) + if ((OffAlign0 + *MUC0.NumBytes) <= OffAlign1 || + (OffAlign1 + *MUC1.NumBytes) <= OffAlign0) return false; } @@ -19105,17 +20529,16 @@ bool DAGCombiner::isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) const { UseAA = false; #endif - if (UseAA && AA && - Op0->getMemOperand()->getValue() && Op1->getMemOperand()->getValue()) { + if (UseAA && AA && MUC0.MMO->getValue() && MUC1.MMO->getValue()) { // Use alias analysis information. int64_t MinOffset = std::min(SrcValOffset0, SrcValOffset1); - int64_t Overlap0 = NumBytes0 + SrcValOffset0 - MinOffset; - int64_t Overlap1 = NumBytes1 + SrcValOffset1 - MinOffset; - AliasResult AAResult = - AA->alias(MemoryLocation(Op0->getMemOperand()->getValue(), Overlap0, - UseTBAA ? Op0->getAAInfo() : AAMDNodes()), - MemoryLocation(Op1->getMemOperand()->getValue(), Overlap1, - UseTBAA ? Op1->getAAInfo() : AAMDNodes()) ); + int64_t Overlap0 = *MUC0.NumBytes + SrcValOffset0 - MinOffset; + int64_t Overlap1 = *MUC1.NumBytes + SrcValOffset1 - MinOffset; + AliasResult AAResult = AA->alias( + MemoryLocation(MUC0.MMO->getValue(), Overlap0, + UseTBAA ? MUC0.MMO->getAAInfo() : AAMDNodes()), + MemoryLocation(MUC1.MMO->getValue(), Overlap1, + UseTBAA ? MUC1.MMO->getAAInfo() : AAMDNodes())); if (AAResult == NoAlias) return false; } @@ -19132,18 +20555,64 @@ void DAGCombiner::GatherAllAliases(SDNode *N, SDValue OriginalChain, SmallPtrSet Visited; // Visited node set. // Get alias information for node. - bool IsLoad = isa(N) && !cast(N)->isVolatile(); + const bool IsLoad = isa(N) && !cast(N)->isVolatile(); // Starting off. Chains.push_back(OriginalChain); unsigned Depth = 0; + // Attempt to improve chain by a single step + std::function ImproveChain = [&](SDValue &C) -> bool { + switch (C.getOpcode()) { + case ISD::EntryToken: + // No need to mark EntryToken. + C = SDValue(); + return true; + case ISD::LOAD: + case ISD::STORE: { + // Get alias information for C. + bool IsOpLoad = isa(C.getNode()) && + !cast(C.getNode())->isVolatile(); + if ((IsLoad && IsOpLoad) || !isAlias(N, C.getNode())) { + // Look further up the chain. + C = C.getOperand(0); + return true; + } + // Alias, so stop here. + return false; + } + + case ISD::CopyFromReg: + // Always forward past past CopyFromReg. + C = C.getOperand(0); + return true; + + case ISD::LIFETIME_START: + case ISD::LIFETIME_END: { + // We can forward past any lifetime start/end that can be proven not to + // alias the memory access. + if (!isAlias(N, C.getNode())) { + // Look further up the chain. + C = C.getOperand(0); + return true; + } + return false; + } + default: + return false; + } + }; + // Look at each chain and determine if it is an alias. If so, add it to the // aliases list. If not, then continue up the chain looking for the next // candidate. while (!Chains.empty()) { SDValue Chain = Chains.pop_back_val(); + // Don't bother if we've seen Chain before. + if (!Visited.insert(Chain.getNode()).second) + continue; + // For TokenFactor nodes, look at each operand and only continue up the // chain until we reach the depth limit. // @@ -19156,58 +20625,30 @@ void DAGCombiner::GatherAllAliases(SDNode *N, SDValue OriginalChain, return; } - // Don't bother if we've been before. - if (!Visited.insert(Chain.getNode()).second) - continue; - - switch (Chain.getOpcode()) { - case ISD::EntryToken: - // Entry token is ideal chain operand, but handled in FindBetterChain. - break; - - case ISD::LOAD: - case ISD::STORE: { - // Get alias information for Chain. - bool IsOpLoad = isa(Chain.getNode()) && - !cast(Chain.getNode())->isVolatile(); - - // If chain is alias then stop here. - if (!(IsLoad && IsOpLoad) && - isAlias(cast(N), cast(Chain.getNode()))) { - Aliases.push_back(Chain); - } else { - // Look further up the chain. - Chains.push_back(Chain.getOperand(0)); - ++Depth; - } - break; - } - - case ISD::TokenFactor: + if (Chain.getOpcode() == ISD::TokenFactor) { // We have to check each of the operands of the token factor for "small" // token factors, so we queue them up. Adding the operands to the queue // (stack) in reverse order maintains the original order and increases the // likelihood that getNode will find a matching token factor (CSE.) if (Chain.getNumOperands() > 16) { Aliases.push_back(Chain); - break; + continue; } for (unsigned n = Chain.getNumOperands(); n;) Chains.push_back(Chain.getOperand(--n)); ++Depth; - break; - - case ISD::CopyFromReg: - // Forward past CopyFromReg. - Chains.push_back(Chain.getOperand(0)); + continue; + } + // Everything else + if (ImproveChain(Chain)) { + // Updated Chain Found, Consider new chain if one exists. + if (Chain.getNode()) + Chains.push_back(Chain); ++Depth; - break; - - default: - // For all other instructions we will just have to take what we can get. - Aliases.push_back(Chain); - break; + continue; } + // No Improved Chain Possible, treat as Alias. + Aliases.push_back(Chain); } } @@ -19232,13 +20673,15 @@ SDValue DAGCombiner::FindBetterChain(SDNode *N, SDValue OldChain) { return Aliases[0]; // Construct a custom tailored token factor. - return DAG.getNode(ISD::TokenFactor, SDLoc(N), MVT::Other, Aliases); + return DAG.getTokenFactor(SDLoc(N), Aliases); } +namespace { // TODO: Replace with with std::monostate when we move to C++17. struct UnitT { } Unit; bool operator==(const UnitT &, const UnitT &) { return true; } bool operator!=(const UnitT &, const UnitT &) { return false; } +} // namespace // This function tries to collect a bunch of potentially interesting // nodes to improve the chains of, all at once. This might seem @@ -19349,7 +20792,7 @@ bool DAGCombiner::parallelizeChainedStores(StoreSDNode *St) { if (AddNewChain) TFOps.insert(TFOps.begin(), NewChain); - SDValue TF = DAG.getNode(ISD::TokenFactor, SDLoc(STChain), MVT::Other, TFOps); + SDValue TF = DAG.getTokenFactor(SDLoc(STChain), TFOps); CombineTo(St, TF); AddToWorklist(STChain); diff --git a/lib/CodeGen/SelectionDAG/FastISel.cpp b/lib/CodeGen/SelectionDAG/FastISel.cpp index a9a3c44ea0c9..22c23ba877e8 100644 --- a/lib/CodeGen/SelectionDAG/FastISel.cpp +++ b/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -1,9 +1,8 @@ //===- FastISel.cpp - Implementation of the FastISel class ----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -782,7 +781,7 @@ bool FastISel::addStackMapLiveVars(SmallVectorImpl &Ops, unsigned Reg = getRegForValue(Val); if (!Reg) return false; - Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false)); + Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/false)); } } return true; @@ -831,8 +830,8 @@ bool FastISel::selectStackmap(const CallInst *I) { const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC); for (unsigned i = 0; ScratchRegs[i]; ++i) Ops.push_back(MachineOperand::CreateReg( - ScratchRegs[i], /*IsDef=*/true, /*IsImp=*/true, /*IsKill=*/false, - /*IsDead=*/false, /*IsUndef=*/false, /*IsEarlyClobber=*/true)); + ScratchRegs[i], /*isDef=*/true, /*isImp=*/true, /*isKill=*/false, + /*isDead=*/false, /*isUndef=*/false, /*isEarlyClobber=*/true)); // Issue CALLSEQ_START unsigned AdjStackDown = TII.getCallFrameSetupOpcode(); @@ -942,7 +941,7 @@ bool FastISel::selectPatchpoint(const CallInst *I) { assert(CLI.NumResultRegs == 0 && "Unexpected result register."); CLI.ResultReg = createResultReg(TLI.getRegClassFor(MVT::i64)); CLI.NumResultRegs = 1; - Ops.push_back(MachineOperand::CreateReg(CLI.ResultReg, /*IsDef=*/true)); + Ops.push_back(MachineOperand::CreateReg(CLI.ResultReg, /*isDef=*/true)); } // Add the and constants. @@ -991,13 +990,13 @@ bool FastISel::selectPatchpoint(const CallInst *I) { unsigned Reg = getRegForValue(I->getArgOperand(i)); if (!Reg) return false; - Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false)); + Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/false)); } } // Push the arguments from the call instruction. for (auto Reg : CLI.OutRegs) - Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/false)); + Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/false)); // Push live variables for the stack map. if (!addStackMapLiveVars(Ops, I, NumMetaOpers + NumArgs)) @@ -1011,13 +1010,13 @@ bool FastISel::selectPatchpoint(const CallInst *I) { const MCPhysReg *ScratchRegs = TLI.getScratchRegisters(CC); for (unsigned i = 0; ScratchRegs[i]; ++i) Ops.push_back(MachineOperand::CreateReg( - ScratchRegs[i], /*IsDef=*/true, /*IsImp=*/true, /*IsKill=*/false, - /*IsDead=*/false, /*IsUndef=*/false, /*IsEarlyClobber=*/true)); + ScratchRegs[i], /*isDef=*/true, /*isImp=*/true, /*isKill=*/false, + /*isDead=*/false, /*isUndef=*/false, /*isEarlyClobber=*/true)); // Add implicit defs (return values). for (auto Reg : CLI.InRegs) - Ops.push_back(MachineOperand::CreateReg(Reg, /*IsDef=*/true, - /*IsImpl=*/true)); + Ops.push_back(MachineOperand::CreateReg(Reg, /*isDef=*/true, + /*isImp=*/true)); // Insert the patchpoint instruction before the call generated by the target. MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, CLI.Call, DbgLoc, @@ -1045,9 +1044,9 @@ bool FastISel::selectXRayCustomEvent(const CallInst *I) { return true; // don't do anything to this instruction. SmallVector Ops; Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(0)), - /*IsDef=*/false)); + /*isDef=*/false)); Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(1)), - /*IsDef=*/false)); + /*isDef=*/false)); MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::PATCHABLE_EVENT_CALL)); @@ -1064,11 +1063,11 @@ bool FastISel::selectXRayTypedEvent(const CallInst *I) { return true; // don't do anything to this instruction. SmallVector Ops; Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(0)), - /*IsDef=*/false)); + /*isDef=*/false)); Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(1)), - /*IsDef=*/false)); + /*isDef=*/false)); Ops.push_back(MachineOperand::CreateReg(getRegForValue(I->getArgOperand(2)), - /*IsDef=*/false)); + /*isDef=*/false)); MachineInstrBuilder MIB = BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::PATCHABLE_TYPED_EVENT_CALL)); @@ -1205,9 +1204,11 @@ bool FastISel::lowerCallTo(CallLoweringInfo &CLI) { if (Arg.IsByVal || Arg.IsInAlloca) { PointerType *Ty = cast(Arg.Ty); Type *ElementTy = Ty->getElementType(); - unsigned FrameSize = DL.getTypeAllocSize(ElementTy); - // For ByVal, alignment should come from FE. BE will guess if this info is - // not there, but there are cases it cannot get right. + unsigned FrameSize = + DL.getTypeAllocSize(Arg.ByValType ? Arg.ByValType : ElementTy); + + // For ByVal, alignment should come from FE. BE will guess if this info + // is not there, but there are cases it cannot get right. unsigned FrameAlign = Arg.Alignment; if (!FrameAlign) FrameAlign = TLI.getByValTypeAlignment(ElementTy, DL); @@ -1235,6 +1236,12 @@ bool FastISel::lowerCallTo(CallLoweringInfo &CLI) { if (CLI.NumResultRegs && CLI.CS) updateValueMap(CLI.CS->getInstruction(), CLI.ResultReg, CLI.NumResultRegs); + // Set labels for heapallocsite call. + if (CLI.CS && CLI.CS->getInstruction()->getMetadata("heapallocsite")) { + MDNode *MD = CLI.CS->getInstruction()->getMetadata("heapallocsite"); + MF->addCodeViewHeapAllocSite(CLI.Call, MD); + } + return true; } @@ -1304,9 +1311,6 @@ bool FastISel::selectCall(const User *I) { return true; } - MachineModuleInfo &MMI = FuncInfo.MF->getMMI(); - computeUsesVAFloatArgument(*Call, MMI); - // Handle intrinsic function calls. if (const auto *II = dyn_cast(Call)) return selectIntrinsicCall(II); @@ -1710,14 +1714,11 @@ void FastISel::finishCondBranch(const BasicBlock *BranchBB, } /// Emit an FNeg operation. -bool FastISel::selectFNeg(const User *I) { - Value *X; - if (!match(I, m_FNeg(m_Value(X)))) - return false; - unsigned OpReg = getRegForValue(X); +bool FastISel::selectFNeg(const User *I, const Value *In) { + unsigned OpReg = getRegForValue(In); if (!OpReg) return false; - bool OpRegIsKill = hasTrivialKill(I); + bool OpRegIsKill = hasTrivialKill(In); // If the target has ISD::FNEG, use it. EVT VT = TLI.getValueType(DL, I->getType()); @@ -1804,9 +1805,13 @@ bool FastISel::selectOperator(const User *I, unsigned Opcode) { return selectBinaryOp(I, ISD::FADD); case Instruction::Sub: return selectBinaryOp(I, ISD::SUB); - case Instruction::FSub: + case Instruction::FSub: { // FNeg is currently represented in LLVM IR as a special case of FSub. - return selectFNeg(I) || selectBinaryOp(I, ISD::FSUB); + Value *X; + if (match(I, m_FNeg(m_Value(X)))) + return selectFNeg(I, X); + return selectBinaryOp(I, ISD::FSUB); + } case Instruction::Mul: return selectBinaryOp(I, ISD::MUL); case Instruction::FMul: @@ -1836,6 +1841,9 @@ bool FastISel::selectOperator(const User *I, unsigned Opcode) { case Instruction::Xor: return selectBinaryOp(I, ISD::XOR); + case Instruction::FNeg: + return selectFNeg(I, I->getOperand(0)); + case Instruction::GetElementPtr: return selectGetElementPtr(I); @@ -1869,6 +1877,13 @@ bool FastISel::selectOperator(const User *I, unsigned Opcode) { return false; case Instruction::Call: + // On AIX, call lowering uses the DAG-ISEL path currently so that the + // callee of the direct function call instruction will be mapped to the + // symbol for the function's entry point, which is distinct from the + // function descriptor symbol. The latter is the symbol whose XCOFF symbol + // name is the C-linkage name of the source level function. + if (TM.getTargetTriple().isOSAIX()) + return false; return selectCall(I); case Instruction::BitCast: diff --git a/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp b/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp index fba728625b07..8b1759246b76 100644 --- a/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp +++ b/lib/CodeGen/SelectionDAG/FunctionLoweringInfo.cpp @@ -1,9 +1,8 @@ //===-- FunctionLoweringInfo.cpp ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -86,6 +85,7 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf, RegInfo = &MF->getRegInfo(); const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering(); unsigned StackAlign = TFI->getStackAlignment(); + DA = DAG->getDivergenceAnalysis(); // Check whether the function can return without sret-demotion. SmallVector Outs; @@ -151,7 +151,7 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf, auto Iter = CatchObjects.find(AI); if (Iter != CatchObjects.end() && TLI->needsFixedCatchObjects()) { FrameIndex = MF->getFrameInfo().CreateFixedObject( - TySize, 0, /*Immutable=*/false, /*isAliased=*/true); + TySize, 0, /*IsImmutable=*/false, /*isAliased=*/true); MF->getFrameInfo().setObjectAlignment(FrameIndex, Align); } else { FrameIndex = @@ -322,13 +322,6 @@ void FunctionLoweringInfo::set(const Function &fn, MachineFunction &mf, NewMap[MBBMap[Src]] = MBBMap[Dst]; } EHInfo.EHPadUnwindMap = std::move(NewMap); - NewMap.clear(); - for (auto &KV : EHInfo.ThrowUnwindMap) { - const auto *Src = KV.first.get(); - const auto *Dst = KV.second.get(); - NewMap[MBBMap[Src]] = MBBMap[Dst]; - } - EHInfo.ThrowUnwindMap = std::move(NewMap); } } @@ -343,6 +336,7 @@ void FunctionLoweringInfo::clear() { LiveOutRegInfo.clear(); VisitedBBs.clear(); ArgDbgValues.clear(); + DescribedArgs.clear(); ByValArgFrameIndexMap.clear(); RegFixups.clear(); RegsWithFixups.clear(); @@ -352,9 +346,9 @@ void FunctionLoweringInfo::clear() { } /// CreateReg - Allocate a single virtual register for the given type. -unsigned FunctionLoweringInfo::CreateReg(MVT VT) { +unsigned FunctionLoweringInfo::CreateReg(MVT VT, bool isDivergent) { return RegInfo->createVirtualRegister( - MF->getSubtarget().getTargetLowering()->getRegClassFor(VT)); + MF->getSubtarget().getTargetLowering()->getRegClassFor(VT, isDivergent)); } /// CreateRegs - Allocate the appropriate number of virtual registers of @@ -364,7 +358,7 @@ unsigned FunctionLoweringInfo::CreateReg(MVT VT) { /// In the case that the given value has struct or array type, this function /// will assign registers for each member or element. /// -unsigned FunctionLoweringInfo::CreateRegs(Type *Ty) { +unsigned FunctionLoweringInfo::CreateRegs(Type *Ty, bool isDivergent) { const TargetLowering *TLI = MF->getSubtarget().getTargetLowering(); SmallVector ValueVTs; @@ -377,13 +371,18 @@ unsigned FunctionLoweringInfo::CreateRegs(Type *Ty) { unsigned NumRegs = TLI->getNumRegisters(Ty->getContext(), ValueVT); for (unsigned i = 0; i != NumRegs; ++i) { - unsigned R = CreateReg(RegisterVT); + unsigned R = CreateReg(RegisterVT, isDivergent); if (!FirstReg) FirstReg = R; } } return FirstReg; } +unsigned FunctionLoweringInfo::CreateRegs(const Value *V) { + return CreateRegs(V->getType(), DA && !TLI->requiresUniformRegister(*MF, V) && + DA->isDivergent(V)); +} + /// GetLiveOutRegInfo - Gets LiveOutInfo for a register, returning NULL if the /// register is a PHI destination and the PHI's LiveOutInfo is not valid. If /// the register's LiveOutInfo is for a smaller bit width, it is extended to @@ -400,7 +399,7 @@ FunctionLoweringInfo::GetLiveOutRegInfo(unsigned Reg, unsigned BitWidth) { if (BitWidth > LOI->Known.getBitWidth()) { LOI->NumSignBits = 1; - LOI->Known = LOI->Known.zextOrTrunc(BitWidth); + LOI->Known = LOI->Known.zext(BitWidth, false /* => any extend */); } return LOI; @@ -526,56 +525,6 @@ unsigned FunctionLoweringInfo::getCatchPadExceptionPointerVReg( return VReg; } -unsigned -FunctionLoweringInfo::getOrCreateSwiftErrorVReg(const MachineBasicBlock *MBB, - const Value *Val) { - auto Key = std::make_pair(MBB, Val); - auto It = SwiftErrorVRegDefMap.find(Key); - // If this is the first use of this swifterror value in this basic block, - // create a new virtual register. - // After we processed all basic blocks we will satisfy this "upwards exposed - // use" by inserting a copy or phi at the beginning of this block. - if (It == SwiftErrorVRegDefMap.end()) { - auto &DL = MF->getDataLayout(); - const TargetRegisterClass *RC = TLI->getRegClassFor(TLI->getPointerTy(DL)); - auto VReg = MF->getRegInfo().createVirtualRegister(RC); - SwiftErrorVRegDefMap[Key] = VReg; - SwiftErrorVRegUpwardsUse[Key] = VReg; - return VReg; - } else return It->second; -} - -void FunctionLoweringInfo::setCurrentSwiftErrorVReg( - const MachineBasicBlock *MBB, const Value *Val, unsigned VReg) { - SwiftErrorVRegDefMap[std::make_pair(MBB, Val)] = VReg; -} - -std::pair -FunctionLoweringInfo::getOrCreateSwiftErrorVRegDefAt(const Instruction *I) { - auto Key = PointerIntPair(I, true); - auto It = SwiftErrorVRegDefUses.find(Key); - if (It == SwiftErrorVRegDefUses.end()) { - auto &DL = MF->getDataLayout(); - const TargetRegisterClass *RC = TLI->getRegClassFor(TLI->getPointerTy(DL)); - unsigned VReg = MF->getRegInfo().createVirtualRegister(RC); - SwiftErrorVRegDefUses[Key] = VReg; - return std::make_pair(VReg, true); - } - return std::make_pair(It->second, false); -} - -std::pair -FunctionLoweringInfo::getOrCreateSwiftErrorVRegUseAt(const Instruction *I, const MachineBasicBlock *MBB, const Value *Val) { - auto Key = PointerIntPair(I, false); - auto It = SwiftErrorVRegDefUses.find(Key); - if (It == SwiftErrorVRegDefUses.end()) { - unsigned VReg = getOrCreateSwiftErrorVReg(MBB, Val); - SwiftErrorVRegDefUses[Key] = VReg; - return std::make_pair(VReg, true); - } - return std::make_pair(It->second, false); -} - const Value * FunctionLoweringInfo::getValueFromVirtualReg(unsigned Vreg) { if (VirtReg2Value.empty()) { diff --git a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp index 6a6114677cc2..9bc07d35dfc5 100644 --- a/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ b/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -1,9 +1,8 @@ //==--- InstrEmitter.cpp - Emit MachineInstrs for the SelectionDAG class ---==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -106,7 +105,7 @@ EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone, bool IsCloned, // Stick to the preferred register classes for legal types. if (TLI->isTypeLegal(VT)) - UseRC = TLI->getRegClassFor(VT); + UseRC = TLI->getRegClassFor(VT, Node->isDivergent()); if (!IsClone && !IsCloned) for (SDNode *User : Node->uses()) { @@ -165,7 +164,7 @@ EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone, bool IsCloned, "Incompatible phys register def and uses!"); DstRC = UseRC; } else { - DstRC = TLI->getRegClassFor(VT); + DstRC = TLI->getRegClassFor(VT, Node->isDivergent()); } // If all uses are reading from the src physical register and copying the @@ -187,24 +186,6 @@ EmitCopyFromReg(SDNode *Node, unsigned ResNo, bool IsClone, bool IsCloned, assert(isNew && "Node emitted out of order - early"); } -/// getDstOfCopyToRegUse - If the only use of the specified result number of -/// node is a CopyToReg, return its destination register. Return 0 otherwise. -unsigned InstrEmitter::getDstOfOnlyCopyToRegUse(SDNode *Node, - unsigned ResNo) const { - if (!Node->hasOneUse()) - return 0; - - SDNode *User = *Node->use_begin(); - if (User->getOpcode() == ISD::CopyToReg && - User->getOperand(2).getNode() == Node && - User->getOperand(2).getResNo() == ResNo) { - unsigned Reg = cast(User->getOperand(1))->getReg(); - if (TargetRegisterInfo::isVirtualRegister(Reg)) - return Reg; - } - return 0; -} - void InstrEmitter::CreateVirtualRegisters(SDNode *Node, MachineInstrBuilder &MIB, const MCInstrDesc &II, @@ -226,8 +207,9 @@ void InstrEmitter::CreateVirtualRegisters(SDNode *Node, // type correctly. For example, a 64-bit float (X86::FR64) can't live in // the 32-bit float super-class (X86::FR32). if (i < NumResults && TLI->isTypeLegal(Node->getSimpleValueType(i))) { - const TargetRegisterClass *VTRC = - TLI->getRegClassFor(Node->getSimpleValueType(i)); + const TargetRegisterClass *VTRC = TLI->getRegClassFor( + Node->getSimpleValueType(i), + (Node->isDivergent() || (RC && TRI->isDivergentRegClass(RC)))); if (RC) VTRC = TRI->getCommonSubClass(RC, VTRC); if (VTRC) @@ -286,14 +268,11 @@ unsigned InstrEmitter::getVR(SDValue Op, if (Op.isMachineOpcode() && Op.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF) { // Add an IMPLICIT_DEF instruction before every use. - unsigned VReg = getDstOfOnlyCopyToRegUse(Op.getNode(), Op.getResNo()); // IMPLICIT_DEF can produce any type of result so its MCInstrDesc // does not include operand register class info. - if (!VReg) { - const TargetRegisterClass *RC = - TLI->getRegClassFor(Op.getSimpleValueType()); - VReg = MRI->createVirtualRegister(RC); - } + const TargetRegisterClass *RC = TLI->getRegClassFor( + Op.getSimpleValueType(), Op.getNode()->isDivergent()); + unsigned VReg = MRI->createVirtualRegister(RC); BuildMI(*MBB, InsertPos, Op.getDebugLoc(), TII->get(TargetOpcode::IMPLICIT_DEF), VReg); return VReg; @@ -396,11 +375,15 @@ void InstrEmitter::AddOperand(MachineInstrBuilder &MIB, } else if (RegisterSDNode *R = dyn_cast(Op)) { unsigned VReg = R->getReg(); MVT OpVT = Op.getSimpleValueType(); - const TargetRegisterClass *OpRC = - TLI->isTypeLegal(OpVT) ? TLI->getRegClassFor(OpVT) : nullptr; const TargetRegisterClass *IIRC = II ? TRI->getAllocatableClass(TII->getRegClass(*II, IIOpNum, TRI, *MF)) : nullptr; + const TargetRegisterClass *OpRC = + TLI->isTypeLegal(OpVT) + ? TLI->getRegClassFor(OpVT, + Op.getNode()->isDivergent() || + (IIRC && TRI->isDivergentRegClass(IIRC))) + : nullptr; if (OpRC && IIRC && OpRC != IIRC && TargetRegisterInfo::isVirtualRegister(VReg)) { @@ -465,7 +448,7 @@ void InstrEmitter::AddOperand(MachineInstrBuilder &MIB, } unsigned InstrEmitter::ConstrainForSubReg(unsigned VReg, unsigned SubIdx, - MVT VT, const DebugLoc &DL) { + MVT VT, bool isDivergent, const DebugLoc &DL) { const TargetRegisterClass *VRC = MRI->getRegClass(VReg); const TargetRegisterClass *RC = TRI->getSubClassWithSubReg(VRC, SubIdx); @@ -480,7 +463,7 @@ unsigned InstrEmitter::ConstrainForSubReg(unsigned VReg, unsigned SubIdx, // VReg couldn't be reasonably constrained. Emit a COPY to a new virtual // register instead. - RC = TRI->getSubClassWithSubReg(TLI->getRegClassFor(VT), SubIdx); + RC = TRI->getSubClassWithSubReg(TLI->getRegClassFor(VT, isDivergent), SubIdx); assert(RC && "No legal register class for VT supports that SubIdx"); unsigned NewReg = MRI->createVirtualRegister(RC); BuildMI(*MBB, InsertPos, DL, TII->get(TargetOpcode::COPY), NewReg) @@ -515,7 +498,7 @@ void InstrEmitter::EmitSubregNode(SDNode *Node, // classes. unsigned SubIdx = cast(Node->getOperand(1))->getZExtValue(); const TargetRegisterClass *TRC = - TLI->getRegClassFor(Node->getSimpleValueType(0)); + TLI->getRegClassFor(Node->getSimpleValueType(0), Node->isDivergent()); unsigned Reg; MachineInstr *DefMI; @@ -549,8 +532,7 @@ void InstrEmitter::EmitSubregNode(SDNode *Node, if (TargetRegisterInfo::isVirtualRegister(Reg)) Reg = ConstrainForSubReg(Reg, SubIdx, Node->getOperand(0).getSimpleValueType(), - Node->getDebugLoc()); - + Node->isDivergent(), Node->getDebugLoc()); // Create the destreg if it is missing. if (VRBase == 0) VRBase = MRI->createVirtualRegister(TRC); @@ -585,7 +567,8 @@ void InstrEmitter::EmitSubregNode(SDNode *Node, // // There is no constraint on the %src register class. // - const TargetRegisterClass *SRC = TLI->getRegClassFor(Node->getSimpleValueType(0)); + const TargetRegisterClass *SRC = + TLI->getRegClassFor(Node->getSimpleValueType(0), Node->isDivergent()); SRC = TRI->getSubClassWithSubReg(SRC, SubIdx); assert(SRC && "No register class supports VT and SubIdx for INSERT_SUBREG"); @@ -900,6 +883,9 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, if (Flags.hasExact()) MI->setFlag(MachineInstr::MIFlag::IsExact); + + if (Flags.hasFPExcept()) + MI->setFlag(MachineInstr::MIFlag::FPExcept); } // Emit all of the actual operands of this instruction, adding them to the @@ -1007,14 +993,23 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, case ISD::TokenFactor: // fall thru break; case ISD::CopyToReg: { - unsigned SrcReg; + unsigned DestReg = cast(Node->getOperand(1))->getReg(); SDValue SrcVal = Node->getOperand(2); + if (TargetRegisterInfo::isVirtualRegister(DestReg) && + SrcVal.isMachineOpcode() && + SrcVal.getMachineOpcode() == TargetOpcode::IMPLICIT_DEF) { + // Instead building a COPY to that vreg destination, build an + // IMPLICIT_DEF instruction instead. + BuildMI(*MBB, InsertPos, Node->getDebugLoc(), + TII->get(TargetOpcode::IMPLICIT_DEF), DestReg); + break; + } + unsigned SrcReg; if (RegisterSDNode *R = dyn_cast(SrcVal)) SrcReg = R->getReg(); else SrcReg = getVR(SrcVal, VRBaseMap); - unsigned DestReg = cast(Node->getOperand(1))->getReg(); if (SrcReg == DestReg) // Coalesced away the copy? Ignore. break; @@ -1049,14 +1044,18 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, break; } - case ISD::INLINEASM: { + case ISD::INLINEASM: + case ISD::INLINEASM_BR: { unsigned NumOps = Node->getNumOperands(); if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue) --NumOps; // Ignore the glue operand. // Create the inline asm machine instruction. - MachineInstrBuilder MIB = BuildMI(*MF, Node->getDebugLoc(), - TII->get(TargetOpcode::INLINEASM)); + unsigned TgtOpc = Node->getOpcode() == ISD::INLINEASM_BR + ? TargetOpcode::INLINEASM_BR + : TargetOpcode::INLINEASM; + MachineInstrBuilder MIB = + BuildMI(*MF, Node->getDebugLoc(), TII->get(TgtOpc)); // Add the asm string as an external symbol operand. SDValue AsmStrV = Node->getOperand(InlineAsm::Op_AsmString); @@ -1137,7 +1136,8 @@ EmitSpecialNode(SDNode *Node, bool IsClone, bool IsCloned, // then remove the early-clobber flag. for (unsigned Reg : ECRegs) { if (MIB->readsRegister(Reg, TRI)) { - MachineOperand *MO = MIB->findRegisterDefOperand(Reg, false, TRI); + MachineOperand *MO = + MIB->findRegisterDefOperand(Reg, false, false, TRI); assert(MO && "No def operand for clobbered register?"); MO->setIsEarlyClobber(false); } diff --git a/lib/CodeGen/SelectionDAG/InstrEmitter.h b/lib/CodeGen/SelectionDAG/InstrEmitter.h index 701b6368690b..cfe99dd977b5 100644 --- a/lib/CodeGen/SelectionDAG/InstrEmitter.h +++ b/lib/CodeGen/SelectionDAG/InstrEmitter.h @@ -1,9 +1,8 @@ //===- InstrEmitter.h - Emit MachineInstrs for the SelectionDAG -*- C++ -*--==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -43,11 +42,6 @@ class LLVM_LIBRARY_VISIBILITY InstrEmitter { unsigned SrcReg, DenseMap &VRBaseMap); - /// getDstOfCopyToRegUse - If the only use of the specified result number of - /// node is a CopyToReg, return its destination register. Return 0 otherwise. - unsigned getDstOfOnlyCopyToRegUse(SDNode *Node, - unsigned ResNo) const; - void CreateVirtualRegisters(SDNode *Node, MachineInstrBuilder &MIB, const MCInstrDesc &II, @@ -84,7 +78,7 @@ class LLVM_LIBRARY_VISIBILITY InstrEmitter { /// supports SubIdx sub-registers. Emit a copy if that isn't possible. /// Return the virtual register to use. unsigned ConstrainForSubReg(unsigned VReg, unsigned SubIdx, MVT VT, - const DebugLoc &DL); + bool isDivergent, const DebugLoc &DL); /// EmitSubregNode - Generate machine code for subreg nodes. /// diff --git a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index d3aea37f944d..bf817f00f83d 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -1,9 +1,8 @@ //===- LegalizeDAG.cpp - Implement SelectionDAG::Legalize -----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -137,8 +136,6 @@ private: bool &NeedInvert, const SDLoc &dl); SDValue ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned); - SDValue ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, const SDValue *Ops, - unsigned NumOps, bool isSigned, const SDLoc &dl); std::pair ExpandChainLibCall(RTLIB::Libcall LC, SDNode *Node, bool isSigned); @@ -152,11 +149,17 @@ private: RTLIB::Libcall Call_I32, RTLIB::Libcall Call_I64, RTLIB::Libcall Call_I128); + SDValue ExpandArgFPLibCall(SDNode *Node, + RTLIB::Libcall Call_F32, RTLIB::Libcall Call_F64, + RTLIB::Libcall Call_F80, RTLIB::Libcall Call_F128, + RTLIB::Libcall Call_PPCF128); void ExpandDivRemLibCall(SDNode *Node, SmallVectorImpl &Results); void ExpandSinCosLibCall(SDNode *Node, SmallVectorImpl &Results); SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, const SDLoc &dl); + SDValue EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, + const SDLoc &dl, SDValue ChainIn); SDValue ExpandBUILD_VECTOR(SDNode *Node); SDValue ExpandSCALAR_TO_VECTOR(SDNode *Node); void ExpandDYNAMIC_STACKALLOC(SDNode *Node, @@ -489,10 +492,9 @@ void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) { // If this is an unaligned store and the target doesn't support it, // expand it. EVT MemVT = ST->getMemoryVT(); - unsigned AS = ST->getAddressSpace(); - unsigned Align = ST->getAlignment(); const DataLayout &DL = DAG.getDataLayout(); - if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) { + if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, + *ST->getMemOperand())) { LLVM_DEBUG(dbgs() << "Expanding unsupported unaligned store\n"); SDValue Result = TLI.expandUnalignedStore(ST, DAG); ReplaceNode(SDValue(ST, 0), Result); @@ -542,7 +544,9 @@ void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) { } else if (StWidth & (StWidth - 1)) { // If not storing a power-of-2 number of bits, expand as two stores. assert(!StVT.isVector() && "Unsupported truncstore!"); - unsigned RoundWidth = 1 << Log2_32(StWidth); + unsigned LogStWidth = Log2_32(StWidth); + assert(LogStWidth < 32); + unsigned RoundWidth = 1 << LogStWidth; assert(RoundWidth < StWidth); unsigned ExtraWidth = StWidth - RoundWidth; assert(ExtraWidth < RoundWidth); @@ -602,11 +606,10 @@ void SelectionDAGLegalize::LegalizeStoreOps(SDNode *Node) { default: llvm_unreachable("This action is not supported yet!"); case TargetLowering::Legal: { EVT MemVT = ST->getMemoryVT(); - unsigned AS = ST->getAddressSpace(); - unsigned Align = ST->getAlignment(); // If this is an unaligned store and the target doesn't support it, // expand it. - if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) { + if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, + *ST->getMemOperand())) { SDValue Result = TLI.expandUnalignedStore(ST, DAG); ReplaceNode(SDValue(ST, 0), Result); } @@ -663,13 +666,12 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { default: llvm_unreachable("This action is not supported yet!"); case TargetLowering::Legal: { EVT MemVT = LD->getMemoryVT(); - unsigned AS = LD->getAddressSpace(); - unsigned Align = LD->getAlignment(); const DataLayout &DL = DAG.getDataLayout(); // If this is an unaligned load and the target doesn't support it, // expand it. - if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) { - std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG); + if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, + *LD->getMemOperand())) { + std::tie(RVal, RChain) = TLI.expandUnalignedLoad(LD, DAG); } break; } @@ -756,7 +758,9 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { } else if (SrcWidth & (SrcWidth - 1)) { // If not loading a power-of-2 number of bits, expand as two loads. assert(!SrcVT.isVector() && "Unsupported extload!"); - unsigned RoundWidth = 1 << Log2_32(SrcWidth); + unsigned LogSrcWidth = Log2_32(SrcWidth); + assert(LogSrcWidth < 32); + unsigned RoundWidth = 1 << LogSrcWidth; assert(RoundWidth < SrcWidth); unsigned ExtraWidth = SrcWidth - RoundWidth; assert(ExtraWidth < RoundWidth); @@ -853,10 +857,9 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) { // If this is an unaligned load and the target doesn't support it, // expand it. EVT MemVT = LD->getMemoryVT(); - unsigned AS = LD->getAddressSpace(); - unsigned Align = LD->getAlignment(); const DataLayout &DL = DAG.getDataLayout(); - if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, AS, Align)) { + if (!TLI.allowsMemoryAccess(*DAG.getContext(), DL, MemVT, + *LD->getMemOperand())) { std::tie(Value, Chain) = TLI.expandUnalignedLoad(LD, DAG); } } @@ -994,6 +997,10 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { case ISD::SINT_TO_FP: case ISD::UINT_TO_FP: case ISD::EXTRACT_VECTOR_ELT: + case ISD::LROUND: + case ISD::LLROUND: + case ISD::LRINT: + case ISD::LLRINT: Action = TLI.getOperationAction(Node->getOpcode(), Node->getOperand(0).getValueType()); break; @@ -1114,6 +1121,8 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { case ISD::STRICT_FFLOOR: case ISD::STRICT_FROUND: case ISD::STRICT_FTRUNC: + case ISD::STRICT_FP_ROUND: + case ISD::STRICT_FP_EXTEND: // These pseudo-ops get legalized as if they were their non-strict // equivalent. For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT // is also legal, but if ISD::FSQRT requires expansion then so does @@ -1128,7 +1137,9 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); break; } - case ISD::SMULFIX: { + case ISD::SMULFIX: + case ISD::SMULFIXSAT: + case ISD::UMULFIX: { unsigned Scale = Node->getConstantOperandVal(2); Action = TLI.getFixedPointOperationAction(Node->getOpcode(), Node->getValueType(0), Scale); @@ -1142,6 +1153,22 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) { Action = TLI.getOperationAction(Node->getOpcode(), cast(Node)->getValue().getValueType()); break; + case ISD::VECREDUCE_FADD: + case ISD::VECREDUCE_FMUL: + case ISD::VECREDUCE_ADD: + case ISD::VECREDUCE_MUL: + case ISD::VECREDUCE_AND: + case ISD::VECREDUCE_OR: + case ISD::VECREDUCE_XOR: + case ISD::VECREDUCE_SMAX: + case ISD::VECREDUCE_SMIN: + case ISD::VECREDUCE_UMAX: + case ISD::VECREDUCE_UMIN: + case ISD::VECREDUCE_FMAX: + case ISD::VECREDUCE_FMIN: + Action = TLI.getOperationAction( + Node->getOpcode(), Node->getOperand(0).getValueType()); + break; default: if (Node->getOpcode() >= ISD::BUILTIN_OP_END) { Action = TargetLowering::Legal; @@ -1386,6 +1413,7 @@ SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) { // Emit a store of each element to the stack slot. SmallVector Stores; unsigned TypeByteSize = EltVT.getSizeInBits() / 8; + assert(TypeByteSize > 0 && "Vector element type too small for stack store!"); // Store (in the right endianness) the elements to memory. for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) { // Ignore undef elements. @@ -1723,6 +1751,12 @@ bool SelectionDAGLegalize::LegalizeSetCCCondCode(EVT VT, SDValue &LHS, /// The resultant code need not be legal. SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT, EVT DestVT, const SDLoc &dl) { + return EmitStackConvert(SrcOp, SlotVT, DestVT, dl, DAG.getEntryNode()); +} + +SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT, + EVT DestVT, const SDLoc &dl, + SDValue Chain) { // Create the stack frame object. unsigned SrcAlign = DAG.getDataLayout().getPrefTypeAlignment( SrcOp.getValueType().getTypeForEVT(*DAG.getContext())); @@ -1743,19 +1777,19 @@ SDValue SelectionDAGLegalize::EmitStackConvert(SDValue SrcOp, EVT SlotVT, // later than DestVT. SDValue Store; - if (SrcSize > SlotSize) - Store = DAG.getTruncStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo, + if (SrcSize > SlotSize) + Store = DAG.getTruncStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SlotVT, SrcAlign); else { assert(SrcSize == SlotSize && "Invalid store"); Store = - DAG.getStore(DAG.getEntryNode(), dl, SrcOp, FIPtr, PtrInfo, SrcAlign); + DAG.getStore(Chain, dl, SrcOp, FIPtr, PtrInfo, SrcAlign); } // Result is a load from the stack slot. if (SlotSize == DestSize) return DAG.getLoad(DestVT, dl, Store, FIPtr, PtrInfo, DestAlign); - + assert(SlotSize < DestSize && "Unknown extension!"); return DAG.getExtLoad(ISD::EXTLOAD, dl, DestVT, Store, FIPtr, PtrInfo, SlotVT, DestAlign); @@ -2049,41 +2083,6 @@ SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, SDNode *Node, return CallInfo.first; } -/// Generate a libcall taking the given operands as arguments -/// and returning a result of type RetVT. -SDValue SelectionDAGLegalize::ExpandLibCall(RTLIB::Libcall LC, EVT RetVT, - const SDValue *Ops, unsigned NumOps, - bool isSigned, const SDLoc &dl) { - TargetLowering::ArgListTy Args; - Args.reserve(NumOps); - - TargetLowering::ArgListEntry Entry; - for (unsigned i = 0; i != NumOps; ++i) { - Entry.Node = Ops[i]; - Entry.Ty = Entry.Node.getValueType().getTypeForEVT(*DAG.getContext()); - Entry.IsSExt = isSigned; - Entry.IsZExt = !isSigned; - Args.push_back(Entry); - } - SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(LC), - TLI.getPointerTy(DAG.getDataLayout())); - - Type *RetTy = RetVT.getTypeForEVT(*DAG.getContext()); - - TargetLowering::CallLoweringInfo CLI(DAG); - CLI.setDebugLoc(dl) - .setChain(DAG.getEntryNode()) - .setLibCallee(TLI.getLibcallCallingConv(LC), RetTy, Callee, - std::move(Args)) - .setSExtResult(isSigned) - .setZExtResult(!isSigned) - .setIsPostTypeLegalization(true); - - std::pair CallInfo = TLI.LowerCallTo(CLI); - - return CallInfo.first; -} - // Expand a node into a call to a libcall. Similar to // ExpandLibCall except that the first operand is the in-chain. std::pair @@ -2160,6 +2159,27 @@ SDValue SelectionDAGLegalize::ExpandIntLibCall(SDNode* Node, bool isSigned, return ExpandLibCall(LC, Node, isSigned); } +/// Expand the node to a libcall based on first argument type (for instance +/// lround and its variant). +SDValue SelectionDAGLegalize::ExpandArgFPLibCall(SDNode* Node, + RTLIB::Libcall Call_F32, + RTLIB::Libcall Call_F64, + RTLIB::Libcall Call_F80, + RTLIB::Libcall Call_F128, + RTLIB::Libcall Call_PPCF128) { + RTLIB::Libcall LC; + switch (Node->getOperand(0).getValueType().getSimpleVT().SimpleTy) { + default: llvm_unreachable("Unexpected request for libcall!"); + case MVT::f32: LC = Call_F32; break; + case MVT::f64: LC = Call_F64; break; + case MVT::f80: LC = Call_F80; break; + case MVT::f128: LC = Call_F128; break; + case MVT::ppcf128: LC = Call_PPCF128; break; + } + + return ExpandLibCall(LC, Node, false); +} + /// Issue libcalls to __{u}divmod to compute div / rem pairs. void SelectionDAGLegalize::ExpandDivRemLibCall(SDNode *Node, @@ -2530,16 +2550,12 @@ SDValue SelectionDAGLegalize::ExpandBITREVERSE(SDValue Op, const SDLoc &dl) { // TODO: We can easily support i4/i2 legal types if any target ever does. if (Sz >= 8 && isPowerOf2_32(Sz)) { // Create the masks - repeating the pattern every byte. - APInt MaskHi4(Sz, 0), MaskHi2(Sz, 0), MaskHi1(Sz, 0); - APInt MaskLo4(Sz, 0), MaskLo2(Sz, 0), MaskLo1(Sz, 0); - for (unsigned J = 0; J != Sz; J += 8) { - MaskHi4 = MaskHi4 | (0xF0ull << J); - MaskLo4 = MaskLo4 | (0x0Full << J); - MaskHi2 = MaskHi2 | (0xCCull << J); - MaskLo2 = MaskLo2 | (0x33ull << J); - MaskHi1 = MaskHi1 | (0xAAull << J); - MaskLo1 = MaskLo1 | (0x55ull << J); - } + APInt MaskHi4 = APInt::getSplat(Sz, APInt(8, 0xF0)); + APInt MaskHi2 = APInt::getSplat(Sz, APInt(8, 0xCC)); + APInt MaskHi1 = APInt::getSplat(Sz, APInt(8, 0xAA)); + APInt MaskLo4 = APInt::getSplat(Sz, APInt(8, 0x0F)); + APInt MaskLo2 = APInt::getSplat(Sz, APInt(8, 0x33)); + APInt MaskLo1 = APInt::getSplat(Sz, APInt(8, 0x55)); // BSWAP if the type is wider than a single byte. Tmp = (Sz > 8 ? DAG.getNode(ISD::BSWAP, dl, VT, Op) : Op); @@ -2593,9 +2609,8 @@ SDValue SelectionDAGLegalize::ExpandBSWAP(SDValue Op, const SDLoc &dl) { switch (VT.getSimpleVT().getScalarType().SimpleTy) { default: llvm_unreachable("Unhandled Expand type in BSWAP!"); case MVT::i16: - Tmp2 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); - Tmp1 = DAG.getNode(ISD::SRL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); - return DAG.getNode(ISD::OR, dl, VT, Tmp1, Tmp2); + // Use a rotate by 8. This can be further expanded if necessary. + return DAG.getNode(ISD::ROTL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); case MVT::i32: Tmp4 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(24, dl, SHVT)); Tmp3 = DAG.getNode(ISD::SHL, dl, VT, Op, DAG.getConstant(8, dl, SHVT)); @@ -2799,12 +2814,27 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { } break; } + case ISD::STRICT_FP_ROUND: + Tmp1 = EmitStackConvert(Node->getOperand(1), + Node->getValueType(0), + Node->getValueType(0), dl, Node->getOperand(0)); + ReplaceNode(Node, Tmp1.getNode()); + LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_ROUND node\n"); + return true; case ISD::FP_ROUND: case ISD::BITCAST: - Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getValueType(0), + Tmp1 = EmitStackConvert(Node->getOperand(0), + Node->getValueType(0), Node->getValueType(0), dl); Results.push_back(Tmp1); break; + case ISD::STRICT_FP_EXTEND: + Tmp1 = EmitStackConvert(Node->getOperand(1), + Node->getOperand(1).getValueType(), + Node->getValueType(0), dl, Node->getOperand(0)); + ReplaceNode(Node, Tmp1.getNode()); + LLVM_DEBUG(dbgs() << "Successfully expanded STRICT_FP_EXTEND node\n"); + return true; case ISD::FP_EXTEND: Tmp1 = EmitStackConvert(Node->getOperand(0), Node->getOperand(0).getValueType(), @@ -2875,6 +2905,30 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { if (TLI.expandFP_TO_UINT(Node, Tmp1, DAG)) Results.push_back(Tmp1); break; + case ISD::LROUND: + Results.push_back(ExpandArgFPLibCall(Node, RTLIB::LROUND_F32, + RTLIB::LROUND_F64, RTLIB::LROUND_F80, + RTLIB::LROUND_F128, + RTLIB::LROUND_PPCF128)); + break; + case ISD::LLROUND: + Results.push_back(ExpandArgFPLibCall(Node, RTLIB::LLROUND_F32, + RTLIB::LLROUND_F64, RTLIB::LLROUND_F80, + RTLIB::LLROUND_F128, + RTLIB::LLROUND_PPCF128)); + break; + case ISD::LRINT: + Results.push_back(ExpandArgFPLibCall(Node, RTLIB::LRINT_F32, + RTLIB::LRINT_F64, RTLIB::LRINT_F80, + RTLIB::LRINT_F128, + RTLIB::LRINT_PPCF128)); + break; + case ISD::LLRINT: + Results.push_back(ExpandArgFPLibCall(Node, RTLIB::LLRINT_F32, + RTLIB::LLRINT_F64, RTLIB::LLRINT_F80, + RTLIB::LLRINT_F128, + RTLIB::LLRINT_PPCF128)); + break; case ISD::VAARG: Results.push_back(DAG.expandVAArg(Node)); Results.push_back(Results[0].getValue(1)); @@ -3117,7 +3171,8 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { ConstantFPSDNode *CFP = cast(Node); // Check to see if this FP immediate is already legal. // If this is a legal constant, turn it into a TargetConstantFP node. - if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0))) + if (!TLI.isFPImmLegal(CFP->getValueAPF(), Node->getValueType(0), + DAG.getMachineFunction().getFunction().hasOptSize())) Results.push_back(ExpandConstantFP(CFP, true)); break; } @@ -3291,176 +3346,75 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { Results.push_back(TLI.expandAddSubSat(Node, DAG)); break; case ISD::SMULFIX: - Results.push_back(TLI.getExpandedFixedPointMultiplication(Node, DAG)); + case ISD::SMULFIXSAT: + case ISD::UMULFIX: + Results.push_back(TLI.expandFixedPointMul(Node, DAG)); break; - case ISD::SADDO: - case ISD::SSUBO: { + case ISD::ADDCARRY: + case ISD::SUBCARRY: { SDValue LHS = Node->getOperand(0); SDValue RHS = Node->getOperand(1); - SDValue Sum = DAG.getNode(Node->getOpcode() == ISD::SADDO ? - ISD::ADD : ISD::SUB, dl, LHS.getValueType(), - LHS, RHS); - Results.push_back(Sum); - EVT ResultType = Node->getValueType(1); - EVT OType = getSetCCResultType(Node->getValueType(0)); - - SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType()); - - // LHSSign -> LHS >= 0 - // RHSSign -> RHS >= 0 - // SumSign -> Sum >= 0 - // - // Add: - // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) - // Sub: - // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) - SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE); - SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE); - SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign, - Node->getOpcode() == ISD::SADDO ? - ISD::SETEQ : ISD::SETNE); - - SDValue SumSign = DAG.getSetCC(dl, OType, Sum, Zero, ISD::SETGE); - SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE); - - SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE); - Results.push_back(DAG.getBoolExtOrTrunc(Cmp, dl, ResultType, ResultType)); - break; - } - case ISD::UADDO: - case ISD::USUBO: { - SDValue LHS = Node->getOperand(0); - SDValue RHS = Node->getOperand(1); - bool IsAdd = Node->getOpcode() == ISD::UADDO; - // If ADD/SUBCARRY is legal, use that instead. - unsigned OpcCarry = IsAdd ? ISD::ADDCARRY : ISD::SUBCARRY; - if (TLI.isOperationLegalOrCustom(OpcCarry, Node->getValueType(0))) { - SDValue CarryIn = DAG.getConstant(0, dl, Node->getValueType(1)); - SDValue NodeCarry = DAG.getNode(OpcCarry, dl, Node->getVTList(), - { LHS, RHS, CarryIn }); - Results.push_back(SDValue(NodeCarry.getNode(), 0)); - Results.push_back(SDValue(NodeCarry.getNode(), 1)); - break; - } + SDValue Carry = Node->getOperand(2); + + bool IsAdd = Node->getOpcode() == ISD::ADDCARRY; - SDValue Sum = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl, - LHS.getValueType(), LHS, RHS); - Results.push_back(Sum); + // Initial add of the 2 operands. + unsigned Op = IsAdd ? ISD::ADD : ISD::SUB; + EVT VT = LHS.getValueType(); + SDValue Sum = DAG.getNode(Op, dl, VT, LHS, RHS); - EVT ResultType = Node->getValueType(1); + // Initial check for overflow. + EVT CarryType = Node->getValueType(1); EVT SetCCType = getSetCCResultType(Node->getValueType(0)); ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT; - SDValue SetCC = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC); - - Results.push_back(DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType)); + SDValue Overflow = DAG.getSetCC(dl, SetCCType, Sum, LHS, CC); + + // Add of the sum and the carry. + SDValue CarryExt = + DAG.getZeroExtendInReg(DAG.getZExtOrTrunc(Carry, dl, VT), dl, MVT::i1); + SDValue Sum2 = DAG.getNode(Op, dl, VT, Sum, CarryExt); + + // Second check for overflow. If we are adding, we can only overflow if the + // initial sum is all 1s ang the carry is set, resulting in a new sum of 0. + // If we are subtracting, we can only overflow if the initial sum is 0 and + // the carry is set, resulting in a new sum of all 1s. + SDValue Zero = DAG.getConstant(0, dl, VT); + SDValue Overflow2 = + IsAdd ? DAG.getSetCC(dl, SetCCType, Sum2, Zero, ISD::SETEQ) + : DAG.getSetCC(dl, SetCCType, Sum, Zero, ISD::SETEQ); + Overflow2 = DAG.getNode(ISD::AND, dl, SetCCType, Overflow2, + DAG.getZExtOrTrunc(Carry, dl, SetCCType)); + + SDValue ResultCarry = + DAG.getNode(ISD::OR, dl, SetCCType, Overflow, Overflow2); + + Results.push_back(Sum2); + Results.push_back(DAG.getBoolExtOrTrunc(ResultCarry, dl, CarryType, VT)); + break; + } + case ISD::SADDO: + case ISD::SSUBO: { + SDValue Result, Overflow; + TLI.expandSADDSUBO(Node, Result, Overflow, DAG); + Results.push_back(Result); + Results.push_back(Overflow); + break; + } + case ISD::UADDO: + case ISD::USUBO: { + SDValue Result, Overflow; + TLI.expandUADDSUBO(Node, Result, Overflow, DAG); + Results.push_back(Result); + Results.push_back(Overflow); break; } case ISD::UMULO: case ISD::SMULO: { - EVT VT = Node->getValueType(0); - EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits() * 2); - SDValue LHS = Node->getOperand(0); - SDValue RHS = Node->getOperand(1); - SDValue BottomHalf; - SDValue TopHalf; - static const unsigned Ops[2][3] = - { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND }, - { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }}; - bool isSigned = Node->getOpcode() == ISD::SMULO; - if (TLI.isOperationLegalOrCustom(Ops[isSigned][0], VT)) { - BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); - TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS); - } else if (TLI.isOperationLegalOrCustom(Ops[isSigned][1], VT)) { - BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS, - RHS); - TopHalf = BottomHalf.getValue(1); - } else if (TLI.isTypeLegal(WideVT)) { - LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS); - RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS); - Tmp1 = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS); - BottomHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1, - DAG.getIntPtrConstant(0, dl)); - TopHalf = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, VT, Tmp1, - DAG.getIntPtrConstant(1, dl)); - } else { - // We can fall back to a libcall with an illegal type for the MUL if we - // have a libcall big enough. - // Also, we can fall back to a division in some cases, but that's a big - // performance hit in the general case. - RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; - if (WideVT == MVT::i16) - LC = RTLIB::MUL_I16; - else if (WideVT == MVT::i32) - LC = RTLIB::MUL_I32; - else if (WideVT == MVT::i64) - LC = RTLIB::MUL_I64; - else if (WideVT == MVT::i128) - LC = RTLIB::MUL_I128; - assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!"); - - SDValue HiLHS; - SDValue HiRHS; - if (isSigned) { - // The high part is obtained by SRA'ing all but one of the bits of low - // part. - unsigned LoSize = VT.getSizeInBits(); - HiLHS = - DAG.getNode(ISD::SRA, dl, VT, LHS, - DAG.getConstant(LoSize - 1, dl, - TLI.getPointerTy(DAG.getDataLayout()))); - HiRHS = - DAG.getNode(ISD::SRA, dl, VT, RHS, - DAG.getConstant(LoSize - 1, dl, - TLI.getPointerTy(DAG.getDataLayout()))); - } else { - HiLHS = DAG.getConstant(0, dl, VT); - HiRHS = DAG.getConstant(0, dl, VT); - } - - // Here we're passing the 2 arguments explicitly as 4 arguments that are - // pre-lowered to the correct types. This all depends upon WideVT not - // being a legal type for the architecture and thus has to be split to - // two arguments. - SDValue Ret; - if(DAG.getDataLayout().isLittleEndian()) { - // Halves of WideVT are packed into registers in different order - // depending on platform endianness. This is usually handled by - // the C calling convention, but we can't defer to it in - // the legalizer. - SDValue Args[] = { LHS, HiLHS, RHS, HiRHS }; - Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl); - } else { - SDValue Args[] = { HiLHS, LHS, HiRHS, RHS }; - Ret = ExpandLibCall(LC, WideVT, Args, 4, isSigned, dl); - } - assert(Ret.getOpcode() == ISD::MERGE_VALUES && - "Ret value is a collection of constituent nodes holding result."); - BottomHalf = Ret.getOperand(0); - TopHalf = Ret.getOperand(1); + SDValue Result, Overflow; + if (TLI.expandMULO(Node, Result, Overflow, DAG)) { + Results.push_back(Result); + Results.push_back(Overflow); } - - if (isSigned) { - Tmp1 = DAG.getConstant( - VT.getSizeInBits() - 1, dl, - TLI.getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout())); - Tmp1 = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, Tmp1); - TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, Tmp1, - ISD::SETNE); - } else { - TopHalf = DAG.getSetCC(dl, getSetCCResultType(VT), TopHalf, - DAG.getConstant(0, dl, VT), ISD::SETNE); - } - - // Truncate the result if SetCC returns a larger type than needed. - EVT RType = Node->getValueType(1); - if (RType.getSizeInBits() < TopHalf.getValueSizeInBits()) - TopHalf = DAG.getNode(ISD::TRUNCATE, dl, RType, TopHalf); - - assert(RType.getSizeInBits() == TopHalf.getValueSizeInBits() && - "Unexpected result type for S/UMULO legalization"); - - Results.push_back(BottomHalf); - Results.push_back(TopHalf); break; } case ISD::BUILD_PAIR: { @@ -3487,6 +3441,7 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { DAG.getConstant(0, dl, Tmp1.getValueType()), Tmp2, Tmp3, ISD::SETNE); } + Tmp1->setFlags(Node->getFlags()); Results.push_back(Tmp1); break; case ISD::BR_JT: { @@ -3570,7 +3525,7 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { // condition code, create a new SETCC node. if (Tmp3.getNode()) Tmp1 = DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), - Tmp1, Tmp2, Tmp3); + Tmp1, Tmp2, Tmp3, Node->getFlags()); // If we expanded the SETCC by inverting the condition code, then wrap // the existing SETCC in a NOT to restore the intended condition. @@ -3598,6 +3553,7 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { DAG.getConstant(TrueValue, dl, VT), DAG.getConstant(0, dl, VT), Tmp3); + Tmp1->setFlags(Node->getFlags()); Results.push_back(Tmp1); break; } @@ -3617,9 +3573,8 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { assert(!TLI.isOperationExpand(ISD::SELECT, VT) && "Cannot expand ISD::SELECT_CC when ISD::SELECT also needs to be " "expanded."); - EVT CCVT = - TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT); - SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC); + EVT CCVT = getSetCCResultType(CmpVT); + SDValue Cond = DAG.getNode(ISD::SETCC, dl, CCVT, Tmp1, Tmp2, CC, Node->getFlags()); Results.push_back(DAG.getSelect(dl, VT, Cond, Tmp3, Tmp4)); break; } @@ -3635,6 +3590,7 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { // Use the new condition code and swap true and false Legalized = true; Tmp1 = DAG.getSelectCC(dl, Tmp1, Tmp2, Tmp4, Tmp3, InvCC); + Tmp1->setFlags(Node->getFlags()); } else { // If The inverse is not legal, then try to swap the arguments using // the inverse condition code. @@ -3644,6 +3600,7 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { // lhs and rhs. Legalized = true; Tmp1 = DAG.getSelectCC(dl, Tmp2, Tmp1, Tmp4, Tmp3, SwapInvCC); + Tmp1->setFlags(Node->getFlags()); } } @@ -3670,6 +3627,7 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { Tmp1 = DAG.getNode(ISD::SELECT_CC, dl, Node->getValueType(0), Tmp1, Tmp2, Tmp3, Tmp4, CC); } + Tmp1->setFlags(Node->getFlags()); } Results.push_back(Tmp1); break; @@ -3729,6 +3687,21 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) { ReplaceNode(SDValue(Node, 0), Result); break; } + case ISD::VECREDUCE_FADD: + case ISD::VECREDUCE_FMUL: + case ISD::VECREDUCE_ADD: + case ISD::VECREDUCE_MUL: + case ISD::VECREDUCE_AND: + case ISD::VECREDUCE_OR: + case ISD::VECREDUCE_XOR: + case ISD::VECREDUCE_SMAX: + case ISD::VECREDUCE_SMIN: + case ISD::VECREDUCE_UMAX: + case ISD::VECREDUCE_UMIN: + case ISD::VECREDUCE_FMAX: + case ISD::VECREDUCE_FMIN: + Results.push_back(TLI.expandVecReduce(Node, DAG)); + break; case ISD::GLOBAL_OFFSET_TABLE: case ISD::GlobalAddress: case ISD::GlobalTLSAddress: @@ -4273,6 +4246,7 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) { Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2)); // Perform the larger operation, then round down. Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3); + Tmp1->setFlags(Node->getFlags()); if (TruncOp != ISD::FP_ROUND) Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1); else @@ -4303,8 +4277,8 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) { } Tmp1 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(0)); Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1)); - Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), - Tmp1, Tmp2, Node->getOperand(2))); + Results.push_back(DAG.getNode(ISD::SETCC, dl, Node->getValueType(0), Tmp1, + Tmp2, Node->getOperand(2), Node->getFlags())); break; } case ISD::BR_CC: { @@ -4532,6 +4506,24 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) { Results.push_back(CvtVec); break; } + case ISD::ATOMIC_SWAP: { + AtomicSDNode *AM = cast(Node); + SDLoc SL(Node); + SDValue CastVal = DAG.getNode(ISD::BITCAST, SL, NVT, AM->getVal()); + assert(NVT.getSizeInBits() == OVT.getSizeInBits() && + "unexpected promotion type"); + assert(AM->getMemoryVT().getSizeInBits() == NVT.getSizeInBits() && + "unexpected atomic_swap with illegal type"); + + SDValue NewAtomic + = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, NVT, + DAG.getVTList(NVT, MVT::Other), + { AM->getChain(), AM->getBasePtr(), CastVal }, + AM->getMemOperand()); + Results.push_back(DAG.getNode(ISD::BITCAST, SL, OVT, NewAtomic)); + Results.push_back(NewAtomic.getValue(1)); + break; + } } // Replace the original node with the legalized result. diff --git a/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp index 4644e9588e7b..b4849b2881e6 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp @@ -1,9 +1,8 @@ //===-------- LegalizeFloatTypes.cpp - Legalization of float types --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -104,6 +103,7 @@ bool DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) { case ISD::FSUB: R = SoftenFloatRes_FSUB(N); break; case ISD::FTRUNC: R = SoftenFloatRes_FTRUNC(N); break; case ISD::LOAD: R = SoftenFloatRes_LOAD(N, ResNo); break; + case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; case ISD::SELECT: R = SoftenFloatRes_SELECT(N, ResNo); break; case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N, ResNo); break; case ISD::SINT_TO_FP: @@ -440,6 +440,15 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_FNEG(SDNode *N, unsigned ResNo) { return SDValue(N, ResNo); EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); SDLoc dl(N); + + EVT FloatVT = N->getValueType(ResNo); + if (FloatVT == MVT::f32 || FloatVT == MVT::f64 || FloatVT == MVT::f128) { + // Expand Y = FNEG(X) -> Y = X ^ sign mask + APInt SignMask = APInt::getSignMask(NVT.getSizeInBits()); + return DAG.getNode(ISD::XOR, dl, NVT, GetSoftenedFloat(N->getOperand(0)), + DAG.getConstant(SignMask, dl, NVT)); + } + // Expand Y = FNEG(X) -> Y = SUB -0.0, X SDValue Ops[2] = { DAG.getConstantFP(-0.0, dl, N->getValueType(0)), GetSoftenedFloat(N->getOperand(0)) }; @@ -763,6 +772,10 @@ bool DAGTypeLegalizer::SoftenFloatOperand(SDNode *N, unsigned OpNo) { case ISD::FP_ROUND: Res = SoftenFloatOp_FP_ROUND(N); break; case ISD::FP_TO_SINT: case ISD::FP_TO_UINT: Res = SoftenFloatOp_FP_TO_XINT(N); break; + case ISD::LROUND: Res = SoftenFloatOp_LROUND(N); break; + case ISD::LLROUND: Res = SoftenFloatOp_LLROUND(N); break; + case ISD::LRINT: Res = SoftenFloatOp_LRINT(N); break; + case ISD::LLRINT: Res = SoftenFloatOp_LLRINT(N); break; case ISD::SELECT: Res = SoftenFloatOp_SELECT(N); break; case ISD::SELECT_CC: Res = SoftenFloatOp_SELECT_CC(N); break; case ISD::SETCC: Res = SoftenFloatOp_SETCC(N); break; @@ -1029,6 +1042,61 @@ SDValue DAGTypeLegalizer::SoftenFloatOp_STORE(SDNode *N, unsigned OpNo) { ST->getMemOperand()); } +SDValue DAGTypeLegalizer::SoftenFloatOp_LROUND(SDNode *N) { + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); + + SDValue Op = GetSoftenedFloat(N->getOperand(0)); + EVT RetVT = N->getOperand(0).getValueType().getSimpleVT().SimpleTy; + return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, + RTLIB::LROUND_F32, + RTLIB::LROUND_F64, + RTLIB::LROUND_F80, + RTLIB::LROUND_F128, + RTLIB::LROUND_PPCF128), + NVT, Op, false, SDLoc(N)).first; +} + +SDValue DAGTypeLegalizer::SoftenFloatOp_LLROUND(SDNode *N) { + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); + + SDValue Op = GetSoftenedFloat(N->getOperand(0)); + EVT RetVT = N->getOperand(0).getValueType().getSimpleVT().SimpleTy; + return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, + RTLIB::LLROUND_F32, + RTLIB::LLROUND_F64, + RTLIB::LLROUND_F80, + RTLIB::LLROUND_F128, + RTLIB::LLROUND_PPCF128), + NVT, Op, false, SDLoc(N)).first; +} + +SDValue DAGTypeLegalizer::SoftenFloatOp_LRINT(SDNode *N) { + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); + + SDValue Op = GetSoftenedFloat(N->getOperand(0)); + EVT RetVT = N->getOperand(0).getValueType().getSimpleVT().SimpleTy; + return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, + RTLIB::LRINT_F32, + RTLIB::LRINT_F64, + RTLIB::LRINT_F80, + RTLIB::LRINT_F128, + RTLIB::LRINT_PPCF128), + NVT, Op, false, SDLoc(N)).first; +} + +SDValue DAGTypeLegalizer::SoftenFloatOp_LLRINT(SDNode *N) { + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); + + SDValue Op = GetSoftenedFloat(N->getOperand(0)); + EVT RetVT = N->getOperand(0).getValueType().getSimpleVT().SimpleTy; + return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, + RTLIB::LLRINT_F32, + RTLIB::LLRINT_F64, + RTLIB::LLRINT_F80, + RTLIB::LLRINT_F128, + RTLIB::LLRINT_PPCF128), + NVT, Op, false, SDLoc(N)).first; +} //===----------------------------------------------------------------------===// // Float Result Expansion @@ -1562,6 +1630,10 @@ bool DAGTypeLegalizer::ExpandFloatOperand(SDNode *N, unsigned OpNo) { case ISD::FP_ROUND: Res = ExpandFloatOp_FP_ROUND(N); break; case ISD::FP_TO_SINT: Res = ExpandFloatOp_FP_TO_SINT(N); break; case ISD::FP_TO_UINT: Res = ExpandFloatOp_FP_TO_UINT(N); break; + case ISD::LROUND: Res = ExpandFloatOp_LROUND(N); break; + case ISD::LLROUND: Res = ExpandFloatOp_LLROUND(N); break; + case ISD::LRINT: Res = ExpandFloatOp_LRINT(N); break; + case ISD::LLRINT: Res = ExpandFloatOp_LLRINT(N); break; case ISD::SELECT_CC: Res = ExpandFloatOp_SELECT_CC(N); break; case ISD::SETCC: Res = ExpandFloatOp_SETCC(N); break; case ISD::STORE: Res = ExpandFloatOp_STORE(cast(N), @@ -1732,6 +1804,54 @@ SDValue DAGTypeLegalizer::ExpandFloatOp_STORE(SDNode *N, unsigned OpNo) { ST->getMemoryVT(), ST->getMemOperand()); } +SDValue DAGTypeLegalizer::ExpandFloatOp_LROUND(SDNode *N) { + EVT RVT = N->getValueType(0); + EVT RetVT = N->getOperand(0).getValueType().getSimpleVT().SimpleTy; + return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, + RTLIB::LROUND_F32, + RTLIB::LROUND_F64, + RTLIB::LROUND_F80, + RTLIB::LROUND_F128, + RTLIB::LROUND_PPCF128), + RVT, N->getOperand(0), false, SDLoc(N)).first; +} + +SDValue DAGTypeLegalizer::ExpandFloatOp_LLROUND(SDNode *N) { + EVT RVT = N->getValueType(0); + EVT RetVT = N->getOperand(0).getValueType().getSimpleVT().SimpleTy; + return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, + RTLIB::LLROUND_F32, + RTLIB::LLROUND_F64, + RTLIB::LLROUND_F80, + RTLIB::LLROUND_F128, + RTLIB::LLROUND_PPCF128), + RVT, N->getOperand(0), false, SDLoc(N)).first; +} + +SDValue DAGTypeLegalizer::ExpandFloatOp_LRINT(SDNode *N) { + EVT RVT = N->getValueType(0); + EVT RetVT = N->getOperand(0).getValueType().getSimpleVT().SimpleTy; + return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, + RTLIB::LRINT_F32, + RTLIB::LRINT_F64, + RTLIB::LRINT_F80, + RTLIB::LRINT_F128, + RTLIB::LRINT_PPCF128), + RVT, N->getOperand(0), false, SDLoc(N)).first; +} + +SDValue DAGTypeLegalizer::ExpandFloatOp_LLRINT(SDNode *N) { + EVT RVT = N->getValueType(0); + EVT RetVT = N->getOperand(0).getValueType().getSimpleVT().SimpleTy; + return TLI.makeLibCall(DAG, GetFPLibCall(RetVT, + RTLIB::LLRINT_F32, + RTLIB::LLRINT_F64, + RTLIB::LLRINT_F80, + RTLIB::LLRINT_F128, + RTLIB::LLRINT_PPCF128), + RVT, N->getOperand(0), false, SDLoc(N)).first; +} + //===----------------------------------------------------------------------===// // Float Operand Promotion //===----------------------------------------------------------------------===// @@ -1748,6 +1868,8 @@ static ISD::NodeType GetPromotionOpcode(EVT OpVT, EVT RetVT) { } bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) { + LLVM_DEBUG(dbgs() << "Promote float operand " << OpNo << ": "; N->dump(&DAG); + dbgs() << "\n"); SDValue R = SDValue(); if (CustomLowerNode(N, N->getOperand(OpNo).getValueType(), false)) { @@ -1762,6 +1884,10 @@ bool DAGTypeLegalizer::PromoteFloatOperand(SDNode *N, unsigned OpNo) { // a part of PromoteFloatResult. switch (N->getOpcode()) { default: + #ifndef NDEBUG + dbgs() << "PromoteFloatOperand Op #" << OpNo << ": "; + N->dump(&DAG); dbgs() << "\n"; + #endif llvm_unreachable("Do not know how to promote this operator's operand!"); case ISD::BITCAST: R = PromoteFloatOp_BITCAST(N, OpNo); break; @@ -1872,6 +1998,8 @@ SDValue DAGTypeLegalizer::PromoteFloatOp_STORE(SDNode *N, unsigned OpNo) { //===----------------------------------------------------------------------===// void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) { + LLVM_DEBUG(dbgs() << "Promote float result " << ResNo << ": "; N->dump(&DAG); + dbgs() << "\n"); SDValue R = SDValue(); switch (N->getOpcode()) { @@ -1880,6 +2008,10 @@ void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) { case ISD::FP16_TO_FP: case ISD::FP_TO_FP16: default: +#ifndef NDEBUG + dbgs() << "PromoteFloatResult #" << ResNo << ": "; + N->dump(&DAG); dbgs() << "\n"; +#endif llvm_unreachable("Do not know how to promote this operator's result!"); case ISD::BITCAST: R = PromoteFloatRes_BITCAST(N); break; @@ -1932,7 +2064,7 @@ void DAGTypeLegalizer::PromoteFloatResult(SDNode *N, unsigned ResNo) { case ISD::SINT_TO_FP: case ISD::UINT_TO_FP: R = PromoteFloatRes_XINT_TO_FP(N); break; case ISD::UNDEF: R = PromoteFloatRes_UNDEF(N); break; - + case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break; } if (R.getNode()) @@ -2166,3 +2298,29 @@ SDValue DAGTypeLegalizer::PromoteFloatRes_UNDEF(SDNode *N) { N->getValueType(0))); } +SDValue DAGTypeLegalizer::BitcastToInt_ATOMIC_SWAP(SDNode *N) { + EVT VT = N->getValueType(0); + EVT NFPVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); + + AtomicSDNode *AM = cast(N); + SDLoc SL(N); + + SDValue CastVal = BitConvertToInteger(AM->getVal()); + EVT CastVT = CastVal.getValueType(); + + SDValue NewAtomic + = DAG.getAtomic(ISD::ATOMIC_SWAP, SL, CastVT, + DAG.getVTList(CastVT, MVT::Other), + { AM->getChain(), AM->getBasePtr(), CastVal }, + AM->getMemOperand()); + + SDValue ResultCast = DAG.getNode(GetPromotionOpcode(VT, NFPVT), SL, NFPVT, + NewAtomic); + // Legalize the chain result by replacing uses of the old value chain with the + // new one + ReplaceValueWith(SDValue(N, 1), NewAtomic.getValue(1)); + + return ResultCast; + +} + diff --git a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp index 5fbc70fce60d..15ac45c37c66 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp @@ -1,9 +1,8 @@ //===----- LegalizeIntegerTypes.cpp - Legalization of integer types -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -149,7 +148,10 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { case ISD::UADDSAT: case ISD::SSUBSAT: case ISD::USUBSAT: Res = PromoteIntRes_ADDSUBSAT(N); break; - case ISD::SMULFIX: Res = PromoteIntRes_SMULFIX(N); break; + case ISD::SMULFIX: + case ISD::SMULFIXSAT: + case ISD::UMULFIX: Res = PromoteIntRes_MULFIX(N); break; + case ISD::ABS: Res = PromoteIntRes_ABS(N); break; case ISD::ATOMIC_LOAD: Res = PromoteIntRes_Atomic0(cast(N)); break; @@ -172,6 +174,18 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) { case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: Res = PromoteIntRes_AtomicCmpSwap(cast(N), ResNo); break; + + case ISD::VECREDUCE_ADD: + case ISD::VECREDUCE_MUL: + case ISD::VECREDUCE_AND: + case ISD::VECREDUCE_OR: + case ISD::VECREDUCE_XOR: + case ISD::VECREDUCE_SMAX: + case ISD::VECREDUCE_SMIN: + case ISD::VECREDUCE_UMAX: + case ISD::VECREDUCE_UMIN: + Res = PromoteIntRes_VECREDUCE(N); + break; } // If the result is null then the sub-method took care of registering it. @@ -293,21 +307,24 @@ SDValue DAGTypeLegalizer::PromoteIntRes_BITCAST(SDNode *N) { BitConvertToInteger(GetScalarizedVector(InOp))); break; case TargetLowering::TypeSplitVector: { - // For example, i32 = BITCAST v2i16 on alpha. Convert the split - // pieces of the input into integers and reassemble in the final type. - SDValue Lo, Hi; - GetSplitVector(N->getOperand(0), Lo, Hi); - Lo = BitConvertToInteger(Lo); - Hi = BitConvertToInteger(Hi); - - if (DAG.getDataLayout().isBigEndian()) - std::swap(Lo, Hi); - - InOp = DAG.getNode(ISD::ANY_EXTEND, dl, - EVT::getIntegerVT(*DAG.getContext(), - NOutVT.getSizeInBits()), - JoinIntegers(Lo, Hi)); - return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp); + if (!NOutVT.isVector()) { + // For example, i32 = BITCAST v2i16 on alpha. Convert the split + // pieces of the input into integers and reassemble in the final type. + SDValue Lo, Hi; + GetSplitVector(N->getOperand(0), Lo, Hi); + Lo = BitConvertToInteger(Lo); + Hi = BitConvertToInteger(Hi); + + if (DAG.getDataLayout().isBigEndian()) + std::swap(Lo, Hi); + + InOp = DAG.getNode(ISD::ANY_EXTEND, dl, + EVT::getIntegerVT(*DAG.getContext(), + NOutVT.getSizeInBits()), + JoinIntegers(Lo, Hi)); + return DAG.getNode(ISD::BITCAST, dl, NOutVT, InOp); + } + break; } case TargetLowering::TypeWidenVector: // The input is widened to the same size. Convert to the widened value. @@ -555,7 +572,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_MLOAD(MaskedLoadSDNode *N) { SDLoc dl(N); SDValue Res = DAG.getMaskedLoad(NVT, dl, N->getChain(), N->getBasePtr(), N->getMask(), ExtPassThru, N->getMemoryVT(), - N->getMemOperand(), ISD::SEXTLOAD); + N->getMemOperand(), ISD::EXTLOAD); // Legalize the chain result - switch anything that used the old chain to // use the new one. ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); @@ -582,23 +599,27 @@ SDValue DAGTypeLegalizer::PromoteIntRes_MGATHER(MaskedGatherSDNode *N) { /// Promote the overflow flag of an overflowing arithmetic node. SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) { - // Simply change the return type of the boolean result. + // Change the return type of the boolean result while obeying + // getSetCCResultType. EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(1)); - EVT ValueVTs[] = { N->getValueType(0), NVT }; + EVT VT = N->getValueType(0); + EVT SVT = getSetCCResultType(VT); SDValue Ops[3] = { N->getOperand(0), N->getOperand(1) }; unsigned NumOps = N->getNumOperands(); assert(NumOps <= 3 && "Too many operands"); if (NumOps == 3) Ops[2] = N->getOperand(2); - SDValue Res = DAG.getNode(N->getOpcode(), SDLoc(N), - DAG.getVTList(ValueVTs), makeArrayRef(Ops, NumOps)); + SDLoc dl(N); + SDValue Res = DAG.getNode(N->getOpcode(), dl, DAG.getVTList(VT, SVT), + makeArrayRef(Ops, NumOps)); // Modified the sum result - switch anything that used the old sum to use // the new one. ReplaceValueWith(SDValue(N, 0), Res); - return SDValue(Res.getNode(), 1); + // Convert to the expected type. + return DAG.getBoolExtOrTrunc(Res.getValue(1), dl, NVT, VT); } SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSAT(SDNode *N) { @@ -646,12 +667,39 @@ SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBSAT(SDNode *N) { return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount); } -SDValue DAGTypeLegalizer::PromoteIntRes_SMULFIX(SDNode *N) { +SDValue DAGTypeLegalizer::PromoteIntRes_MULFIX(SDNode *N) { // Can just promote the operands then continue with operation. SDLoc dl(N); - SDValue Op1Promoted = SExtPromotedInteger(N->getOperand(0)); - SDValue Op2Promoted = SExtPromotedInteger(N->getOperand(1)); + SDValue Op1Promoted, Op2Promoted; + bool Signed = + N->getOpcode() == ISD::SMULFIX || N->getOpcode() == ISD::SMULFIXSAT; + if (Signed) { + Op1Promoted = SExtPromotedInteger(N->getOperand(0)); + Op2Promoted = SExtPromotedInteger(N->getOperand(1)); + } else { + Op1Promoted = ZExtPromotedInteger(N->getOperand(0)); + Op2Promoted = ZExtPromotedInteger(N->getOperand(1)); + } + EVT OldType = N->getOperand(0).getValueType(); EVT PromotedType = Op1Promoted.getValueType(); + unsigned DiffSize = + PromotedType.getScalarSizeInBits() - OldType.getScalarSizeInBits(); + + bool Saturating = N->getOpcode() == ISD::SMULFIXSAT; + if (Saturating) { + // Promoting the operand and result values changes the saturation width, + // which is extends the values that we clamp to on saturation. This could be + // resolved by shifting one of the operands the same amount, which would + // also shift the result we compare against, then shifting back. + EVT ShiftTy = TLI.getShiftAmountTy(PromotedType, DAG.getDataLayout()); + Op1Promoted = DAG.getNode(ISD::SHL, dl, PromotedType, Op1Promoted, + DAG.getConstant(DiffSize, dl, ShiftTy)); + SDValue Result = DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, + Op2Promoted, N->getOperand(2)); + unsigned ShiftOp = Signed ? ISD::SRA : ISD::SRL; + return DAG.getNode(ShiftOp, dl, PromotedType, Result, + DAG.getConstant(DiffSize, dl, ShiftTy)); + } return DAG.getNode(N->getOpcode(), dl, PromotedType, Op1Promoted, Op2Promoted, N->getOperand(2)); } @@ -875,7 +923,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_UADDSUBO(SDNode *N, unsigned ResNo) { // Calculate the overflow flag: zero extend the arithmetic result from // the original type. - SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT); + SDValue Ofl = DAG.getZeroExtendInReg(Res, dl, OVT.getScalarType()); // Overflowed if and only if this is not equal to Res. Ofl = DAG.getSetCC(dl, N->getValueType(1), Ofl, Res, ISD::SETNE); @@ -917,6 +965,11 @@ SDValue DAGTypeLegalizer::PromoteIntRes_ADDSUBCARRY(SDNode *N, unsigned ResNo) { return SDValue(Res.getNode(), 0); } +SDValue DAGTypeLegalizer::PromoteIntRes_ABS(SDNode *N) { + SDValue Op0 = SExtPromotedInteger(N->getOperand(0)); + return DAG.getNode(ISD::ABS, SDLoc(N), Op0.getValueType(), Op0); +} + SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { // Promote the overflow bit trivially. if (ResNo == 1) @@ -946,9 +999,11 @@ SDValue DAGTypeLegalizer::PromoteIntRes_XMULO(SDNode *N, unsigned ResNo) { SDValue Overflow; if (N->getOpcode() == ISD::UMULO) { // Unsigned overflow occurred if the high part is non-zero. + unsigned Shift = SmallVT.getScalarSizeInBits(); + EVT ShiftTy = getShiftAmountTyForConstant(Shift, Mul.getValueType(), + TLI, DAG); SDValue Hi = DAG.getNode(ISD::SRL, DL, Mul.getValueType(), Mul, - DAG.getIntPtrConstant(SmallVT.getSizeInBits(), - DL)); + DAG.getConstant(Shift, DL, ShiftTy)); Overflow = DAG.getSetCC(DL, N->getValueType(1), Hi, DAG.getConstant(0, DL, Hi.getValueType()), ISD::SETNE); @@ -1091,7 +1146,21 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) { case ISD::PREFETCH: Res = PromoteIntOp_PREFETCH(N, OpNo); break; - case ISD::SMULFIX: Res = PromoteIntOp_SMULFIX(N); break; + case ISD::SMULFIX: + case ISD::SMULFIXSAT: + case ISD::UMULFIX: Res = PromoteIntOp_MULFIX(N); break; + + case ISD::FPOWI: Res = PromoteIntOp_FPOWI(N); break; + + case ISD::VECREDUCE_ADD: + case ISD::VECREDUCE_MUL: + case ISD::VECREDUCE_AND: + case ISD::VECREDUCE_OR: + case ISD::VECREDUCE_XOR: + case ISD::VECREDUCE_SMAX: + case ISD::VECREDUCE_SMIN: + case ISD::VECREDUCE_UMAX: + case ISD::VECREDUCE_UMIN: Res = PromoteIntOp_VECREDUCE(N); break; } // If the result is null, the sub-method took care of registering results etc. @@ -1434,24 +1503,12 @@ SDValue DAGTypeLegalizer::PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo) { SDValue Carry = N->getOperand(2); SDLoc DL(N); - auto VT = getSetCCResultType(LHS.getValueType()); - TargetLoweringBase::BooleanContent BoolType = TLI.getBooleanContents(VT); - switch (BoolType) { - case TargetLoweringBase::UndefinedBooleanContent: - Carry = DAG.getAnyExtOrTrunc(Carry, DL, VT); - break; - case TargetLoweringBase::ZeroOrOneBooleanContent: - Carry = DAG.getZExtOrTrunc(Carry, DL, VT); - break; - case TargetLoweringBase::ZeroOrNegativeOneBooleanContent: - Carry = DAG.getSExtOrTrunc(Carry, DL, VT); - break; - } + Carry = PromoteTargetBoolean(Carry, LHS.getValueType()); return SDValue(DAG.UpdateNodeOperands(N, LHS, RHS, Carry), 0); } -SDValue DAGTypeLegalizer::PromoteIntOp_SMULFIX(SDNode *N) { +SDValue DAGTypeLegalizer::PromoteIntOp_MULFIX(SDNode *N) { SDValue Op2 = ZExtPromotedInteger(N->getOperand(2)); return SDValue( DAG.UpdateNodeOperands(N, N->getOperand(0), N->getOperand(1), Op2), 0); @@ -1475,6 +1532,44 @@ SDValue DAGTypeLegalizer::PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo) { 0); } +SDValue DAGTypeLegalizer::PromoteIntOp_FPOWI(SDNode *N) { + SDValue Op = SExtPromotedInteger(N->getOperand(1)); + return SDValue(DAG.UpdateNodeOperands(N, N->getOperand(0), Op), 0); +} + +SDValue DAGTypeLegalizer::PromoteIntOp_VECREDUCE(SDNode *N) { + SDLoc dl(N); + SDValue Op; + switch (N->getOpcode()) { + default: llvm_unreachable("Expected integer vector reduction"); + case ISD::VECREDUCE_ADD: + case ISD::VECREDUCE_MUL: + case ISD::VECREDUCE_AND: + case ISD::VECREDUCE_OR: + case ISD::VECREDUCE_XOR: + Op = GetPromotedInteger(N->getOperand(0)); + break; + case ISD::VECREDUCE_SMAX: + case ISD::VECREDUCE_SMIN: + Op = SExtPromotedInteger(N->getOperand(0)); + break; + case ISD::VECREDUCE_UMAX: + case ISD::VECREDUCE_UMIN: + Op = ZExtPromotedInteger(N->getOperand(0)); + break; + } + + EVT EltVT = Op.getValueType().getVectorElementType(); + EVT VT = N->getValueType(0); + if (VT.bitsGE(EltVT)) + return DAG.getNode(N->getOpcode(), SDLoc(N), VT, Op); + + // Result size must be >= element size. If this is not the case after + // promotion, also promote the result type and then truncate. + SDValue Reduce = DAG.getNode(N->getOpcode(), dl, EltVT, Op); + return DAG.getNode(ISD::TRUNCATE, dl, VT, Reduce); +} + //===----------------------------------------------------------------------===// // Integer Result Expansion //===----------------------------------------------------------------------===// @@ -1499,7 +1594,8 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { dbgs() << "ExpandIntegerResult #" << ResNo << ": "; N->dump(&DAG); dbgs() << "\n"; #endif - llvm_unreachable("Do not know how to expand the result of this operator!"); + report_fatal_error("Do not know how to expand the result of this " + "operator!"); case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break; case ISD::SELECT: SplitRes_SELECT(N, Lo, Hi); break; @@ -1518,6 +1614,7 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { case ISD::BITREVERSE: ExpandIntRes_BITREVERSE(N, Lo, Hi); break; case ISD::BSWAP: ExpandIntRes_BSWAP(N, Lo, Hi); break; case ISD::Constant: ExpandIntRes_Constant(N, Lo, Hi); break; + case ISD::ABS: ExpandIntRes_ABS(N, Lo, Hi); break; case ISD::CTLZ_ZERO_UNDEF: case ISD::CTLZ: ExpandIntRes_CTLZ(N, Lo, Hi); break; case ISD::CTPOP: ExpandIntRes_CTPOP(N, Lo, Hi); break; @@ -1526,6 +1623,8 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { case ISD::FLT_ROUNDS_: ExpandIntRes_FLT_ROUNDS(N, Lo, Hi); break; case ISD::FP_TO_SINT: ExpandIntRes_FP_TO_SINT(N, Lo, Hi); break; case ISD::FP_TO_UINT: ExpandIntRes_FP_TO_UINT(N, Lo, Hi); break; + case ISD::LLROUND: ExpandIntRes_LLROUND(N, Lo, Hi); break; + case ISD::LLRINT: ExpandIntRes_LLRINT(N, Lo, Hi); break; case ISD::LOAD: ExpandIntRes_LOAD(cast(N), Lo, Hi); break; case ISD::MUL: ExpandIntRes_MUL(N, Lo, Hi); break; case ISD::READCYCLECOUNTER: ExpandIntRes_READCYCLECOUNTER(N, Lo, Hi); break; @@ -1613,7 +1712,20 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) { case ISD::UADDSAT: case ISD::SSUBSAT: case ISD::USUBSAT: ExpandIntRes_ADDSUBSAT(N, Lo, Hi); break; - case ISD::SMULFIX: ExpandIntRes_SMULFIX(N, Lo, Hi); break; + + case ISD::SMULFIX: + case ISD::SMULFIXSAT: + case ISD::UMULFIX: ExpandIntRes_MULFIX(N, Lo, Hi); break; + + case ISD::VECREDUCE_ADD: + case ISD::VECREDUCE_MUL: + case ISD::VECREDUCE_AND: + case ISD::VECREDUCE_OR: + case ISD::VECREDUCE_XOR: + case ISD::VECREDUCE_SMAX: + case ISD::VECREDUCE_SMIN: + case ISD::VECREDUCE_UMAX: + case ISD::VECREDUCE_UMIN: ExpandIntRes_VECREDUCE(N, Lo, Hi); break; } // If Lo/Hi is null, the sub-method took care of registering results etc. @@ -2267,6 +2379,25 @@ void DAGTypeLegalizer::ExpandIntRes_Constant(SDNode *N, IsOpaque); } +void DAGTypeLegalizer::ExpandIntRes_ABS(SDNode *N, SDValue &Lo, SDValue &Hi) { + SDLoc dl(N); + + // abs(HiLo) -> (Hi < 0 ? -HiLo : HiLo) + EVT VT = N->getValueType(0); + SDValue N0 = N->getOperand(0); + SDValue Neg = DAG.getNode(ISD::SUB, dl, VT, + DAG.getConstant(0, dl, VT), N0); + SDValue NegLo, NegHi; + SplitInteger(Neg, NegLo, NegHi); + + GetExpandedInteger(N0, Lo, Hi); + EVT NVT = Lo.getValueType(); + SDValue HiIsNeg = DAG.getSetCC(dl, getSetCCResultType(NVT), + DAG.getConstant(0, dl, NVT), Hi, ISD::SETGT); + Lo = DAG.getSelect(dl, NVT, HiIsNeg, NegLo, Lo); + Hi = DAG.getSelect(dl, NVT, HiIsNeg, NegHi, Hi); +} + void DAGTypeLegalizer::ExpandIntRes_CTLZ(SDNode *N, SDValue &Lo, SDValue &Hi) { SDLoc dl(N); @@ -2361,6 +2492,58 @@ void DAGTypeLegalizer::ExpandIntRes_FP_TO_UINT(SDNode *N, SDValue &Lo, Lo, Hi); } +void DAGTypeLegalizer::ExpandIntRes_LLROUND(SDNode *N, SDValue &Lo, + SDValue &Hi) { + RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; + EVT VT = N->getOperand(0).getValueType().getSimpleVT().SimpleTy; + if (VT == MVT::f32) + LC = RTLIB::LLROUND_F32; + else if (VT == MVT::f64) + LC = RTLIB::LLROUND_F64; + else if (VT == MVT::f80) + LC = RTLIB::LLROUND_F80; + else if (VT == MVT::f128) + LC = RTLIB::LLROUND_F128; + else if (VT == MVT::ppcf128) + LC = RTLIB::LLROUND_PPCF128; + assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llround input type!"); + + SDValue Op = N->getOperand(0); + if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) + Op = GetPromotedFloat(Op); + + SDLoc dl(N); + EVT RetVT = N->getValueType(0); + SplitInteger(TLI.makeLibCall(DAG, LC, RetVT, Op, true/*irrelevant*/, dl).first, + Lo, Hi); +} + +void DAGTypeLegalizer::ExpandIntRes_LLRINT(SDNode *N, SDValue &Lo, + SDValue &Hi) { + RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; + EVT VT = N->getOperand(0).getValueType().getSimpleVT().SimpleTy; + if (VT == MVT::f32) + LC = RTLIB::LLRINT_F32; + else if (VT == MVT::f64) + LC = RTLIB::LLRINT_F64; + else if (VT == MVT::f80) + LC = RTLIB::LLRINT_F80; + else if (VT == MVT::f128) + LC = RTLIB::LLRINT_F128; + else if (VT == MVT::ppcf128) + LC = RTLIB::LLRINT_PPCF128; + assert(LC != RTLIB::UNKNOWN_LIBCALL && "Unexpected llrint input type!"); + + SDValue Op = N->getOperand(0); + if (getTypeAction(Op.getValueType()) == TargetLowering::TypePromoteFloat) + Op = GetPromotedFloat(Op); + + SDLoc dl(N); + EVT RetVT = N->getValueType(0); + SplitInteger(TLI.makeLibCall(DAG, LC, RetVT, Op, true/*irrelevant*/, dl).first, + Lo, Hi); +} + void DAGTypeLegalizer::ExpandIntRes_LOAD(LoadSDNode *N, SDValue &Lo, SDValue &Hi) { if (ISD::isNormalLoad(N)) { @@ -2581,15 +2764,39 @@ void DAGTypeLegalizer::ExpandIntRes_ADDSUBSAT(SDNode *N, SDValue &Lo, SplitInteger(Result, Lo, Hi); } -void DAGTypeLegalizer::ExpandIntRes_SMULFIX(SDNode *N, SDValue &Lo, - SDValue &Hi) { +/// This performs an expansion of the integer result for a fixed point +/// multiplication. The default expansion performs rounding down towards +/// negative infinity, though targets that do care about rounding should specify +/// a target hook for rounding and provide their own expansion or lowering of +/// fixed point multiplication to be consistent with rounding. +void DAGTypeLegalizer::ExpandIntRes_MULFIX(SDNode *N, SDValue &Lo, + SDValue &Hi) { SDLoc dl(N); EVT VT = N->getValueType(0); + unsigned VTSize = VT.getScalarSizeInBits(); SDValue LHS = N->getOperand(0); SDValue RHS = N->getOperand(1); uint64_t Scale = N->getConstantOperandVal(2); + bool Saturating = N->getOpcode() == ISD::SMULFIXSAT; + EVT BoolVT = getSetCCResultType(VT); + SDValue Zero = DAG.getConstant(0, dl, VT); if (!Scale) { - SDValue Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); + SDValue Result; + if (!Saturating) { + Result = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); + } else { + Result = DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); + SDValue Product = Result.getValue(0); + SDValue Overflow = Result.getValue(1); + + APInt MinVal = APInt::getSignedMinValue(VTSize); + APInt MaxVal = APInt::getSignedMaxValue(VTSize); + SDValue SatMin = DAG.getConstant(MinVal, dl, VT); + SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); + SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Product, Zero, ISD::SETLT); + Result = DAG.getSelect(dl, VT, ProdNeg, SatMax, SatMin); + Result = DAG.getSelect(dl, VT, Overflow, Result, Product); + } SplitInteger(Result, Lo, Hi); return; } @@ -2600,15 +2807,19 @@ void DAGTypeLegalizer::ExpandIntRes_SMULFIX(SDNode *N, SDValue &Lo, GetExpandedInteger(RHS, RL, RH); SmallVector Result; - if (!TLI.expandMUL_LOHI(ISD::SMUL_LOHI, VT, dl, LHS, RHS, Result, NVT, DAG, + bool Signed = (N->getOpcode() == ISD::SMULFIX || + N->getOpcode() == ISD::SMULFIXSAT); + unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI; + if (!TLI.expandMUL_LOHI(LoHiOp, VT, dl, LHS, RHS, Result, NVT, DAG, TargetLowering::MulExpansionKind::OnlyLegalOrCustom, LL, LH, RL, RH)) { - report_fatal_error("Unable to expand SMUL_FIX using SMUL_LOHI."); + report_fatal_error("Unable to expand MUL_FIX using MUL_LOHI."); return; } - unsigned VTSize = VT.getScalarSizeInBits(); unsigned NVTSize = NVT.getScalarSizeInBits(); + assert((VTSize == NVTSize * 2) && "Expected the new value type to be half " + "the size of the current value type"); EVT ShiftTy = TLI.getShiftAmountTy(NVT, DAG.getDataLayout()); // Shift whole amount by scale. @@ -2617,6 +2828,11 @@ void DAGTypeLegalizer::ExpandIntRes_SMULFIX(SDNode *N, SDValue &Lo, SDValue ResultHL = Result[2]; SDValue ResultHH = Result[3]; + SDValue SatMax, SatMin; + SDValue NVTZero = DAG.getConstant(0, dl, NVT); + SDValue NVTNeg1 = DAG.getConstant(-1, dl, NVT); + EVT BoolNVT = getSetCCResultType(NVT); + // After getting the multplication result in 4 parts, we need to perform a // shift right by the amount of the scale to get the result in that scale. // Let's say we multiply 2 64 bit numbers. The resulting value can be held in @@ -2645,11 +2861,60 @@ void DAGTypeLegalizer::ExpandIntRes_SMULFIX(SDNode *N, SDValue &Lo, Hi = DAG.getNode(ISD::SRL, dl, NVT, ResultLH, SRLAmnt); Hi = DAG.getNode(ISD::OR, dl, NVT, Hi, DAG.getNode(ISD::SHL, dl, NVT, ResultHL, SHLAmnt)); + + // We cannot overflow past HH when multiplying 2 ints of size VTSize, so the + // highest bit of HH determines saturation direction in the event of + // saturation. + // The number of overflow bits we can check are VTSize - Scale + 1 (we + // include the sign bit). If these top bits are > 0, then we overflowed past + // the max value. If these top bits are < -1, then we overflowed past the + // min value. Otherwise, we did not overflow. + if (Saturating) { + unsigned OverflowBits = VTSize - Scale + 1; + assert(OverflowBits <= VTSize && OverflowBits > NVTSize && + "Extent of overflow bits must start within HL"); + SDValue HLHiMask = DAG.getConstant( + APInt::getHighBitsSet(NVTSize, OverflowBits - NVTSize), dl, NVT); + SDValue HLLoMask = DAG.getConstant( + APInt::getLowBitsSet(NVTSize, VTSize - OverflowBits), dl, NVT); + + // HH > 0 or HH == 0 && HL > HLLoMask + SDValue HHPos = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT); + SDValue HHZero = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ); + SDValue HLPos = + DAG.getSetCC(dl, BoolNVT, ResultHL, HLLoMask, ISD::SETUGT); + SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHPos, + DAG.getNode(ISD::AND, dl, BoolNVT, HHZero, HLPos)); + + // HH < -1 or HH == -1 && HL < HLHiMask + SDValue HHNeg = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT); + SDValue HHNeg1 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ); + SDValue HLNeg = + DAG.getSetCC(dl, BoolNVT, ResultHL, HLHiMask, ISD::SETULT); + SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHNeg, + DAG.getNode(ISD::AND, dl, BoolNVT, HHNeg1, HLNeg)); + } } else if (Scale == NVTSize) { // If the scales are equal, Lo and Hi are ResultLH and Result HL, // respectively. Avoid shifting to prevent undefined behavior. Lo = ResultLH; Hi = ResultHL; + + // We overflow max if HH > 0 or HH == 0 && HL sign bit is 1. + // We overflow min if HH < -1 or HH == -1 && HL sign bit is 0. + if (Saturating) { + SDValue HHPos = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETGT); + SDValue HHZero = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTZero, ISD::SETEQ); + SDValue HLNeg = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETLT); + SatMax = DAG.getNode(ISD::OR, dl, BoolNVT, HHPos, + DAG.getNode(ISD::AND, dl, BoolNVT, HHZero, HLNeg)); + + SDValue HHNeg = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETLT); + SDValue HHNeg1 = DAG.getSetCC(dl, BoolNVT, ResultHH, NVTNeg1, ISD::SETEQ); + SDValue HLPos = DAG.getSetCC(dl, BoolNVT, ResultHL, NVTZero, ISD::SETGE); + SatMin = DAG.getNode(ISD::OR, dl, BoolNVT, HHNeg, + DAG.getNode(ISD::AND, dl, BoolNVT, HHNeg1, HLPos)); + } } else if (Scale < VTSize) { // If the scale is instead less than the old VT size, but greater than or // equal to the expanded VT size, the first part of the result (ResultLL) is @@ -2664,9 +2929,39 @@ void DAGTypeLegalizer::ExpandIntRes_SMULFIX(SDNode *N, SDValue &Lo, Hi = DAG.getNode(ISD::SRL, dl, NVT, ResultHL, SRLAmnt); Hi = DAG.getNode(ISD::OR, dl, NVT, Hi, DAG.getNode(ISD::SHL, dl, NVT, ResultHH, SHLAmnt)); + + // This is similar to the case when we saturate if Scale < NVTSize, but we + // only need to chech HH. + if (Saturating) { + unsigned OverflowBits = VTSize - Scale + 1; + SDValue HHHiMask = DAG.getConstant( + APInt::getHighBitsSet(NVTSize, OverflowBits), dl, NVT); + SDValue HHLoMask = DAG.getConstant( + APInt::getLowBitsSet(NVTSize, NVTSize - OverflowBits), dl, NVT); + + SatMax = DAG.getSetCC(dl, BoolNVT, ResultHH, HHLoMask, ISD::SETGT); + SatMin = DAG.getSetCC(dl, BoolNVT, ResultHH, HHHiMask, ISD::SETLT); + } + } else if (Scale == VTSize) { + assert( + !Signed && + "Only unsigned types can have a scale equal to the operand bit width"); + + Lo = ResultHL; + Hi = ResultHH; } else { - llvm_unreachable( - "Expected the scale to be less than the width of the operands"); + llvm_unreachable("Expected the scale to be less than or equal to the width " + "of the operands"); + } + + if (Saturating) { + APInt LHMax = APInt::getSignedMaxValue(NVTSize); + APInt LLMax = APInt::getAllOnesValue(NVTSize); + APInt LHMin = APInt::getSignedMinValue(NVTSize); + Hi = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(LHMax, dl, NVT), Hi); + Hi = DAG.getSelect(dl, NVT, SatMin, DAG.getConstant(LHMin, dl, NVT), Hi); + Lo = DAG.getSelect(dl, NVT, SatMax, DAG.getConstant(LLMax, dl, NVT), Lo); + Lo = DAG.getSelect(dl, NVT, SatMin, NVTZero, Lo); } } @@ -2765,11 +3060,15 @@ void DAGTypeLegalizer::ExpandIntRes_Shift(SDNode *N, } // Next check to see if the target supports this SHL_PARTS operation or if it - // will custom expand it. + // will custom expand it. Don't lower this to SHL_PARTS when we optimise for + // size, but create a libcall instead. EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); TargetLowering::LegalizeAction Action = TLI.getOperationAction(PartsOpc, NVT); - if ((Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || - Action == TargetLowering::Custom) { + const bool LegalOrCustom = + (Action == TargetLowering::Legal && TLI.isTypeLegal(NVT)) || + Action == TargetLowering::Custom; + + if (LegalOrCustom && TLI.shouldExpandShift(DAG, N)) { // Expand the subcomponents. SDValue LHSL, LHSH; GetExpandedInteger(N->getOperand(0), LHSL, LHSH); @@ -3145,6 +3444,14 @@ void DAGTypeLegalizer::ExpandIntRes_ATOMIC_LOAD(SDNode *N, ReplaceValueWith(SDValue(N, 1), Swap.getValue(2)); } +void DAGTypeLegalizer::ExpandIntRes_VECREDUCE(SDNode *N, + SDValue &Lo, SDValue &Hi) { + // TODO For VECREDUCE_(AND|OR|XOR) we could split the vector and calculate + // both halves independently. + SDValue Res = TLI.expandVecReduce(N, DAG); + SplitInteger(Res, Lo, Hi); +} + //===----------------------------------------------------------------------===// // Integer Operand Expansion //===----------------------------------------------------------------------===// @@ -3167,7 +3474,7 @@ bool DAGTypeLegalizer::ExpandIntegerOperand(SDNode *N, unsigned OpNo) { dbgs() << "ExpandIntegerOperand Op #" << OpNo << ": "; N->dump(&DAG); dbgs() << "\n"; #endif - llvm_unreachable("Do not know how to expand this operator's operand!"); + report_fatal_error("Do not know how to expand this operator's operand!"); case ISD::BITCAST: Res = ExpandOp_BITCAST(N); break; case ISD::BR_CC: Res = ExpandIntOp_BR_CC(N); break; @@ -3632,8 +3939,6 @@ SDValue DAGTypeLegalizer::ExpandIntOp_ATOMIC_STORE(SDNode *N) { SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { - SDValue InOp0 = N->getOperand(0); - EVT InVT = InOp0.getValueType(); EVT OutVT = N->getValueType(0); EVT NOutVT = TLI.getTypeToTransformTo(*DAG.getContext(), OutVT); @@ -3644,6 +3949,12 @@ SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { SDLoc dl(N); SDValue BaseIdx = N->getOperand(1); + SDValue InOp0 = N->getOperand(0); + if (getTypeAction(InOp0.getValueType()) == TargetLowering::TypePromoteInteger) + InOp0 = GetPromotedInteger(N->getOperand(0)); + + EVT InVT = InOp0.getValueType(); + SmallVector Ops; Ops.reserve(OutNumElems); for (unsigned i = 0; i != OutNumElems; ++i) { @@ -3654,7 +3965,7 @@ SDValue DAGTypeLegalizer::PromoteIntRes_EXTRACT_SUBVECTOR(SDNode *N) { SDValue Ext = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InVT.getVectorElementType(), N->getOperand(0), Index); - SDValue Op = DAG.getNode(ISD::ANY_EXTEND, dl, NOutVTElem, Ext); + SDValue Op = DAG.getAnyExtOrTrunc(Ext, dl, NOutVTElem); // Insert the converted element to the new vector. Ops.push_back(Op); } @@ -3809,6 +4120,14 @@ SDValue DAGTypeLegalizer::PromoteIntRes_INSERT_VECTOR_ELT(SDNode *N) { V0, ConvElem, N->getOperand(2)); } +SDValue DAGTypeLegalizer::PromoteIntRes_VECREDUCE(SDNode *N) { + // The VECREDUCE result size may be larger than the element size, so + // we can simply change the result type. + SDLoc dl(N); + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); + return DAG.getNode(N->getOpcode(), dl, NVT, N->getOperand(0)); +} + SDValue DAGTypeLegalizer::PromoteIntOp_EXTRACT_VECTOR_ELT(SDNode *N) { SDLoc dl(N); SDValue V0 = GetPromotedInteger(N->getOperand(0)); diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp index a9f144c06e9a..14fd5be23ccb 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.cpp @@ -1,9 +1,8 @@ //===-- LegalizeTypes.cpp - Common code for DAG type legalizer ------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -708,6 +707,7 @@ void DAGTypeLegalizer::SetPromotedInteger(SDValue Op, SDValue Result) { auto &OpIdEntry = PromotedIntegers[getTableId(Op)]; assert((OpIdEntry == 0) && "Node is already promoted!"); OpIdEntry = getTableId(Result); + Result->setFlags(Op->getFlags()); DAG.transferDbgValues(Op, Result); } diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/lib/CodeGen/SelectionDAG/LegalizeTypes.h index 032000f6cb79..1d489b1b3a33 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -1,9 +1,8 @@ //===-- LegalizeTypes.h - DAG Type Legalizer class definition ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -345,8 +344,10 @@ private: SDValue PromoteIntRes_VAARG(SDNode *N); SDValue PromoteIntRes_XMULO(SDNode *N, unsigned ResNo); SDValue PromoteIntRes_ADDSUBSAT(SDNode *N); - SDValue PromoteIntRes_SMULFIX(SDNode *N); + SDValue PromoteIntRes_MULFIX(SDNode *N); SDValue PromoteIntRes_FLT_ROUNDS(SDNode *N); + SDValue PromoteIntRes_VECREDUCE(SDNode *N); + SDValue PromoteIntRes_ABS(SDNode *N); // Integer Operand Promotion. bool PromoteIntegerOperand(SDNode *N, unsigned OpNo); @@ -379,7 +380,9 @@ private: SDValue PromoteIntOp_ADDSUBCARRY(SDNode *N, unsigned OpNo); SDValue PromoteIntOp_FRAMERETURNADDR(SDNode *N); SDValue PromoteIntOp_PREFETCH(SDNode *N, unsigned OpNo); - SDValue PromoteIntOp_SMULFIX(SDNode *N); + SDValue PromoteIntOp_MULFIX(SDNode *N); + SDValue PromoteIntOp_FPOWI(SDNode *N); + SDValue PromoteIntOp_VECREDUCE(SDNode *N); void PromoteSetCCOperands(SDValue &LHS,SDValue &RHS, ISD::CondCode Code); @@ -402,6 +405,7 @@ private: void ExpandIntRes_AssertSext (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_AssertZext (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_Constant (SDNode *N, SDValue &Lo, SDValue &Hi); + void ExpandIntRes_ABS (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_CTLZ (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_CTPOP (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_CTTZ (SDNode *N, SDValue &Lo, SDValue &Hi); @@ -414,6 +418,8 @@ private: void ExpandIntRes_FLT_ROUNDS (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_FP_TO_SINT (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_FP_TO_UINT (SDNode *N, SDValue &Lo, SDValue &Hi); + void ExpandIntRes_LLROUND (SDNode *N, SDValue &Lo, SDValue &Hi); + void ExpandIntRes_LLRINT (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_Logical (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_ADDSUB (SDNode *N, SDValue &Lo, SDValue &Hi); @@ -435,9 +441,10 @@ private: void ExpandIntRes_UADDSUBO (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_XMULO (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_ADDSUBSAT (SDNode *N, SDValue &Lo, SDValue &Hi); - void ExpandIntRes_SMULFIX (SDNode *N, SDValue &Lo, SDValue &Hi); + void ExpandIntRes_MULFIX (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandIntRes_ATOMIC_LOAD (SDNode *N, SDValue &Lo, SDValue &Hi); + void ExpandIntRes_VECREDUCE (SDNode *N, SDValue &Lo, SDValue &Hi); void ExpandShiftByConstant(SDNode *N, const APInt &Amt, SDValue &Lo, SDValue &Hi); @@ -548,6 +555,10 @@ private: SDValue SoftenFloatOp_FP_EXTEND(SDNode *N); SDValue SoftenFloatOp_FP_ROUND(SDNode *N); SDValue SoftenFloatOp_FP_TO_XINT(SDNode *N); + SDValue SoftenFloatOp_LROUND(SDNode *N); + SDValue SoftenFloatOp_LLROUND(SDNode *N); + SDValue SoftenFloatOp_LRINT(SDNode *N); + SDValue SoftenFloatOp_LLRINT(SDNode *N); SDValue SoftenFloatOp_SELECT(SDNode *N); SDValue SoftenFloatOp_SELECT_CC(SDNode *N); SDValue SoftenFloatOp_SETCC(SDNode *N); @@ -607,6 +618,10 @@ private: SDValue ExpandFloatOp_FP_ROUND(SDNode *N); SDValue ExpandFloatOp_FP_TO_SINT(SDNode *N); SDValue ExpandFloatOp_FP_TO_UINT(SDNode *N); + SDValue ExpandFloatOp_LROUND(SDNode *N); + SDValue ExpandFloatOp_LLROUND(SDNode *N); + SDValue ExpandFloatOp_LRINT(SDNode *N); + SDValue ExpandFloatOp_LLRINT(SDNode *N); SDValue ExpandFloatOp_SELECT_CC(SDNode *N); SDValue ExpandFloatOp_SETCC(SDNode *N); SDValue ExpandFloatOp_STORE(SDNode *N, unsigned OpNo); @@ -640,6 +655,7 @@ private: SDValue PromoteFloatRes_SELECT_CC(SDNode *N); SDValue PromoteFloatRes_UnaryOp(SDNode *N); SDValue PromoteFloatRes_UNDEF(SDNode *N); + SDValue BitcastToInt_ATOMIC_SWAP(SDNode *N); SDValue PromoteFloatRes_XINT_TO_FP(SDNode *N); bool PromoteFloatOperand(SDNode *N, unsigned OpNo); @@ -673,6 +689,7 @@ private: SDValue ScalarizeVecRes_TernaryOp(SDNode *N); SDValue ScalarizeVecRes_UnaryOp(SDNode *N); SDValue ScalarizeVecRes_StrictFPOp(SDNode *N); + SDValue ScalarizeVecRes_OverflowOp(SDNode *N, unsigned ResNo); SDValue ScalarizeVecRes_InregOp(SDNode *N); SDValue ScalarizeVecRes_VecInregOp(SDNode *N); @@ -680,6 +697,7 @@ private: SDValue ScalarizeVecRes_BUILD_VECTOR(SDNode *N); SDValue ScalarizeVecRes_EXTRACT_SUBVECTOR(SDNode *N); SDValue ScalarizeVecRes_FP_ROUND(SDNode *N); + SDValue ScalarizeVecRes_STRICT_FP_ROUND(SDNode *N); SDValue ScalarizeVecRes_FPOWI(SDNode *N); SDValue ScalarizeVecRes_INSERT_VECTOR_ELT(SDNode *N); SDValue ScalarizeVecRes_LOAD(LoadSDNode *N); @@ -691,7 +709,7 @@ private: SDValue ScalarizeVecRes_UNDEF(SDNode *N); SDValue ScalarizeVecRes_VECTOR_SHUFFLE(SDNode *N); - SDValue ScalarizeVecRes_SMULFIX(SDNode *N); + SDValue ScalarizeVecRes_MULFIX(SDNode *N); // Vector Operand Scalarization: <1 x ty> -> ty. bool ScalarizeVectorOperand(SDNode *N, unsigned OpNo); @@ -703,6 +721,8 @@ private: SDValue ScalarizeVecOp_VSETCC(SDNode *N); SDValue ScalarizeVecOp_STORE(StoreSDNode *N, unsigned OpNo); SDValue ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo); + SDValue ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N, unsigned OpNo); + SDValue ScalarizeVecOp_VECREDUCE(SDNode *N); //===--------------------------------------------------------------------===// // Vector Splitting Support: LegalizeVectorTypes.cpp @@ -727,8 +747,10 @@ private: void SplitVecRes_InregOp(SDNode *N, SDValue &Lo, SDValue &Hi); void SplitVecRes_ExtVecInRegOp(SDNode *N, SDValue &Lo, SDValue &Hi); void SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo, SDValue &Hi); + void SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo, + SDValue &Lo, SDValue &Hi); - void SplitVecRes_SMULFIX(SDNode *N, SDValue &Lo, SDValue &Hi); + void SplitVecRes_MULFIX(SDNode *N, SDValue &Lo, SDValue &Hi); void SplitVecRes_BITCAST(SDNode *N, SDValue &Lo, SDValue &Hi); void SplitVecRes_BUILD_VECTOR(SDNode *N, SDValue &Lo, SDValue &Hi); @@ -745,6 +767,7 @@ private: void SplitVecRes_SETCC(SDNode *N, SDValue &Lo, SDValue &Hi); void SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, SDValue &Lo, SDValue &Hi); + void SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi); // Vector Operand Splitting: <128 x ty> -> 2 x <64 x ty>. bool SplitVectorOperand(SDNode *N, unsigned OpNo); @@ -808,7 +831,9 @@ private: SDValue WidenVecRes_Binary(SDNode *N); SDValue WidenVecRes_BinaryCanTrap(SDNode *N); SDValue WidenVecRes_StrictFP(SDNode *N); + SDValue WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo); SDValue WidenVecRes_Convert(SDNode *N); + SDValue WidenVecRes_Convert_StrictFP(SDNode *N); SDValue WidenVecRes_FCOPYSIGN(SDNode *N); SDValue WidenVecRes_POWI(SDNode *N); SDValue WidenVecRes_Shift(SDNode *N); @@ -827,9 +852,16 @@ private: SDValue WidenVecOp_MGATHER(SDNode* N, unsigned OpNo); SDValue WidenVecOp_MSCATTER(SDNode* N, unsigned OpNo); SDValue WidenVecOp_SETCC(SDNode* N); + SDValue WidenVecOp_VSELECT(SDNode *N); SDValue WidenVecOp_Convert(SDNode *N); SDValue WidenVecOp_FCOPYSIGN(SDNode *N); + SDValue WidenVecOp_VECREDUCE(SDNode *N); + + /// Helper function to generate a set of operations to perform + /// a vector operation for a wider type. + /// + SDValue UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE); //===--------------------------------------------------------------------===// // Vector Widening Utilities Support: LegalizeVectorTypes.cpp diff --git a/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp b/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp index b9d370441c3e..943f63f46c47 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp @@ -1,9 +1,8 @@ //===-------- LegalizeTypesGeneric.cpp - Generic type legalization --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp index 4923a529c21b..10b8b705869e 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp @@ -1,9 +1,8 @@ //===- LegalizeVectorOps.cpp - Implement SelectionDAG::LegalizeVectors ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -141,7 +140,11 @@ class VectorLegalizer { SDValue ExpandFunnelShift(SDValue Op); SDValue ExpandROT(SDValue Op); SDValue ExpandFMINNUM_FMAXNUM(SDValue Op); + SDValue ExpandUADDSUBO(SDValue Op); + SDValue ExpandSADDSUBO(SDValue Op); + SDValue ExpandMULO(SDValue Op); SDValue ExpandAddSubSat(SDValue Op); + SDValue ExpandFixedPointMul(SDValue Op); SDValue ExpandStrictFPOp(SDValue Op); /// Implements vector promotion. @@ -263,7 +266,7 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) { LLVM_FALLTHROUGH; case TargetLowering::Expand: Changed = true; - return LegalizeOp(ExpandLoad(Op)); + return ExpandLoad(Op); } } } else if (Op.getOpcode() == ISD::STORE) { @@ -288,17 +291,18 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) { } case TargetLowering::Expand: Changed = true; - return LegalizeOp(ExpandStore(Op)); + return ExpandStore(Op); } } } - bool HasVectorValue = false; - for (SDNode::value_iterator J = Node->value_begin(), E = Node->value_end(); - J != E; - ++J) - HasVectorValue |= J->isVector(); - if (!HasVectorValue) + bool HasVectorValueOrOp = false; + for (auto J = Node->value_begin(), E = Node->value_end(); J != E; ++J) + HasVectorValueOrOp |= J->isVector(); + for (const SDValue &Op : Node->op_values()) + HasVectorValueOrOp |= Op.getValueType().isVector(); + + if (!HasVectorValueOrOp) return TranslateLegalizeResults(Op, Result); TargetLowering::LegalizeAction Action = TargetLowering::Legal; @@ -329,6 +333,8 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) { case ISD::STRICT_FFLOOR: case ISD::STRICT_FROUND: case ISD::STRICT_FTRUNC: + case ISD::STRICT_FP_ROUND: + case ISD::STRICT_FP_EXTEND: // These pseudo-ops get legalized as if they were their non-strict // equivalent. For instance, if ISD::FSQRT is legal then ISD::STRICT_FSQRT // is also legal, but if ISD::FSQRT requires expansion then so does @@ -418,6 +424,12 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) { case ISD::UMAX: case ISD::SMUL_LOHI: case ISD::UMUL_LOHI: + case ISD::SADDO: + case ISD::UADDO: + case ISD::SSUBO: + case ISD::USUBO: + case ISD::SMULO: + case ISD::UMULO: case ISD::FCANONICALIZE: case ISD::SADDSAT: case ISD::UADDSAT: @@ -425,7 +437,9 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) { case ISD::USUBSAT: Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0)); break; - case ISD::SMULFIX: { + case ISD::SMULFIX: + case ISD::SMULFIXSAT: + case ISD::UMULFIX: { unsigned Scale = Node->getConstantOperandVal(2); Action = TLI.getFixedPointOperationAction(Node->getOpcode(), Node->getValueType(0), Scale); @@ -437,6 +451,19 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) { break; case ISD::SINT_TO_FP: case ISD::UINT_TO_FP: + case ISD::VECREDUCE_ADD: + case ISD::VECREDUCE_MUL: + case ISD::VECREDUCE_AND: + case ISD::VECREDUCE_OR: + case ISD::VECREDUCE_XOR: + case ISD::VECREDUCE_SMAX: + case ISD::VECREDUCE_SMIN: + case ISD::VECREDUCE_UMAX: + case ISD::VECREDUCE_UMIN: + case ISD::VECREDUCE_FADD: + case ISD::VECREDUCE_FMUL: + case ISD::VECREDUCE_FMAX: + case ISD::VECREDUCE_FMIN: Action = TLI.getOperationAction(Node->getOpcode(), Node->getOperand(0).getValueType()); break; @@ -650,23 +677,21 @@ SDValue VectorLegalizer::ExpandLoad(SDValue Op) { LoadChains.push_back(ScalarLoad.getValue(1)); } - // Extract bits, pack and extend/trunc them into destination type. - unsigned SrcEltBits = SrcEltVT.getSizeInBits(); - SDValue SrcEltBitMask = DAG.getConstant((1U << SrcEltBits) - 1, dl, WideVT); - unsigned BitOffset = 0; unsigned WideIdx = 0; unsigned WideBits = WideVT.getSizeInBits(); + // Extract bits, pack and extend/trunc them into destination type. + unsigned SrcEltBits = SrcEltVT.getSizeInBits(); + SDValue SrcEltBitMask = DAG.getConstant( + APInt::getLowBitsSet(WideBits, SrcEltBits), dl, WideVT); + for (unsigned Idx = 0; Idx != NumElem; ++Idx) { - SDValue Lo, Hi, ShAmt; + assert(BitOffset < WideBits && "Unexpected offset!"); - if (BitOffset < WideBits) { - ShAmt = DAG.getConstant( - BitOffset, dl, TLI.getShiftAmountTy(WideVT, DAG.getDataLayout())); - Lo = DAG.getNode(ISD::SRL, dl, WideVT, LoadVals[WideIdx], ShAmt); - Lo = DAG.getNode(ISD::AND, dl, WideVT, Lo, SrcEltBitMask); - } + SDValue ShAmt = DAG.getConstant( + BitOffset, dl, TLI.getShiftAmountTy(WideVT, DAG.getDataLayout())); + SDValue Lo = DAG.getNode(ISD::SRL, dl, WideVT, LoadVals[WideIdx], ShAmt); BitOffset += SrcEltBits; if (BitOffset >= WideBits) { @@ -676,13 +701,13 @@ SDValue VectorLegalizer::ExpandLoad(SDValue Op) { ShAmt = DAG.getConstant( SrcEltBits - BitOffset, dl, TLI.getShiftAmountTy(WideVT, DAG.getDataLayout())); - Hi = DAG.getNode(ISD::SHL, dl, WideVT, LoadVals[WideIdx], ShAmt); - Hi = DAG.getNode(ISD::AND, dl, WideVT, Hi, SrcEltBitMask); + SDValue Hi = + DAG.getNode(ISD::SHL, dl, WideVT, LoadVals[WideIdx], ShAmt); + Lo = DAG.getNode(ISD::OR, dl, WideVT, Lo, Hi); } } - if (Hi.getNode()) - Lo = DAG.getNode(ISD::OR, dl, WideVT, Lo, Hi); + Lo = DAG.getNode(ISD::AND, dl, WideVT, Lo, SrcEltBitMask); switch (ExtType) { default: llvm_unreachable("Unknown extended-load op!"); @@ -778,11 +803,23 @@ SDValue VectorLegalizer::Expand(SDValue Op) { case ISD::FMINNUM: case ISD::FMAXNUM: return ExpandFMINNUM_FMAXNUM(Op); + case ISD::UADDO: + case ISD::USUBO: + return ExpandUADDSUBO(Op); + case ISD::SADDO: + case ISD::SSUBO: + return ExpandSADDSUBO(Op); + case ISD::UMULO: + case ISD::SMULO: + return ExpandMULO(Op); case ISD::USUBSAT: case ISD::SSUBSAT: case ISD::UADDSAT: case ISD::SADDSAT: return ExpandAddSubSat(Op); + case ISD::SMULFIX: + case ISD::UMULFIX: + return ExpandFixedPointMul(Op); case ISD::STRICT_FADD: case ISD::STRICT_FSUB: case ISD::STRICT_FMUL: @@ -808,6 +845,20 @@ SDValue VectorLegalizer::Expand(SDValue Op) { case ISD::STRICT_FROUND: case ISD::STRICT_FTRUNC: return ExpandStrictFPOp(Op); + case ISD::VECREDUCE_ADD: + case ISD::VECREDUCE_MUL: + case ISD::VECREDUCE_AND: + case ISD::VECREDUCE_OR: + case ISD::VECREDUCE_XOR: + case ISD::VECREDUCE_SMAX: + case ISD::VECREDUCE_SMIN: + case ISD::VECREDUCE_UMAX: + case ISD::VECREDUCE_UMIN: + case ISD::VECREDUCE_FADD: + case ISD::VECREDUCE_FMUL: + case ISD::VECREDUCE_FMAX: + case ISD::VECREDUCE_FMIN: + return TLI.expandVecReduce(Op.getNode(), DAG); default: return DAG.UnrollVectorOp(Op.getNode()); } @@ -898,6 +949,19 @@ SDValue VectorLegalizer::ExpandANY_EXTEND_VECTOR_INREG(SDValue Op) { EVT SrcVT = Src.getValueType(); int NumSrcElements = SrcVT.getVectorNumElements(); + // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector + // into a larger vector type. + if (SrcVT.bitsLE(VT)) { + assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 && + "ANY_EXTEND_VECTOR_INREG vector size mismatch"); + NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits(); + SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(), + NumSrcElements); + Src = DAG.getNode( + ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT), Src, + DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); + } + // Build a base mask of undef shuffles. SmallVector ShuffleMask; ShuffleMask.resize(NumSrcElements, -1); @@ -945,6 +1009,19 @@ SDValue VectorLegalizer::ExpandZERO_EXTEND_VECTOR_INREG(SDValue Op) { EVT SrcVT = Src.getValueType(); int NumSrcElements = SrcVT.getVectorNumElements(); + // *_EXTEND_VECTOR_INREG SrcVT can be smaller than VT - so insert the vector + // into a larger vector type. + if (SrcVT.bitsLE(VT)) { + assert((VT.getSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 && + "ZERO_EXTEND_VECTOR_INREG vector size mismatch"); + NumSrcElements = VT.getSizeInBits() / SrcVT.getScalarSizeInBits(); + SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcVT.getScalarType(), + NumSrcElements); + Src = DAG.getNode( + ISD::INSERT_SUBVECTOR, DL, SrcVT, DAG.getUNDEF(SrcVT), Src, + DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); + } + // Build up a zero vector to blend into this one. SDValue Zero = DAG.getConstant(0, DL, SrcVT); @@ -1212,12 +1289,58 @@ SDValue VectorLegalizer::ExpandFMINNUM_FMAXNUM(SDValue Op) { return DAG.UnrollVectorOp(Op.getNode()); } +SDValue VectorLegalizer::ExpandUADDSUBO(SDValue Op) { + SDValue Result, Overflow; + TLI.expandUADDSUBO(Op.getNode(), Result, Overflow, DAG); + + if (Op.getResNo() == 0) { + AddLegalizedOperand(Op.getValue(1), LegalizeOp(Overflow)); + return Result; + } else { + AddLegalizedOperand(Op.getValue(0), LegalizeOp(Result)); + return Overflow; + } +} + +SDValue VectorLegalizer::ExpandSADDSUBO(SDValue Op) { + SDValue Result, Overflow; + TLI.expandSADDSUBO(Op.getNode(), Result, Overflow, DAG); + + if (Op.getResNo() == 0) { + AddLegalizedOperand(Op.getValue(1), LegalizeOp(Overflow)); + return Result; + } else { + AddLegalizedOperand(Op.getValue(0), LegalizeOp(Result)); + return Overflow; + } +} + +SDValue VectorLegalizer::ExpandMULO(SDValue Op) { + SDValue Result, Overflow; + if (!TLI.expandMULO(Op.getNode(), Result, Overflow, DAG)) + std::tie(Result, Overflow) = DAG.UnrollVectorOverflowOp(Op.getNode()); + + if (Op.getResNo() == 0) { + AddLegalizedOperand(Op.getValue(1), LegalizeOp(Overflow)); + return Result; + } else { + AddLegalizedOperand(Op.getValue(0), LegalizeOp(Result)); + return Overflow; + } +} + SDValue VectorLegalizer::ExpandAddSubSat(SDValue Op) { if (SDValue Expanded = TLI.expandAddSubSat(Op.getNode(), DAG)) return Expanded; return DAG.UnrollVectorOp(Op.getNode()); } +SDValue VectorLegalizer::ExpandFixedPointMul(SDValue Op) { + if (SDValue Expanded = TLI.expandFixedPointMul(Op.getNode(), DAG)) + return Expanded; + return DAG.UnrollVectorOp(Op.getNode()); +} + SDValue VectorLegalizer::ExpandStrictFPOp(SDValue Op) { EVT VT = Op.getValueType(); EVT EltVT = VT.getVectorElementType(); @@ -1245,7 +1368,7 @@ SDValue VectorLegalizer::ExpandStrictFPOp(SDValue Op) { if (OperVT.isVector()) Oper = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, - EltVT, Oper, Idx); + OperVT.getVectorElementType(), Oper, Idx); Opers.push_back(Oper); } diff --git a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index f367e9358576..7e4d52617977 100644 --- a/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -1,9 +1,8 @@ //===------- LegalizeVectorTypes.cpp - Legalization of vector types -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -51,6 +50,7 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { case ISD::BITCAST: R = ScalarizeVecRes_BITCAST(N); break; case ISD::BUILD_VECTOR: R = ScalarizeVecRes_BUILD_VECTOR(N); break; case ISD::EXTRACT_SUBVECTOR: R = ScalarizeVecRes_EXTRACT_SUBVECTOR(N); break; + case ISD::STRICT_FP_ROUND: R = ScalarizeVecRes_STRICT_FP_ROUND(N); break; case ISD::FP_ROUND: R = ScalarizeVecRes_FP_ROUND(N); break; case ISD::FP_ROUND_INREG: R = ScalarizeVecRes_InregOp(N); break; case ISD::FPOWI: R = ScalarizeVecRes_FPOWI(N); break; @@ -69,6 +69,7 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { case ISD::ZERO_EXTEND_VECTOR_INREG: R = ScalarizeVecRes_VecInregOp(N); break; + case ISD::ABS: case ISD::ANY_EXTEND: case ISD::BITREVERSE: case ISD::BSWAP: @@ -170,10 +171,21 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) { case ISD::STRICT_FFLOOR: case ISD::STRICT_FROUND: case ISD::STRICT_FTRUNC: + case ISD::STRICT_FP_EXTEND: R = ScalarizeVecRes_StrictFPOp(N); break; + case ISD::UADDO: + case ISD::SADDO: + case ISD::USUBO: + case ISD::SSUBO: + case ISD::UMULO: + case ISD::SMULO: + R = ScalarizeVecRes_OverflowOp(N, ResNo); + break; case ISD::SMULFIX: - R = ScalarizeVecRes_SMULFIX(N); + case ISD::SMULFIXSAT: + case ISD::UMULFIX: + R = ScalarizeVecRes_MULFIX(N); break; } @@ -197,7 +209,7 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_TernaryOp(SDNode *N) { Op0.getValueType(), Op0, Op1, Op2); } -SDValue DAGTypeLegalizer::ScalarizeVecRes_SMULFIX(SDNode *N) { +SDValue DAGTypeLegalizer::ScalarizeVecRes_MULFIX(SDNode *N) { SDValue Op0 = GetScalarizedVector(N->getOperand(0)); SDValue Op1 = GetScalarizedVector(N->getOperand(1)); SDValue Op2 = N->getOperand(2); @@ -235,6 +247,43 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_StrictFPOp(SDNode *N) { return Result; } +SDValue DAGTypeLegalizer::ScalarizeVecRes_OverflowOp(SDNode *N, + unsigned ResNo) { + SDLoc DL(N); + EVT ResVT = N->getValueType(0); + EVT OvVT = N->getValueType(1); + + SDValue ScalarLHS, ScalarRHS; + if (getTypeAction(ResVT) == TargetLowering::TypeScalarizeVector) { + ScalarLHS = GetScalarizedVector(N->getOperand(0)); + ScalarRHS = GetScalarizedVector(N->getOperand(1)); + } else { + SmallVector ElemsLHS, ElemsRHS; + DAG.ExtractVectorElements(N->getOperand(0), ElemsLHS); + DAG.ExtractVectorElements(N->getOperand(1), ElemsRHS); + ScalarLHS = ElemsLHS[0]; + ScalarRHS = ElemsRHS[0]; + } + + SDVTList ScalarVTs = DAG.getVTList( + ResVT.getVectorElementType(), OvVT.getVectorElementType()); + SDNode *ScalarNode = DAG.getNode( + N->getOpcode(), DL, ScalarVTs, ScalarLHS, ScalarRHS).getNode(); + + // Replace the other vector result not being explicitly scalarized here. + unsigned OtherNo = 1 - ResNo; + EVT OtherVT = N->getValueType(OtherNo); + if (getTypeAction(OtherVT) == TargetLowering::TypeScalarizeVector) { + SetScalarizedVector(SDValue(N, OtherNo), SDValue(ScalarNode, OtherNo)); + } else { + SDValue OtherVal = DAG.getNode( + ISD::SCALAR_TO_VECTOR, DL, OtherVT, SDValue(ScalarNode, OtherNo)); + ReplaceValueWith(SDValue(N, OtherNo), OtherVal); + } + + return SDValue(ScalarNode, ResNo); +} + SDValue DAGTypeLegalizer::ScalarizeVecRes_MERGE_VALUES(SDNode *N, unsigned ResNo) { SDValue Op = DisintegrateMERGE_VALUES(N, ResNo); @@ -275,6 +324,18 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_FP_ROUND(SDNode *N) { NewVT, Op, N->getOperand(1)); } +SDValue DAGTypeLegalizer::ScalarizeVecRes_STRICT_FP_ROUND(SDNode *N) { + EVT NewVT = N->getValueType(0).getVectorElementType(); + SDValue Op = GetScalarizedVector(N->getOperand(1)); + SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N), + { NewVT, MVT::Other }, + { N->getOperand(0), Op, N->getOperand(2) }); + // Legalize the chain result - switch anything that used the old chain to + // use the new one. + ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); + return Res; +} + SDValue DAGTypeLegalizer::ScalarizeVecRes_FPOWI(SDNode *N) { SDValue Op = GetScalarizedVector(N->getOperand(0)); return DAG.getNode(ISD::FPOWI, SDLoc(N), @@ -558,9 +619,27 @@ bool DAGTypeLegalizer::ScalarizeVectorOperand(SDNode *N, unsigned OpNo) { case ISD::STORE: Res = ScalarizeVecOp_STORE(cast(N), OpNo); break; + case ISD::STRICT_FP_ROUND: + Res = ScalarizeVecOp_STRICT_FP_ROUND(N, OpNo); + break; case ISD::FP_ROUND: Res = ScalarizeVecOp_FP_ROUND(N, OpNo); break; + case ISD::VECREDUCE_FADD: + case ISD::VECREDUCE_FMUL: + case ISD::VECREDUCE_ADD: + case ISD::VECREDUCE_MUL: + case ISD::VECREDUCE_AND: + case ISD::VECREDUCE_OR: + case ISD::VECREDUCE_XOR: + case ISD::VECREDUCE_SMAX: + case ISD::VECREDUCE_SMIN: + case ISD::VECREDUCE_UMAX: + case ISD::VECREDUCE_UMIN: + case ISD::VECREDUCE_FMAX: + case ISD::VECREDUCE_FMIN: + Res = ScalarizeVecOp_VECREDUCE(N); + break; } } @@ -691,6 +770,28 @@ SDValue DAGTypeLegalizer::ScalarizeVecOp_FP_ROUND(SDNode *N, unsigned OpNo) { return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); } +SDValue DAGTypeLegalizer::ScalarizeVecOp_STRICT_FP_ROUND(SDNode *N, + unsigned OpNo) { + assert(OpNo == 1 && "Wrong operand for scalarization!"); + SDValue Elt = GetScalarizedVector(N->getOperand(1)); + SDValue Res = DAG.getNode(ISD::STRICT_FP_ROUND, SDLoc(N), + { N->getValueType(0).getVectorElementType(), + MVT::Other }, + { N->getOperand(0), Elt, N->getOperand(2) }); + // Legalize the chain result - switch anything that used the old chain to + // use the new one. + ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); + return DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(N), N->getValueType(0), Res); +} + +SDValue DAGTypeLegalizer::ScalarizeVecOp_VECREDUCE(SDNode *N) { + SDValue Res = GetScalarizedVector(N->getOperand(0)); + // Result type may be wider than element type. + if (Res.getValueType() != N->getValueType(0)) + Res = DAG.getNode(ISD::ANY_EXTEND, SDLoc(N), N->getValueType(0), Res); + return Res; +} + //===----------------------------------------------------------------------===// // Result Vector Splitting //===----------------------------------------------------------------------===// @@ -748,6 +849,9 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { case ISD::VECTOR_SHUFFLE: SplitVecRes_VECTOR_SHUFFLE(cast(N), Lo, Hi); break; + case ISD::VAARG: + SplitVecRes_VAARG(N, Lo, Hi); + break; case ISD::ANY_EXTEND_VECTOR_INREG: case ISD::SIGN_EXTEND_VECTOR_INREG: @@ -755,6 +859,7 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { SplitVecRes_ExtVecInRegOp(N, Lo, Hi); break; + case ISD::ABS: case ISD::BITREVERSE: case ISD::BSWAP: case ISD::CTLZ: @@ -774,7 +879,9 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { case ISD::FNEARBYINT: case ISD::FNEG: case ISD::FP_EXTEND: + case ISD::STRICT_FP_EXTEND: case ISD::FP_ROUND: + case ISD::STRICT_FP_ROUND: case ISD::FP_TO_SINT: case ISD::FP_TO_UINT: case ISD::FRINT: @@ -859,8 +966,18 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) { case ISD::STRICT_FTRUNC: SplitVecRes_StrictFPOp(N, Lo, Hi); break; + case ISD::UADDO: + case ISD::SADDO: + case ISD::USUBO: + case ISD::SSUBO: + case ISD::UMULO: + case ISD::SMULO: + SplitVecRes_OverflowOp(N, ResNo, Lo, Hi); + break; case ISD::SMULFIX: - SplitVecRes_SMULFIX(N, Lo, Hi); + case ISD::SMULFIXSAT: + case ISD::UMULFIX: + SplitVecRes_MULFIX(N, Lo, Hi); break; } @@ -899,8 +1016,7 @@ void DAGTypeLegalizer::SplitVecRes_TernaryOp(SDNode *N, SDValue &Lo, Op0Hi, Op1Hi, Op2Hi); } -void DAGTypeLegalizer::SplitVecRes_SMULFIX(SDNode *N, SDValue &Lo, - SDValue &Hi) { +void DAGTypeLegalizer::SplitVecRes_MULFIX(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue LHSLo, LHSHi; GetSplitVector(N->getOperand(0), LHSLo, LHSHi); SDValue RHSLo, RHSHi; @@ -1205,6 +1321,104 @@ void DAGTypeLegalizer::SplitVecRes_StrictFPOp(SDNode *N, SDValue &Lo, ReplaceValueWith(SDValue(N, 1), Chain); } +SDValue DAGTypeLegalizer::UnrollVectorOp_StrictFP(SDNode *N, unsigned ResNE) { + SDValue Chain = N->getOperand(0); + EVT VT = N->getValueType(0); + unsigned NE = VT.getVectorNumElements(); + EVT EltVT = VT.getVectorElementType(); + SDLoc dl(N); + + SmallVector Scalars; + SmallVector Operands(N->getNumOperands()); + + // If ResNE is 0, fully unroll the vector op. + if (ResNE == 0) + ResNE = NE; + else if (NE > ResNE) + NE = ResNE; + + //The results of each unrolled operation, including the chain. + EVT ChainVTs[] = {EltVT, MVT::Other}; + SmallVector Chains; + + unsigned i; + for (i = 0; i != NE; ++i) { + Operands[0] = Chain; + for (unsigned j = 1, e = N->getNumOperands(); j != e; ++j) { + SDValue Operand = N->getOperand(j); + EVT OperandVT = Operand.getValueType(); + if (OperandVT.isVector()) { + EVT OperandEltVT = OperandVT.getVectorElementType(); + Operands[j] = + DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, OperandEltVT, Operand, + DAG.getConstant(i, dl, TLI.getVectorIdxTy( + DAG.getDataLayout()))); + } else { + Operands[j] = Operand; + } + } + SDValue Scalar = DAG.getNode(N->getOpcode(), dl, ChainVTs, Operands); + Scalar.getNode()->setFlags(N->getFlags()); + + //Add in the scalar as well as its chain value to the + //result vectors. + Scalars.push_back(Scalar); + Chains.push_back(Scalar.getValue(1)); + } + + for (; i < ResNE; ++i) + Scalars.push_back(DAG.getUNDEF(EltVT)); + + // Build a new factor node to connect the chain back together. + Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Chains); + ReplaceValueWith(SDValue(N, 1), Chain); + + // Create a new BUILD_VECTOR node + EVT VecVT = EVT::getVectorVT(*DAG.getContext(), EltVT, ResNE); + return DAG.getBuildVector(VecVT, dl, Scalars); +} + +void DAGTypeLegalizer::SplitVecRes_OverflowOp(SDNode *N, unsigned ResNo, + SDValue &Lo, SDValue &Hi) { + SDLoc dl(N); + EVT ResVT = N->getValueType(0); + EVT OvVT = N->getValueType(1); + EVT LoResVT, HiResVT, LoOvVT, HiOvVT; + std::tie(LoResVT, HiResVT) = DAG.GetSplitDestVTs(ResVT); + std::tie(LoOvVT, HiOvVT) = DAG.GetSplitDestVTs(OvVT); + + SDValue LoLHS, HiLHS, LoRHS, HiRHS; + if (getTypeAction(ResVT) == TargetLowering::TypeSplitVector) { + GetSplitVector(N->getOperand(0), LoLHS, HiLHS); + GetSplitVector(N->getOperand(1), LoRHS, HiRHS); + } else { + std::tie(LoLHS, HiLHS) = DAG.SplitVectorOperand(N, 0); + std::tie(LoRHS, HiRHS) = DAG.SplitVectorOperand(N, 1); + } + + unsigned Opcode = N->getOpcode(); + SDVTList LoVTs = DAG.getVTList(LoResVT, LoOvVT); + SDVTList HiVTs = DAG.getVTList(HiResVT, HiOvVT); + SDNode *LoNode = DAG.getNode(Opcode, dl, LoVTs, LoLHS, LoRHS).getNode(); + SDNode *HiNode = DAG.getNode(Opcode, dl, HiVTs, HiLHS, HiRHS).getNode(); + + Lo = SDValue(LoNode, ResNo); + Hi = SDValue(HiNode, ResNo); + + // Replace the other vector result not being explicitly split here. + unsigned OtherNo = 1 - ResNo; + EVT OtherVT = N->getValueType(OtherNo); + if (getTypeAction(OtherVT) == TargetLowering::TypeSplitVector) { + SetSplitVector(SDValue(N, OtherNo), + SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo)); + } else { + SDValue OtherVal = DAG.getNode( + ISD::CONCAT_VECTORS, dl, OtherVT, + SDValue(LoNode, OtherNo), SDValue(HiNode, OtherNo)); + ReplaceValueWith(SDValue(N, OtherNo), OtherVal); + } +} + void DAGTypeLegalizer::SplitVecRes_INSERT_VECTOR_ELT(SDNode *N, SDValue &Lo, SDValue &Hi) { SDValue Vec = N->getOperand(0); @@ -1344,12 +1558,6 @@ void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD, unsigned Alignment = MLD->getOriginalAlignment(); ISD::LoadExtType ExtType = MLD->getExtensionType(); - // if Alignment is equal to the vector size, - // take the half of it for the second part - unsigned SecondHalfAlignment = - (Alignment == MLD->getValueType(0).getSizeInBits()/8) ? - Alignment/2 : Alignment; - // Split Mask operand SDValue MaskLo, MaskHi; if (getTypeAction(Mask.getValueType()) == TargetLowering::TypeSplitVector) @@ -1381,7 +1589,7 @@ void DAGTypeLegalizer::SplitVecRes_MLOAD(MaskedLoadSDNode *MLD, MMO = DAG.getMachineFunction().getMachineMemOperand( MLD->getPointerInfo().getWithOffset(HiOffset), MachineMemOperand::MOLoad, - HiMemVT.getStoreSize(), SecondHalfAlignment, MLD->getAAInfo(), + HiMemVT.getStoreSize(), Alignment, MLD->getAAInfo(), MLD->getRanges()); Hi = DAG.getMaskedLoad(HiVT, dl, Ch, Ptr, MaskHi, PassThruHi, HiMemVT, MMO, @@ -1496,15 +1704,34 @@ void DAGTypeLegalizer::SplitVecRes_UnaryOp(SDNode *N, SDValue &Lo, // If the input also splits, handle it directly for a compile time speedup. // Otherwise split it by hand. - EVT InVT = N->getOperand(0).getValueType(); + unsigned OpNo = N->isStrictFPOpcode() ? 1 : 0; + EVT InVT = N->getOperand(OpNo).getValueType(); if (getTypeAction(InVT) == TargetLowering::TypeSplitVector) - GetSplitVector(N->getOperand(0), Lo, Hi); + GetSplitVector(N->getOperand(OpNo), Lo, Hi); else - std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0); + std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, OpNo); if (N->getOpcode() == ISD::FP_ROUND) { Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo, N->getOperand(1)); Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi, N->getOperand(1)); + } else if (N->getOpcode() == ISD::STRICT_FP_ROUND) { + Lo = DAG.getNode(N->getOpcode(), dl, { LoVT, MVT::Other }, + { N->getOperand(0), Lo, N->getOperand(2) }); + Hi = DAG.getNode(N->getOpcode(), dl, { HiVT, MVT::Other }, + { N->getOperand(0), Hi, N->getOperand(2) }); + SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, + Lo.getValue(1), Hi.getValue(1)); + ReplaceValueWith(SDValue(N, 1), NewChain); + } else if (N->isStrictFPOpcode()) { + Lo = DAG.getNode(N->getOpcode(), dl, { LoVT, MVT::Other }, + { N->getOperand(0), Lo }); + Hi = DAG.getNode(N->getOpcode(), dl, { HiVT, MVT::Other }, + { N->getOperand(0), Hi }); + // Legalize the chain result - switch anything that used the old chain to + // use the new one. + SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, + Lo.getValue(1), Hi.getValue(1)); + ReplaceValueWith(SDValue(N, 1), NewChain); } else { Lo = DAG.getNode(N->getOpcode(), dl, LoVT, Lo); Hi = DAG.getNode(N->getOpcode(), dl, HiVT, Hi); @@ -1669,6 +1896,26 @@ void DAGTypeLegalizer::SplitVecRes_VECTOR_SHUFFLE(ShuffleVectorSDNode *N, } } +void DAGTypeLegalizer::SplitVecRes_VAARG(SDNode *N, SDValue &Lo, SDValue &Hi) { + EVT OVT = N->getValueType(0); + EVT NVT = OVT.getHalfNumVectorElementsVT(*DAG.getContext()); + SDValue Chain = N->getOperand(0); + SDValue Ptr = N->getOperand(1); + SDValue SV = N->getOperand(2); + SDLoc dl(N); + + const unsigned Alignment = DAG.getDataLayout().getABITypeAlignment( + NVT.getTypeForEVT(*DAG.getContext())); + + Lo = DAG.getVAArg(NVT, dl, Chain, Ptr, SV, Alignment); + Hi = DAG.getVAArg(NVT, dl, Lo.getValue(1), Ptr, SV, Alignment); + Chain = Hi.getValue(1); + + // Modified the chain - switch anything that used the old chain to use + // the new one. + ReplaceValueWith(SDValue(N, 1), Chain); +} + //===----------------------------------------------------------------------===// // Operand Vector Splitting @@ -1705,6 +1952,7 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { case ISD::TRUNCATE: Res = SplitVecOp_TruncateHelper(N); break; + case ISD::STRICT_FP_ROUND: case ISD::FP_ROUND: Res = SplitVecOp_FP_ROUND(N); break; case ISD::FCOPYSIGN: Res = SplitVecOp_FCOPYSIGN(N); break; case ISD::STORE: @@ -1734,6 +1982,7 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { case ISD::CTTZ: case ISD::CTLZ: case ISD::CTPOP: + case ISD::STRICT_FP_EXTEND: case ISD::FP_EXTEND: case ISD::SIGN_EXTEND: case ISD::ZERO_EXTEND: @@ -1775,7 +2024,11 @@ bool DAGTypeLegalizer::SplitVectorOperand(SDNode *N, unsigned OpNo) { if (Res.getNode() == N) return true; - assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && + if (N->isStrictFPOpcode()) + assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 && + "Invalid operand expansion"); + else + assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && "Invalid operand expansion"); ReplaceValueWith(SDValue(N, 0), Res); @@ -1863,14 +2116,30 @@ SDValue DAGTypeLegalizer::SplitVecOp_UnaryOp(SDNode *N) { EVT ResVT = N->getValueType(0); SDValue Lo, Hi; SDLoc dl(N); - GetSplitVector(N->getOperand(0), Lo, Hi); + GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi); EVT InVT = Lo.getValueType(); EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), InVT.getVectorNumElements()); - Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo); - Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi); + if (N->isStrictFPOpcode()) { + Lo = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other }, + { N->getOperand(0), Lo }); + Hi = DAG.getNode(N->getOpcode(), dl, { OutVT, MVT::Other }, + { N->getOperand(0), Hi }); + + // Build a factor node to remember that this operation is independent + // of the other one. + SDValue Ch = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Lo.getValue(1), + Hi.getValue(1)); + + // Legalize the chain result - switch anything that used the old chain to + // use the new one. + ReplaceValueWith(SDValue(N, 1), Ch); + } else { + Lo = DAG.getNode(N->getOpcode(), dl, OutVT, Lo); + Hi = DAG.getNode(N->getOpcode(), dl, OutVT, Hi); + } return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi); } @@ -1920,7 +2189,6 @@ SDValue DAGTypeLegalizer::SplitVecOp_EXTRACT_VECTOR_ELT(SDNode *N) { if (isa(Idx)) { uint64_t IdxVal = cast(Idx)->getZExtValue(); - assert(IdxVal < VecVT.getVectorNumElements() && "Invalid vector index!"); SDValue Lo, Hi; GetSplitVector(Vec, Lo, Hi); @@ -2079,12 +2347,6 @@ SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N, else std::tie(MaskLo, MaskHi) = DAG.SplitVector(Mask, DL); - // if Alignment is equal to the vector size, - // take the half of it for the second part - unsigned SecondHalfAlignment = - (Alignment == Data->getValueType(0).getSizeInBits()/8) ? - Alignment/2 : Alignment; - SDValue Lo, Hi; MachineMemOperand *MMO = DAG.getMachineFunction(). getMachineMemOperand(N->getPointerInfo(), @@ -2101,7 +2363,7 @@ SDValue DAGTypeLegalizer::SplitVecOp_MSTORE(MaskedStoreSDNode *N, MMO = DAG.getMachineFunction().getMachineMemOperand( N->getPointerInfo().getWithOffset(HiOffset), MachineMemOperand::MOStore, - HiMemVT.getStoreSize(), SecondHalfAlignment, N->getAAInfo(), + HiMemVT.getStoreSize(), Alignment, N->getAAInfo(), N->getRanges()); Hi = DAG.getMaskedStore(Ch, DL, DataHi, Ptr, MaskHi, HiMemVT, MMO, @@ -2343,14 +2605,26 @@ SDValue DAGTypeLegalizer::SplitVecOp_FP_ROUND(SDNode *N) { EVT ResVT = N->getValueType(0); SDValue Lo, Hi; SDLoc DL(N); - GetSplitVector(N->getOperand(0), Lo, Hi); + GetSplitVector(N->getOperand(N->isStrictFPOpcode() ? 1 : 0), Lo, Hi); EVT InVT = Lo.getValueType(); EVT OutVT = EVT::getVectorVT(*DAG.getContext(), ResVT.getVectorElementType(), InVT.getVectorNumElements()); - Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1)); - Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1)); + if (N->isStrictFPOpcode()) { + Lo = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other }, + { N->getOperand(0), Lo, N->getOperand(2) }); + Hi = DAG.getNode(N->getOpcode(), DL, { OutVT, MVT::Other }, + { N->getOperand(0), Hi, N->getOperand(2) }); + // Legalize the chain result - switch anything that used the old chain to + // use the new one. + SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, + Lo.getValue(1), Hi.getValue(1)); + ReplaceValueWith(SDValue(N, 1), NewChain); + } else { + Lo = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Lo, N->getOperand(1)); + Hi = DAG.getNode(ISD::FP_ROUND, DL, OutVT, Hi, N->getOperand(1)); + } return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi); } @@ -2472,6 +2746,15 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { Res = WidenVecRes_StrictFP(N); break; + case ISD::UADDO: + case ISD::SADDO: + case ISD::USUBO: + case ISD::SSUBO: + case ISD::UMULO: + case ISD::SMULO: + Res = WidenVecRes_OverflowOp(N, ResNo); + break; + case ISD::FCOPYSIGN: Res = WidenVecRes_FCOPYSIGN(N); break; @@ -2505,6 +2788,11 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { Res = WidenVecRes_Convert(N); break; + case ISD::STRICT_FP_EXTEND: + case ISD::STRICT_FP_ROUND: + Res = WidenVecRes_Convert_StrictFP(N); + break; + case ISD::FABS: case ISD::FCEIL: case ISD::FCOS: @@ -2523,13 +2811,11 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { // We're going to widen this vector op to a legal type by padding with undef // elements. If the wide vector op is eventually going to be expanded to // scalar libcalls, then unroll into scalar ops now to avoid unnecessary - // libcalls on the undef elements. We are assuming that if the scalar op - // requires expanding, then the vector op needs expanding too. + // libcalls on the undef elements. EVT VT = N->getValueType(0); - if (TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) { - EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); - assert(!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) && - "Target supports vector op, but scalar requires expansion?"); + EVT WideVecVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); + if (!TLI.isOperationLegalOrCustom(N->getOpcode(), WideVecVT) && + TLI.isOperationExpand(N->getOpcode(), VT.getScalarType())) { Res = DAG.UnrollVectorOp(N, WideVecVT.getVectorNumElements()); break; } @@ -2539,11 +2825,14 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) { // any other unary ops. LLVM_FALLTHROUGH; + case ISD::ABS: case ISD::BITREVERSE: case ISD::BSWAP: case ISD::CTLZ: + case ISD::CTLZ_ZERO_UNDEF: case ISD::CTPOP: case ISD::CTTZ: + case ISD::CTTZ_ZERO_UNDEF: case ISD::FNEG: case ISD::FCANONICALIZE: Res = WidenVecRes_Unary(N); @@ -2593,14 +2882,13 @@ static SDValue CollectOpsToWiden(SelectionDAG &DAG, const TargetLowering &TLI, SDLoc dl(ConcatOps[0]); EVT WidenEltVT = WidenVT.getVectorElementType(); - int Idx = 0; // while (Some element of ConcatOps is not of type MaxVT) { // From the end of ConcatOps, collect elements of the same type and put // them into an op of the next larger supported type // } while (ConcatOps[ConcatEnd-1].getValueType() != MaxVT) { - Idx = ConcatEnd - 1; + int Idx = ConcatEnd - 1; VT = ConcatOps[Idx--].getValueType(); while (Idx >= 0 && ConcatOps[Idx].getValueType() == VT) Idx--; @@ -2750,7 +3038,7 @@ SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) { // No legal vector version so unroll the vector operation and then widen. if (NumElts == 1) - return DAG.UnrollVectorOp(N, WidenVT.getVectorNumElements()); + return UnrollVectorOp_StrictFP(N, WidenVT.getVectorNumElements()); // Since the operation can trap, apply operation on the original vector. EVT MaxVT = VT; @@ -2846,6 +3134,58 @@ SDValue DAGTypeLegalizer::WidenVecRes_StrictFP(SDNode *N) { return CollectOpsToWiden(DAG, TLI, ConcatOps, ConcatEnd, VT, MaxVT, WidenVT); } +SDValue DAGTypeLegalizer::WidenVecRes_OverflowOp(SDNode *N, unsigned ResNo) { + SDLoc DL(N); + EVT ResVT = N->getValueType(0); + EVT OvVT = N->getValueType(1); + EVT WideResVT, WideOvVT; + SDValue WideLHS, WideRHS; + + // TODO: This might result in a widen/split loop. + if (ResNo == 0) { + WideResVT = TLI.getTypeToTransformTo(*DAG.getContext(), ResVT); + WideOvVT = EVT::getVectorVT( + *DAG.getContext(), OvVT.getVectorElementType(), + WideResVT.getVectorNumElements()); + + WideLHS = GetWidenedVector(N->getOperand(0)); + WideRHS = GetWidenedVector(N->getOperand(1)); + } else { + WideOvVT = TLI.getTypeToTransformTo(*DAG.getContext(), OvVT); + WideResVT = EVT::getVectorVT( + *DAG.getContext(), ResVT.getVectorElementType(), + WideOvVT.getVectorNumElements()); + + SDValue Zero = DAG.getConstant( + 0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())); + WideLHS = DAG.getNode( + ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT), + N->getOperand(0), Zero); + WideRHS = DAG.getNode( + ISD::INSERT_SUBVECTOR, DL, WideResVT, DAG.getUNDEF(WideResVT), + N->getOperand(1), Zero); + } + + SDVTList WideVTs = DAG.getVTList(WideResVT, WideOvVT); + SDNode *WideNode = DAG.getNode( + N->getOpcode(), DL, WideVTs, WideLHS, WideRHS).getNode(); + + // Replace the other vector result not being explicitly widened here. + unsigned OtherNo = 1 - ResNo; + EVT OtherVT = N->getValueType(OtherNo); + if (getTypeAction(OtherVT) == TargetLowering::TypeWidenVector) { + SetWidenedVector(SDValue(N, OtherNo), SDValue(WideNode, OtherNo)); + } else { + SDValue Zero = DAG.getConstant( + 0, DL, TLI.getVectorIdxTy(DAG.getDataLayout())); + SDValue OtherVal = DAG.getNode( + ISD::EXTRACT_SUBVECTOR, DL, OtherVT, SDValue(WideNode, OtherNo), Zero); + ReplaceValueWith(SDValue(N, OtherNo), OtherVal); + } + + return SDValue(WideNode, ResNo); +} + SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { SDValue InOp = N->getOperand(0); SDLoc DL(N); @@ -2929,6 +3269,43 @@ SDValue DAGTypeLegalizer::WidenVecRes_Convert(SDNode *N) { return DAG.getBuildVector(WidenVT, DL, Ops); } +SDValue DAGTypeLegalizer::WidenVecRes_Convert_StrictFP(SDNode *N) { + SDValue InOp = N->getOperand(1); + SDLoc DL(N); + SmallVector NewOps(N->op_begin(), N->op_end()); + + EVT WidenVT = TLI.getTypeToTransformTo(*DAG.getContext(), N->getValueType(0)); + unsigned WidenNumElts = WidenVT.getVectorNumElements(); + SmallVector WidenVTs = { WidenVT, MVT::Other }; + + EVT InVT = InOp.getValueType(); + EVT InEltVT = InVT.getVectorElementType(); + + unsigned Opcode = N->getOpcode(); + + // FIXME: Optimizations need to be implemented here. + + // Otherwise unroll into some nasty scalar code and rebuild the vector. + EVT EltVT = WidenVT.getVectorElementType(); + SmallVector EltVTs = { EltVT, MVT::Other }; + SmallVector Ops(WidenNumElts, DAG.getUNDEF(EltVT)); + SmallVector OpChains; + // Use the original element count so we don't do more scalar opts than + // necessary. + unsigned MinElts = N->getValueType(0).getVectorNumElements(); + for (unsigned i=0; i < MinElts; ++i) { + NewOps[1] = DAG.getNode( + ISD::EXTRACT_VECTOR_ELT, DL, InEltVT, InOp, + DAG.getConstant(i, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); + Ops[i] = DAG.getNode(Opcode, DL, EltVTs, NewOps); + OpChains.push_back(Ops[i].getValue(1)); + } + SDValue NewChain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OpChains); + ReplaceValueWith(SDValue(N, 1), NewChain); + + return DAG.getBuildVector(WidenVT, DL, Ops); +} + SDValue DAGTypeLegalizer::WidenVecRes_EXTEND_VECTOR_INREG(SDNode *N) { unsigned Opcode = N->getOpcode(); SDValue InOp = N->getOperand(0); @@ -3654,8 +4031,15 @@ SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) { return Res; } - InOp1 = GetWidenedVector(InOp1); - SDValue InOp2 = GetWidenedVector(N->getOperand(1)); + // If the inputs also widen, handle them directly. Otherwise widen by hand. + SDValue InOp2 = N->getOperand(1); + if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { + InOp1 = GetWidenedVector(InOp1); + InOp2 = GetWidenedVector(InOp2); + } else { + InOp1 = DAG.WidenVector(InOp1, SDLoc(N)); + InOp2 = DAG.WidenVector(InOp2, SDLoc(N)); + } // Assume that the input and output will be widen appropriately. If not, // we will have to unroll it at some point. @@ -3698,6 +4082,7 @@ bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) { case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break; case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break; case ISD::SETCC: Res = WidenVecOp_SETCC(N); break; + case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break; case ISD::FCOPYSIGN: Res = WidenVecOp_FCOPYSIGN(N); break; case ISD::ANY_EXTEND: @@ -3707,6 +4092,7 @@ bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) { break; case ISD::FP_EXTEND: + case ISD::STRICT_FP_EXTEND: case ISD::FP_TO_SINT: case ISD::FP_TO_UINT: case ISD::SINT_TO_FP: @@ -3714,6 +4100,22 @@ bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) { case ISD::TRUNCATE: Res = WidenVecOp_Convert(N); break; + + case ISD::VECREDUCE_FADD: + case ISD::VECREDUCE_FMUL: + case ISD::VECREDUCE_ADD: + case ISD::VECREDUCE_MUL: + case ISD::VECREDUCE_AND: + case ISD::VECREDUCE_OR: + case ISD::VECREDUCE_XOR: + case ISD::VECREDUCE_SMAX: + case ISD::VECREDUCE_SMIN: + case ISD::VECREDUCE_UMAX: + case ISD::VECREDUCE_UMIN: + case ISD::VECREDUCE_FMAX: + case ISD::VECREDUCE_FMIN: + Res = WidenVecOp_VECREDUCE(N); + break; } // If Res is null, the sub-method took care of registering the result. @@ -3725,8 +4127,12 @@ bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) { return true; - assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && - "Invalid operand expansion"); + if (N->isStrictFPOpcode()) + assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 2 && + "Invalid operand expansion"); + else + assert(Res.getValueType() == N->getValueType(0) && N->getNumValues() == 1 && + "Invalid operand expansion"); ReplaceValueWith(SDValue(N, 0), Res); return false; @@ -3806,7 +4212,7 @@ SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { EVT EltVT = VT.getVectorElementType(); SDLoc dl(N); unsigned NumElts = VT.getVectorNumElements(); - SDValue InOp = N->getOperand(0); + SDValue InOp = N->getOperand(N->isStrictFPOpcode() ? 1 : 0); assert(getTypeAction(InOp.getValueType()) == TargetLowering::TypeWidenVector && "Unexpected type action"); @@ -3815,10 +4221,19 @@ SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { unsigned Opcode = N->getOpcode(); // See if a widened result type would be legal, if so widen the node. + // FIXME: This isn't safe for StrictFP. Other optimization here is needed. EVT WideVT = EVT::getVectorVT(*DAG.getContext(), EltVT, InVT.getVectorNumElements()); - if (TLI.isTypeLegal(WideVT)) { - SDValue Res = DAG.getNode(Opcode, dl, WideVT, InOp); + if (TLI.isTypeLegal(WideVT) && !N->isStrictFPOpcode()) { + SDValue Res; + if (N->isStrictFPOpcode()) { + Res = DAG.getNode(Opcode, dl, { WideVT, MVT::Other }, + { N->getOperand(0), InOp }); + // Legalize the chain result - switch anything that used the old chain to + // use the new one. + ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); + } else + Res = DAG.getNode(Opcode, dl, WideVT, InOp); return DAG.getNode( ISD::EXTRACT_SUBVECTOR, dl, VT, Res, DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); @@ -3828,12 +4243,26 @@ SDValue DAGTypeLegalizer::WidenVecOp_Convert(SDNode *N) { // Unroll the convert into some scalar code and create a nasty build vector. SmallVector Ops(NumElts); - for (unsigned i=0; i < NumElts; ++i) - Ops[i] = DAG.getNode( - Opcode, dl, EltVT, - DAG.getNode( - ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, - DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())))); + if (N->isStrictFPOpcode()) { + SmallVector NewOps(N->op_begin(), N->op_end()); + SmallVector OpChains; + for (unsigned i=0; i < NumElts; ++i) { + NewOps[1] = DAG.getNode( + ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, + DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); + Ops[i] = DAG.getNode(Opcode, dl, { EltVT, MVT::Other }, NewOps); + OpChains.push_back(Ops[i].getValue(1)); + } + SDValue NewChain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OpChains); + ReplaceValueWith(SDValue(N, 1), NewChain); + } else { + for (unsigned i = 0; i < NumElts; ++i) + Ops[i] = DAG.getNode( + Opcode, dl, EltVT, + DAG.getNode( + ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, InOp, + DAG.getConstant(i, dl, TLI.getVectorIdxTy(DAG.getDataLayout())))); + } return DAG.getBuildVector(VT, dl, Ops); } @@ -3859,6 +4288,24 @@ SDValue DAGTypeLegalizer::WidenVecOp_BITCAST(SDNode *N) { } } + // Handle a case like bitcast v12i8 -> v3i32. Normally that would get widened + // to v16i8 -> v4i32, but for a target where v3i32 is legal but v12i8 is not, + // we end up here. Handling the case here with EXTRACT_SUBVECTOR avoids + // having to copy via memory. + if (VT.isVector()) { + EVT EltVT = VT.getVectorElementType(); + unsigned EltSize = EltVT.getSizeInBits(); + if (InWidenSize % EltSize == 0) { + unsigned NewNumElts = InWidenSize / EltSize; + EVT NewVT = EVT::getVectorVT(*DAG.getContext(), EltVT, NewNumElts); + if (TLI.isTypeLegal(NewVT)) { + SDValue BitOp = DAG.getNode(ISD::BITCAST, dl, NewVT, InOp); + return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, BitOp, + DAG.getConstant(0, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); + } + } + } + return CreateStackStoreLoad(InOp, VT); } @@ -4000,10 +4447,9 @@ SDValue DAGTypeLegalizer::WidenVecOp_MSCATTER(SDNode *N, unsigned OpNo) { SDValue Index = MSC->getIndex(); SDValue Scale = MSC->getScale(); - unsigned NumElts; if (OpNo == 1) { DataOp = GetWidenedVector(DataOp); - NumElts = DataOp.getValueType().getVectorNumElements(); + unsigned NumElts = DataOp.getValueType().getVectorNumElements(); // Widen index. EVT IndexVT = Index.getValueType(); @@ -4041,8 +4487,7 @@ SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) { // Get a new SETCC node to compare the newly widened operands. // Only some of the compared elements are legal. - EVT SVT = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), - InOp0.getValueType()); + EVT SVT = getSetCCResultType(InOp0.getValueType()); // The result type is legal, if its vXi1, keep vXi1 for the new SETCC. if (VT.getScalarType() == MVT::i1) SVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, @@ -4062,6 +4507,80 @@ SDValue DAGTypeLegalizer::WidenVecOp_SETCC(SDNode *N) { return PromoteTargetBoolean(CC, VT); } +SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) { + SDLoc dl(N); + SDValue Op = GetWidenedVector(N->getOperand(0)); + EVT OrigVT = N->getOperand(0).getValueType(); + EVT WideVT = Op.getValueType(); + EVT ElemVT = OrigVT.getVectorElementType(); + + SDValue NeutralElem; + switch (N->getOpcode()) { + case ISD::VECREDUCE_ADD: + case ISD::VECREDUCE_OR: + case ISD::VECREDUCE_XOR: + case ISD::VECREDUCE_UMAX: + NeutralElem = DAG.getConstant(0, dl, ElemVT); + break; + case ISD::VECREDUCE_MUL: + NeutralElem = DAG.getConstant(1, dl, ElemVT); + break; + case ISD::VECREDUCE_AND: + case ISD::VECREDUCE_UMIN: + NeutralElem = DAG.getAllOnesConstant(dl, ElemVT); + break; + case ISD::VECREDUCE_SMAX: + NeutralElem = DAG.getConstant( + APInt::getSignedMinValue(ElemVT.getSizeInBits()), dl, ElemVT); + break; + case ISD::VECREDUCE_SMIN: + NeutralElem = DAG.getConstant( + APInt::getSignedMaxValue(ElemVT.getSizeInBits()), dl, ElemVT); + break; + case ISD::VECREDUCE_FADD: + NeutralElem = DAG.getConstantFP(0.0, dl, ElemVT); + break; + case ISD::VECREDUCE_FMUL: + NeutralElem = DAG.getConstantFP(1.0, dl, ElemVT); + break; + case ISD::VECREDUCE_FMAX: + NeutralElem = DAG.getConstantFP( + std::numeric_limits::infinity(), dl, ElemVT); + break; + case ISD::VECREDUCE_FMIN: + NeutralElem = DAG.getConstantFP( + -std::numeric_limits::infinity(), dl, ElemVT); + break; + } + + // Pad the vector with the neutral element. + unsigned OrigElts = OrigVT.getVectorNumElements(); + unsigned WideElts = WideVT.getVectorNumElements(); + for (unsigned Idx = OrigElts; Idx < WideElts; Idx++) + Op = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, WideVT, Op, NeutralElem, + DAG.getConstant(Idx, dl, TLI.getVectorIdxTy(DAG.getDataLayout()))); + + return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Op, N->getFlags()); +} + +SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) { + // This only gets called in the case that the left and right inputs and + // result are of a legal odd vector type, and the condition is illegal i1 of + // the same odd width that needs widening. + EVT VT = N->getValueType(0); + assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT)); + + SDValue Cond = GetWidenedVector(N->getOperand(0)); + SDValue LeftIn = DAG.WidenVector(N->getOperand(1), SDLoc(N)); + SDValue RightIn = DAG.WidenVector(N->getOperand(2), SDLoc(N)); + SDLoc DL(N); + + SDValue Select = DAG.getNode(N->getOpcode(), DL, LeftIn.getValueType(), Cond, + LeftIn, RightIn); + return DAG.getNode( + ISD::EXTRACT_SUBVECTOR, DL, VT, Select, + DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); +} //===----------------------------------------------------------------------===// // Vector Widening Utilities @@ -4102,6 +4621,8 @@ static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI, isPowerOf2_32(WidenWidth / MemVTWidth) && (MemVTWidth <= Width || (Align!=0 && MemVTWidth<=AlignInBits && MemVTWidth<=Width+WidenEx))) { + if (MemVTWidth == WidenWidth) + return MemVT; RetVT = MemVT; break; } @@ -4113,7 +4634,10 @@ static EVT FindMemType(SelectionDAG& DAG, const TargetLowering &TLI, VT >= (unsigned)MVT::FIRST_VECTOR_VALUETYPE; --VT) { EVT MemVT = (MVT::SimpleValueType) VT; unsigned MemVTWidth = MemVT.getSizeInBits(); - if (TLI.isTypeLegal(MemVT) && WidenEltVT == MemVT.getVectorElementType() && + auto Action = TLI.getTypeAction(*DAG.getContext(), MemVT); + if ((Action == TargetLowering::TypeLegal || + Action == TargetLowering::TypePromoteInteger) && + WidenEltVT == MemVT.getVectorElementType() && (WidenWidth % MemVTWidth) == 0 && isPowerOf2_32(WidenWidth / MemVTWidth) && (MemVTWidth <= Width || diff --git a/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp b/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp index 7f369c746d24..34660e3a48ec 100644 --- a/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp +++ b/lib/CodeGen/SelectionDAG/ResourcePriorityQueue.cpp @@ -1,9 +1,8 @@ //===- ResourcePriorityQueue.cpp - A DFA-oriented priority queue -*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -85,6 +84,7 @@ ResourcePriorityQueue::numberRCValPredInSU(SUnit *SU, unsigned RCId) { case ISD::CopyFromReg: NumberDeps++; break; case ISD::CopyToReg: break; case ISD::INLINEASM: break; + case ISD::INLINEASM_BR: break; } if (!ScegN->isMachineOpcode()) continue; @@ -121,6 +121,7 @@ unsigned ResourcePriorityQueue::numberRCValSuccInSU(SUnit *SU, case ISD::CopyFromReg: break; case ISD::CopyToReg: NumberDeps++; break; case ISD::INLINEASM: break; + case ISD::INLINEASM_BR: break; } if (!ScegN->isMachineOpcode()) continue; @@ -446,6 +447,7 @@ int ResourcePriorityQueue::SUSchedulingCost(SUnit *SU) { break; case ISD::INLINEASM: + case ISD::INLINEASM_BR: ResCount += PriorityThree; break; } @@ -548,6 +550,7 @@ void ResourcePriorityQueue::initNumRegDefsLeft(SUnit *SU) { NodeNumDefs++; break; case ISD::INLINEASM: + case ISD::INLINEASM_BR: NodeNumDefs++; break; } diff --git a/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h b/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h index f7566b246f32..65b9d017fc5c 100644 --- a/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h +++ b/lib/CodeGen/SelectionDAG/SDNodeDbgValue.h @@ -1,9 +1,8 @@ //===-- llvm/CodeGen/SDNodeDbgValue.h - SelectionDAG dbg_value --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -136,7 +135,8 @@ public: /// dbg.addr is emitted twice. void clearIsEmitted() { Emitted = false; } - LLVM_DUMP_METHOD void dump(raw_ostream &OS) const; + LLVM_DUMP_METHOD void dump() const; + LLVM_DUMP_METHOD void print(raw_ostream &OS) const; }; /// Holds the information from a dbg_label node through SDISel. diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp index 90e109b022fd..2cb850fa1a3d 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp @@ -1,9 +1,8 @@ //===----- ScheduleDAGFast.cpp - Fast poor list scheduler -----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -480,7 +479,8 @@ bool ScheduleDAGFast::DelayForLiveRegsBottomUp(SUnit *SU, } for (SDNode *Node = SU->getNode(); Node; Node = Node->getGluedNode()) { - if (Node->getOpcode() == ISD::INLINEASM) { + if (Node->getOpcode() == ISD::INLINEASM || + Node->getOpcode() == ISD::INLINEASM_BR) { // Inline asm can clobber physical defs. unsigned NumOps = Node->getNumOperands(); if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue) diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp index 8d75b8133a30..34b4c8502353 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp @@ -1,9 +1,8 @@ //===- ScheduleDAGRRList.cpp - Reg pressure reduction list scheduler ------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -220,6 +219,14 @@ public: return Topo.WillCreateCycle(SU, TargetSU); } + /// AddPredQueued - Queues and update to add a predecessor edge to SUnit SU. + /// This returns true if this is a new predecessor. + /// Does *NOT* update the topological ordering! It just queues an update. + void AddPredQueued(SUnit *SU, const SDep &D) { + Topo.AddPredQueued(SU, D.getSUnit()); + SU->addPred(D); + } + /// AddPred - adds a predecessor edge to SUnit SU. /// This returns true if this is a new predecessor. /// Updates the topological ordering if required. @@ -267,24 +274,22 @@ private: void ListScheduleBottomUp(); /// CreateNewSUnit - Creates a new SUnit and returns a pointer to it. - /// Updates the topological ordering if required. SUnit *CreateNewSUnit(SDNode *N) { unsigned NumSUnits = SUnits.size(); SUnit *NewNode = newSUnit(N); // Update the topological ordering. if (NewNode->NodeNum >= NumSUnits) - Topo.InitDAGTopologicalSorting(); + Topo.MarkDirty(); return NewNode; } /// CreateClone - Creates a new SUnit from an existing one. - /// Updates the topological ordering if required. SUnit *CreateClone(SUnit *N) { unsigned NumSUnits = SUnits.size(); SUnit *NewNode = Clone(N); // Update the topological ordering. if (NewNode->NodeNum >= NumSUnits) - Topo.InitDAGTopologicalSorting(); + Topo.MarkDirty(); return NewNode; } @@ -366,7 +371,7 @@ void ScheduleDAGRRList::Schedule() { BuildSchedGraph(nullptr); LLVM_DEBUG(dump()); - Topo.InitDAGTopologicalSorting(); + Topo.MarkDirty(); AvailableQueue->initNodes(SUnits); @@ -709,6 +714,7 @@ void ScheduleDAGRRList::EmitNode(SUnit *SU) { // removed. return; case ISD::INLINEASM: + case ISD::INLINEASM_BR: // For inline asm, clear the pipeline state. HazardRec->Reset(); return; @@ -1017,8 +1023,9 @@ SUnit *ScheduleDAGRRList::TryUnfoldSU(SUnit *SU) { NewSU = &SUnits[N->getNodeId()]; // If NewSU has already been scheduled, we need to clone it, but this // negates the benefit to unfolding so just return SU. - if (NewSU->isScheduled) + if (NewSU->isScheduled) { return SU; + } isNewN = false; } else { NewSU = CreateNewSUnit(N); @@ -1071,23 +1078,23 @@ SUnit *ScheduleDAGRRList::TryUnfoldSU(SUnit *SU) { for (const SDep &Pred : ChainPreds) { RemovePred(SU, Pred); if (isNewLoad) - AddPred(LoadSU, Pred); + AddPredQueued(LoadSU, Pred); } for (const SDep &Pred : LoadPreds) { RemovePred(SU, Pred); if (isNewLoad) - AddPred(LoadSU, Pred); + AddPredQueued(LoadSU, Pred); } for (const SDep &Pred : NodePreds) { RemovePred(SU, Pred); - AddPred(NewSU, Pred); + AddPredQueued(NewSU, Pred); } for (SDep D : NodeSuccs) { SUnit *SuccDep = D.getSUnit(); D.setSUnit(SU); RemovePred(SuccDep, D); D.setSUnit(NewSU); - AddPred(SuccDep, D); + AddPredQueued(SuccDep, D); // Balance register pressure. if (AvailableQueue->tracksRegPressure() && SuccDep->isScheduled && !D.isCtrl() && NewSU->NumRegDefsLeft > 0) @@ -1099,7 +1106,7 @@ SUnit *ScheduleDAGRRList::TryUnfoldSU(SUnit *SU) { RemovePred(SuccDep, D); if (isNewLoad) { D.setSUnit(LoadSU); - AddPred(SuccDep, D); + AddPredQueued(SuccDep, D); } } @@ -1107,7 +1114,7 @@ SUnit *ScheduleDAGRRList::TryUnfoldSU(SUnit *SU) { // by LoadSU. SDep D(LoadSU, SDep::Data, 0); D.setLatency(LoadSU->Latency); - AddPred(NewSU, D); + AddPredQueued(NewSU, D); if (isNewLoad) AvailableQueue->addNode(LoadSU); @@ -1179,7 +1186,7 @@ SUnit *ScheduleDAGRRList::CopyAndMoveSuccessors(SUnit *SU) { // New SUnit has the exact same predecessors. for (SDep &Pred : SU->Preds) if (!Pred.isArtificial()) - AddPred(NewSU, Pred); + AddPredQueued(NewSU, Pred); // Only copy scheduled successors. Cut them from old node's successor // list and move them over. @@ -1191,7 +1198,7 @@ SUnit *ScheduleDAGRRList::CopyAndMoveSuccessors(SUnit *SU) { if (SuccSU->isScheduled) { SDep D = Succ; D.setSUnit(NewSU); - AddPred(SuccSU, D); + AddPredQueued(SuccSU, D); D.setSUnit(SU); DelDeps.push_back(std::make_pair(SuccSU, D)); } @@ -1230,14 +1237,14 @@ void ScheduleDAGRRList::InsertCopiesAndMoveSuccs(SUnit *SU, unsigned Reg, if (SuccSU->isScheduled) { SDep D = Succ; D.setSUnit(CopyToSU); - AddPred(SuccSU, D); + AddPredQueued(SuccSU, D); DelDeps.push_back(std::make_pair(SuccSU, Succ)); } else { // Avoid scheduling the def-side copy before other successors. Otherwise // we could introduce another physreg interference on the copy and // continue inserting copies indefinitely. - AddPred(SuccSU, SDep(CopyFromSU, SDep::Artificial)); + AddPredQueued(SuccSU, SDep(CopyFromSU, SDep::Artificial)); } } for (auto &DelDep : DelDeps) @@ -1245,10 +1252,10 @@ void ScheduleDAGRRList::InsertCopiesAndMoveSuccs(SUnit *SU, unsigned Reg, SDep FromDep(SU, SDep::Data, Reg); FromDep.setLatency(SU->Latency); - AddPred(CopyFromSU, FromDep); + AddPredQueued(CopyFromSU, FromDep); SDep ToDep(CopyFromSU, SDep::Data, 0); ToDep.setLatency(CopyFromSU->Latency); - AddPred(CopyToSU, ToDep); + AddPredQueued(CopyToSU, ToDep); AvailableQueue->updateNode(SU); AvailableQueue->addNode(CopyFromSU); @@ -1348,7 +1355,8 @@ DelayForLiveRegsBottomUp(SUnit *SU, SmallVectorImpl &LRegs) { } for (SDNode *Node = SU->getNode(); Node; Node = Node->getGluedNode()) { - if (Node->getOpcode() == ISD::INLINEASM) { + if (Node->getOpcode() == ISD::INLINEASM || + Node->getOpcode() == ISD::INLINEASM_BR) { // Inline asm can clobber physical defs. unsigned NumOps = Node->getNumOperands(); if (Node->getOperand(NumOps-1).getValueType() == MVT::Glue) @@ -1477,6 +1485,11 @@ SUnit *ScheduleDAGRRList::PickNodeToScheduleBottomUp() { if (CurSU) return CurSU; + // We query the topological order in the loop body, so make sure outstanding + // updates are applied before entering it (we only enter the loop if there + // are some interferences). If we make changes to the ordering, we exit + // the loop. + // All candidates are delayed due to live physical reg dependencies. // Try backtracking, code duplication, or inserting cross class copies // to resolve it. @@ -1506,7 +1519,7 @@ SUnit *ScheduleDAGRRList::PickNodeToScheduleBottomUp() { } LLVM_DEBUG(dbgs() << "ARTIFICIAL edge from SU(" << BtSU->NodeNum << ") to SU(" << TrySU->NodeNum << ")\n"); - AddPred(TrySU, SDep(BtSU, SDep::Artificial)); + AddPredQueued(TrySU, SDep(BtSU, SDep::Artificial)); // If one or more successors has been unscheduled, then the current // node is no longer available. @@ -1560,14 +1573,14 @@ SUnit *ScheduleDAGRRList::PickNodeToScheduleBottomUp() { InsertCopiesAndMoveSuccs(LRDef, Reg, DestRC, RC, Copies); LLVM_DEBUG(dbgs() << " Adding an edge from SU #" << TrySU->NodeNum << " to SU #" << Copies.front()->NodeNum << "\n"); - AddPred(TrySU, SDep(Copies.front(), SDep::Artificial)); + AddPredQueued(TrySU, SDep(Copies.front(), SDep::Artificial)); NewDef = Copies.back(); } LLVM_DEBUG(dbgs() << " Adding an edge from SU #" << NewDef->NodeNum << " to SU #" << TrySU->NodeNum << "\n"); LiveRegDefs[Reg] = NewDef; - AddPred(NewDef, SDep(TrySU, SDep::Artificial)); + AddPredQueued(NewDef, SDep(TrySU, SDep::Artificial)); TrySU->isAvailable = false; CurSU = NewDef; } @@ -2939,6 +2952,29 @@ void RegReductionPQBase::PrescheduleNodesWithMultipleUses() { (cast(N->getOperand(1))->getReg())) continue; + SDNode *PredFrameSetup = nullptr; + for (const SDep &Pred : SU.Preds) + if (Pred.isCtrl() && Pred.getSUnit()) { + // Find the predecessor which is not data dependence. + SDNode *PredND = Pred.getSUnit()->getNode(); + + // If PredND is FrameSetup, we should not pre-scheduled the node, + // or else, when bottom up scheduling, ADJCALLSTACKDOWN and + // ADJCALLSTACKUP may hold CallResource too long and make other + // calls can't be scheduled. If there's no other available node + // to schedule, the schedular will try to rename the register by + // creating copy to avoid the conflict which will fail because + // CallResource is not a real physical register. + if (PredND && PredND->isMachineOpcode() && + (PredND->getMachineOpcode() == TII->getCallFrameSetupOpcode())) { + PredFrameSetup = PredND; + break; + } + } + // Skip the node has FrameSetup parent. + if (PredFrameSetup != nullptr) + continue; + // Locate the single data predecessor. SUnit *PredSU = nullptr; for (const SDep &Pred : SU.Preds) @@ -2993,9 +3029,9 @@ void RegReductionPQBase::PrescheduleNodesWithMultipleUses() { if (SuccSU != &SU) { Edge.setSUnit(PredSU); scheduleDAG->RemovePred(SuccSU, Edge); - scheduleDAG->AddPred(&SU, Edge); + scheduleDAG->AddPredQueued(&SU, Edge); Edge.setSUnit(&SU); - scheduleDAG->AddPred(SuccSU, Edge); + scheduleDAG->AddPredQueued(SuccSU, Edge); --i; } } @@ -3077,7 +3113,7 @@ void RegReductionPQBase::AddPseudoTwoAddrDeps() { LLVM_DEBUG(dbgs() << " Adding a pseudo-two-addr edge from SU #" << SU.NodeNum << " to SU #" << SuccSU->NodeNum << "\n"); - scheduleDAG->AddPred(&SU, SDep(SuccSU, SDep::Artificial)); + scheduleDAG->AddPredQueued(&SU, SDep(SuccSU, SDep::Artificial)); } } } diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp index e258f0a218a5..568c6191e512 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp @@ -1,9 +1,8 @@ //===--- ScheduleDAGSDNodes.cpp - Implement the ScheduleDAGSDNodes class --===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -206,6 +205,19 @@ void ScheduleDAGSDNodes::ClusterNeighboringLoads(SDNode *Node) { if (!Chain) return; + // Skip any load instruction that has a tied input. There may be an additional + // dependency requiring a different order than by increasing offsets, and the + // added glue may introduce a cycle. + auto hasTiedInput = [this](const SDNode *N) { + const MCInstrDesc &MCID = TII->get(N->getMachineOpcode()); + for (unsigned I = 0; I != MCID.getNumOperands(); ++I) { + if (MCID.getOperandConstraint(I, MCOI::TIED_TO) != -1) + return true; + } + + return false; + }; + // Look for other loads of the same chain. Find loads that are loading from // the same base pointer and different offsets. SmallPtrSet Visited; @@ -213,6 +225,10 @@ void ScheduleDAGSDNodes::ClusterNeighboringLoads(SDNode *Node) { DenseMap O2SMap; // Map from offset to SDNode. bool Cluster = false; SDNode *Base = Node; + + if (hasTiedInput(Base)) + return; + // This algorithm requires a reasonably low use count before finding a match // to avoid uselessly blowing up compile time in large blocks. unsigned UseCount = 0; @@ -223,10 +239,12 @@ void ScheduleDAGSDNodes::ClusterNeighboringLoads(SDNode *Node) { continue; int64_t Offset1, Offset2; if (!TII->areLoadsFromSameBasePtr(Base, User, Offset1, Offset2) || - Offset1 == Offset2) + Offset1 == Offset2 || + hasTiedInput(User)) { // FIXME: Should be ok if they addresses are identical. But earlier // optimizations really should have eliminated one of the loads. continue; + } if (O2SMap.insert(std::make_pair(Offset1, Base)).second) Offsets.push_back(Offset1); O2SMap.insert(std::make_pair(Offset2, User)); @@ -741,28 +759,27 @@ ProcessSDDbgValues(SDNode *N, SelectionDAG *DAG, InstrEmitter &Emitter, static void ProcessSourceNode(SDNode *N, SelectionDAG *DAG, InstrEmitter &Emitter, DenseMap &VRBaseMap, - SmallVectorImpl > &Orders, - SmallSet &Seen) { + SmallVectorImpl> &Orders, + SmallSet &Seen, MachineInstr *NewInsn) { unsigned Order = N->getIROrder(); - if (!Order || !Seen.insert(Order).second) { + if (!Order || Seen.count(Order)) { // Process any valid SDDbgValues even if node does not have any order // assigned. ProcessSDDbgValues(N, DAG, Emitter, Orders, VRBaseMap, 0); return; } - MachineBasicBlock *BB = Emitter.getBlock(); - auto IP = Emitter.getInsertPos(); - if (IP == BB->begin() || BB->back().isPHI() || - // Fast-isel may have inserted some instructions, in which case the - // BB->back().isPHI() test will not fire when we want it to. - std::prev(IP)->isPHI()) { - // Did not insert any instruction. - Orders.push_back({Order, (MachineInstr *)nullptr}); - return; + // If a new instruction was generated for this Order number, record it. + // Otherwise, leave this order number unseen: we will either find later + // instructions for it, or leave it unseen if there were no instructions at + // all. + if (NewInsn) { + Seen.insert(Order); + Orders.push_back({Order, NewInsn}); } - Orders.push_back({Order, &*std::prev(IP)}); + // Even if no instruction was generated, a Value may have become defined via + // earlier nodes. Try to process them now. ProcessSDDbgValues(N, DAG, Emitter, Orders, VRBaseMap, Order); } @@ -815,6 +832,43 @@ EmitSchedule(MachineBasicBlock::iterator &InsertPos) { SmallSet Seen; bool HasDbg = DAG->hasDebugValues(); + // Emit a node, and determine where its first instruction is for debuginfo. + // Zero, one, or multiple instructions can be created when emitting a node. + auto EmitNode = + [&](SDNode *Node, bool IsClone, bool IsCloned, + DenseMap &VRBaseMap) -> MachineInstr * { + // Fetch instruction prior to this, or end() if nonexistant. + auto GetPrevInsn = [&](MachineBasicBlock::iterator I) { + if (I == BB->begin()) + return BB->end(); + else + return std::prev(Emitter.getInsertPos()); + }; + + MachineBasicBlock::iterator Before = GetPrevInsn(Emitter.getInsertPos()); + Emitter.EmitNode(Node, IsClone, IsCloned, VRBaseMap); + MachineBasicBlock::iterator After = GetPrevInsn(Emitter.getInsertPos()); + + // If the iterator did not change, no instructions were inserted. + if (Before == After) + return nullptr; + + MachineInstr *MI; + if (Before == BB->end()) { + // There were no prior instructions; the new ones must start at the + // beginning of the block. + MI = &Emitter.getBlock()->instr_front(); + } else { + // Return first instruction after the pre-existing instructions. + MI = &*std::next(Before); + } + + if (MI->isCall() && DAG->getTarget().Options.EnableDebugEntryValues) + MF.addCallArgsForwardingRegs(MI, DAG->getSDCallSiteInfo(Node)); + + return MI; + }; + // If this is the first BB, emit byval parameter dbg_value's. if (HasDbg && BB->getParent()->begin() == MachineFunction::iterator(BB)) { SDDbgInfo::DbgIterator PDI = DAG->ByvalParmDbgBegin(); @@ -851,18 +905,18 @@ EmitSchedule(MachineBasicBlock::iterator &InsertPos) { GluedNodes.push_back(N); while (!GluedNodes.empty()) { SDNode *N = GluedNodes.back(); - Emitter.EmitNode(N, SU->OrigNode != SU, SU->isCloned, VRBaseMap); + auto NewInsn = EmitNode(N, SU->OrigNode != SU, SU->isCloned, VRBaseMap); // Remember the source order of the inserted instruction. if (HasDbg) - ProcessSourceNode(N, DAG, Emitter, VRBaseMap, Orders, Seen); + ProcessSourceNode(N, DAG, Emitter, VRBaseMap, Orders, Seen, NewInsn); GluedNodes.pop_back(); } - Emitter.EmitNode(SU->getNode(), SU->OrigNode != SU, SU->isCloned, - VRBaseMap); + auto NewInsn = + EmitNode(SU->getNode(), SU->OrigNode != SU, SU->isCloned, VRBaseMap); // Remember the source order of the inserted instruction. if (HasDbg) - ProcessSourceNode(SU->getNode(), DAG, Emitter, VRBaseMap, Orders, - Seen); + ProcessSourceNode(SU->getNode(), DAG, Emitter, VRBaseMap, Orders, Seen, + NewInsn); } // Insert all the dbg_values which have not already been inserted in source @@ -873,7 +927,7 @@ EmitSchedule(MachineBasicBlock::iterator &InsertPos) { // Sort the source order instructions and use the order to insert debug // values. Use stable_sort so that DBG_VALUEs are inserted in the same order // regardless of the host's implementation fo std::sort. - std::stable_sort(Orders.begin(), Orders.end(), less_first()); + llvm::stable_sort(Orders, less_first()); std::stable_sort(DAG->DbgBegin(), DAG->DbgEnd(), [](const SDDbgValue *LHS, const SDDbgValue *RHS) { return LHS->getOrder() < RHS->getOrder(); @@ -887,8 +941,7 @@ EmitSchedule(MachineBasicBlock::iterator &InsertPos) { unsigned Order = Orders[i].first; MachineInstr *MI = Orders[i].second; // Insert all SDDbgValue's whose order(s) are before "Order". - if (!MI) - continue; + assert(MI); for (; DI != DE; ++DI) { if ((*DI)->getOrder() < LastOrder || (*DI)->getOrder() >= Order) break; diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h index 3fa7ad895725..5163b4fa4fd3 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h @@ -1,9 +1,8 @@ //===---- ScheduleDAGSDNodes.h - SDNode Scheduling --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp b/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp index 416061475b1a..ab06b55b49fd 100644 --- a/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp +++ b/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp @@ -1,9 +1,8 @@ //===- ScheduleDAGVLIW.cpp - SelectionDAG list scheduler for VLIW -*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 647496c1afcb..5852e693fa9f 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -1,9 +1,8 @@ //===- SelectionDAG.cpp - Implement the SelectionDAG data structures ------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -86,6 +85,7 @@ static SDVTList makeVTList(const EVT *VTs, unsigned NumVTs) { // Default null implementations of the callbacks. void SelectionDAG::DAGUpdateListener::NodeDeleted(SDNode*, SDNode*) {} void SelectionDAG::DAGUpdateListener::NodeUpdated(SDNode*) {} +void SelectionDAG::DAGUpdateListener::NodeInserted(SDNode *) {} void SelectionDAG::DAGNodeDeletedListener::anchor() {} @@ -262,12 +262,7 @@ bool ISD::allOperandsUndef(const SDNode *N) { // is probably the desired behavior. if (N->getNumOperands() == 0) return false; - - for (const SDValue &Op : N->op_values()) - if (!Op.isUndef()) - return false; - - return true; + return all_of(N->op_values(), [](SDValue Op) { return Op.isUndef(); }); } bool ISD::matchUnaryPredicate(SDValue Op, @@ -299,8 +294,8 @@ bool ISD::matchUnaryPredicate(SDValue Op, bool ISD::matchBinaryPredicate( SDValue LHS, SDValue RHS, std::function Match, - bool AllowUndefs) { - if (LHS.getValueType() != RHS.getValueType()) + bool AllowUndefs, bool AllowTypeMismatch) { + if (!AllowTypeMismatch && LHS.getValueType() != RHS.getValueType()) return false; // TODO: Add support for scalar UNDEF cases? @@ -323,8 +318,8 @@ bool ISD::matchBinaryPredicate( auto *RHSCst = dyn_cast(RHSOp); if ((!LHSCst && !LHSUndef) || (!RHSCst && !RHSUndef)) return false; - if (LHSOp.getValueType() != SVT || - LHSOp.getValueType() != RHSOp.getValueType()) + if (!AllowTypeMismatch && (LHSOp.getValueType() != SVT || + LHSOp.getValueType() != RHSOp.getValueType())) return false; if (!Match(LHSCst, RHSCst)) return false; @@ -518,6 +513,13 @@ static void AddNodeIDCustom(FoldingSetNodeID &ID, const SDNode *N) { case ISD::TargetFrameIndex: ID.AddInteger(cast(N)->getIndex()); break; + case ISD::LIFETIME_START: + case ISD::LIFETIME_END: + if (cast(N)->hasOffset()) { + ID.AddInteger(cast(N)->getSize()); + ID.AddInteger(cast(N)->getOffset()); + } + break; case ISD::JumpTable: case ISD::TargetJumpTable: ID.AddInteger(cast(N)->getIndex()); @@ -834,6 +836,8 @@ void SelectionDAG::InsertNode(SDNode *N) { N->PersistentId = NextPersistentId++; VerifySDNode(N); #endif + for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next) + DUL->NodeInserted(N); } /// RemoveNodeFromCSEMaps - Take the specified node out of the CSE map that @@ -1136,6 +1140,18 @@ SDValue SelectionDAG::getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) { getConstant(Imm, DL, Op.getValueType())); } +SDValue SelectionDAG::getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT) { + // Only unsigned pointer semantics are supported right now. In the future this + // might delegate to TLI to check pointer signedness. + return getZExtOrTrunc(Op, DL, VT); +} + +SDValue SelectionDAG::getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT) { + // Only unsigned pointer semantics are supported right now. In the future this + // might delegate to TLI to check pointer signedness. + return getZeroExtendInReg(Op, DL, VT); +} + /// getNOT - Create a bitwise NOT operation as (XOR Val, -1). SDValue SelectionDAG::getNOT(const SDLoc &DL, SDValue Val, EVT VT) { EVT EltVT = VT.getScalarType(); @@ -1274,6 +1290,12 @@ SDValue SelectionDAG::getIntPtrConstant(uint64_t Val, const SDLoc &DL, return getConstant(Val, DL, TLI->getPointerTy(getDataLayout()), isTarget); } +SDValue SelectionDAG::getShiftAmountConstant(uint64_t Val, EVT VT, + const SDLoc &DL, bool LegalTypes) { + EVT ShiftVT = TLI->getShiftAmountTy(VT, getDataLayout(), LegalTypes); + return getConstant(Val, DL, ShiftVT); +} + SDValue SelectionDAG::getConstantFP(const APFloat &V, const SDLoc &DL, EVT VT, bool isTarget) { return getConstantFP(*ConstantFP::get(*getContext(), V), DL, VT, isTarget); @@ -1403,7 +1425,7 @@ SDValue SelectionDAG::getConstantPool(const Constant *C, EVT VT, assert((TargetFlags == 0 || isTarget) && "Cannot set target flags on target-independent globals"); if (Alignment == 0) - Alignment = MF->getFunction().optForSize() + Alignment = MF->getFunction().hasOptSize() ? getDataLayout().getABITypeAlignment(C->getType()) : getDataLayout().getPrefTypeAlignment(C->getType()); unsigned Opc = isTarget ? ISD::TargetConstantPool : ISD::ConstantPool; @@ -1770,7 +1792,8 @@ SDValue SelectionDAG::getLabelNode(unsigned Opcode, const SDLoc &dl, if (SDNode *E = FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - auto *N = newSDNode(dl.getIROrder(), dl.getDebugLoc(), Label); + auto *N = + newSDNode(Opcode, dl.getIROrder(), dl.getDebugLoc(), Label); createOperands(N, Ops); CSEMap.InsertNode(N, IP); @@ -1965,10 +1988,30 @@ SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2, case ISD::SETUO: case ISD::SETUEQ: case ISD::SETUNE: - assert(!N1.getValueType().isInteger() && "Illegal setcc for integer!"); + assert(!OpVT.isInteger() && "Illegal setcc for integer!"); break; } + if (OpVT.isInteger()) { + // For EQ and NE, we can always pick a value for the undef to make the + // predicate pass or fail, so we can return undef. + // Matches behavior in llvm::ConstantFoldCompareInstruction. + // icmp eq/ne X, undef -> undef. + if ((N1.isUndef() || N2.isUndef()) && + (Cond == ISD::SETEQ || Cond == ISD::SETNE)) + return getUNDEF(VT); + + // If both operands are undef, we can return undef for int comparison. + // icmp undef, undef -> undef. + if (N1.isUndef() && N2.isUndef()) + return getUNDEF(VT); + + // icmp X, X -> true/false + // icmp X, undef -> true/false because undef could be X. + if (N1 == N2) + return getBoolConstant(ISD::isTrueWhenEqual(Cond), dl, VT, OpVT); + } + if (ConstantSDNode *N2C = dyn_cast(N2)) { const APInt &C2 = N2C->getAPIntValue(); if (ConstantSDNode *N1C = dyn_cast(N1)) { @@ -1989,71 +2032,88 @@ SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2, } } } - if (ConstantFPSDNode *N1C = dyn_cast(N1)) { - if (ConstantFPSDNode *N2C = dyn_cast(N2)) { - APFloat::cmpResult R = N1C->getValueAPF().compare(N2C->getValueAPF()); - switch (Cond) { - default: break; - case ISD::SETEQ: if (R==APFloat::cmpUnordered) - return getUNDEF(VT); - LLVM_FALLTHROUGH; - case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT, - OpVT); - case ISD::SETNE: if (R==APFloat::cmpUnordered) - return getUNDEF(VT); - LLVM_FALLTHROUGH; - case ISD::SETONE: return getBoolConstant(R==APFloat::cmpGreaterThan || - R==APFloat::cmpLessThan, dl, VT, - OpVT); - case ISD::SETLT: if (R==APFloat::cmpUnordered) - return getUNDEF(VT); - LLVM_FALLTHROUGH; - case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT, - OpVT); - case ISD::SETGT: if (R==APFloat::cmpUnordered) - return getUNDEF(VT); - LLVM_FALLTHROUGH; - case ISD::SETOGT: return getBoolConstant(R==APFloat::cmpGreaterThan, dl, - VT, OpVT); - case ISD::SETLE: if (R==APFloat::cmpUnordered) - return getUNDEF(VT); - LLVM_FALLTHROUGH; - case ISD::SETOLE: return getBoolConstant(R==APFloat::cmpLessThan || - R==APFloat::cmpEqual, dl, VT, - OpVT); - case ISD::SETGE: if (R==APFloat::cmpUnordered) - return getUNDEF(VT); - LLVM_FALLTHROUGH; - case ISD::SETOGE: return getBoolConstant(R==APFloat::cmpGreaterThan || - R==APFloat::cmpEqual, dl, VT, OpVT); - case ISD::SETO: return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT, - OpVT); - case ISD::SETUO: return getBoolConstant(R==APFloat::cmpUnordered, dl, VT, - OpVT); - case ISD::SETUEQ: return getBoolConstant(R==APFloat::cmpUnordered || - R==APFloat::cmpEqual, dl, VT, - OpVT); - case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT, - OpVT); - case ISD::SETULT: return getBoolConstant(R==APFloat::cmpUnordered || - R==APFloat::cmpLessThan, dl, VT, - OpVT); - case ISD::SETUGT: return getBoolConstant(R==APFloat::cmpGreaterThan || - R==APFloat::cmpUnordered, dl, VT, - OpVT); - case ISD::SETULE: return getBoolConstant(R!=APFloat::cmpGreaterThan, dl, - VT, OpVT); - case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT, - OpVT); - } - } else { - // Ensure that the constant occurs on the RHS. - ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond); - MVT CompVT = N1.getValueType().getSimpleVT(); - if (!TLI->isCondCodeLegal(SwappedCond, CompVT)) - return SDValue(); - return getSetCC(dl, VT, N2, N1, SwappedCond); + auto *N1CFP = dyn_cast(N1); + auto *N2CFP = dyn_cast(N2); + + if (N1CFP && N2CFP) { + APFloat::cmpResult R = N1CFP->getValueAPF().compare(N2CFP->getValueAPF()); + switch (Cond) { + default: break; + case ISD::SETEQ: if (R==APFloat::cmpUnordered) + return getUNDEF(VT); + LLVM_FALLTHROUGH; + case ISD::SETOEQ: return getBoolConstant(R==APFloat::cmpEqual, dl, VT, + OpVT); + case ISD::SETNE: if (R==APFloat::cmpUnordered) + return getUNDEF(VT); + LLVM_FALLTHROUGH; + case ISD::SETONE: return getBoolConstant(R==APFloat::cmpGreaterThan || + R==APFloat::cmpLessThan, dl, VT, + OpVT); + case ISD::SETLT: if (R==APFloat::cmpUnordered) + return getUNDEF(VT); + LLVM_FALLTHROUGH; + case ISD::SETOLT: return getBoolConstant(R==APFloat::cmpLessThan, dl, VT, + OpVT); + case ISD::SETGT: if (R==APFloat::cmpUnordered) + return getUNDEF(VT); + LLVM_FALLTHROUGH; + case ISD::SETOGT: return getBoolConstant(R==APFloat::cmpGreaterThan, dl, + VT, OpVT); + case ISD::SETLE: if (R==APFloat::cmpUnordered) + return getUNDEF(VT); + LLVM_FALLTHROUGH; + case ISD::SETOLE: return getBoolConstant(R==APFloat::cmpLessThan || + R==APFloat::cmpEqual, dl, VT, + OpVT); + case ISD::SETGE: if (R==APFloat::cmpUnordered) + return getUNDEF(VT); + LLVM_FALLTHROUGH; + case ISD::SETOGE: return getBoolConstant(R==APFloat::cmpGreaterThan || + R==APFloat::cmpEqual, dl, VT, OpVT); + case ISD::SETO: return getBoolConstant(R!=APFloat::cmpUnordered, dl, VT, + OpVT); + case ISD::SETUO: return getBoolConstant(R==APFloat::cmpUnordered, dl, VT, + OpVT); + case ISD::SETUEQ: return getBoolConstant(R==APFloat::cmpUnordered || + R==APFloat::cmpEqual, dl, VT, + OpVT); + case ISD::SETUNE: return getBoolConstant(R!=APFloat::cmpEqual, dl, VT, + OpVT); + case ISD::SETULT: return getBoolConstant(R==APFloat::cmpUnordered || + R==APFloat::cmpLessThan, dl, VT, + OpVT); + case ISD::SETUGT: return getBoolConstant(R==APFloat::cmpGreaterThan || + R==APFloat::cmpUnordered, dl, VT, + OpVT); + case ISD::SETULE: return getBoolConstant(R!=APFloat::cmpGreaterThan, dl, + VT, OpVT); + case ISD::SETUGE: return getBoolConstant(R!=APFloat::cmpLessThan, dl, VT, + OpVT); + } + } else if (N1CFP && OpVT.isSimple() && !N2.isUndef()) { + // Ensure that the constant occurs on the RHS. + ISD::CondCode SwappedCond = ISD::getSetCCSwappedOperands(Cond); + if (!TLI->isCondCodeLegal(SwappedCond, OpVT.getSimpleVT())) + return SDValue(); + return getSetCC(dl, VT, N2, N1, SwappedCond); + } else if ((N2CFP && N2CFP->getValueAPF().isNaN()) || + (OpVT.isFloatingPoint() && (N1.isUndef() || N2.isUndef()))) { + // If an operand is known to be a nan (or undef that could be a nan), we can + // fold it. + // Choosing NaN for the undef will always make unordered comparison succeed + // and ordered comparison fails. + // Matches behavior in llvm::ConstantFoldCompareInstruction. + switch (ISD::getUnorderedFlavor(Cond)) { + default: + llvm_unreachable("Unknown flavor!"); + case 0: // Known false. + return getBoolConstant(false, dl, VT, OpVT); + case 1: // Known true. + return getBoolConstant(true, dl, VT, OpVT); + case 2: // Undefined. + return getUNDEF(VT); } } @@ -2062,16 +2122,32 @@ SDValue SelectionDAG::FoldSetCC(EVT VT, SDValue N1, SDValue N2, } /// See if the specified operand can be simplified with the knowledge that only -/// the bits specified by Mask are used. -SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &Mask) { +/// the bits specified by DemandedBits are used. +/// TODO: really we should be making this into the DAG equivalent of +/// SimplifyMultipleUseDemandedBits and not generate any new nodes. +SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits) { + EVT VT = V.getValueType(); + APInt DemandedElts = VT.isVector() + ? APInt::getAllOnesValue(VT.getVectorNumElements()) + : APInt(1, 1); + return GetDemandedBits(V, DemandedBits, DemandedElts); +} + +/// See if the specified operand can be simplified with the knowledge that only +/// the bits specified by DemandedBits are used in the elements specified by +/// DemandedElts. +/// TODO: really we should be making this into the DAG equivalent of +/// SimplifyMultipleUseDemandedBits and not generate any new nodes. +SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &DemandedBits, + const APInt &DemandedElts) { switch (V.getOpcode()) { default: break; case ISD::Constant: { - const ConstantSDNode *CV = cast(V.getNode()); + auto *CV = cast(V.getNode()); assert(CV && "Const value should be ConstSDNode."); const APInt &CVal = CV->getAPIntValue(); - APInt NewVal = CVal & Mask; + APInt NewVal = CVal & DemandedBits; if (NewVal != CVal) return getConstant(NewVal, SDLoc(V), V.getValueType()); break; @@ -2079,44 +2155,51 @@ SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &Mask) { case ISD::OR: case ISD::XOR: // If the LHS or RHS don't contribute bits to the or, drop them. - if (MaskedValueIsZero(V.getOperand(0), Mask)) + if (MaskedValueIsZero(V.getOperand(0), DemandedBits)) return V.getOperand(1); - if (MaskedValueIsZero(V.getOperand(1), Mask)) + if (MaskedValueIsZero(V.getOperand(1), DemandedBits)) return V.getOperand(0); break; case ISD::SRL: // Only look at single-use SRLs. if (!V.getNode()->hasOneUse()) break; - if (ConstantSDNode *RHSC = dyn_cast(V.getOperand(1))) { + if (auto *RHSC = dyn_cast(V.getOperand(1))) { // See if we can recursively simplify the LHS. unsigned Amt = RHSC->getZExtValue(); // Watch out for shift count overflow though. - if (Amt >= Mask.getBitWidth()) + if (Amt >= DemandedBits.getBitWidth()) break; - APInt NewMask = Mask << Amt; - if (SDValue SimplifyLHS = GetDemandedBits(V.getOperand(0), NewMask)) + APInt SrcDemandedBits = DemandedBits << Amt; + if (SDValue SimplifyLHS = + GetDemandedBits(V.getOperand(0), SrcDemandedBits)) return getNode(ISD::SRL, SDLoc(V), V.getValueType(), SimplifyLHS, V.getOperand(1)); } break; case ISD::AND: { // X & -1 -> X (ignoring bits which aren't demanded). - ConstantSDNode *AndVal = isConstOrConstSplat(V.getOperand(1)); - if (AndVal && Mask.isSubsetOf(AndVal->getAPIntValue())) - return V.getOperand(0); + // Also handle the case where masked out bits in X are known to be zero. + if (ConstantSDNode *RHSC = isConstOrConstSplat(V.getOperand(1))) { + const APInt &AndVal = RHSC->getAPIntValue(); + if (DemandedBits.isSubsetOf(AndVal) || + DemandedBits.isSubsetOf(computeKnownBits(V.getOperand(0)).Zero | + AndVal)) + return V.getOperand(0); + } break; } case ISD::ANY_EXTEND: { SDValue Src = V.getOperand(0); unsigned SrcBitWidth = Src.getScalarValueSizeInBits(); // Being conservative here - only peek through if we only demand bits in the - // non-extended source (even though the extended bits are technically undef). - if (Mask.getActiveBits() > SrcBitWidth) + // non-extended source (even though the extended bits are technically + // undef). + if (DemandedBits.getActiveBits() > SrcBitWidth) break; - APInt SrcMask = Mask.trunc(SrcBitWidth); - if (SDValue DemandedSrc = GetDemandedBits(Src, SrcMask)) + APInt SrcDemandedBits = DemandedBits.trunc(SrcBitWidth); + if (SDValue DemandedSrc = GetDemandedBits(Src, SrcDemandedBits)) return getNode(ISD::ANY_EXTEND, SDLoc(V), V.getValueType(), DemandedSrc); break; } @@ -2125,7 +2208,7 @@ SDValue SelectionDAG::GetDemandedBits(SDValue V, const APInt &Mask) { unsigned ExVTBits = ExVT.getScalarSizeInBits(); // If none of the extended bits are demanded, eliminate the sextinreg. - if (Mask.getActiveBits() <= ExVTBits) + if (DemandedBits.getActiveBits() <= ExVTBits) return V.getOperand(0); break; @@ -2143,9 +2226,28 @@ bool SelectionDAG::SignBitIsZero(SDValue Op, unsigned Depth) const { /// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero. We use /// this predicate to simplify operations downstream. Mask is known to be zero /// for bits that V cannot have. -bool SelectionDAG::MaskedValueIsZero(SDValue Op, const APInt &Mask, +bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask, + unsigned Depth) const { + EVT VT = V.getValueType(); + APInt DemandedElts = VT.isVector() + ? APInt::getAllOnesValue(VT.getVectorNumElements()) + : APInt(1, 1); + return MaskedValueIsZero(V, Mask, DemandedElts, Depth); +} + +/// MaskedValueIsZero - Return true if 'V & Mask' is known to be zero in +/// DemandedElts. We use this predicate to simplify operations downstream. +/// Mask is known to be zero for bits that V cannot have. +bool SelectionDAG::MaskedValueIsZero(SDValue V, const APInt &Mask, + const APInt &DemandedElts, unsigned Depth) const { - return Mask.isSubsetOf(computeKnownBits(Op, Depth).Zero); + return Mask.isSubsetOf(computeKnownBits(V, DemandedElts, Depth).Zero); +} + +/// MaskedValueIsAllOnes - Return true if '(Op & Mask) == Mask'. +bool SelectionDAG::MaskedValueIsAllOnes(SDValue V, const APInt &Mask, + unsigned Depth) const { + return Mask.isSubsetOf(computeKnownBits(V, Depth).One); } /// isSplatValue - Return true if the vector V has the same value @@ -2244,28 +2346,50 @@ bool SelectionDAG::isSplatValue(SDValue V, bool AllowUndefs) { (AllowUndefs || !UndefElts); } -/// Helper function that checks to see if a node is a constant or a -/// build vector of splat constants at least within the demanded elts. -static ConstantSDNode *isConstOrDemandedConstSplat(SDValue N, - const APInt &DemandedElts) { - if (ConstantSDNode *CN = dyn_cast(N)) - return CN; - if (N.getOpcode() != ISD::BUILD_VECTOR) - return nullptr; - EVT VT = N.getValueType(); - ConstantSDNode *Cst = nullptr; - unsigned NumElts = VT.getVectorNumElements(); - assert(DemandedElts.getBitWidth() == NumElts && "Unexpected vector size"); - for (unsigned i = 0; i != NumElts; ++i) { - if (!DemandedElts[i]) - continue; - ConstantSDNode *C = dyn_cast(N.getOperand(i)); - if (!C || (Cst && Cst->getAPIntValue() != C->getAPIntValue()) || - C->getValueType(0) != VT.getScalarType()) - return nullptr; - Cst = C; +SDValue SelectionDAG::getSplatSourceVector(SDValue V, int &SplatIdx) { + V = peekThroughExtractSubvectors(V); + + EVT VT = V.getValueType(); + unsigned Opcode = V.getOpcode(); + switch (Opcode) { + default: { + APInt UndefElts; + APInt DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements()); + if (isSplatValue(V, DemandedElts, UndefElts)) { + // Handle case where all demanded elements are UNDEF. + if (DemandedElts.isSubsetOf(UndefElts)) { + SplatIdx = 0; + return getUNDEF(VT); + } + SplatIdx = (UndefElts & DemandedElts).countTrailingOnes(); + return V; + } + break; + } + case ISD::VECTOR_SHUFFLE: { + // Check if this is a shuffle node doing a splat. + // TODO - remove this and rely purely on SelectionDAG::isSplatValue, + // getTargetVShiftNode currently struggles without the splat source. + auto *SVN = cast(V); + if (!SVN->isSplat()) + break; + int Idx = SVN->getSplatIndex(); + int NumElts = V.getValueType().getVectorNumElements(); + SplatIdx = Idx % NumElts; + return V.getOperand(Idx / NumElts); } - return Cst; + } + + return SDValue(); +} + +SDValue SelectionDAG::getSplatValue(SDValue V) { + int SplatIdx; + if (SDValue SrcVector = getSplatSourceVector(V, SplatIdx)) + return getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), + SrcVector.getValueType().getScalarType(), SrcVector, + getIntPtrConstant(SplatIdx, SDLoc(V))); + return SDValue(); } /// If a SHL/SRA/SRL node has a constant or splat constant shift amount that @@ -2708,8 +2832,7 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, break; case ISD::FSHL: case ISD::FSHR: - if (ConstantSDNode *C = - isConstOrDemandedConstSplat(Op.getOperand(2), DemandedElts)) { + if (ConstantSDNode *C = isConstOrConstSplat(Op.getOperand(2), DemandedElts)) { unsigned Amt = C->getAPIntValue().urem(BitWidth); // For fshl, 0-shift returns the 1st arg. @@ -2801,8 +2924,59 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, } case ISD::LOAD: { LoadSDNode *LD = cast(Op); - // If this is a ZEXTLoad and we are looking at the loaded value. - if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) { + const Constant *Cst = TLI->getTargetConstantFromLoad(LD); + if (ISD::isNON_EXTLoad(LD) && Cst) { + // Determine any common known bits from the loaded constant pool value. + Type *CstTy = Cst->getType(); + if ((NumElts * BitWidth) == CstTy->getPrimitiveSizeInBits()) { + // If its a vector splat, then we can (quickly) reuse the scalar path. + // NOTE: We assume all elements match and none are UNDEF. + if (CstTy->isVectorTy()) { + if (const Constant *Splat = Cst->getSplatValue()) { + Cst = Splat; + CstTy = Cst->getType(); + } + } + // TODO - do we need to handle different bitwidths? + if (CstTy->isVectorTy() && BitWidth == CstTy->getScalarSizeInBits()) { + // Iterate across all vector elements finding common known bits. + Known.One.setAllBits(); + Known.Zero.setAllBits(); + for (unsigned i = 0; i != NumElts; ++i) { + if (!DemandedElts[i]) + continue; + if (Constant *Elt = Cst->getAggregateElement(i)) { + if (auto *CInt = dyn_cast(Elt)) { + const APInt &Value = CInt->getValue(); + Known.One &= Value; + Known.Zero &= ~Value; + continue; + } + if (auto *CFP = dyn_cast(Elt)) { + APInt Value = CFP->getValueAPF().bitcastToAPInt(); + Known.One &= Value; + Known.Zero &= ~Value; + continue; + } + } + Known.One.clearAllBits(); + Known.Zero.clearAllBits(); + break; + } + } else if (BitWidth == CstTy->getPrimitiveSizeInBits()) { + if (auto *CInt = dyn_cast(Cst)) { + const APInt &Value = CInt->getValue(); + Known.One = Value; + Known.Zero = ~Value; + } else if (auto *CFP = dyn_cast(Cst)) { + APInt Value = CFP->getValueAPF().bitcastToAPInt(); + Known.One = Value; + Known.Zero = ~Value; + } + } + } + } else if (ISD::isZEXTLoad(Op.getNode()) && Op.getResNo() == 0) { + // If this is a ZEXTLoad and we are looking at the loaded value. EVT VT = LD->getMemoryVT(); unsigned MemBits = VT.getScalarSizeInBits(); Known.Zero.setBitsFrom(MemBits); @@ -2816,15 +2990,12 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, EVT InVT = Op.getOperand(0).getValueType(); APInt InDemandedElts = DemandedElts.zextOrSelf(InVT.getVectorNumElements()); Known = computeKnownBits(Op.getOperand(0), InDemandedElts, Depth + 1); - Known = Known.zext(BitWidth); - Known.Zero.setBitsFrom(InVT.getScalarSizeInBits()); + Known = Known.zext(BitWidth, true /* ExtendedBitsAreKnownZero */); break; } case ISD::ZERO_EXTEND: { - EVT InVT = Op.getOperand(0).getValueType(); Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); - Known = Known.zext(BitWidth); - Known.Zero.setBitsFrom(InVT.getScalarSizeInBits()); + Known = Known.zext(BitWidth, true /* ExtendedBitsAreKnownZero */); break; } case ISD::SIGN_EXTEND_VECTOR_INREG: { @@ -2845,7 +3016,7 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, } case ISD::ANY_EXTEND: { Known = computeKnownBits(Op.getOperand(0), Depth+1); - Known = Known.zext(BitWidth); + Known = Known.zext(BitWidth, false /* ExtendedBitsAreKnownZero */); break; } case ISD::TRUNCATE: { @@ -2878,39 +3049,10 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, LLVM_FALLTHROUGH; case ISD::SUB: case ISD::SUBC: { - if (ConstantSDNode *CLHS = isConstOrConstSplat(Op.getOperand(0))) { - // We know that the top bits of C-X are clear if X contains less bits - // than C (i.e. no wrap-around can happen). For example, 20-X is - // positive if we can prove that X is >= 0 and < 16. - if (CLHS->getAPIntValue().isNonNegative()) { - unsigned NLZ = (CLHS->getAPIntValue()+1).countLeadingZeros(); - // NLZ can't be BitWidth with no sign bit - APInt MaskV = APInt::getHighBitsSet(BitWidth, NLZ+1); - Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, - Depth + 1); - - // If all of the MaskV bits are known to be zero, then we know the - // output top bits are zero, because we now know that the output is - // from [0-C]. - if ((Known2.Zero & MaskV) == MaskV) { - unsigned NLZ2 = CLHS->getAPIntValue().countLeadingZeros(); - // Top bits known zero. - Known.Zero.setHighBits(NLZ2); - } - } - } - - // If low bits are know to be zero in both operands, then we know they are - // going to be 0 in the result. Both addition and complement operations - // preserve the low zero bits. - Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); - unsigned KnownZeroLow = Known2.countMinTrailingZeros(); - if (KnownZeroLow == 0) - break; - + Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); - KnownZeroLow = std::min(KnownZeroLow, Known2.countMinTrailingZeros()); - Known.Zero.setLowBits(KnownZeroLow); + Known = KnownBits::computeForAddSub(/* Add */ false, /* NSW */ false, + Known, Known2); break; } case ISD::UADDO: @@ -2928,34 +3070,26 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, case ISD::ADD: case ISD::ADDC: case ISD::ADDE: { - // Output known-0 bits are known if clear or set in both the low clear bits - // common to both LHS & RHS. For example, 8+(X<<3) is known to have the - // low 3 bits clear. - // Output known-0 bits are also known if the top bits of each input are - // known to be clear. For example, if one input has the top 10 bits clear - // and the other has the top 8 bits clear, we know the top 7 bits of the - // output must be clear. - Known2 = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); - unsigned KnownZeroHigh = Known2.countMinLeadingZeros(); - unsigned KnownZeroLow = Known2.countMinTrailingZeros(); + assert(Op.getResNo() == 0 && "We only compute knownbits for the sum here."); + + // With ADDE and ADDCARRY, a carry bit may be added in. + KnownBits Carry(1); + if (Opcode == ISD::ADDE) + // Can't track carry from glue, set carry to unknown. + Carry.resetAll(); + else if (Opcode == ISD::ADDCARRY) + // TODO: Compute known bits for the carry operand. Not sure if it is worth + // the trouble (how often will we find a known carry bit). And I haven't + // tested this very much yet, but something like this might work: + // Carry = computeKnownBits(Op.getOperand(2), DemandedElts, Depth + 1); + // Carry = Carry.zextOrTrunc(1, false); + Carry.resetAll(); + else + Carry.setAllZero(); + Known = computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1); Known2 = computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1); - KnownZeroHigh = std::min(KnownZeroHigh, Known2.countMinLeadingZeros()); - KnownZeroLow = std::min(KnownZeroLow, Known2.countMinTrailingZeros()); - - if (Opcode == ISD::ADDE || Opcode == ISD::ADDCARRY) { - // With ADDE and ADDCARRY, a carry bit may be added in, so we can only - // use this information if we know (at least) that the low two bits are - // clear. We then return to the caller that the low bit is unknown but - // that other bits are known zero. - if (KnownZeroLow >= 2) - Known.Zero.setBits(1, KnownZeroLow); - break; - } - - Known.Zero.setLowBits(KnownZeroLow); - if (KnownZeroHigh > 1) - Known.Zero.setHighBits(KnownZeroHigh - 1); + Known = KnownBits::computeForAddCarry(Known, Known2, Carry); break; } case ISD::SREM: @@ -3010,21 +3144,20 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, case ISD::EXTRACT_ELEMENT: { Known = computeKnownBits(Op.getOperand(0), Depth+1); const unsigned Index = Op.getConstantOperandVal(1); - const unsigned BitWidth = Op.getValueSizeInBits(); + const unsigned EltBitWidth = Op.getValueSizeInBits(); // Remove low part of known bits mask - Known.Zero = Known.Zero.getHiBits(Known.Zero.getBitWidth() - Index * BitWidth); - Known.One = Known.One.getHiBits(Known.One.getBitWidth() - Index * BitWidth); + Known.Zero = Known.Zero.getHiBits(Known.getBitWidth() - Index * EltBitWidth); + Known.One = Known.One.getHiBits(Known.getBitWidth() - Index * EltBitWidth); // Remove high part of known bit mask - Known = Known.trunc(BitWidth); + Known = Known.trunc(EltBitWidth); break; } case ISD::EXTRACT_VECTOR_ELT: { SDValue InVec = Op.getOperand(0); SDValue EltNo = Op.getOperand(1); EVT VecVT = InVec.getValueType(); - const unsigned BitWidth = Op.getValueSizeInBits(); const unsigned EltBitWidth = VecVT.getScalarSizeInBits(); const unsigned NumSrcElts = VecVT.getVectorNumElements(); // If BitWidth > EltBitWidth the value is anyext:ed. So we do not know @@ -3042,7 +3175,7 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, Known = computeKnownBits(InVec, Depth + 1); } if (BitWidth > EltBitWidth) - Known = Known.zext(BitWidth); + Known = Known.zext(BitWidth, false /* => any extend */); break; } case ISD::INSERT_VECTOR_ELT: { @@ -3146,10 +3279,10 @@ KnownBits SelectionDAG::computeKnownBits(SDValue Op, const APInt &DemandedElts, // the minimum of the clamp min/max range. bool IsMax = (Opcode == ISD::SMAX); ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr; - if ((CstLow = isConstOrDemandedConstSplat(Op.getOperand(1), DemandedElts))) + if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts))) if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX)) - CstHigh = isConstOrDemandedConstSplat(Op.getOperand(0).getOperand(1), - DemandedElts); + CstHigh = + isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts); if (CstLow && CstHigh) { if (!IsMax) std::swap(CstLow, CstHigh); @@ -3430,7 +3563,7 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1); // SRA X, C -> adds C sign bits. if (ConstantSDNode *C = - isConstOrDemandedConstSplat(Op.getOperand(1), DemandedElts)) { + isConstOrConstSplat(Op.getOperand(1), DemandedElts)) { APInt ShiftVal = C->getAPIntValue(); ShiftVal += Tmp; Tmp = ShiftVal.uge(VTBits) ? VTBits : ShiftVal.getZExtValue(); @@ -3438,7 +3571,7 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, return Tmp; case ISD::SHL: if (ConstantSDNode *C = - isConstOrDemandedConstSplat(Op.getOperand(1), DemandedElts)) { + isConstOrConstSplat(Op.getOperand(1), DemandedElts)) { // shl destroys sign bits. Tmp = ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth+1); if (C->getAPIntValue().uge(VTBits) || // Bad shift. @@ -3478,10 +3611,10 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, // the minimum of the clamp min/max range. bool IsMax = (Opcode == ISD::SMAX); ConstantSDNode *CstLow = nullptr, *CstHigh = nullptr; - if ((CstLow = isConstOrDemandedConstSplat(Op.getOperand(1), DemandedElts))) + if ((CstLow = isConstOrConstSplat(Op.getOperand(1), DemandedElts))) if (Op.getOperand(0).getOpcode() == (IsMax ? ISD::SMIN : ISD::SMAX)) - CstHigh = isConstOrDemandedConstSplat(Op.getOperand(0).getOperand(1), - DemandedElts); + CstHigh = + isConstOrConstSplat(Op.getOperand(0).getOperand(1), DemandedElts); if (CstLow && CstHigh) { if (!IsMax) std::swap(CstLow, CstHigh); @@ -3621,7 +3754,6 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, SDValue InVec = Op.getOperand(0); SDValue InVal = Op.getOperand(1); SDValue EltNo = Op.getOperand(2); - unsigned NumElts = InVec.getValueType().getVectorNumElements(); ConstantSDNode *CEltNo = dyn_cast(EltNo); if (CEltNo && CEltNo->getAPIntValue().ult(NumElts)) { @@ -3752,13 +3884,43 @@ unsigned SelectionDAG::ComputeNumSignBits(SDValue Op, const APInt &DemandedElts, if (LoadSDNode *LD = dyn_cast(Op)) { unsigned ExtType = LD->getExtensionType(); switch (ExtType) { - default: break; - case ISD::SEXTLOAD: // '17' bits known - Tmp = LD->getMemoryVT().getScalarSizeInBits(); - return VTBits-Tmp+1; - case ISD::ZEXTLOAD: // '16' bits known - Tmp = LD->getMemoryVT().getScalarSizeInBits(); - return VTBits-Tmp; + default: break; + case ISD::SEXTLOAD: // e.g. i16->i32 = '17' bits known. + Tmp = LD->getMemoryVT().getScalarSizeInBits(); + return VTBits - Tmp + 1; + case ISD::ZEXTLOAD: // e.g. i16->i32 = '16' bits known. + Tmp = LD->getMemoryVT().getScalarSizeInBits(); + return VTBits - Tmp; + case ISD::NON_EXTLOAD: + if (const Constant *Cst = TLI->getTargetConstantFromLoad(LD)) { + // We only need to handle vectors - computeKnownBits should handle + // scalar cases. + Type *CstTy = Cst->getType(); + if (CstTy->isVectorTy() && + (NumElts * VTBits) == CstTy->getPrimitiveSizeInBits()) { + Tmp = VTBits; + for (unsigned i = 0; i != NumElts; ++i) { + if (!DemandedElts[i]) + continue; + if (Constant *Elt = Cst->getAggregateElement(i)) { + if (auto *CInt = dyn_cast(Elt)) { + const APInt &Value = CInt->getValue(); + Tmp = std::min(Tmp, Value.getNumSignBits()); + continue; + } + if (auto *CFP = dyn_cast(Elt)) { + APInt Value = CFP->getValueAPF().bitcastToAPInt(); + Tmp = std::min(Tmp, Value.getNumSignBits()); + continue; + } + } + // Unknown type. Conservatively assume no bits match sign bit. + return 1; + } + return Tmp; + } + } + break; } } } @@ -3803,8 +3965,7 @@ bool SelectionDAG::isBaseWithConstantOffset(SDValue Op) const { return false; if (Op.getOpcode() == ISD::OR && - !MaskedValueIsZero(Op.getOperand(0), - cast(Op.getOperand(1))->getAPIntValue())) + !MaskedValueIsZero(Op.getOperand(0), Op.getConstantOperandAPInt(1))) return false; return true; @@ -4013,7 +4174,9 @@ static SDValue FoldBUILD_VECTOR(const SDLoc &DL, EVT VT, return SDValue(); } -static SDValue FoldCONCAT_VECTORS(const SDLoc &DL, EVT VT, +/// Try to simplify vector concatenation to an input value, undef, or build +/// vector. +static SDValue foldCONCAT_VECTORS(const SDLoc &DL, EVT VT, ArrayRef Ops, SelectionDAG &DAG) { assert(!Ops.empty() && "Can't concatenate an empty list of vectors!"); @@ -4033,6 +4196,31 @@ static SDValue FoldCONCAT_VECTORS(const SDLoc &DL, EVT VT, if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); })) return DAG.getUNDEF(VT); + // Scan the operands and look for extract operations from a single source + // that correspond to insertion at the same location via this concatenation: + // concat (extract X, 0*subvec_elts), (extract X, 1*subvec_elts), ... + SDValue IdentitySrc; + bool IsIdentity = true; + for (unsigned i = 0, e = Ops.size(); i != e; ++i) { + SDValue Op = Ops[i]; + unsigned IdentityIndex = i * Op.getValueType().getVectorNumElements(); + if (Op.getOpcode() != ISD::EXTRACT_SUBVECTOR || + Op.getOperand(0).getValueType() != VT || + (IdentitySrc && Op.getOperand(0) != IdentitySrc) || + !isa(Op.getOperand(1)) || + Op.getConstantOperandVal(1) != IdentityIndex) { + IsIdentity = false; + break; + } + assert((!IdentitySrc || IdentitySrc == Op.getOperand(0)) && + "Unexpected identity source vector for concat of extracts"); + IdentitySrc = Op.getOperand(0); + } + if (IsIdentity) { + assert(IdentitySrc && "Failed to set source vector of extracts"); + return IdentitySrc; + } + // A CONCAT_VECTOR with all UNDEF/BUILD_VECTOR operands can be // simplified to one big BUILD_VECTOR. // FIXME: Add support for SCALAR_TO_VECTOR as well. @@ -4288,9 +4476,23 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, if (Operand.isUndef()) return getUNDEF(VT); break; + case ISD::FP_TO_SINT: + case ISD::FP_TO_UINT: + if (Operand.isUndef()) + return getUNDEF(VT); + break; + case ISD::SINT_TO_FP: + case ISD::UINT_TO_FP: + // [us]itofp(undef) = 0, because the result value is bounded. + if (Operand.isUndef()) + return getConstantFP(0.0, DL, VT); + break; case ISD::SIGN_EXTEND: assert(VT.isInteger() && Operand.getValueType().isInteger() && "Invalid SIGN_EXTEND!"); + assert(VT.isVector() == Operand.getValueType().isVector() && + "SIGN_EXTEND result type type should be vector iff the operand " + "type is vector!"); if (Operand.getValueType() == VT) return Operand; // noop extension assert((!VT.isVector() || VT.getVectorNumElements() == @@ -4307,6 +4509,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, case ISD::ZERO_EXTEND: assert(VT.isInteger() && Operand.getValueType().isInteger() && "Invalid ZERO_EXTEND!"); + assert(VT.isVector() == Operand.getValueType().isVector() && + "ZERO_EXTEND result type type should be vector iff the operand " + "type is vector!"); if (Operand.getValueType() == VT) return Operand; // noop extension assert((!VT.isVector() || VT.getVectorNumElements() == @@ -4323,6 +4528,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, case ISD::ANY_EXTEND: assert(VT.isInteger() && Operand.getValueType().isInteger() && "Invalid ANY_EXTEND!"); + assert(VT.isVector() == Operand.getValueType().isVector() && + "ANY_EXTEND result type type should be vector iff the operand " + "type is vector!"); if (Operand.getValueType() == VT) return Operand; // noop extension assert((!VT.isVector() || VT.getVectorNumElements() == @@ -4350,6 +4558,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, case ISD::TRUNCATE: assert(VT.isInteger() && Operand.getValueType().isInteger() && "Invalid TRUNCATE!"); + assert(VT.isVector() == Operand.getValueType().isVector() && + "TRUNCATE result type type should be vector iff the operand " + "type is vector!"); if (Operand.getValueType() == VT) return Operand; // noop truncate assert((!VT.isVector() || VT.getVectorNumElements() == @@ -4429,6 +4640,10 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, return Operand.getOperand(0); break; case ISD::FNEG: + // Negation of an unknown bag of bits is still completely undefined. + if (OpOpcode == ISD::UNDEF) + return getUNDEF(VT); + // -(X-Y) -> (Y-X) is unsafe because when X==Y, -0.0 != +0.0 if ((getTarget().Options.UnsafeFPMath || Flags.hasNoSignedZeros()) && OpOpcode == ISD::FSUB) @@ -4513,13 +4728,13 @@ static std::pair FoldValue(unsigned Opcode, const APInt &C1, } SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, - EVT VT, const ConstantSDNode *Cst1, - const ConstantSDNode *Cst2) { - if (Cst1->isOpaque() || Cst2->isOpaque()) + EVT VT, const ConstantSDNode *C1, + const ConstantSDNode *C2) { + if (C1->isOpaque() || C2->isOpaque()) return SDValue(); - std::pair Folded = FoldValue(Opcode, Cst1->getAPIntValue(), - Cst2->getAPIntValue()); + std::pair Folded = FoldValue(Opcode, C1->getAPIntValue(), + C2->getAPIntValue()); if (!Folded.second) return SDValue(); return getConstant(Folded.first, DL, VT); @@ -4532,16 +4747,16 @@ SDValue SelectionDAG::FoldSymbolOffset(unsigned Opcode, EVT VT, return SDValue(); if (!TLI->isOffsetFoldingLegal(GA)) return SDValue(); - const ConstantSDNode *Cst2 = dyn_cast(N2); - if (!Cst2) + auto *C2 = dyn_cast(N2); + if (!C2) return SDValue(); - int64_t Offset = Cst2->getSExtValue(); + int64_t Offset = C2->getSExtValue(); switch (Opcode) { case ISD::ADD: break; case ISD::SUB: Offset = -uint64_t(Offset); break; default: return SDValue(); } - return getGlobalAddress(GA->getGlobal(), SDLoc(Cst2), VT, + return getGlobalAddress(GA->getGlobal(), SDLoc(C2), VT, GA->getOffset() + uint64_t(Offset)); } @@ -4571,21 +4786,20 @@ bool SelectionDAG::isUndef(unsigned Opcode, ArrayRef Ops) { } SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, - EVT VT, SDNode *Cst1, - SDNode *Cst2) { + EVT VT, SDNode *N1, SDNode *N2) { // If the opcode is a target-specific ISD node, there's nothing we can // do here and the operand rules may not line up with the below, so // bail early. if (Opcode >= ISD::BUILTIN_OP_END) return SDValue(); - if (isUndef(Opcode, {SDValue(Cst1, 0), SDValue(Cst2, 0)})) + if (isUndef(Opcode, {SDValue(N1, 0), SDValue(N2, 0)})) return getUNDEF(VT); // Handle the case of two scalars. - if (const ConstantSDNode *Scalar1 = dyn_cast(Cst1)) { - if (const ConstantSDNode *Scalar2 = dyn_cast(Cst2)) { - SDValue Folded = FoldConstantArithmetic(Opcode, DL, VT, Scalar1, Scalar2); + if (auto *C1 = dyn_cast(N1)) { + if (auto *C2 = dyn_cast(N2)) { + SDValue Folded = FoldConstantArithmetic(Opcode, DL, VT, C1, C2); assert((!Folded || !VT.isVector()) && "Can't fold vectors ops with scalar operands"); return Folded; @@ -4593,19 +4807,19 @@ SDValue SelectionDAG::FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, } // fold (add Sym, c) -> Sym+c - if (GlobalAddressSDNode *GA = dyn_cast(Cst1)) - return FoldSymbolOffset(Opcode, VT, GA, Cst2); + if (GlobalAddressSDNode *GA = dyn_cast(N1)) + return FoldSymbolOffset(Opcode, VT, GA, N2); if (TLI->isCommutativeBinOp(Opcode)) - if (GlobalAddressSDNode *GA = dyn_cast(Cst2)) - return FoldSymbolOffset(Opcode, VT, GA, Cst1); + if (GlobalAddressSDNode *GA = dyn_cast(N2)) + return FoldSymbolOffset(Opcode, VT, GA, N1); // For vectors, extract each constant element and fold them individually. // Either input may be an undef value. - auto *BV1 = dyn_cast(Cst1); - if (!BV1 && !Cst1->isUndef()) + auto *BV1 = dyn_cast(N1); + if (!BV1 && !N1->isUndef()) return SDValue(); - auto *BV2 = dyn_cast(Cst2); - if (!BV2 && !Cst2->isUndef()) + auto *BV2 = dyn_cast(N2); + if (!BV2 && !N2->isUndef()) return SDValue(); // If both operands are undef, that's handled the same way as scalars. if (!BV1 && !BV2) @@ -4755,6 +4969,64 @@ SDValue SelectionDAG::FoldConstantVectorArithmetic(unsigned Opcode, return V; } +SDValue SelectionDAG::foldConstantFPMath(unsigned Opcode, const SDLoc &DL, + EVT VT, SDValue N1, SDValue N2) { + // TODO: We don't do any constant folding for strict FP opcodes here, but we + // should. That will require dealing with a potentially non-default + // rounding mode, checking the "opStatus" return value from the APFloat + // math calculations, and possibly other variations. + auto *N1CFP = dyn_cast(N1.getNode()); + auto *N2CFP = dyn_cast(N2.getNode()); + if (N1CFP && N2CFP) { + APFloat C1 = N1CFP->getValueAPF(), C2 = N2CFP->getValueAPF(); + switch (Opcode) { + case ISD::FADD: + C1.add(C2, APFloat::rmNearestTiesToEven); + return getConstantFP(C1, DL, VT); + case ISD::FSUB: + C1.subtract(C2, APFloat::rmNearestTiesToEven); + return getConstantFP(C1, DL, VT); + case ISD::FMUL: + C1.multiply(C2, APFloat::rmNearestTiesToEven); + return getConstantFP(C1, DL, VT); + case ISD::FDIV: + C1.divide(C2, APFloat::rmNearestTiesToEven); + return getConstantFP(C1, DL, VT); + case ISD::FREM: + C1.mod(C2); + return getConstantFP(C1, DL, VT); + case ISD::FCOPYSIGN: + C1.copySign(C2); + return getConstantFP(C1, DL, VT); + default: break; + } + } + if (N1CFP && Opcode == ISD::FP_ROUND) { + APFloat C1 = N1CFP->getValueAPF(); // make copy + bool Unused; + // This can return overflow, underflow, or inexact; we don't care. + // FIXME need to be more flexible about rounding mode. + (void) C1.convert(EVTToAPFloatSemantics(VT), APFloat::rmNearestTiesToEven, + &Unused); + return getConstantFP(C1, DL, VT); + } + + switch (Opcode) { + case ISD::FADD: + case ISD::FSUB: + case ISD::FMUL: + case ISD::FDIV: + case ISD::FREM: + // If both operands are undef, the result is undef. If 1 operand is undef, + // the result is NaN. This should match the behavior of the IR optimizer. + if (N1.isUndef() && N2.isUndef()) + return getUNDEF(VT); + if (N1.isUndef() || N2.isUndef()) + return getConstantFP(APFloat::getNaN(EVTToAPFloatSemantics(VT)), DL, VT); + } + return SDValue(); +} + SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1, SDValue N2, const SDNodeFlags Flags) { ConstantSDNode *N1C = dyn_cast(N1); @@ -4791,9 +5063,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, break; } case ISD::CONCAT_VECTORS: { - // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF. SDValue Ops[] = {N1, N2}; - if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this)) + if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this)) return V; break; } @@ -4847,6 +5118,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, assert(VT.isFloatingPoint() && "This operator only applies to FP types!"); assert(N1.getValueType() == N2.getValueType() && N1.getValueType() == VT && "Binary operator types must match!"); + if (SDValue V = simplifyFPBinop(Opcode, N1, N2)) + return V; break; case ISD::FCOPYSIGN: // N1 and result must match. N1/N2 need not match. assert(N1.getValueType() == VT && @@ -5100,73 +5373,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, FoldConstantArithmetic(Opcode, DL, VT, N1.getNode(), N2.getNode())) return SV; - // Constant fold FP operations. - bool HasFPExceptions = TLI->hasFloatingPointExceptions(); - if (N1CFP) { - if (N2CFP) { - APFloat V1 = N1CFP->getValueAPF(), V2 = N2CFP->getValueAPF(); - APFloat::opStatus s; - switch (Opcode) { - case ISD::FADD: - s = V1.add(V2, APFloat::rmNearestTiesToEven); - if (!HasFPExceptions || s != APFloat::opInvalidOp) - return getConstantFP(V1, DL, VT); - break; - case ISD::FSUB: - s = V1.subtract(V2, APFloat::rmNearestTiesToEven); - if (!HasFPExceptions || s!=APFloat::opInvalidOp) - return getConstantFP(V1, DL, VT); - break; - case ISD::FMUL: - s = V1.multiply(V2, APFloat::rmNearestTiesToEven); - if (!HasFPExceptions || s!=APFloat::opInvalidOp) - return getConstantFP(V1, DL, VT); - break; - case ISD::FDIV: - s = V1.divide(V2, APFloat::rmNearestTiesToEven); - if (!HasFPExceptions || (s!=APFloat::opInvalidOp && - s!=APFloat::opDivByZero)) { - return getConstantFP(V1, DL, VT); - } - break; - case ISD::FREM : - s = V1.mod(V2); - if (!HasFPExceptions || (s!=APFloat::opInvalidOp && - s!=APFloat::opDivByZero)) { - return getConstantFP(V1, DL, VT); - } - break; - case ISD::FCOPYSIGN: - V1.copySign(V2); - return getConstantFP(V1, DL, VT); - default: break; - } - } - - if (Opcode == ISD::FP_ROUND) { - APFloat V = N1CFP->getValueAPF(); // make copy - bool ignored; - // This can return overflow, underflow, or inexact; we don't care. - // FIXME need to be more flexible about rounding mode. - (void)V.convert(EVTToAPFloatSemantics(VT), - APFloat::rmNearestTiesToEven, &ignored); - return getConstantFP(V, DL, VT); - } - } - - switch (Opcode) { - case ISD::FADD: - case ISD::FSUB: - case ISD::FMUL: - case ISD::FDIV: - case ISD::FREM: - // If both operands are undef, the result is undef. If 1 operand is undef, - // the result is NaN. This should match the behavior of the IR optimizer. - if (N1.isUndef() && N2.isUndef()) - return getUNDEF(VT); - if (N1.isUndef() || N2.isUndef()) - return getConstantFP(APFloat::getNaN(EVTToAPFloatSemantics(VT)), DL, VT); - } + if (SDValue V = foldConstantFPMath(Opcode, DL, VT, N1, N2)) + return V; // Canonicalize an UNDEF to the RHS, even over a constant. if (N1.isUndef()) { @@ -5261,10 +5469,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, APFloat V1 = N1CFP->getValueAPF(); const APFloat &V2 = N2CFP->getValueAPF(); const APFloat &V3 = N3CFP->getValueAPF(); - APFloat::opStatus s = - V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven); - if (!TLI->hasFloatingPointExceptions() || s != APFloat::opInvalidOp) - return getConstantFP(V1, DL, VT); + V1.fusedMultiplyAdd(V2, V3, APFloat::rmNearestTiesToEven); + return getConstantFP(V1, DL, VT); } break; } @@ -5276,9 +5482,8 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, break; } case ISD::CONCAT_VECTORS: { - // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF. SDValue Ops[] = {N1, N2, N3}; - if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this)) + if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this)) return V; break; } @@ -5317,6 +5522,9 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, break; } case ISD::INSERT_SUBVECTOR: { + // Inserting undef into undef is still undef. + if (N1.isUndef() && N2.isUndef()) + return getUNDEF(VT); SDValue Index = N3; if (VT.isSimple() && N1.getValueType().isSimple() && N2.getValueType().isSimple()) { @@ -5337,6 +5545,12 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, // Trivial insertion. if (VT.getSimpleVT() == N2.getSimpleValueType()) return N2; + + // If this is an insert of an extracted vector into an undef vector, we + // can just use the input to the extract. + if (N1.isUndef() && N2.getOpcode() == ISD::EXTRACT_SUBVECTOR && + N2.getOperand(1) == N3 && N2.getOperand(0).getValueType() == VT) + return N2.getOperand(0); } break; } @@ -5521,116 +5735,12 @@ static bool isMemSrcFromConstant(SDValue Src, ConstantDataArraySlice &Slice) { SrcDelta + G->getOffset()); } -/// Determines the optimal series of memory ops to replace the memset / memcpy. -/// Return true if the number of memory ops is below the threshold (Limit). -/// It returns the types of the sequence of memory ops to perform -/// memset / memcpy by reference. -static bool FindOptimalMemOpLowering(std::vector &MemOps, - unsigned Limit, uint64_t Size, - unsigned DstAlign, unsigned SrcAlign, - bool IsMemset, - bool ZeroMemset, - bool MemcpyStrSrc, - bool AllowOverlap, - unsigned DstAS, unsigned SrcAS, - SelectionDAG &DAG, - const TargetLowering &TLI) { - assert((SrcAlign == 0 || SrcAlign >= DstAlign) && - "Expecting memcpy / memset source to meet alignment requirement!"); - // If 'SrcAlign' is zero, that means the memory operation does not need to - // load the value, i.e. memset or memcpy from constant string. Otherwise, - // it's the inferred alignment of the source. 'DstAlign', on the other hand, - // is the specified alignment of the memory operation. If it is zero, that - // means it's possible to change the alignment of the destination. - // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does - // not need to be loaded. - EVT VT = TLI.getOptimalMemOpType(Size, DstAlign, SrcAlign, - IsMemset, ZeroMemset, MemcpyStrSrc, - DAG.getMachineFunction()); - - if (VT == MVT::Other) { - // Use the largest integer type whose alignment constraints are satisfied. - // We only need to check DstAlign here as SrcAlign is always greater or - // equal to DstAlign (or zero). - VT = MVT::i64; - while (DstAlign && DstAlign < VT.getSizeInBits() / 8 && - !TLI.allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign)) - VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1); - assert(VT.isInteger()); - - // Find the largest legal integer type. - MVT LVT = MVT::i64; - while (!TLI.isTypeLegal(LVT)) - LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1); - assert(LVT.isInteger()); - - // If the type we've chosen is larger than the largest legal integer type - // then use that instead. - if (VT.bitsGT(LVT)) - VT = LVT; - } - - unsigned NumMemOps = 0; - while (Size != 0) { - unsigned VTSize = VT.getSizeInBits() / 8; - while (VTSize > Size) { - // For now, only use non-vector load / store's for the left-over pieces. - EVT NewVT = VT; - unsigned NewVTSize; - - bool Found = false; - if (VT.isVector() || VT.isFloatingPoint()) { - NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32; - if (TLI.isOperationLegalOrCustom(ISD::STORE, NewVT) && - TLI.isSafeMemOpType(NewVT.getSimpleVT())) - Found = true; - else if (NewVT == MVT::i64 && - TLI.isOperationLegalOrCustom(ISD::STORE, MVT::f64) && - TLI.isSafeMemOpType(MVT::f64)) { - // i64 is usually not legal on 32-bit targets, but f64 may be. - NewVT = MVT::f64; - Found = true; - } - } - - if (!Found) { - do { - NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1); - if (NewVT == MVT::i8) - break; - } while (!TLI.isSafeMemOpType(NewVT.getSimpleVT())); - } - NewVTSize = NewVT.getSizeInBits() / 8; - - // If the new VT cannot cover all of the remaining bits, then consider - // issuing a (or a pair of) unaligned and overlapping load / store. - bool Fast; - if (NumMemOps && AllowOverlap && NewVTSize < Size && - TLI.allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign, &Fast) && - Fast) - VTSize = Size; - else { - VT = NewVT; - VTSize = NewVTSize; - } - } - - if (++NumMemOps > Limit) - return false; - - MemOps.push_back(VT); - Size -= VTSize; - } - - return true; -} - static bool shouldLowerMemFuncForSize(const MachineFunction &MF) { // On Darwin, -Os means optimize for size without hurting performance, so // only really optimize for size when -Oz (MinSize) is used. if (MF.getTarget().getTargetTriple().isOSDarwin()) - return MF.getFunction().optForMinSize(); - return MF.getFunction().optForSize(); + return MF.getFunction().hasMinSize(); + return MF.getFunction().hasOptSize(); } static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl, @@ -5665,6 +5775,7 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) { // Turn a memcpy of undef to nop. + // FIXME: We need to honor volatile even is Src is undef. if (Src.isUndef()) return Chain; @@ -5691,13 +5802,12 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, bool isZeroConstant = CopyFromConstant && Slice.Array == nullptr; unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemcpy(OptSize); - if (!FindOptimalMemOpLowering(MemOps, Limit, Size, - (DstAlignCanChange ? 0 : Align), - (isZeroConstant ? 0 : SrcAlign), - false, false, CopyFromConstant, true, - DstPtrInfo.getAddrSpace(), - SrcPtrInfo.getAddrSpace(), - DAG, TLI)) + if (!TLI.findOptimalMemOpLowering( + MemOps, Limit, Size, (DstAlignCanChange ? 0 : Align), + (isZeroConstant ? 0 : SrcAlign), /*IsMemset=*/false, + /*ZeroMemset=*/false, /*MemcpyStrSrc=*/CopyFromConstant, + /*AllowOverlap=*/!isVol, DstPtrInfo.getAddrSpace(), + SrcPtrInfo.getAddrSpace(), MF.getFunction().getAttributes())) return SDValue(); if (DstAlignCanChange) { @@ -5851,6 +5961,7 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) { // Turn a memmove of undef to nop. + // FIXME: We need to honor volatile even is Src is undef. if (Src.isUndef()) return Chain; @@ -5871,13 +5982,15 @@ static SDValue getMemmoveLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, if (Align > SrcAlign) SrcAlign = Align; unsigned Limit = AlwaysInline ? ~0U : TLI.getMaxStoresPerMemmove(OptSize); - - if (!FindOptimalMemOpLowering(MemOps, Limit, Size, - (DstAlignCanChange ? 0 : Align), SrcAlign, - false, false, false, false, - DstPtrInfo.getAddrSpace(), - SrcPtrInfo.getAddrSpace(), - DAG, TLI)) + // FIXME: `AllowOverlap` should really be `!isVol` but there is a bug in + // findOptimalMemOpLowering. Meanwhile, setting it to `false` produces the + // correct code. + bool AllowOverlap = false; + if (!TLI.findOptimalMemOpLowering( + MemOps, Limit, Size, (DstAlignCanChange ? 0 : Align), SrcAlign, + /*IsMemset=*/false, /*ZeroMemset=*/false, /*MemcpyStrSrc=*/false, + AllowOverlap, DstPtrInfo.getAddrSpace(), SrcPtrInfo.getAddrSpace(), + MF.getFunction().getAttributes())) return SDValue(); if (DstAlignCanChange) { @@ -5956,6 +6069,7 @@ static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl, uint64_t Size, unsigned Align, bool isVol, MachinePointerInfo DstPtrInfo) { // Turn a memset of undef to nop. + // FIXME: We need to honor volatile even is Src is undef. if (Src.isUndef()) return Chain; @@ -5972,11 +6086,12 @@ static SDValue getMemsetStores(SelectionDAG &DAG, const SDLoc &dl, DstAlignCanChange = true; bool IsZeroVal = isa(Src) && cast(Src)->isNullValue(); - if (!FindOptimalMemOpLowering(MemOps, TLI.getMaxStoresPerMemset(OptSize), - Size, (DstAlignCanChange ? 0 : Align), 0, - true, IsZeroVal, false, true, - DstPtrInfo.getAddrSpace(), ~0u, - DAG, TLI)) + if (!TLI.findOptimalMemOpLowering( + MemOps, TLI.getMaxStoresPerMemset(OptSize), Size, + (DstAlignCanChange ? 0 : Align), 0, /*IsMemset=*/true, + /*ZeroMemset=*/IsZeroVal, /*MemcpyStrSrc=*/false, + /*AllowOverlap=*/!isVol, DstPtrInfo.getAddrSpace(), ~0u, + MF.getFunction().getAttributes())) return SDValue(); if (DstAlignCanChange) { @@ -6097,9 +6212,11 @@ SDValue SelectionDAG::getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, // Emit a library call. TargetLowering::ArgListTy Args; TargetLowering::ArgListEntry Entry; - Entry.Ty = getDataLayout().getIntPtrType(*getContext()); + Entry.Ty = Type::getInt8PtrTy(*getContext()); Entry.Node = Dst; Args.push_back(Entry); Entry.Node = Src; Args.push_back(Entry); + + Entry.Ty = getDataLayout().getIntPtrType(*getContext()); Entry.Node = Size; Args.push_back(Entry); // FIXME: pass in SDLoc TargetLowering::CallLoweringInfo CLI(*this); @@ -6199,9 +6316,11 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, // Emit a library call. TargetLowering::ArgListTy Args; TargetLowering::ArgListEntry Entry; - Entry.Ty = getDataLayout().getIntPtrType(*getContext()); + Entry.Ty = Type::getInt8PtrTy(*getContext()); Entry.Node = Dst; Args.push_back(Entry); Entry.Node = Src; Args.push_back(Entry); + + Entry.Ty = getDataLayout().getIntPtrType(*getContext()); Entry.Node = Size; Args.push_back(Entry); // FIXME: pass in SDLoc TargetLowering::CallLoweringInfo CLI(*this); @@ -6294,16 +6413,15 @@ SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace()); // Emit a library call. - Type *IntPtrTy = getDataLayout().getIntPtrType(*getContext()); TargetLowering::ArgListTy Args; TargetLowering::ArgListEntry Entry; - Entry.Node = Dst; Entry.Ty = IntPtrTy; + Entry.Node = Dst; Entry.Ty = Type::getInt8PtrTy(*getContext()); Args.push_back(Entry); Entry.Node = Src; Entry.Ty = Src.getValueType().getTypeForEVT(*getContext()); Args.push_back(Entry); Entry.Node = Size; - Entry.Ty = IntPtrTy; + Entry.Ty = getDataLayout().getIntPtrType(*getContext()); Args.push_back(Entry); // FIXME: pass in SDLoc @@ -6384,32 +6502,6 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, return SDValue(N, 0); } -SDValue SelectionDAG::getAtomicCmpSwap( - unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, - SDValue Ptr, SDValue Cmp, SDValue Swp, MachinePointerInfo PtrInfo, - unsigned Alignment, AtomicOrdering SuccessOrdering, - AtomicOrdering FailureOrdering, SyncScope::ID SSID) { - assert(Opcode == ISD::ATOMIC_CMP_SWAP || - Opcode == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS); - assert(Cmp.getValueType() == Swp.getValueType() && "Invalid Atomic Op Types"); - - if (Alignment == 0) // Ensure that codegen never sees alignment 0 - Alignment = getEVTAlignment(MemVT); - - MachineFunction &MF = getMachineFunction(); - - // FIXME: Volatile isn't really correct; we should keep track of atomic - // orderings in the memoperand. - auto Flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad | - MachineMemOperand::MOStore; - MachineMemOperand *MMO = - MF.getMachineMemOperand(PtrInfo, Flags, MemVT.getStoreSize(), Alignment, - AAMDNodes(), nullptr, SSID, SuccessOrdering, - FailureOrdering); - - return getAtomicCmpSwap(Opcode, dl, MemVT, VTs, Chain, Ptr, Cmp, Swp, MMO); -} - SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDVTList VTs, SDValue Chain, SDValue Ptr, SDValue Cmp, SDValue Swp, @@ -6422,35 +6514,6 @@ SDValue SelectionDAG::getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, return getAtomic(Opcode, dl, MemVT, VTs, Ops, MMO); } -SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, - SDValue Chain, SDValue Ptr, SDValue Val, - const Value *PtrVal, unsigned Alignment, - AtomicOrdering Ordering, - SyncScope::ID SSID) { - if (Alignment == 0) // Ensure that codegen never sees alignment 0 - Alignment = getEVTAlignment(MemVT); - - MachineFunction &MF = getMachineFunction(); - // An atomic store does not load. An atomic load does not store. - // (An atomicrmw obviously both loads and stores.) - // For now, atomics are considered to be volatile always, and they are - // chained as such. - // FIXME: Volatile isn't really correct; we should keep track of atomic - // orderings in the memoperand. - auto Flags = MachineMemOperand::MOVolatile; - if (Opcode != ISD::ATOMIC_STORE) - Flags |= MachineMemOperand::MOLoad; - if (Opcode != ISD::ATOMIC_LOAD) - Flags |= MachineMemOperand::MOStore; - - MachineMemOperand *MMO = - MF.getMachineMemOperand(MachinePointerInfo(PtrVal), Flags, - MemVT.getStoreSize(), Alignment, AAMDNodes(), - nullptr, SSID, Ordering); - - return getAtomic(Opcode, dl, MemVT, Chain, Ptr, Val, MMO); -} - SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain, SDValue Ptr, SDValue Val, MachineMemOperand *MMO) { @@ -6465,6 +6528,8 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, Opcode == ISD::ATOMIC_LOAD_MAX || Opcode == ISD::ATOMIC_LOAD_UMIN || Opcode == ISD::ATOMIC_LOAD_UMAX || + Opcode == ISD::ATOMIC_LOAD_FADD || + Opcode == ISD::ATOMIC_LOAD_FSUB || Opcode == ISD::ATOMIC_SWAP || Opcode == ISD::ATOMIC_STORE) && "Invalid Atomic Op"); @@ -6502,7 +6567,7 @@ SDValue SelectionDAG::getMergeValues(ArrayRef Ops, const SDLoc &dl) { SDValue SelectionDAG::getMemIntrinsicNode( unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef Ops, EVT MemVT, MachinePointerInfo PtrInfo, unsigned Align, - MachineMemOperand::Flags Flags, unsigned Size) { + MachineMemOperand::Flags Flags, unsigned Size, const AAMDNodes &AAInfo) { if (Align == 0) // Ensure that codegen never sees alignment 0 Align = getEVTAlignment(MemVT); @@ -6511,7 +6576,7 @@ SDValue SelectionDAG::getMemIntrinsicNode( MachineFunction &MF = getMachineFunction(); MachineMemOperand *MMO = - MF.getMachineMemOperand(PtrInfo, Flags, Size, Align); + MF.getMachineMemOperand(PtrInfo, Flags, Size, Align, AAInfo); return getMemIntrinsicNode(Opcode, dl, VTList, Ops, MemVT, MMO); } @@ -6557,6 +6622,36 @@ SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, return SDValue(N, 0); } +SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl, + SDValue Chain, int FrameIndex, + int64_t Size, int64_t Offset) { + const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END; + const auto VTs = getVTList(MVT::Other); + SDValue Ops[2] = { + Chain, + getFrameIndex(FrameIndex, + getTargetLoweringInfo().getFrameIndexTy(getDataLayout()), + true)}; + + FoldingSetNodeID ID; + AddNodeIDNode(ID, Opcode, VTs, Ops); + ID.AddInteger(FrameIndex); + ID.AddInteger(Size); + ID.AddInteger(Offset); + void *IP = nullptr; + if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) + return SDValue(E, 0); + + LifetimeSDNode *N = newSDNode( + Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, Size, Offset); + createOperands(N, Ops); + CSEMap.InsertNode(N, IP); + InsertNode(N); + SDValue V(N, 0); + NewSDValueDbgMsg(V, "Creating new node: ", this); + return V; +} + /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a /// MachinePointerInfo record from it. This is particularly useful because the /// code generator has many cases where it doesn't bother passing in a @@ -6875,7 +6970,7 @@ SDValue SelectionDAG::getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ops[] = { Chain, Ptr, Mask, PassThru }; FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MLOAD, VTs, Ops); - ID.AddInteger(VT.getRawBits()); + ID.AddInteger(MemVT.getRawBits()); ID.AddInteger(getSyntheticNodeSubclassData( dl.getIROrder(), VTs, ExtTy, isExpanding, MemVT, MMO)); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); @@ -6901,12 +6996,11 @@ SDValue SelectionDAG::getMaskedStore(SDValue Chain, const SDLoc &dl, bool IsTruncating, bool IsCompressing) { assert(Chain.getValueType() == MVT::Other && "Invalid chain type"); - EVT VT = Val.getValueType(); SDVTList VTs = getVTList(MVT::Other); SDValue Ops[] = { Chain, Val, Ptr, Mask }; FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MSTORE, VTs, Ops); - ID.AddInteger(VT.getRawBits()); + ID.AddInteger(MemVT.getRawBits()); ID.AddInteger(getSyntheticNodeSubclassData( dl.getIROrder(), VTs, IsTruncating, IsCompressing, MemVT, MMO)); ID.AddInteger(MMO->getPointerInfo().getAddrSpace()); @@ -7057,6 +7151,31 @@ SDValue SelectionDAG::simplifyShift(SDValue X, SDValue Y) { return SDValue(); } +// TODO: Use fast-math-flags to enable more simplifications. +SDValue SelectionDAG::simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y) { + ConstantFPSDNode *YC = isConstOrConstSplatFP(Y, /* AllowUndefs */ true); + if (!YC) + return SDValue(); + + // X + -0.0 --> X + if (Opcode == ISD::FADD) + if (YC->getValueAPF().isNegZero()) + return X; + + // X - +0.0 --> X + if (Opcode == ISD::FSUB) + if (YC->getValueAPF().isPosZero()) + return X; + + // X * 1.0 --> X + // X / 1.0 --> X + if (Opcode == ISD::FMUL || Opcode == ISD::FDIV) + if (YC->getValueAPF().isExactlyValue(1.0)) + return X; + + return SDValue(); +} + SDValue SelectionDAG::getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue SV, unsigned Align) { SDValue Ops[] = { Chain, Ptr, SV, getTargetConstant(Align, dl, MVT::i32) }; @@ -7098,8 +7217,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT, return V; break; case ISD::CONCAT_VECTORS: - // Attempt to fold CONCAT_VECTORS into BUILD_VECTOR or UNDEF. - if (SDValue V = FoldCONCAT_VECTORS(DL, VT, Ops, *this)) + if (SDValue V = foldCONCAT_VECTORS(DL, VT, Ops, *this)) return V; break; case ISD::SELECT_CC: @@ -7629,56 +7747,50 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, SDNode* SelectionDAG::mutateStrictFPToFP(SDNode *Node) { unsigned OrigOpc = Node->getOpcode(); unsigned NewOpc; - bool IsUnary = false; - bool IsTernary = false; switch (OrigOpc) { default: llvm_unreachable("mutateStrictFPToFP called with unexpected opcode!"); - case ISD::STRICT_FADD: NewOpc = ISD::FADD; break; - case ISD::STRICT_FSUB: NewOpc = ISD::FSUB; break; - case ISD::STRICT_FMUL: NewOpc = ISD::FMUL; break; - case ISD::STRICT_FDIV: NewOpc = ISD::FDIV; break; - case ISD::STRICT_FREM: NewOpc = ISD::FREM; break; - case ISD::STRICT_FMA: NewOpc = ISD::FMA; IsTernary = true; break; - case ISD::STRICT_FSQRT: NewOpc = ISD::FSQRT; IsUnary = true; break; - case ISD::STRICT_FPOW: NewOpc = ISD::FPOW; break; - case ISD::STRICT_FPOWI: NewOpc = ISD::FPOWI; break; - case ISD::STRICT_FSIN: NewOpc = ISD::FSIN; IsUnary = true; break; - case ISD::STRICT_FCOS: NewOpc = ISD::FCOS; IsUnary = true; break; - case ISD::STRICT_FEXP: NewOpc = ISD::FEXP; IsUnary = true; break; - case ISD::STRICT_FEXP2: NewOpc = ISD::FEXP2; IsUnary = true; break; - case ISD::STRICT_FLOG: NewOpc = ISD::FLOG; IsUnary = true; break; - case ISD::STRICT_FLOG10: NewOpc = ISD::FLOG10; IsUnary = true; break; - case ISD::STRICT_FLOG2: NewOpc = ISD::FLOG2; IsUnary = true; break; - case ISD::STRICT_FRINT: NewOpc = ISD::FRINT; IsUnary = true; break; - case ISD::STRICT_FNEARBYINT: - NewOpc = ISD::FNEARBYINT; - IsUnary = true; - break; - case ISD::STRICT_FMAXNUM: NewOpc = ISD::FMAXNUM; break; - case ISD::STRICT_FMINNUM: NewOpc = ISD::FMINNUM; break; - case ISD::STRICT_FCEIL: NewOpc = ISD::FCEIL; IsUnary = true; break; - case ISD::STRICT_FFLOOR: NewOpc = ISD::FFLOOR; IsUnary = true; break; - case ISD::STRICT_FROUND: NewOpc = ISD::FROUND; IsUnary = true; break; - case ISD::STRICT_FTRUNC: NewOpc = ISD::FTRUNC; IsUnary = true; break; - } + case ISD::STRICT_FADD: NewOpc = ISD::FADD; break; + case ISD::STRICT_FSUB: NewOpc = ISD::FSUB; break; + case ISD::STRICT_FMUL: NewOpc = ISD::FMUL; break; + case ISD::STRICT_FDIV: NewOpc = ISD::FDIV; break; + case ISD::STRICT_FREM: NewOpc = ISD::FREM; break; + case ISD::STRICT_FMA: NewOpc = ISD::FMA; break; + case ISD::STRICT_FSQRT: NewOpc = ISD::FSQRT; break; + case ISD::STRICT_FPOW: NewOpc = ISD::FPOW; break; + case ISD::STRICT_FPOWI: NewOpc = ISD::FPOWI; break; + case ISD::STRICT_FSIN: NewOpc = ISD::FSIN; break; + case ISD::STRICT_FCOS: NewOpc = ISD::FCOS; break; + case ISD::STRICT_FEXP: NewOpc = ISD::FEXP; break; + case ISD::STRICT_FEXP2: NewOpc = ISD::FEXP2; break; + case ISD::STRICT_FLOG: NewOpc = ISD::FLOG; break; + case ISD::STRICT_FLOG10: NewOpc = ISD::FLOG10; break; + case ISD::STRICT_FLOG2: NewOpc = ISD::FLOG2; break; + case ISD::STRICT_FRINT: NewOpc = ISD::FRINT; break; + case ISD::STRICT_FNEARBYINT: NewOpc = ISD::FNEARBYINT; break; + case ISD::STRICT_FMAXNUM: NewOpc = ISD::FMAXNUM; break; + case ISD::STRICT_FMINNUM: NewOpc = ISD::FMINNUM; break; + case ISD::STRICT_FCEIL: NewOpc = ISD::FCEIL; break; + case ISD::STRICT_FFLOOR: NewOpc = ISD::FFLOOR; break; + case ISD::STRICT_FROUND: NewOpc = ISD::FROUND; break; + case ISD::STRICT_FTRUNC: NewOpc = ISD::FTRUNC; break; + case ISD::STRICT_FP_ROUND: NewOpc = ISD::FP_ROUND; break; + case ISD::STRICT_FP_EXTEND: NewOpc = ISD::FP_EXTEND; break; + } + + assert(Node->getNumValues() == 2 && "Unexpected number of results!"); // We're taking this node out of the chain, so we need to re-link things. SDValue InputChain = Node->getOperand(0); SDValue OutputChain = SDValue(Node, 1); ReplaceAllUsesOfValueWith(OutputChain, InputChain); - SDVTList VTs = getVTList(Node->getOperand(1).getValueType()); - SDNode *Res = nullptr; - if (IsUnary) - Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1) }); - else if (IsTernary) - Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1), - Node->getOperand(2), - Node->getOperand(3)}); - else - Res = MorphNodeTo(Node, NewOpc, VTs, { Node->getOperand(1), - Node->getOperand(2) }); + SmallVector Ops; + for (unsigned i = 1, e = Node->getNumOperands(); i != e; ++i) + Ops.push_back(Node->getOperand(i)); + + SDVTList VTs = getVTList(Node->getValueType(0)); + SDNode *Res = MorphNodeTo(Node, NewOpc, VTs, Ops); // MorphNodeTo can operate in two ways: if an existing node with the // specified operands exists, it can just return it. Otherwise, it @@ -7980,9 +8092,8 @@ void SelectionDAG::salvageDebugInfo(SDNode &N) { // DIExpression, we need to mark the expression with a // DW_OP_stack_value. auto *DIExpr = DV->getExpression(); - DIExpr = DIExpression::prepend(DIExpr, DIExpression::NoDeref, Offset, - DIExpression::NoDeref, - DIExpression::WithStackValue); + DIExpr = + DIExpression::prepend(DIExpr, DIExpression::StackValue, Offset); SDDbgValue *Clone = getDbgValue(DV->getVariable(), DIExpr, N0.getNode(), N0.getResNo(), DV->isIndirect(), DV->getDebugLoc(), DV->getOrder()); @@ -8288,19 +8399,17 @@ void SelectionDAG::updateDivergence(SDNode * N) } } - -void SelectionDAG::CreateTopologicalOrder(std::vector& Order) { +void SelectionDAG::CreateTopologicalOrder(std::vector &Order) { DenseMap Degree; Order.reserve(AllNodes.size()); - for (auto & N : allnodes()) { + for (auto &N : allnodes()) { unsigned NOps = N.getNumOperands(); Degree[&N] = NOps; if (0 == NOps) Order.push_back(&N); } - for (std::vector::iterator I = Order.begin(); - I!=Order.end();++I) { - SDNode * N = *I; + for (size_t I = 0; I != Order.size(); ++I) { + SDNode *N = Order[I]; for (auto U : N->uses()) { unsigned &UnsortedOps = Degree[U]; if (0 == --UnsortedOps) @@ -8310,9 +8419,8 @@ void SelectionDAG::CreateTopologicalOrder(std::vector& Order) { } #ifndef NDEBUG -void SelectionDAG::VerifyDAGDiverence() -{ - std::vector TopoOrder; +void SelectionDAG::VerifyDAGDiverence() { + std::vector TopoOrder; CreateTopologicalOrder(TopoOrder); const TargetLowering &TLI = getTargetLoweringInfo(); DenseMap DivergenceMap; @@ -8338,7 +8446,6 @@ void SelectionDAG::VerifyDAGDiverence() } #endif - /// ReplaceAllUsesOfValuesWith - Replace any uses of From with To, leaving /// uses of other values produced by From.getNode() alone. The same value /// may appear in both the From and To list. The Deleted vector is @@ -8584,14 +8691,24 @@ SDValue llvm::peekThroughOneUseBitcasts(SDValue V) { return V; } -bool llvm::isBitwiseNot(SDValue V) { +SDValue llvm::peekThroughExtractSubvectors(SDValue V) { + while (V.getOpcode() == ISD::EXTRACT_SUBVECTOR) + V = V.getOperand(0); + return V; +} + +bool llvm::isBitwiseNot(SDValue V, bool AllowUndefs) { if (V.getOpcode() != ISD::XOR) return false; - ConstantSDNode *C = isConstOrConstSplat(peekThroughBitcasts(V.getOperand(1))); - return C && C->isAllOnesValue(); + V = peekThroughBitcasts(V.getOperand(1)); + unsigned NumBits = V.getScalarValueSizeInBits(); + ConstantSDNode *C = + isConstOrConstSplat(V, AllowUndefs, /*AllowTruncation*/ true); + return C && (C->getAPIntValue().countTrailingOnes() >= NumBits); } -ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs) { +ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs, + bool AllowTruncation) { if (ConstantSDNode *CN = dyn_cast(N)) return CN; @@ -8599,10 +8716,39 @@ ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, bool AllowUndefs) { BitVector UndefElements; ConstantSDNode *CN = BV->getConstantSplatNode(&UndefElements); - // BuildVectors can truncate their operands. Ignore that case here. - if (CN && (UndefElements.none() || AllowUndefs) && - CN->getValueType(0) == N.getValueType().getScalarType()) - return CN; + // BuildVectors can truncate their operands. Ignore that case here unless + // AllowTruncation is set. + if (CN && (UndefElements.none() || AllowUndefs)) { + EVT CVT = CN->getValueType(0); + EVT NSVT = N.getValueType().getScalarType(); + assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension"); + if (AllowTruncation || (CVT == NSVT)) + return CN; + } + } + + return nullptr; +} + +ConstantSDNode *llvm::isConstOrConstSplat(SDValue N, const APInt &DemandedElts, + bool AllowUndefs, + bool AllowTruncation) { + if (ConstantSDNode *CN = dyn_cast(N)) + return CN; + + if (BuildVectorSDNode *BV = dyn_cast(N)) { + BitVector UndefElements; + ConstantSDNode *CN = BV->getConstantSplatNode(DemandedElts, &UndefElements); + + // BuildVectors can truncate their operands. Ignore that case here unless + // AllowTruncation is set. + if (CN && (UndefElements.none() || AllowUndefs)) { + EVT CVT = CN->getValueType(0); + EVT NSVT = N.getValueType().getScalarType(); + assert(CVT.bitsGE(NSVT) && "Illegal build vector element extension"); + if (AllowTruncation || (CVT == NSVT)) + return CN; + } } return nullptr; @@ -8622,9 +8768,26 @@ ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, bool AllowUndefs) { return nullptr; } -bool llvm::isNullOrNullSplat(SDValue N) { +ConstantFPSDNode *llvm::isConstOrConstSplatFP(SDValue N, + const APInt &DemandedElts, + bool AllowUndefs) { + if (ConstantFPSDNode *CN = dyn_cast(N)) + return CN; + + if (BuildVectorSDNode *BV = dyn_cast(N)) { + BitVector UndefElements; + ConstantFPSDNode *CN = + BV->getConstantFPSplatNode(DemandedElts, &UndefElements); + if (CN && (UndefElements.none() || AllowUndefs)) + return CN; + } + + return nullptr; +} + +bool llvm::isNullOrNullSplat(SDValue N, bool AllowUndefs) { // TODO: may want to use peekThroughBitcast() here. - ConstantSDNode *C = isConstOrConstSplat(N); + ConstantSDNode *C = isConstOrConstSplat(N, AllowUndefs); return C && C->isNullValue(); } @@ -8773,17 +8936,12 @@ bool SDNode::areOnlyUsersOf(ArrayRef Nodes, const SDNode *N) { /// isOperand - Return true if this node is an operand of N. bool SDValue::isOperandOf(const SDNode *N) const { - for (const SDValue &Op : N->op_values()) - if (*this == Op) - return true; - return false; + return any_of(N->op_values(), [this](SDValue Op) { return *this == Op; }); } bool SDNode::isOperandOf(const SDNode *N) const { - for (const SDValue &Op : N->op_values()) - if (this == Op.getNode()) - return true; - return false; + return any_of(N->op_values(), + [this](SDValue Op) { return this == Op.getNode(); }); } /// reachesChainWithoutSideEffects - Return true if this operand (which must @@ -8973,6 +9131,56 @@ SDValue SelectionDAG::UnrollVectorOp(SDNode *N, unsigned ResNE) { return getBuildVector(VecVT, dl, Scalars); } +std::pair SelectionDAG::UnrollVectorOverflowOp( + SDNode *N, unsigned ResNE) { + unsigned Opcode = N->getOpcode(); + assert((Opcode == ISD::UADDO || Opcode == ISD::SADDO || + Opcode == ISD::USUBO || Opcode == ISD::SSUBO || + Opcode == ISD::UMULO || Opcode == ISD::SMULO) && + "Expected an overflow opcode"); + + EVT ResVT = N->getValueType(0); + EVT OvVT = N->getValueType(1); + EVT ResEltVT = ResVT.getVectorElementType(); + EVT OvEltVT = OvVT.getVectorElementType(); + SDLoc dl(N); + + // If ResNE is 0, fully unroll the vector op. + unsigned NE = ResVT.getVectorNumElements(); + if (ResNE == 0) + ResNE = NE; + else if (NE > ResNE) + NE = ResNE; + + SmallVector LHSScalars; + SmallVector RHSScalars; + ExtractVectorElements(N->getOperand(0), LHSScalars, 0, NE); + ExtractVectorElements(N->getOperand(1), RHSScalars, 0, NE); + + EVT SVT = TLI->getSetCCResultType(getDataLayout(), *getContext(), ResEltVT); + SDVTList VTs = getVTList(ResEltVT, SVT); + SmallVector ResScalars; + SmallVector OvScalars; + for (unsigned i = 0; i < NE; ++i) { + SDValue Res = getNode(Opcode, dl, VTs, LHSScalars[i], RHSScalars[i]); + SDValue Ov = + getSelect(dl, OvEltVT, Res.getValue(1), + getBoolConstant(true, dl, OvEltVT, ResVT), + getConstant(0, dl, OvEltVT)); + + ResScalars.push_back(Res); + OvScalars.push_back(Ov); + } + + ResScalars.append(ResNE - NE, getUNDEF(ResEltVT)); + OvScalars.append(ResNE - NE, getUNDEF(OvEltVT)); + + EVT NewResVT = EVT::getVectorVT(*getContext(), ResEltVT, ResNE); + EVT NewOvVT = EVT::getVectorVT(*getContext(), OvEltVT, ResNE); + return std::make_pair(getBuildVector(NewResVT, dl, ResScalars), + getBuildVector(NewOvVT, dl, OvScalars)); +} + bool SelectionDAG::areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base, unsigned Bytes, @@ -9014,7 +9222,7 @@ unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { // If this is a direct reference to a stack slot, use information about the // stack slot's alignment. - int FrameIdx = 1 << 31; + int FrameIdx = INT_MIN; int64_t FrameOffset = 0; if (FrameIndexSDNode *FI = dyn_cast(Ptr)) { FrameIdx = FI->getIndex(); @@ -9025,7 +9233,7 @@ unsigned SelectionDAG::InferPtrAlignment(SDValue Ptr) const { FrameOffset = Ptr.getConstantOperandVal(1); } - if (FrameIdx != (1 << 31)) { + if (FrameIdx != INT_MIN) { const MachineFrameInfo &MFI = getMachineFunction().getFrameInfo(); unsigned FIInfoAlign = MinAlign(MFI.getObjectAlignment(FrameIdx), FrameOffset); @@ -9065,6 +9273,15 @@ SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, return std::make_pair(Lo, Hi); } +/// Widen the vector up to the next power of two using INSERT_SUBVECTOR. +SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) { + EVT VT = N.getValueType(); + EVT WideVT = EVT::getVectorVT(*getContext(), VT.getVectorElementType(), + NextPowerOf2(VT.getVectorNumElements())); + return getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, getUNDEF(WideVT), N, + getConstant(0, DL, TLI->getVectorIdxTy(getDataLayout()))); +} + void SelectionDAG::ExtractVectorElements(SDValue Op, SmallVectorImpl &Args, unsigned Start, unsigned Count) { @@ -9158,13 +9375,20 @@ bool BuildVectorSDNode::isConstantSplat(APInt &SplatValue, APInt &SplatUndef, return true; } -SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const { +SDValue BuildVectorSDNode::getSplatValue(const APInt &DemandedElts, + BitVector *UndefElements) const { if (UndefElements) { UndefElements->clear(); UndefElements->resize(getNumOperands()); } + assert(getNumOperands() == DemandedElts.getBitWidth() && + "Unexpected vector size"); + if (!DemandedElts) + return SDValue(); SDValue Splatted; for (unsigned i = 0, e = getNumOperands(); i != e; ++i) { + if (!DemandedElts[i]) + continue; SDValue Op = getOperand(i); if (Op.isUndef()) { if (UndefElements) @@ -9177,19 +9401,39 @@ SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const { } if (!Splatted) { - assert(getOperand(0).isUndef() && + unsigned FirstDemandedIdx = DemandedElts.countTrailingZeros(); + assert(getOperand(FirstDemandedIdx).isUndef() && "Can only have a splat without a constant for all undefs."); - return getOperand(0); + return getOperand(FirstDemandedIdx); } return Splatted; } +SDValue BuildVectorSDNode::getSplatValue(BitVector *UndefElements) const { + APInt DemandedElts = APInt::getAllOnesValue(getNumOperands()); + return getSplatValue(DemandedElts, UndefElements); +} + +ConstantSDNode * +BuildVectorSDNode::getConstantSplatNode(const APInt &DemandedElts, + BitVector *UndefElements) const { + return dyn_cast_or_null( + getSplatValue(DemandedElts, UndefElements)); +} + ConstantSDNode * BuildVectorSDNode::getConstantSplatNode(BitVector *UndefElements) const { return dyn_cast_or_null(getSplatValue(UndefElements)); } +ConstantFPSDNode * +BuildVectorSDNode::getConstantFPSplatNode(const APInt &DemandedElts, + BitVector *UndefElements) const { + return dyn_cast_or_null( + getSplatValue(DemandedElts, UndefElements)); +} + ConstantFPSDNode * BuildVectorSDNode::getConstantFPSplatNode(BitVector *UndefElements) const { return dyn_cast_or_null(getSplatValue(UndefElements)); @@ -9228,7 +9472,10 @@ bool ShuffleVectorSDNode::isSplatMask(const int *Mask, EVT VT) { for (i = 0, e = VT.getVectorNumElements(); i != e && Mask[i] < 0; ++i) /* search */; - assert(i != e && "VECTOR_SHUFFLE node with all undef indices!"); + // If all elements are undefined, this shuffle can be considered a splat + // (although it should eventually get simplified away completely). + if (i == e) + return true; // Make sure all remaining elements are either undef or the same as the first // non-undef value. @@ -9266,8 +9513,7 @@ SDNode *SelectionDAG::isConstantFPBuildVectorOrConstantFP(SDValue N) { void SelectionDAG::createOperands(SDNode *Node, ArrayRef Vals) { assert(!Node->OperandList && "Node already has operands"); - assert(std::numeric_limits::max() >= - Vals.size() && + assert(SDNode::getMaxNumOperands() >= Vals.size() && "too many operands to fit into SDNode"); SDUse *Ops = OperandRecycler.allocate( ArrayRecycler::Capacity::get(Vals.size()), OperandAllocator); @@ -9287,6 +9533,19 @@ void SelectionDAG::createOperands(SDNode *Node, ArrayRef Vals) { checkForCycles(Node); } +SDValue SelectionDAG::getTokenFactor(const SDLoc &DL, + SmallVectorImpl &Vals) { + size_t Limit = SDNode::getMaxNumOperands(); + while (Vals.size() > Limit) { + unsigned SliceIdx = Vals.size() - Limit; + auto ExtractedTFs = ArrayRef(Vals).slice(SliceIdx, Limit); + SDValue NewTF = getNode(ISD::TokenFactor, DL, MVT::Other, ExtractedTFs); + Vals.erase(Vals.begin() + SliceIdx, Vals.end()); + Vals.emplace_back(NewTF); + } + return getNode(ISD::TokenFactor, DL, MVT::Other, Vals); +} + #ifndef NDEBUG static void checkForCyclesHelper(const SDNode *N, SmallPtrSetImpl &Visited, diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp index 488bac1a9a80..9592bc30a4e1 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp @@ -1,9 +1,8 @@ //==- llvm/CodeGen/SelectionDAGAddressAnalysis.cpp - DAG Address Analysis --==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -25,8 +24,10 @@ bool BaseIndexOffset::equalBaseIndex(const BaseIndexOffset &Other, // Conservatively fail if we a match failed.. if (!Base.getNode() || !Other.Base.getNode()) return false; + if (!hasValidOffset() || !Other.hasValidOffset()) + return false; // Initial Offset difference. - Off = Other.Offset - Offset; + Off = *Other.Offset - *Offset; if ((Other.Index == Index) && (Other.IsIndexSignExt == IsIndexSignExt)) { // Trivial match. @@ -60,24 +61,110 @@ bool BaseIndexOffset::equalBaseIndex(const BaseIndexOffset &Other, const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); - // Match non-equal FrameIndexes - If both frame indices are fixed - // we know their relative offsets and can compare them. Otherwise - // we must be conservative. + // Match FrameIndexes. if (auto *A = dyn_cast(Base)) - if (auto *B = dyn_cast(Other.Base)) + if (auto *B = dyn_cast(Other.Base)) { + // Equal FrameIndexes - offsets are directly comparable. + if (A->getIndex() == B->getIndex()) + return true; + // Non-equal FrameIndexes - If both frame indices are fixed + // we know their relative offsets and can compare them. Otherwise + // we must be conservative. if (MFI.isFixedObjectIndex(A->getIndex()) && MFI.isFixedObjectIndex(B->getIndex())) { Off += MFI.getObjectOffset(B->getIndex()) - MFI.getObjectOffset(A->getIndex()); return true; } + } } return false; } +bool BaseIndexOffset::computeAliasing(const SDNode *Op0, + const Optional NumBytes0, + const SDNode *Op1, + const Optional NumBytes1, + const SelectionDAG &DAG, bool &IsAlias) { + + BaseIndexOffset BasePtr0 = match(Op0, DAG); + BaseIndexOffset BasePtr1 = match(Op1, DAG); + + if (!(BasePtr0.getBase().getNode() && BasePtr1.getBase().getNode())) + return false; + int64_t PtrDiff; + if (NumBytes0.hasValue() && NumBytes1.hasValue() && + BasePtr0.equalBaseIndex(BasePtr1, DAG, PtrDiff)) { + // BasePtr1 is PtrDiff away from BasePtr0. They alias if none of the + // following situations arise: + IsAlias = !( + // [----BasePtr0----] + // [---BasePtr1--] + // ========PtrDiff========> + (*NumBytes0 <= PtrDiff) || + // [----BasePtr0----] + // [---BasePtr1--] + // =====(-PtrDiff)====> + (PtrDiff + *NumBytes1 <= 0)); // i.e. *NumBytes1 < -PtrDiff. + return true; + } + // If both BasePtr0 and BasePtr1 are FrameIndexes, we will not be + // able to calculate their relative offset if at least one arises + // from an alloca. However, these allocas cannot overlap and we + // can infer there is no alias. + if (auto *A = dyn_cast(BasePtr0.getBase())) + if (auto *B = dyn_cast(BasePtr1.getBase())) { + MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); + // If the base are the same frame index but the we couldn't find a + // constant offset, (indices are different) be conservative. + if (A != B && (!MFI.isFixedObjectIndex(A->getIndex()) || + !MFI.isFixedObjectIndex(B->getIndex()))) { + IsAlias = false; + return true; + } + } + + bool IsFI0 = isa(BasePtr0.getBase()); + bool IsFI1 = isa(BasePtr1.getBase()); + bool IsGV0 = isa(BasePtr0.getBase()); + bool IsGV1 = isa(BasePtr1.getBase()); + bool IsCV0 = isa(BasePtr0.getBase()); + bool IsCV1 = isa(BasePtr1.getBase()); + + // If of mismatched base types or checkable indices we can check + // they do not alias. + if ((BasePtr0.getIndex() == BasePtr1.getIndex() || (IsFI0 != IsFI1) || + (IsGV0 != IsGV1) || (IsCV0 != IsCV1)) && + (IsFI0 || IsGV0 || IsCV0) && (IsFI1 || IsGV1 || IsCV1)) { + IsAlias = false; + return true; + } + return false; // Cannot determine whether the pointers alias. +} + +bool BaseIndexOffset::contains(const SelectionDAG &DAG, int64_t BitSize, + const BaseIndexOffset &Other, + int64_t OtherBitSize, int64_t &BitOffset) const { + int64_t Offset; + if (!equalBaseIndex(Other, DAG, Offset)) + return false; + if (Offset >= 0) { + // Other is after *this: + // [-------*this---------] + // [---Other--] + // ==Offset==> + BitOffset = 8 * Offset; + return BitOffset + OtherBitSize <= BitSize; + } + // Other starts strictly before *this, it cannot be fully contained. + // [-------*this---------] + // [--Other--] + return false; +} + /// Parses tree in Ptr for base, index, offset addresses. -BaseIndexOffset BaseIndexOffset::match(const LSBaseSDNode *N, - const SelectionDAG &DAG) { +static BaseIndexOffset matchLSNode(const LSBaseSDNode *N, + const SelectionDAG &DAG) { SDValue Ptr = N->getBasePtr(); // (((B + I*M) + c)) + c ... @@ -178,3 +265,33 @@ BaseIndexOffset BaseIndexOffset::match(const LSBaseSDNode *N, } return BaseIndexOffset(Base, Index, Offset, IsIndexSignExt); } + +BaseIndexOffset BaseIndexOffset::match(const SDNode *N, + const SelectionDAG &DAG) { + if (const auto *LS0 = dyn_cast(N)) + return matchLSNode(LS0, DAG); + if (const auto *LN = dyn_cast(N)) { + if (LN->hasOffset()) + return BaseIndexOffset(LN->getOperand(1), SDValue(), LN->getOffset(), + false); + return BaseIndexOffset(LN->getOperand(1), SDValue(), false); + } + return BaseIndexOffset(); +} + +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) + +LLVM_DUMP_METHOD void BaseIndexOffset::dump() const { + print(dbgs()); +} + +void BaseIndexOffset::print(raw_ostream& OS) const { + OS << "BaseIndexOffset base=["; + Base->print(OS); + OS << "] index=["; + if (Index) + Index->print(OS); + OS << "] offset=" << Offset; +} + +#endif diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 871ab9b29881..e818dd27c05e 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -1,9 +1,8 @@ //===- SelectionDAGBuilder.cpp - Selection-DAG building -------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -55,6 +54,7 @@ #include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/CodeGen/SelectionDAGTargetInfo.h" #include "llvm/CodeGen/StackMaps.h" +#include "llvm/CodeGen/SwiftErrorValueTracking.h" #include "llvm/CodeGen/TargetFrameLowering.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetLowering.h" @@ -109,6 +109,7 @@ #include "llvm/Target/TargetIntrinsicInfo.h" #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" +#include "llvm/Transforms/Utils/Local.h" #include #include #include @@ -123,6 +124,7 @@ using namespace llvm; using namespace PatternMatch; +using namespace SwitchCG; #define DEBUG_TYPE "isel" @@ -215,8 +217,8 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, unsigned ValueBits = ValueVT.getSizeInBits(); // Assemble the power of 2 part. - unsigned RoundParts = NumParts & (NumParts - 1) ? - 1 << Log2_32(NumParts) : NumParts; + unsigned RoundParts = + (NumParts & (NumParts - 1)) ? 1 << Log2_32(NumParts) : NumParts; unsigned RoundBits = PartBits * RoundParts; EVT RoundVT = RoundBits == ValueBits ? ValueVT : EVT::getIntegerVT(*DAG.getContext(), RoundBits); @@ -322,7 +324,15 @@ static SDValue getCopyFromParts(SelectionDAG &DAG, const SDLoc &DL, return DAG.getNode(ISD::FP_EXTEND, DL, ValueVT, Val); } - llvm_unreachable("Unknown mismatch!"); + // Handle MMX to a narrower integer type by bitcasting MMX to integer and + // then truncating. + if (PartEVT == MVT::x86mmx && ValueVT.isInteger() && + ValueVT.bitsLT(PartEVT)) { + Val = DAG.getNode(ISD::BITCAST, DL, MVT::i64, Val); + return DAG.getNode(ISD::TRUNCATE, DL, ValueVT, Val); + } + + report_fatal_error("Unknown mismatch in getCopyFromParts!"); } static void diagnosePossiblyInvalidConstraint(LLVMContext &Ctx, const Value *V, @@ -573,7 +583,8 @@ static void getCopyToParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val, unsigned RoundBits = RoundParts * PartBits; unsigned OddParts = NumParts - RoundParts; SDValue OddVal = DAG.getNode(ISD::SRL, DL, ValueVT, Val, - DAG.getIntPtrConstant(RoundBits, DL)); + DAG.getShiftAmountConstant(RoundBits, ValueVT, DL, /*LegalTypes*/false)); + getCopyToParts(DAG, DL, OddVal, Parts + RoundParts, OddParts, PartVT, V, CallConv); @@ -1003,6 +1014,7 @@ void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa, DL = &DAG.getDataLayout(); Context = DAG.getContext(); LPadToCallSiteMap.clear(); + SL->init(DAG.getTargetLoweringInfo(), TM, DAG.getDataLayout()); } void SelectionDAGBuilder::clear() { @@ -1032,19 +1044,7 @@ SDValue SelectionDAGBuilder::getRoot() { } // Otherwise, we have to make a token factor node. - // If we have >= 2^16 loads then split across multiple token factors as - // there's a 64k limit on the number of SDNode operands. - SDValue Root; - size_t Limit = (1 << 16) - 1; - while (PendingLoads.size() > Limit) { - unsigned SliceIdx = PendingLoads.size() - Limit; - auto ExtractedTFs = ArrayRef(PendingLoads).slice(SliceIdx, Limit); - SDValue NewTF = - DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, ExtractedTFs); - PendingLoads.erase(PendingLoads.begin() + SliceIdx, PendingLoads.end()); - PendingLoads.emplace_back(NewTF); - } - Root = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, PendingLoads); + SDValue Root = DAG.getTokenFactor(getCurSDLoc(), PendingLoads); PendingLoads.clear(); DAG.setRoot(Root); return Root; @@ -1144,6 +1144,13 @@ void SelectionDAGBuilder::dropDanglingDebugInfo(const DILocalVariable *Variable, for (auto &DDIMI : DanglingDebugInfoMap) { DanglingDebugInfoVector &DDIV = DDIMI.second; + + // If debug info is to be dropped, run it through final checks to see + // whether it can be salvaged. + for (auto &DDI : DDIV) + if (isMatchingDbgValue(DDI)) + salvageUnresolvedDbgValue(DDI); + DDIV.erase(remove_if(DDIV, isMatchingDbgValue), DDIV.end()); } } @@ -1169,6 +1176,12 @@ void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V, "Expected inlined-at fields to agree"); SDDbgValue *SDV; if (Val.getNode()) { + // FIXME: I doubt that it is correct to resolve a dangling DbgValue as a + // FuncArgumentDbgValue (it would be hoisted to the function entry, and if + // we couldn't resolve it directly when examining the DbgValue intrinsic + // in the first place we should not be more successful here). Unless we + // have some test case that prove this to be correct we should avoid + // calling EmitFuncArgumentDbgValue here. if (!EmitFuncArgumentDbgValue(V, Variable, Expr, dl, false, Val)) { LLVM_DEBUG(dbgs() << "Resolve dangling debug info [order=" << DbgSDNodeOrder << "] for:\n " << *DI << "\n"); @@ -1186,12 +1199,173 @@ void SelectionDAGBuilder::resolveDanglingDebugInfo(const Value *V, } else LLVM_DEBUG(dbgs() << "Resolved dangling debug info for " << *DI << "in EmitFuncArgumentDbgValue\n"); - } else + } else { LLVM_DEBUG(dbgs() << "Dropping debug info for " << *DI << "\n"); + auto Undef = + UndefValue::get(DDI.getDI()->getVariableLocation()->getType()); + auto SDV = + DAG.getConstantDbgValue(Variable, Expr, Undef, dl, DbgSDNodeOrder); + DAG.AddDbgValue(SDV, nullptr, false); + } } DDIV.clear(); } +void SelectionDAGBuilder::salvageUnresolvedDbgValue(DanglingDebugInfo &DDI) { + Value *V = DDI.getDI()->getValue(); + DILocalVariable *Var = DDI.getDI()->getVariable(); + DIExpression *Expr = DDI.getDI()->getExpression(); + DebugLoc DL = DDI.getdl(); + DebugLoc InstDL = DDI.getDI()->getDebugLoc(); + unsigned SDOrder = DDI.getSDNodeOrder(); + + // Currently we consider only dbg.value intrinsics -- we tell the salvager + // that DW_OP_stack_value is desired. + assert(isa(DDI.getDI())); + bool StackValue = true; + + // Can this Value can be encoded without any further work? + if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder)) + return; + + // Attempt to salvage back through as many instructions as possible. Bail if + // a non-instruction is seen, such as a constant expression or global + // variable. FIXME: Further work could recover those too. + while (isa(V)) { + Instruction &VAsInst = *cast(V); + DIExpression *NewExpr = salvageDebugInfoImpl(VAsInst, Expr, StackValue); + + // If we cannot salvage any further, and haven't yet found a suitable debug + // expression, bail out. + if (!NewExpr) + break; + + // New value and expr now represent this debuginfo. + V = VAsInst.getOperand(0); + Expr = NewExpr; + + // Some kind of simplification occurred: check whether the operand of the + // salvaged debug expression can be encoded in this DAG. + if (handleDebugValue(V, Var, Expr, DL, InstDL, SDOrder)) { + LLVM_DEBUG(dbgs() << "Salvaged debug location info for:\n " + << DDI.getDI() << "\nBy stripping back to:\n " << V); + return; + } + } + + // This was the final opportunity to salvage this debug information, and it + // couldn't be done. Place an undef DBG_VALUE at this location to terminate + // any earlier variable location. + auto Undef = UndefValue::get(DDI.getDI()->getVariableLocation()->getType()); + auto SDV = DAG.getConstantDbgValue(Var, Expr, Undef, DL, SDNodeOrder); + DAG.AddDbgValue(SDV, nullptr, false); + + LLVM_DEBUG(dbgs() << "Dropping debug value info for:\n " << DDI.getDI() + << "\n"); + LLVM_DEBUG(dbgs() << " Last seen at:\n " << *DDI.getDI()->getOperand(0) + << "\n"); +} + +bool SelectionDAGBuilder::handleDebugValue(const Value *V, DILocalVariable *Var, + DIExpression *Expr, DebugLoc dl, + DebugLoc InstDL, unsigned Order) { + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + SDDbgValue *SDV; + if (isa(V) || isa(V) || isa(V) || + isa(V)) { + SDV = DAG.getConstantDbgValue(Var, Expr, V, dl, SDNodeOrder); + DAG.AddDbgValue(SDV, nullptr, false); + return true; + } + + // If the Value is a frame index, we can create a FrameIndex debug value + // without relying on the DAG at all. + if (const AllocaInst *AI = dyn_cast(V)) { + auto SI = FuncInfo.StaticAllocaMap.find(AI); + if (SI != FuncInfo.StaticAllocaMap.end()) { + auto SDV = + DAG.getFrameIndexDbgValue(Var, Expr, SI->second, + /*IsIndirect*/ false, dl, SDNodeOrder); + // Do not attach the SDNodeDbgValue to an SDNode: this variable location + // is still available even if the SDNode gets optimized out. + DAG.AddDbgValue(SDV, nullptr, false); + return true; + } + } + + // Do not use getValue() in here; we don't want to generate code at + // this point if it hasn't been done yet. + SDValue N = NodeMap[V]; + if (!N.getNode() && isa(V)) // Check unused arguments map. + N = UnusedArgNodeMap[V]; + if (N.getNode()) { + if (EmitFuncArgumentDbgValue(V, Var, Expr, dl, false, N)) + return true; + SDV = getDbgValue(N, Var, Expr, dl, SDNodeOrder); + DAG.AddDbgValue(SDV, N.getNode(), false); + return true; + } + + // Special rules apply for the first dbg.values of parameter variables in a + // function. Identify them by the fact they reference Argument Values, that + // they're parameters, and they are parameters of the current function. We + // need to let them dangle until they get an SDNode. + bool IsParamOfFunc = isa(V) && Var->isParameter() && + !InstDL.getInlinedAt(); + if (!IsParamOfFunc) { + // The value is not used in this block yet (or it would have an SDNode). + // We still want the value to appear for the user if possible -- if it has + // an associated VReg, we can refer to that instead. + auto VMI = FuncInfo.ValueMap.find(V); + if (VMI != FuncInfo.ValueMap.end()) { + unsigned Reg = VMI->second; + // If this is a PHI node, it may be split up into several MI PHI nodes + // (in FunctionLoweringInfo::set). + RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, + V->getType(), None); + if (RFV.occupiesMultipleRegs()) { + unsigned Offset = 0; + unsigned BitsToDescribe = 0; + if (auto VarSize = Var->getSizeInBits()) + BitsToDescribe = *VarSize; + if (auto Fragment = Expr->getFragmentInfo()) + BitsToDescribe = Fragment->SizeInBits; + for (auto RegAndSize : RFV.getRegsAndSizes()) { + unsigned RegisterSize = RegAndSize.second; + // Bail out if all bits are described already. + if (Offset >= BitsToDescribe) + break; + unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe) + ? BitsToDescribe - Offset + : RegisterSize; + auto FragmentExpr = DIExpression::createFragmentExpression( + Expr, Offset, FragmentSize); + if (!FragmentExpr) + continue; + SDV = DAG.getVRegDbgValue(Var, *FragmentExpr, RegAndSize.first, + false, dl, SDNodeOrder); + DAG.AddDbgValue(SDV, nullptr, false); + Offset += RegisterSize; + } + } else { + SDV = DAG.getVRegDbgValue(Var, Expr, Reg, false, dl, SDNodeOrder); + DAG.AddDbgValue(SDV, nullptr, false); + } + return true; + } + } + + return false; +} + +void SelectionDAGBuilder::resolveOrClearDbgInfo() { + // Try to fixup any remaining dangling debug info -- and drop it if we can't. + for (auto &Pair : DanglingDebugInfoMap) + for (auto &DDI : Pair.second) + salvageUnresolvedDbgValue(DDI); + clearDanglingDebugInfo(); +} + /// getCopyFromRegs - If there was virtual register allocated for the value V /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise. SDValue SelectionDAGBuilder::getCopyFromRegs(const Value *V, Type *Ty) { @@ -1469,6 +1643,36 @@ void SelectionDAGBuilder::visitCleanupPad(const CleanupPadInst &CPI) { } } +// For wasm, there's alwyas a single catch pad attached to a catchswitch, and +// the control flow always stops at the single catch pad, as it does for a +// cleanup pad. In case the exception caught is not of the types the catch pad +// catches, it will be rethrown by a rethrow. +static void findWasmUnwindDestinations( + FunctionLoweringInfo &FuncInfo, const BasicBlock *EHPadBB, + BranchProbability Prob, + SmallVectorImpl> + &UnwindDests) { + while (EHPadBB) { + const Instruction *Pad = EHPadBB->getFirstNonPHI(); + if (isa(Pad)) { + // Stop on cleanup pads. + UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob); + UnwindDests.back().first->setIsEHScopeEntry(); + break; + } else if (auto *CatchSwitch = dyn_cast(Pad)) { + // Add the catchpad handlers to the possible destinations. We don't + // continue to the unwind destination of the catchswitch for wasm. + for (const BasicBlock *CatchPadBB : CatchSwitch->handlers()) { + UnwindDests.emplace_back(FuncInfo.MBBMap[CatchPadBB], Prob); + UnwindDests.back().first->setIsEHScopeEntry(); + } + break; + } else { + continue; + } + } +} + /// When an invoke or a cleanupret unwinds to the next EH pad, there are /// many places it could ultimately go. In the IR, we have a single unwind /// destination, but in the machine CFG, we enumerate all the possible blocks. @@ -1489,6 +1693,13 @@ static void findUnwindDestinations( bool IsWasmCXX = Personality == EHPersonality::Wasm_CXX; bool IsSEH = isAsynchronousEHPersonality(Personality); + if (IsWasmCXX) { + findWasmUnwindDestinations(FuncInfo, EHPadBB, Prob, UnwindDests); + assert(UnwindDests.size() <= 1 && + "There should be at most one unwind destination for wasm"); + return; + } + while (EHPadBB) { const Instruction *Pad = EHPadBB->getFirstNonPHI(); BasicBlock *NewEHPadBB = nullptr; @@ -1501,8 +1712,7 @@ static void findUnwindDestinations( // personalities. UnwindDests.emplace_back(FuncInfo.MBBMap[EHPadBB], Prob); UnwindDests.back().first->setIsEHScopeEntry(); - if (!IsWasmCXX) - UnwindDests.back().first->setIsEHFuncletEntry(); + UnwindDests.back().first->setIsEHFuncletEntry(); break; } else if (auto *CatchSwitch = dyn_cast(Pad)) { // Add the catchpad handlers to the possible destinations. @@ -1588,9 +1798,10 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) { DemoteReg, PtrValueVTs[0]); SDValue RetOp = getValue(I.getOperand(0)); - SmallVector ValueVTs; + SmallVector ValueVTs, MemVTs; SmallVector Offsets; - ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &Offsets); + ComputeValueVTs(TLI, DL, I.getOperand(0)->getType(), ValueVTs, &MemVTs, + &Offsets); unsigned NumValues = ValueVTs.size(); SmallVector Chains(NumValues); @@ -1598,8 +1809,11 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) { // An aggregate return value cannot wrap around the address space, so // offsets to its parts don't wrap either. SDValue Ptr = DAG.getObjectPtrOffset(getCurSDLoc(), RetPtr, Offsets[i]); - Chains[i] = DAG.getStore( - Chain, getCurSDLoc(), SDValue(RetOp.getNode(), RetOp.getResNo() + i), + + SDValue Val = RetOp.getValue(i); + if (MemVTs[i] != ValueVTs[i]) + Val = DAG.getPtrExtOrTrunc(Val, getCurSDLoc(), MemVTs[i]); + Chains[i] = DAG.getStore(Chain, getCurSDLoc(), Val, // FIXME: better loc info would be nice. Ptr, MachinePointerInfo::getUnknownStack(DAG.getMachineFunction())); } @@ -1615,6 +1829,10 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) { const Function *F = I.getParent()->getParent(); + bool NeedsRegBlock = TLI.functionArgumentNeedsConsecutiveRegisters( + I.getOperand(0)->getType(), F->getCallingConv(), + /*IsVarArg*/ false); + ISD::NodeType ExtendKind = ISD::ANY_EXTEND; if (F->getAttributes().hasAttribute(AttributeList::ReturnIndex, Attribute::SExt)) @@ -1647,6 +1865,18 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) { if (RetInReg) Flags.setInReg(); + if (I.getOperand(0)->getType()->isPointerTy()) { + Flags.setPointer(); + Flags.setPointerAddrSpace( + cast(I.getOperand(0)->getType())->getAddressSpace()); + } + + if (NeedsRegBlock) { + Flags.setInConsecutiveRegs(); + if (j == NumValues - 1) + Flags.setInConsecutiveRegsLast(); + } + // Propagate extension type if any if (ExtendKind == ISD::SIGN_EXTEND) Flags.setSExt(); @@ -1668,7 +1898,7 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) { const Function *F = I.getParent()->getParent(); if (TLI.supportSwiftError() && F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) { - assert(FuncInfo.SwiftErrorArg && "Need a swift error argument"); + assert(SwiftError.getFunctionArg() && "Need a swift error argument"); ISD::ArgFlagsTy Flags = ISD::ArgFlagsTy(); Flags.setSwiftError(); Outs.push_back(ISD::OutputArg(Flags, EVT(TLI.getPointerTy(DL)) /*vt*/, @@ -1677,8 +1907,8 @@ void SelectionDAGBuilder::visitRet(const ReturnInst &I) { 0 /*partOffs*/)); // Create SDNode for the swifterror virtual register. OutVals.push_back( - DAG.getRegister(FuncInfo.getOrCreateSwiftErrorVRegUseAt( - &I, FuncInfo.MBB, FuncInfo.SwiftErrorArg).first, + DAG.getRegister(SwiftError.getOrCreateVRegUseAt( + &I, FuncInfo.MBB, SwiftError.getFunctionArg()), EVT(TLI.getPointerTy(DL)))); } @@ -1825,7 +2055,7 @@ SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond, CaseBlock CB(Condition, BOp->getOperand(0), BOp->getOperand(1), nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb); - SwitchCases.push_back(CB); + SL->SwitchCases.push_back(CB); return; } } @@ -1834,7 +2064,7 @@ SelectionDAGBuilder::EmitBranchForMergedCondition(const Value *Cond, ISD::CondCode Opc = InvertCond ? ISD::SETNE : ISD::SETEQ; CaseBlock CB(Opc, Cond, ConstantInt::getTrue(*DAG.getContext()), nullptr, TBB, FBB, CurBB, getCurSDLoc(), TProb, FProb); - SwitchCases.push_back(CB); + SL->SwitchCases.push_back(CB); } void SelectionDAGBuilder::FindMergedConditions(const Value *Cond, @@ -2043,27 +2273,27 @@ void SelectionDAGBuilder::visitBr(const BranchInst &I) { // If the compares in later blocks need to use values not currently // exported from this block, export them now. This block should always // be the first entry. - assert(SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!"); + assert(SL->SwitchCases[0].ThisBB == BrMBB && "Unexpected lowering!"); // Allow some cases to be rejected. - if (ShouldEmitAsBranches(SwitchCases)) { - for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) { - ExportFromCurrentBlock(SwitchCases[i].CmpLHS); - ExportFromCurrentBlock(SwitchCases[i].CmpRHS); + if (ShouldEmitAsBranches(SL->SwitchCases)) { + for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) { + ExportFromCurrentBlock(SL->SwitchCases[i].CmpLHS); + ExportFromCurrentBlock(SL->SwitchCases[i].CmpRHS); } // Emit the branch for this block. - visitSwitchCase(SwitchCases[0], BrMBB); - SwitchCases.erase(SwitchCases.begin()); + visitSwitchCase(SL->SwitchCases[0], BrMBB); + SL->SwitchCases.erase(SL->SwitchCases.begin()); return; } // Okay, we decided not to do this, remove any inserted MBB's and clear // SwitchCases. - for (unsigned i = 1, e = SwitchCases.size(); i != e; ++i) - FuncInfo.MF->erase(SwitchCases[i].ThisBB); + for (unsigned i = 1, e = SL->SwitchCases.size(); i != e; ++i) + FuncInfo.MF->erase(SL->SwitchCases[i].ThisBB); - SwitchCases.clear(); + SL->SwitchCases.clear(); } } @@ -2084,6 +2314,20 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB, SDValue CondLHS = getValue(CB.CmpLHS); SDLoc dl = CB.DL; + if (CB.CC == ISD::SETTRUE) { + // Branch or fall through to TrueBB. + addSuccessorWithProb(SwitchBB, CB.TrueBB, CB.TrueProb); + SwitchBB->normalizeSuccProbs(); + if (CB.TrueBB != NextBlock(SwitchBB)) { + DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, getControlRoot(), + DAG.getBasicBlock(CB.TrueBB))); + } + return; + } + + auto &TLI = DAG.getTargetLoweringInfo(); + EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), CB.CmpLHS->getType()); + // Build the setcc now. if (!CB.CmpMHS) { // Fold "(X == true)" to X and "(X == false)" to !X to @@ -2095,8 +2339,18 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB, CB.CC == ISD::SETEQ) { SDValue True = DAG.getConstant(1, dl, CondLHS.getValueType()); Cond = DAG.getNode(ISD::XOR, dl, CondLHS.getValueType(), CondLHS, True); - } else - Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, getValue(CB.CmpRHS), CB.CC); + } else { + SDValue CondRHS = getValue(CB.CmpRHS); + + // If a pointer's DAG type is larger than its memory type then the DAG + // values are zero-extended. This breaks signed comparisons so truncate + // back to the underlying type before doing the compare. + if (CondLHS.getValueType() != MemVT) { + CondLHS = DAG.getPtrExtOrTrunc(CondLHS, getCurSDLoc(), MemVT); + CondRHS = DAG.getPtrExtOrTrunc(CondRHS, getCurSDLoc(), MemVT); + } + Cond = DAG.getSetCC(dl, MVT::i1, CondLHS, CondRHS, CB.CC); + } } else { assert(CB.CC == ISD::SETLE && "Can handle only LE ranges now"); @@ -2147,7 +2401,7 @@ void SelectionDAGBuilder::visitSwitchCase(CaseBlock &CB, } /// visitJumpTable - Emit JumpTable node in the current MBB -void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) { +void SelectionDAGBuilder::visitJumpTable(SwitchCG::JumpTable &JT) { // Emit the code for the jump table assert(JT.Reg != -1U && "Should lower JT Header first!"); EVT PTy = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout()); @@ -2162,14 +2416,12 @@ void SelectionDAGBuilder::visitJumpTable(JumpTable &JT) { /// visitJumpTableHeader - This function emits necessary code to produce index /// in the JumpTable from switch case. -void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT, +void SelectionDAGBuilder::visitJumpTableHeader(SwitchCG::JumpTable &JT, JumpTableHeader &JTH, MachineBasicBlock *SwitchBB) { SDLoc dl = getCurSDLoc(); - // Subtract the lowest switch case value from the value being switched on and - // conditional branch to default mbb if the result is greater than the - // difference between smallest and largest cases. + // Subtract the lowest switch case value from the value being switched on. SDValue SwitchOp = getValue(JTH.SValue); EVT VT = SwitchOp.getValueType(); SDValue Sub = DAG.getNode(ISD::SUB, dl, VT, SwitchOp, @@ -2189,24 +2441,33 @@ void SelectionDAGBuilder::visitJumpTableHeader(JumpTable &JT, JumpTableReg, SwitchOp); JT.Reg = JumpTableReg; - // Emit the range check for the jump table, and branch to the default block - // for the switch statement if the value being switched on exceeds the largest - // case in the switch. - SDValue CMP = DAG.getSetCC( - dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), - Sub.getValueType()), - Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT); - - SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, - MVT::Other, CopyTo, CMP, - DAG.getBasicBlock(JT.Default)); - - // Avoid emitting unnecessary branches to the next block. - if (JT.MBB != NextBlock(SwitchBB)) - BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond, - DAG.getBasicBlock(JT.MBB)); - - DAG.setRoot(BrCond); + if (!JTH.OmitRangeCheck) { + // Emit the range check for the jump table, and branch to the default block + // for the switch statement if the value being switched on exceeds the + // largest case in the switch. + SDValue CMP = DAG.getSetCC( + dl, TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), + Sub.getValueType()), + Sub, DAG.getConstant(JTH.Last - JTH.First, dl, VT), ISD::SETUGT); + + SDValue BrCond = DAG.getNode(ISD::BRCOND, dl, + MVT::Other, CopyTo, CMP, + DAG.getBasicBlock(JT.Default)); + + // Avoid emitting unnecessary branches to the next block. + if (JT.MBB != NextBlock(SwitchBB)) + BrCond = DAG.getNode(ISD::BR, dl, MVT::Other, BrCond, + DAG.getBasicBlock(JT.MBB)); + + DAG.setRoot(BrCond); + } else { + // Avoid emitting unnecessary branches to the next block. + if (JT.MBB != NextBlock(SwitchBB)) + DAG.setRoot(DAG.getNode(ISD::BR, dl, MVT::Other, CopyTo, + DAG.getBasicBlock(JT.MBB))); + else + DAG.setRoot(CopyTo); + } } /// Create a LOAD_STACK_GUARD node, and let it carry the target specific global @@ -2215,6 +2476,7 @@ static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, SDValue &Chain) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout()); + EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout()); MachineFunction &MF = DAG.getMachineFunction(); Value *Global = TLI.getSDagStackGuard(*MF.getFunction().getParent()); MachineSDNode *Node = @@ -2227,6 +2489,8 @@ static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlignment(PtrTy)); DAG.setNodeMemRefs(Node, {MemRef}); } + if (PtrTy != PtrMemTy) + return DAG.getPtrExtOrTrunc(SDValue(Node, 0), DL, PtrMemTy); return SDValue(Node, 0); } @@ -2242,6 +2506,7 @@ void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD, // First create the loads to the guard/stack slot for the comparison. const TargetLowering &TLI = DAG.getTargetLoweringInfo(); EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout()); + EVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout()); MachineFrameInfo &MFI = ParentBB->getParent()->getFrameInfo(); int FI = MFI.getStackProtectorIndex(); @@ -2254,7 +2519,7 @@ void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD, // Generate code to load the content of the guard slot. SDValue GuardVal = DAG.getLoad( - PtrTy, dl, DAG.getEntryNode(), StackSlotPtr, + PtrMemTy, dl, DAG.getEntryNode(), StackSlotPtr, MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), Align, MachineMemOperand::MOVolatile); @@ -2262,27 +2527,26 @@ void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD, GuardVal = TLI.emitStackGuardXorFP(DAG, GuardVal, dl); // Retrieve guard check function, nullptr if instrumentation is inlined. - if (const Value *GuardCheck = TLI.getSSPStackGuardCheck(M)) { + if (const Function *GuardCheckFn = TLI.getSSPStackGuardCheck(M)) { // The target provides a guard check function to validate the guard value. // Generate a call to that function with the content of the guard slot as // argument. - auto *Fn = cast(GuardCheck); - FunctionType *FnTy = Fn->getFunctionType(); + FunctionType *FnTy = GuardCheckFn->getFunctionType(); assert(FnTy->getNumParams() == 1 && "Invalid function signature"); TargetLowering::ArgListTy Args; TargetLowering::ArgListEntry Entry; Entry.Node = GuardVal; Entry.Ty = FnTy->getParamType(0); - if (Fn->hasAttribute(1, Attribute::AttrKind::InReg)) + if (GuardCheckFn->hasAttribute(1, Attribute::AttrKind::InReg)) Entry.IsInReg = true; Args.push_back(Entry); TargetLowering::CallLoweringInfo CLI(DAG); CLI.setDebugLoc(getCurSDLoc()) - .setChain(DAG.getEntryNode()) - .setCallee(Fn->getCallingConv(), FnTy->getReturnType(), - getValue(GuardCheck), std::move(Args)); + .setChain(DAG.getEntryNode()) + .setCallee(GuardCheckFn->getCallingConv(), FnTy->getReturnType(), + getValue(GuardCheckFn), std::move(Args)); std::pair Result = TLI.LowerCallTo(CLI); DAG.setRoot(Result.second); @@ -2298,9 +2562,9 @@ void SelectionDAGBuilder::visitSPDescriptorParent(StackProtectorDescriptor &SPD, const Value *IRGuard = TLI.getSDagStackGuard(M); SDValue GuardPtr = getValue(IRGuard); - Guard = - DAG.getLoad(PtrTy, dl, Chain, GuardPtr, MachinePointerInfo(IRGuard, 0), - Align, MachineMemOperand::MOVolatile); + Guard = DAG.getLoad(PtrMemTy, dl, Chain, GuardPtr, + MachinePointerInfo(IRGuard, 0), Align, + MachineMemOperand::MOVolatile); } // Perform the comparison via a subtract/getsetcc. @@ -2339,6 +2603,12 @@ SelectionDAGBuilder::visitSPDescriptorFailure(StackProtectorDescriptor &SPD) { SDValue Chain = TLI.makeLibCall(DAG, RTLIB::STACKPROTECTOR_CHECK_FAIL, MVT::isVoid, None, false, getCurSDLoc(), false, false).second; + // On PS4, the "return address" must still be within the calling function, + // even if it's at the very end, so emit an explicit TRAP here. + // Passing 'true' for doesNotReturn above won't generate the trap for us. + if (TM.getTargetTriple().isPS4CPU()) + Chain = DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, Chain); + DAG.setRoot(Chain); } @@ -2493,6 +2763,20 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) { case Intrinsic::experimental_gc_statepoint: LowerStatepoint(ImmutableStatepoint(&I), EHPadBB); break; + case Intrinsic::wasm_rethrow_in_catch: { + // This is usually done in visitTargetIntrinsic, but this intrinsic is + // special because it can be invoked, so we manually lower it to a DAG + // node here. + SmallVector Ops; + Ops.push_back(getRoot()); // inchain + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + Ops.push_back( + DAG.getTargetConstant(Intrinsic::wasm_rethrow_in_catch, getCurSDLoc(), + TLI.getPointerTy(DAG.getDataLayout()))); + SDVTList VTs = DAG.getVTList(ArrayRef({MVT::Other})); // outchain + DAG.setRoot(DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops)); + break; + } } } else if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) { // Currently we do not lower any intrinsic calls with deopt operand bundles. @@ -2528,6 +2812,35 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) { InvokeMBB->normalizeSuccProbs(); // Drop into normal successor. + DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(), + DAG.getBasicBlock(Return))); +} + +void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) { + MachineBasicBlock *CallBrMBB = FuncInfo.MBB; + + // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't + // have to do anything here to lower funclet bundles. + assert(!I.hasOperandBundlesOtherThan( + {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && + "Cannot lower callbrs with arbitrary operand bundles yet!"); + + assert(isa(I.getCalledValue()) && + "Only know how to handle inlineasm callbr"); + visitInlineAsm(&I); + + // Retrieve successors. + MachineBasicBlock *Return = FuncInfo.MBBMap[I.getDefaultDest()]; + + // Update successor info. + addSuccessorWithProb(CallBrMBB, Return); + for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) { + MachineBasicBlock *Target = FuncInfo.MBBMap[I.getIndirectDest(i)]; + addSuccessorWithProb(CallBrMBB, Target); + } + CallBrMBB->normalizeSuccProbs(); + + // Drop into default successor. DAG.setRoot(DAG.getNode(ISD::BR, getCurSDLoc(), MVT::Other, getControlRoot(), DAG.getBasicBlock(Return))); @@ -2585,49 +2898,17 @@ void SelectionDAGBuilder::visitLandingPad(const LandingPadInst &LP) { setValue(&LP, Res); } -void SelectionDAGBuilder::sortAndRangeify(CaseClusterVector &Clusters) { -#ifndef NDEBUG - for (const CaseCluster &CC : Clusters) - assert(CC.Low == CC.High && "Input clusters must be single-case"); -#endif - - llvm::sort(Clusters, [](const CaseCluster &a, const CaseCluster &b) { - return a.Low->getValue().slt(b.Low->getValue()); - }); - - // Merge adjacent clusters with the same destination. - const unsigned N = Clusters.size(); - unsigned DstIndex = 0; - for (unsigned SrcIndex = 0; SrcIndex < N; ++SrcIndex) { - CaseCluster &CC = Clusters[SrcIndex]; - const ConstantInt *CaseVal = CC.Low; - MachineBasicBlock *Succ = CC.MBB; - - if (DstIndex != 0 && Clusters[DstIndex - 1].MBB == Succ && - (CaseVal->getValue() - Clusters[DstIndex - 1].High->getValue()) == 1) { - // If this case has the same successor and is a neighbour, merge it into - // the previous cluster. - Clusters[DstIndex - 1].High = CaseVal; - Clusters[DstIndex - 1].Prob += CC.Prob; - } else { - std::memmove(&Clusters[DstIndex++], &Clusters[SrcIndex], - sizeof(Clusters[SrcIndex])); - } - } - Clusters.resize(DstIndex); -} - void SelectionDAGBuilder::UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last) { // Update JTCases. - for (unsigned i = 0, e = JTCases.size(); i != e; ++i) - if (JTCases[i].first.HeaderBB == First) - JTCases[i].first.HeaderBB = Last; + for (unsigned i = 0, e = SL->JTCases.size(); i != e; ++i) + if (SL->JTCases[i].first.HeaderBB == First) + SL->JTCases[i].first.HeaderBB = Last; // Update BitTestCases. - for (unsigned i = 0, e = BitTestCases.size(); i != e; ++i) - if (BitTestCases[i].Parent == First) - BitTestCases[i].Parent = Last; + for (unsigned i = 0, e = SL->BitTestCases.size(); i != e; ++i) + if (SL->BitTestCases[i].Parent == First) + SL->BitTestCases[i].Parent = Last; } void SelectionDAGBuilder::visitIndirectBr(const IndirectBrInst &I) { @@ -2916,6 +3197,18 @@ void SelectionDAGBuilder::visitICmp(const User &I) { SDValue Op2 = getValue(I.getOperand(1)); ISD::CondCode Opcode = getICmpCondCode(predicate); + auto &TLI = DAG.getTargetLoweringInfo(); + EVT MemVT = + TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType()); + + // If a pointer's DAG type is larger than its memory type then the DAG values + // are zero-extended. This breaks signed comparisons so truncate back to the + // underlying type before doing the compare. + if (Op1.getValueType() != MemVT) { + Op1 = DAG.getPtrExtOrTrunc(Op1, getCurSDLoc(), MemVT); + Op2 = DAG.getPtrExtOrTrunc(Op2, getCurSDLoc(), MemVT); + } + EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), I.getType()); setValue(&I, DAG.getSetCC(getCurSDLoc(), DestVT, Op1, Op2, Opcode)); @@ -2963,6 +3256,8 @@ void SelectionDAGBuilder::visitSelect(const User &I) { ISD::NodeType OpCode = Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT; + bool IsUnaryAbs = false; + // Min/max matching is only viable if all output VTs are the same. if (is_splat(ValueVTs)) { EVT VT = ValueVTs[0]; @@ -3023,10 +3318,16 @@ void SelectionDAGBuilder::visitSelect(const User &I) { break; } break; + case SPF_ABS: + IsUnaryAbs = true; + Opc = ISD::ABS; + break; + case SPF_NABS: + // TODO: we need to produce sub(0, abs(X)). default: break; } - if (Opc != ISD::DELETED_NODE && + if (!IsUnaryAbs && Opc != ISD::DELETED_NODE && (TLI.isOperationLegalOrCustom(Opc, VT) || (UseScalarMinMax && TLI.isOperationLegalOrCustom(Opc, VT.getScalarType()))) && @@ -3039,15 +3340,30 @@ void SelectionDAGBuilder::visitSelect(const User &I) { RHSVal = getValue(RHS); BaseOps = {}; } + + if (IsUnaryAbs) { + OpCode = Opc; + LHSVal = getValue(LHS); + BaseOps = {}; + } } - for (unsigned i = 0; i != NumValues; ++i) { - SmallVector Ops(BaseOps.begin(), BaseOps.end()); - Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i)); - Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i)); - Values[i] = DAG.getNode(OpCode, getCurSDLoc(), - LHSVal.getNode()->getValueType(LHSVal.getResNo()+i), - Ops); + if (IsUnaryAbs) { + for (unsigned i = 0; i != NumValues; ++i) { + Values[i] = + DAG.getNode(OpCode, getCurSDLoc(), + LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), + SDValue(LHSVal.getNode(), LHSVal.getResNo() + i)); + } + } else { + for (unsigned i = 0; i != NumValues; ++i) { + SmallVector Ops(BaseOps.begin(), BaseOps.end()); + Ops.push_back(SDValue(LHSVal.getNode(), LHSVal.getResNo() + i)); + Ops.push_back(SDValue(RHSVal.getNode(), RHSVal.getResNo() + i)); + Values[i] = DAG.getNode( + OpCode, getCurSDLoc(), + LHSVal.getNode()->getValueType(LHSVal.getResNo() + i), Ops); + } } setValue(&I, DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), @@ -3135,18 +3451,26 @@ void SelectionDAGBuilder::visitPtrToInt(const User &I) { // What to do depends on the size of the integer and the size of the pointer. // We can either truncate, zero extend, or no-op, accordingly. SDValue N = getValue(I.getOperand(0)); + auto &TLI = DAG.getTargetLoweringInfo(); EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), I.getType()); - setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT)); + EVT PtrMemVT = + TLI.getMemValueType(DAG.getDataLayout(), I.getOperand(0)->getType()); + N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), PtrMemVT); + N = DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT); + setValue(&I, N); } void SelectionDAGBuilder::visitIntToPtr(const User &I) { // What to do depends on the size of the integer and the size of the pointer. // We can either truncate, zero extend, or no-op, accordingly. SDValue N = getValue(I.getOperand(0)); - EVT DestVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(), - I.getType()); - setValue(&I, DAG.getZExtOrTrunc(N, getCurSDLoc(), DestVT)); + auto &TLI = DAG.getTargetLoweringInfo(); + EVT DestVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); + EVT PtrMemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType()); + N = DAG.getZExtOrTrunc(N, getCurSDLoc(), PtrMemVT); + N = DAG.getPtrExtOrTrunc(N, getCurSDLoc(), DestVT); + setValue(&I, N); } void SelectionDAGBuilder::visitBitCast(const User &I) { @@ -3284,12 +3608,8 @@ void SelectionDAGBuilder::visitShuffleVector(const User &I) { MOps1[0] = Src1; MOps2[0] = Src2; - Src1 = Src1.isUndef() - ? DAG.getUNDEF(PaddedVT) - : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1); - Src2 = Src2.isUndef() - ? DAG.getUNDEF(PaddedVT) - : DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2); + Src1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps1); + Src2 = DAG.getNode(ISD::CONCAT_VECTORS, DL, PaddedVT, MOps2); // Readjust mask for new input vector length. SmallVector MappedOps(PaddedMaskNumElts, -1); @@ -3498,6 +3818,9 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) { unsigned AS = Op0->getType()->getScalarType()->getPointerAddressSpace(); SDValue N = getValue(Op0); SDLoc dl = getCurSDLoc(); + auto &TLI = DAG.getTargetLoweringInfo(); + MVT PtrTy = TLI.getPointerTy(DAG.getDataLayout(), AS); + MVT PtrMemTy = TLI.getPointerMemTy(DAG.getDataLayout(), AS); // Normalize Vector GEP - all scalar operands should be converted to the // splat vector. @@ -3555,6 +3878,8 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) { if (Offs.isNonNegative() && cast(I).isInBounds()) Flags.setNoUnsignedWrap(true); + OffsVal = DAG.getSExtOrTrunc(OffsVal, dl, N.getValueType()); + N = DAG.getNode(ISD::ADD, dl, N.getValueType(), N, OffsVal, Flags); continue; } @@ -3580,7 +3905,8 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) { N.getValueType(), IdxN, DAG.getConstant(Amt, dl, IdxN.getValueType())); } else { - SDValue Scale = DAG.getConstant(ElementSize, dl, IdxN.getValueType()); + SDValue Scale = DAG.getConstant(ElementSize.getZExtValue(), dl, + IdxN.getValueType()); IdxN = DAG.getNode(ISD::MUL, dl, N.getValueType(), IdxN, Scale); } @@ -3591,6 +3917,9 @@ void SelectionDAGBuilder::visitGetElementPtr(const User &I) { } } + if (PtrMemTy != PtrTy && !cast(I).isInBounds()) + N = DAG.getPtrExtendInReg(N, dl, PtrMemTy); + setValue(&I, N); } @@ -3675,16 +4004,17 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) { bool isVolatile = I.isVolatile(); bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr; bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr; - bool isDereferenceable = isDereferenceablePointer(SV, DAG.getDataLayout()); + bool isDereferenceable = + isDereferenceablePointer(SV, I.getType(), DAG.getDataLayout()); unsigned Alignment = I.getAlignment(); AAMDNodes AAInfo; I.getAAMetadata(AAInfo); const MDNode *Ranges = I.getMetadata(LLVMContext::MD_range); - SmallVector ValueVTs; + SmallVector ValueVTs, MemVTs; SmallVector Offsets; - ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &Offsets); + ComputeValueVTs(TLI, DAG.getDataLayout(), Ty, ValueVTs, &MemVTs, &Offsets); unsigned NumValues = ValueVTs.size(); if (NumValues == 0) return; @@ -3750,12 +4080,15 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) { MMOFlags |= MachineMemOperand::MODereferenceable; MMOFlags |= TLI.getMMOFlags(I); - SDValue L = DAG.getLoad(ValueVTs[i], dl, Root, A, + SDValue L = DAG.getLoad(MemVTs[i], dl, Root, A, MachinePointerInfo(SV, Offsets[i]), Alignment, MMOFlags, AAInfo, Ranges); + Chains[ChainI] = L.getValue(1); + + if (MemVTs[i] != ValueVTs[i]) + L = DAG.getZExtOrTrunc(L, dl, ValueVTs[i]); Values[i] = L; - Chains[ChainI] = L.getValue(1); } if (!ConstantMemory) { @@ -3785,15 +4118,13 @@ void SelectionDAGBuilder::visitStoreToSwiftError(const StoreInst &I) { SDValue Src = getValue(SrcV); // Create a virtual register, then update the virtual register. - unsigned VReg; bool CreatedVReg; - std::tie(VReg, CreatedVReg) = FuncInfo.getOrCreateSwiftErrorVRegDefAt(&I); + unsigned VReg = + SwiftError.getOrCreateVRegDefAt(&I, FuncInfo.MBB, I.getPointerOperand()); // Chain, DL, Reg, N or Chain, DL, Reg, N, Glue // Chain can be getRoot or getControlRoot. SDValue CopyNode = DAG.getCopyToReg(getRoot(), getCurSDLoc(), VReg, SDValue(Src.getNode(), Src.getResNo())); DAG.setRoot(CopyNode); - if (CreatedVReg) - FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, I.getOperand(1), VReg); } void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) { @@ -3826,8 +4157,7 @@ void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) { // Chain, DL, Reg, VT, Glue or Chain, DL, Reg, VT SDValue L = DAG.getCopyFromReg( getRoot(), getCurSDLoc(), - FuncInfo.getOrCreateSwiftErrorVRegUseAt(&I, FuncInfo.MBB, SV).first, - ValueVTs[0]); + SwiftError.getOrCreateVRegUseAt(&I, FuncInfo.MBB, SV), ValueVTs[0]); setValue(&I, L); } @@ -3854,10 +4184,10 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) { } } - SmallVector ValueVTs; + SmallVector ValueVTs, MemVTs; SmallVector Offsets; ComputeValueVTs(DAG.getTargetLoweringInfo(), DAG.getDataLayout(), - SrcV->getType(), ValueVTs, &Offsets); + SrcV->getType(), ValueVTs, &MemVTs, &Offsets); unsigned NumValues = ValueVTs.size(); if (NumValues == 0) return; @@ -3899,9 +4229,12 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) { } SDValue Add = DAG.getNode(ISD::ADD, dl, PtrVT, Ptr, DAG.getConstant(Offsets[i], dl, PtrVT), Flags); - SDValue St = DAG.getStore( - Root, dl, SDValue(Src.getNode(), Src.getResNo() + i), Add, - MachinePointerInfo(PtrV, Offsets[i]), Alignment, MMOFlags, AAInfo); + SDValue Val = SDValue(Src.getNode(), Src.getResNo() + i); + if (MemVTs[i] != ValueVTs[i]) + Val = DAG.getPtrExtOrTrunc(Val, dl, MemVTs[i]); + SDValue St = + DAG.getStore(Root, dl, Val, Add, MachinePointerInfo(PtrV, Offsets[i]), + Alignment, MMOFlags, AAInfo); Chains[ChainI] = St; } @@ -4181,19 +4514,34 @@ void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) { void SelectionDAGBuilder::visitAtomicCmpXchg(const AtomicCmpXchgInst &I) { SDLoc dl = getCurSDLoc(); - AtomicOrdering SuccessOrder = I.getSuccessOrdering(); - AtomicOrdering FailureOrder = I.getFailureOrdering(); + AtomicOrdering SuccessOrdering = I.getSuccessOrdering(); + AtomicOrdering FailureOrdering = I.getFailureOrdering(); SyncScope::ID SSID = I.getSyncScopeID(); SDValue InChain = getRoot(); MVT MemVT = getValue(I.getCompareOperand()).getSimpleValueType(); SDVTList VTs = DAG.getVTList(MemVT, MVT::i1, MVT::Other); - SDValue L = DAG.getAtomicCmpSwap( - ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, dl, MemVT, VTs, InChain, - getValue(I.getPointerOperand()), getValue(I.getCompareOperand()), - getValue(I.getNewValOperand()), MachinePointerInfo(I.getPointerOperand()), - /*Alignment=*/ 0, SuccessOrder, FailureOrder, SSID); + + auto Alignment = DAG.getEVTAlignment(MemVT); + + auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; + if (I.isVolatile()) + Flags |= MachineMemOperand::MOVolatile; + Flags |= DAG.getTargetLoweringInfo().getMMOFlags(I); + + MachineFunction &MF = DAG.getMachineFunction(); + MachineMemOperand *MMO = + MF.getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()), + Flags, MemVT.getStoreSize(), Alignment, + AAMDNodes(), nullptr, SSID, SuccessOrdering, + FailureOrdering); + + SDValue L = DAG.getAtomicCmpSwap(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, + dl, MemVT, VTs, InChain, + getValue(I.getPointerOperand()), + getValue(I.getCompareOperand()), + getValue(I.getNewValOperand()), MMO); SDValue OutChain = L.getValue(2); @@ -4217,20 +4565,32 @@ void SelectionDAGBuilder::visitAtomicRMW(const AtomicRMWInst &I) { case AtomicRMWInst::Min: NT = ISD::ATOMIC_LOAD_MIN; break; case AtomicRMWInst::UMax: NT = ISD::ATOMIC_LOAD_UMAX; break; case AtomicRMWInst::UMin: NT = ISD::ATOMIC_LOAD_UMIN; break; + case AtomicRMWInst::FAdd: NT = ISD::ATOMIC_LOAD_FADD; break; + case AtomicRMWInst::FSub: NT = ISD::ATOMIC_LOAD_FSUB; break; } - AtomicOrdering Order = I.getOrdering(); + AtomicOrdering Ordering = I.getOrdering(); SyncScope::ID SSID = I.getSyncScopeID(); SDValue InChain = getRoot(); + auto MemVT = getValue(I.getValOperand()).getSimpleValueType(); + auto Alignment = DAG.getEVTAlignment(MemVT); + + auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore; + if (I.isVolatile()) + Flags |= MachineMemOperand::MOVolatile; + Flags |= DAG.getTargetLoweringInfo().getMMOFlags(I); + + MachineFunction &MF = DAG.getMachineFunction(); + MachineMemOperand *MMO = + MF.getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()), Flags, + MemVT.getStoreSize(), Alignment, AAMDNodes(), + nullptr, SSID, Ordering); + SDValue L = - DAG.getAtomic(NT, dl, - getValue(I.getValOperand()).getSimpleValueType(), - InChain, - getValue(I.getPointerOperand()), - getValue(I.getValOperand()), - I.getPointerOperand(), - /* Alignment=*/ 0, Order, SSID); + DAG.getAtomic(NT, dl, MemVT, InChain, + getValue(I.getPointerOperand()), getValue(I.getValOperand()), + MMO); SDValue OutChain = L.getValue(1); @@ -4259,27 +4619,39 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); + EVT MemVT = TLI.getMemValueType(DAG.getDataLayout(), I.getType()); if (!TLI.supportsUnalignedAtomics() && - I.getAlignment() < VT.getStoreSize()) + I.getAlignment() < MemVT.getSizeInBits() / 8) report_fatal_error("Cannot generate unaligned atomic load"); + auto Flags = MachineMemOperand::MOLoad; + if (I.isVolatile()) + Flags |= MachineMemOperand::MOVolatile; + if (I.getMetadata(LLVMContext::MD_invariant_load) != nullptr) + Flags |= MachineMemOperand::MOInvariant; + if (isDereferenceablePointer(I.getPointerOperand(), I.getType(), + DAG.getDataLayout())) + Flags |= MachineMemOperand::MODereferenceable; + + Flags |= TLI.getMMOFlags(I); + MachineMemOperand *MMO = DAG.getMachineFunction(). getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()), - MachineMemOperand::MOVolatile | - MachineMemOperand::MOLoad, - VT.getStoreSize(), + Flags, MemVT.getStoreSize(), I.getAlignment() ? I.getAlignment() : - DAG.getEVTAlignment(VT), + DAG.getEVTAlignment(MemVT), AAMDNodes(), nullptr, SSID, Order); InChain = TLI.prepareVolatileOrAtomicLoad(InChain, dl, DAG); SDValue L = - DAG.getAtomic(ISD::ATOMIC_LOAD, dl, VT, VT, InChain, + DAG.getAtomic(ISD::ATOMIC_LOAD, dl, MemVT, MemVT, InChain, getValue(I.getPointerOperand()), MMO); SDValue OutChain = L.getValue(1); + if (MemVT != VT) + L = DAG.getPtrExtOrTrunc(L, dl, VT); setValue(&I, L); DAG.setRoot(OutChain); @@ -4288,25 +4660,36 @@ void SelectionDAGBuilder::visitAtomicLoad(const LoadInst &I) { void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) { SDLoc dl = getCurSDLoc(); - AtomicOrdering Order = I.getOrdering(); + AtomicOrdering Ordering = I.getOrdering(); SyncScope::ID SSID = I.getSyncScopeID(); SDValue InChain = getRoot(); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); - EVT VT = - TLI.getValueType(DAG.getDataLayout(), I.getValueOperand()->getType()); + EVT MemVT = + TLI.getMemValueType(DAG.getDataLayout(), I.getValueOperand()->getType()); - if (I.getAlignment() < VT.getStoreSize()) + if (I.getAlignment() < MemVT.getSizeInBits() / 8) report_fatal_error("Cannot generate unaligned atomic store"); - SDValue OutChain = - DAG.getAtomic(ISD::ATOMIC_STORE, dl, VT, - InChain, - getValue(I.getPointerOperand()), - getValue(I.getValueOperand()), - I.getPointerOperand(), I.getAlignment(), - Order, SSID); + auto Flags = MachineMemOperand::MOStore; + if (I.isVolatile()) + Flags |= MachineMemOperand::MOVolatile; + Flags |= TLI.getMMOFlags(I); + + MachineFunction &MF = DAG.getMachineFunction(); + MachineMemOperand *MMO = + MF.getMachineMemOperand(MachinePointerInfo(I.getPointerOperand()), Flags, + MemVT.getStoreSize(), I.getAlignment(), AAMDNodes(), + nullptr, SSID, Ordering); + + SDValue Val = getValue(I.getValueOperand()); + if (Val.getValueType() != MemVT) + Val = DAG.getPtrExtOrTrunc(Val, dl, MemVT); + + SDValue OutChain = DAG.getAtomic(ISD::ATOMIC_STORE, dl, MemVT, InChain, + getValue(I.getPointerOperand()), Val, MMO); + DAG.setRoot(OutChain); } @@ -4364,10 +4747,12 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, SDValue Result; if (IsTgtIntrinsic) { // This is target intrinsic that touches memory - Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, - Ops, Info.memVT, - MachinePointerInfo(Info.ptrVal, Info.offset), Info.align, - Info.flags, Info.size); + AAMDNodes AAInfo; + I.getAAMetadata(AAInfo); + Result = + DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops, Info.memVT, + MachinePointerInfo(Info.ptrVal, Info.offset), + Info.align, Info.flags, Info.size, AAInfo); } else if (!HasChain) { Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops); } else if (!I.getType()->isVoidTy()) { @@ -4889,7 +5274,7 @@ static SDValue ExpandPowI(const SDLoc &DL, SDValue LHS, SDValue RHS, return DAG.getConstantFP(1.0, DL, LHS.getValueType()); const Function &F = DAG.getMachineFunction().getFunction(); - if (!F.optForSize() || + if (!F.hasOptSize() || // If optimizing for size, don't insert too many multiplies. // This inserts up to 5 multiplies. countPopulation(Val) + Log2_32(Val) < 7) { @@ -4952,6 +5337,71 @@ bool SelectionDAGBuilder::EmitFuncArgumentDbgValue( if (!Arg) return false; + if (!IsDbgDeclare) { + // ArgDbgValues are hoisted to the beginning of the entry block. So we + // should only emit as ArgDbgValue if the dbg.value intrinsic is found in + // the entry block. + bool IsInEntryBlock = FuncInfo.MBB == &FuncInfo.MF->front(); + if (!IsInEntryBlock) + return false; + + // ArgDbgValues are hoisted to the beginning of the entry block. So we + // should only emit as ArgDbgValue if the dbg.value intrinsic describes a + // variable that also is a param. + // + // Although, if we are at the top of the entry block already, we can still + // emit using ArgDbgValue. This might catch some situations when the + // dbg.value refers to an argument that isn't used in the entry block, so + // any CopyToReg node would be optimized out and the only way to express + // this DBG_VALUE is by using the physical reg (or FI) as done in this + // method. ArgDbgValues are hoisted to the beginning of the entry block. So + // we should only emit as ArgDbgValue if the Variable is an argument to the + // current function, and the dbg.value intrinsic is found in the entry + // block. + bool VariableIsFunctionInputArg = Variable->isParameter() && + !DL->getInlinedAt(); + bool IsInPrologue = SDNodeOrder == LowestSDNodeOrder; + if (!IsInPrologue && !VariableIsFunctionInputArg) + return false; + + // Here we assume that a function argument on IR level only can be used to + // describe one input parameter on source level. If we for example have + // source code like this + // + // struct A { long x, y; }; + // void foo(struct A a, long b) { + // ... + // b = a.x; + // ... + // } + // + // and IR like this + // + // define void @foo(i32 %a1, i32 %a2, i32 %b) { + // entry: + // call void @llvm.dbg.value(metadata i32 %a1, "a", DW_OP_LLVM_fragment + // call void @llvm.dbg.value(metadata i32 %a2, "a", DW_OP_LLVM_fragment + // call void @llvm.dbg.value(metadata i32 %b, "b", + // ... + // call void @llvm.dbg.value(metadata i32 %a1, "b" + // ... + // + // then the last dbg.value is describing a parameter "b" using a value that + // is an argument. But since we already has used %a1 to describe a parameter + // we should not handle that last dbg.value here (that would result in an + // incorrect hoisting of the DBG_VALUE to the function entry). + // Notice that we allow one dbg.value per IR level argument, to accomodate + // for the situation with fragments above. + if (VariableIsFunctionInputArg) { + unsigned ArgNo = Arg->getArgNo(); + if (ArgNo >= FuncInfo.DescribedArgs.size()) + FuncInfo.DescribedArgs.resize(ArgNo + 1, false); + else if (!IsInPrologue && FuncInfo.DescribedArgs.test(ArgNo)) + return false; + FuncInfo.DescribedArgs.set(ArgNo); + } + } + MachineFunction &MF = DAG.getMachineFunction(); const TargetInstrInfo *TII = DAG.getSubtarget().getInstrInfo(); @@ -4976,12 +5426,14 @@ bool SelectionDAGBuilder::EmitFuncArgumentDbgValue( } } - if (!Op && N.getNode()) + if (!Op && N.getNode()) { // Check if frame index is available. - if (LoadSDNode *LNode = dyn_cast(N.getNode())) + SDValue LCandidate = peekThroughBitcasts(N); + if (LoadSDNode *LNode = dyn_cast(LCandidate.getNode())) if (FrameIndexSDNode *FINode = dyn_cast(LNode->getBasePtr().getNode())) Op = MachineOperand::CreateFI(FINode->getIndex()); + } if (!Op) { // Check if ValueMap has reg number. @@ -5055,11 +5507,29 @@ SDDbgValue *SelectionDAGBuilder::getDbgValue(SDValue N, # define setjmp_undefined_for_msvc #endif -/// Lower the call to the specified intrinsic function. If we want to emit this -/// as a call to a named external function, return the name. Otherwise, lower it -/// and return null. -const char * -SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { +static unsigned FixedPointIntrinsicToOpcode(unsigned Intrinsic) { + switch (Intrinsic) { + case Intrinsic::smul_fix: + return ISD::SMULFIX; + case Intrinsic::umul_fix: + return ISD::UMULFIX; + default: + llvm_unreachable("Unhandled fixed point intrinsic"); + } +} + +void SelectionDAGBuilder::lowerCallToExternalSymbol(const CallInst &I, + const char *FunctionName) { + assert(FunctionName && "FunctionName must not be nullptr"); + SDValue Callee = DAG.getExternalSymbol( + FunctionName, + DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout())); + LowerCallTo(&I, Callee, I.isTailCall()); +} + +/// Lower the call to the specified intrinsic function. +void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, + unsigned Intrinsic) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDLoc sdl = getCurSDLoc(); DebugLoc dl = getCurDebugLoc(); @@ -5069,28 +5539,28 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { default: // By default, turn this into a target intrinsic node. visitTargetIntrinsic(I, Intrinsic); - return nullptr; - case Intrinsic::vastart: visitVAStart(I); return nullptr; - case Intrinsic::vaend: visitVAEnd(I); return nullptr; - case Intrinsic::vacopy: visitVACopy(I); return nullptr; + return; + case Intrinsic::vastart: visitVAStart(I); return; + case Intrinsic::vaend: visitVAEnd(I); return; + case Intrinsic::vacopy: visitVACopy(I); return; case Intrinsic::returnaddress: setValue(&I, DAG.getNode(ISD::RETURNADDR, sdl, TLI.getPointerTy(DAG.getDataLayout()), getValue(I.getArgOperand(0)))); - return nullptr; + return; case Intrinsic::addressofreturnaddress: setValue(&I, DAG.getNode(ISD::ADDROFRETURNADDR, sdl, TLI.getPointerTy(DAG.getDataLayout()))); - return nullptr; + return; case Intrinsic::sponentry: setValue(&I, DAG.getNode(ISD::SPONENTRY, sdl, TLI.getPointerTy(DAG.getDataLayout()))); - return nullptr; + return; case Intrinsic::frameaddress: setValue(&I, DAG.getNode(ISD::FRAMEADDR, sdl, TLI.getPointerTy(DAG.getDataLayout()), getValue(I.getArgOperand(0)))); - return nullptr; + return; case Intrinsic::read_register: { Value *Reg = I.getArgOperand(0); SDValue Chain = getRoot(); @@ -5101,7 +5571,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { DAG.getVTList(VT, MVT::Other), Chain, RegName); setValue(&I, Res); DAG.setRoot(Res.getValue(1)); - return nullptr; + return; } case Intrinsic::write_register: { Value *Reg = I.getArgOperand(0); @@ -5111,12 +5581,14 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { DAG.getMDNode(cast(cast(Reg)->getMetadata())); DAG.setRoot(DAG.getNode(ISD::WRITE_REGISTER, sdl, MVT::Other, Chain, RegName, getValue(RegValue))); - return nullptr; + return; } case Intrinsic::setjmp: - return &"_setjmp"[!TLI.usesUnderscoreSetJmp()]; + lowerCallToExternalSymbol(I, &"_setjmp"[!TLI.usesUnderscoreSetJmp()]); + return; case Intrinsic::longjmp: - return &"_longjmp"[!TLI.usesUnderscoreLongJmp()]; + lowerCallToExternalSymbol(I, &"_longjmp"[!TLI.usesUnderscoreLongJmp()]); + return; case Intrinsic::memcpy: { const auto &MCI = cast(I); SDValue Op1 = getValue(I.getArgOperand(0)); @@ -5135,7 +5607,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { MachinePointerInfo(I.getArgOperand(0)), MachinePointerInfo(I.getArgOperand(1))); updateDAGForMaybeTailCall(MC); - return nullptr; + return; } case Intrinsic::memset: { const auto &MSI = cast(I); @@ -5149,7 +5621,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { SDValue MS = DAG.getMemset(getRoot(), sdl, Op1, Op2, Op3, Align, isVol, isTC, MachinePointerInfo(I.getArgOperand(0))); updateDAGForMaybeTailCall(MS); - return nullptr; + return; } case Intrinsic::memmove: { const auto &MMI = cast(I); @@ -5168,7 +5640,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { isTC, MachinePointerInfo(I.getArgOperand(0)), MachinePointerInfo(I.getArgOperand(1))); updateDAGForMaybeTailCall(MM); - return nullptr; + return; } case Intrinsic::memcpy_element_unordered_atomic: { const AtomicMemCpyInst &MI = cast(I); @@ -5186,7 +5658,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { MachinePointerInfo(MI.getRawDest()), MachinePointerInfo(MI.getRawSource())); updateDAGForMaybeTailCall(MC); - return nullptr; + return; } case Intrinsic::memmove_element_unordered_atomic: { auto &MI = cast(I); @@ -5204,7 +5676,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { MachinePointerInfo(MI.getRawDest()), MachinePointerInfo(MI.getRawSource())); updateDAGForMaybeTailCall(MC); - return nullptr; + return; } case Intrinsic::memset_element_unordered_atomic: { auto &MI = cast(I); @@ -5220,7 +5692,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { LengthTy, ElemSz, isTC, MachinePointerInfo(MI.getRawDest())); updateDAGForMaybeTailCall(MC); - return nullptr; + return; } case Intrinsic::dbg_addr: case Intrinsic::dbg_declare: { @@ -5235,7 +5707,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { if (!Address || isa(Address) || (Address->use_empty() && !isa(Address))) { LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI << "\n"); - return nullptr; + return; } bool isParameter = Variable->isParameter() || isa(Address); @@ -5264,7 +5736,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { Variable, Expression, FI, /*IsIndirect*/ true, dl, SDNodeOrder); DAG.AddDbgValue(SDV, getRoot().getNode(), isParameter); } - return nullptr; + return; } SDValue &N = NodeMap[Address]; @@ -5286,7 +5758,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { // Address is an argument, so try to emit its dbg value using // virtual register info from the FuncInfo.ValueMap. EmitFuncArgumentDbgValue(Address, Variable, Expression, dl, true, N); - return nullptr; + return; } else { SDV = DAG.getDbgValue(Variable, Expression, N.getNode(), N.getResNo(), true, dl, SDNodeOrder); @@ -5300,7 +5772,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { LLVM_DEBUG(dbgs() << "Dropping debug info for " << DI << "\n"); } } - return nullptr; + return; } case Intrinsic::dbg_label: { const DbgLabelInst &DI = cast(I); @@ -5310,7 +5782,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { SDDbgLabel *SDV; SDV = DAG.getDbgLabel(Label, dl, SDNodeOrder); DAG.AddDbgLabel(SDV); - return nullptr; + return; } case Intrinsic::dbg_value: { const DbgValueInst &DI = cast(I); @@ -5321,88 +5793,19 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { dropDanglingDebugInfo(Variable, Expression); const Value *V = DI.getValue(); if (!V) - return nullptr; + return; - SDDbgValue *SDV; - if (isa(V) || isa(V) || isa(V) || - isa(V)) { - SDV = DAG.getConstantDbgValue(Variable, Expression, V, dl, SDNodeOrder); - DAG.AddDbgValue(SDV, nullptr, false); - return nullptr; - } - - // Do not use getValue() in here; we don't want to generate code at - // this point if it hasn't been done yet. - SDValue N = NodeMap[V]; - if (!N.getNode() && isa(V)) // Check unused arguments map. - N = UnusedArgNodeMap[V]; - if (N.getNode()) { - if (EmitFuncArgumentDbgValue(V, Variable, Expression, dl, false, N)) - return nullptr; - SDV = getDbgValue(N, Variable, Expression, dl, SDNodeOrder); - DAG.AddDbgValue(SDV, N.getNode(), false); - return nullptr; - } - - // PHI nodes have already been selected, so we should know which VReg that - // is assigns to already. - if (isa(V)) { - auto VMI = FuncInfo.ValueMap.find(V); - if (VMI != FuncInfo.ValueMap.end()) { - unsigned Reg = VMI->second; - // The PHI node may be split up into several MI PHI nodes (in - // FunctionLoweringInfo::set). - RegsForValue RFV(V->getContext(), TLI, DAG.getDataLayout(), Reg, - V->getType(), None); - if (RFV.occupiesMultipleRegs()) { - unsigned Offset = 0; - unsigned BitsToDescribe = 0; - if (auto VarSize = Variable->getSizeInBits()) - BitsToDescribe = *VarSize; - if (auto Fragment = Expression->getFragmentInfo()) - BitsToDescribe = Fragment->SizeInBits; - for (auto RegAndSize : RFV.getRegsAndSizes()) { - unsigned RegisterSize = RegAndSize.second; - // Bail out if all bits are described already. - if (Offset >= BitsToDescribe) - break; - unsigned FragmentSize = (Offset + RegisterSize > BitsToDescribe) - ? BitsToDescribe - Offset - : RegisterSize; - auto FragmentExpr = DIExpression::createFragmentExpression( - Expression, Offset, FragmentSize); - if (!FragmentExpr) - continue; - SDV = DAG.getVRegDbgValue(Variable, *FragmentExpr, RegAndSize.first, - false, dl, SDNodeOrder); - DAG.AddDbgValue(SDV, nullptr, false); - Offset += RegisterSize; - } - } else { - SDV = DAG.getVRegDbgValue(Variable, Expression, Reg, false, dl, - SDNodeOrder); - DAG.AddDbgValue(SDV, nullptr, false); - } - return nullptr; - } - } + if (handleDebugValue(V, Variable, Expression, dl, DI.getDebugLoc(), + SDNodeOrder)) + return; - // TODO: When we get here we will either drop the dbg.value completely, or - // we try to move it forward by letting it dangle for awhile. So we should - // probably add an extra DbgValue to the DAG here, with a reference to - // "noreg", to indicate that we have lost the debug location for the - // variable. + // TODO: Dangling debug info will eventually either be resolved or produce + // an Undef DBG_VALUE. However in the resolution case, a gap may appear + // between the original dbg.value location and its resolved DBG_VALUE, which + // we should ideally fill with an extra Undef DBG_VALUE. - if (!V->use_empty() ) { - // Do not call getValue(V) yet, as we don't want to generate code. - // Remember it for later. - DanglingDebugInfoMap[V].emplace_back(&DI, dl, SDNodeOrder); - return nullptr; - } - - LLVM_DEBUG(dbgs() << "Dropping debug location info for:\n " << DI << "\n"); - LLVM_DEBUG(dbgs() << " Last seen at:\n " << *V << "\n"); - return nullptr; + DanglingDebugInfoMap[V].emplace_back(&DI, dl, SDNodeOrder); + return; } case Intrinsic::eh_typeid_for: { @@ -5411,7 +5814,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { unsigned TypeID = DAG.getMachineFunction().getTypeIDFor(GV); Res = DAG.getConstant(TypeID, sdl, MVT::i32); setValue(&I, Res); - return nullptr; + return; } case Intrinsic::eh_return_i32: @@ -5422,15 +5825,15 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { getControlRoot(), getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)))); - return nullptr; + return; case Intrinsic::eh_unwind_init: DAG.getMachineFunction().setCallsUnwindInit(true); - return nullptr; + return; case Intrinsic::eh_dwarf_cfa: setValue(&I, DAG.getNode(ISD::EH_DWARF_CFA, sdl, TLI.getPointerTy(DAG.getDataLayout()), getValue(I.getArgOperand(0)))); - return nullptr; + return; case Intrinsic::eh_sjlj_callsite: { MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); ConstantInt *CI = dyn_cast(I.getArgOperand(0)); @@ -5438,7 +5841,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { assert(MMI.getCurrentCallSite() == 0 && "Overlapping call sites!"); MMI.setCurrentCallSite(CI->getZExtValue()); - return nullptr; + return; } case Intrinsic::eh_sjlj_functioncontext: { // Get and store the index of the function context. @@ -5447,7 +5850,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { cast(I.getArgOperand(0)->stripPointerCasts()); int FI = FuncInfo.StaticAllocaMap[FnCtx]; MFI.setFunctionContextIndex(FI); - return nullptr; + return; } case Intrinsic::eh_sjlj_setjmp: { SDValue Ops[2]; @@ -5457,34 +5860,34 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { DAG.getVTList(MVT::i32, MVT::Other), Ops); setValue(&I, Op.getValue(0)); DAG.setRoot(Op.getValue(1)); - return nullptr; + return; } case Intrinsic::eh_sjlj_longjmp: DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_LONGJMP, sdl, MVT::Other, getRoot(), getValue(I.getArgOperand(0)))); - return nullptr; + return; case Intrinsic::eh_sjlj_setup_dispatch: DAG.setRoot(DAG.getNode(ISD::EH_SJLJ_SETUP_DISPATCH, sdl, MVT::Other, getRoot())); - return nullptr; + return; case Intrinsic::masked_gather: visitMaskedGather(I); - return nullptr; + return; case Intrinsic::masked_load: visitMaskedLoad(I); - return nullptr; + return; case Intrinsic::masked_scatter: visitMaskedScatter(I); - return nullptr; + return; case Intrinsic::masked_store: visitMaskedStore(I); - return nullptr; + return; case Intrinsic::masked_expandload: visitMaskedLoad(I, true /* IsExpanding */); - return nullptr; + return; case Intrinsic::masked_compressstore: visitMaskedStore(I, true /* IsCompressing */); - return nullptr; + return; case Intrinsic::x86_mmx_pslli_w: case Intrinsic::x86_mmx_pslli_d: case Intrinsic::x86_mmx_pslli_q: @@ -5496,7 +5899,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { SDValue ShAmt = getValue(I.getArgOperand(1)); if (isa(ShAmt)) { visitTargetIntrinsic(I, Intrinsic); - return nullptr; + return; } unsigned NewIntrinsic = 0; EVT ShAmtVT = MVT::v2i32; @@ -5542,31 +5945,31 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { DAG.getConstant(NewIntrinsic, sdl, MVT::i32), getValue(I.getArgOperand(0)), ShAmt); setValue(&I, Res); - return nullptr; + return; } case Intrinsic::powi: setValue(&I, ExpandPowI(sdl, getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), DAG)); - return nullptr; + return; case Intrinsic::log: setValue(&I, expandLog(sdl, getValue(I.getArgOperand(0)), DAG, TLI)); - return nullptr; + return; case Intrinsic::log2: setValue(&I, expandLog2(sdl, getValue(I.getArgOperand(0)), DAG, TLI)); - return nullptr; + return; case Intrinsic::log10: setValue(&I, expandLog10(sdl, getValue(I.getArgOperand(0)), DAG, TLI)); - return nullptr; + return; case Intrinsic::exp: setValue(&I, expandExp(sdl, getValue(I.getArgOperand(0)), DAG, TLI)); - return nullptr; + return; case Intrinsic::exp2: setValue(&I, expandExp2(sdl, getValue(I.getArgOperand(0)), DAG, TLI)); - return nullptr; + return; case Intrinsic::pow: setValue(&I, expandPow(sdl, getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), DAG, TLI)); - return nullptr; + return; case Intrinsic::sqrt: case Intrinsic::fabs: case Intrinsic::sin: @@ -5597,61 +6000,71 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { setValue(&I, DAG.getNode(Opcode, sdl, getValue(I.getArgOperand(0)).getValueType(), getValue(I.getArgOperand(0)))); - return nullptr; + return; + } + case Intrinsic::lround: + case Intrinsic::llround: + case Intrinsic::lrint: + case Intrinsic::llrint: { + unsigned Opcode; + switch (Intrinsic) { + default: llvm_unreachable("Impossible intrinsic"); // Can't reach here. + case Intrinsic::lround: Opcode = ISD::LROUND; break; + case Intrinsic::llround: Opcode = ISD::LLROUND; break; + case Intrinsic::lrint: Opcode = ISD::LRINT; break; + case Intrinsic::llrint: Opcode = ISD::LLRINT; break; + } + + EVT RetVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); + setValue(&I, DAG.getNode(Opcode, sdl, RetVT, + getValue(I.getArgOperand(0)))); + return; } - case Intrinsic::minnum: { - auto VT = getValue(I.getArgOperand(0)).getValueType(); - unsigned Opc = - I.hasNoNaNs() && TLI.isOperationLegalOrCustom(ISD::FMINIMUM, VT) - ? ISD::FMINIMUM - : ISD::FMINNUM; - setValue(&I, DAG.getNode(Opc, sdl, VT, + case Intrinsic::minnum: + setValue(&I, DAG.getNode(ISD::FMINNUM, sdl, + getValue(I.getArgOperand(0)).getValueType(), getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)))); - return nullptr; - } - case Intrinsic::maxnum: { - auto VT = getValue(I.getArgOperand(0)).getValueType(); - unsigned Opc = - I.hasNoNaNs() && TLI.isOperationLegalOrCustom(ISD::FMAXIMUM, VT) - ? ISD::FMAXIMUM - : ISD::FMAXNUM; - setValue(&I, DAG.getNode(Opc, sdl, VT, + return; + case Intrinsic::maxnum: + setValue(&I, DAG.getNode(ISD::FMAXNUM, sdl, + getValue(I.getArgOperand(0)).getValueType(), getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)))); - return nullptr; - } + return; case Intrinsic::minimum: setValue(&I, DAG.getNode(ISD::FMINIMUM, sdl, getValue(I.getArgOperand(0)).getValueType(), getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)))); - return nullptr; + return; case Intrinsic::maximum: setValue(&I, DAG.getNode(ISD::FMAXIMUM, sdl, getValue(I.getArgOperand(0)).getValueType(), getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)))); - return nullptr; + return; case Intrinsic::copysign: setValue(&I, DAG.getNode(ISD::FCOPYSIGN, sdl, getValue(I.getArgOperand(0)).getValueType(), getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)))); - return nullptr; + return; case Intrinsic::fma: setValue(&I, DAG.getNode(ISD::FMA, sdl, getValue(I.getArgOperand(0)).getValueType(), getValue(I.getArgOperand(0)), getValue(I.getArgOperand(1)), getValue(I.getArgOperand(2)))); - return nullptr; + return; case Intrinsic::experimental_constrained_fadd: case Intrinsic::experimental_constrained_fsub: case Intrinsic::experimental_constrained_fmul: case Intrinsic::experimental_constrained_fdiv: case Intrinsic::experimental_constrained_frem: case Intrinsic::experimental_constrained_fma: + case Intrinsic::experimental_constrained_fptrunc: + case Intrinsic::experimental_constrained_fpext: case Intrinsic::experimental_constrained_sqrt: case Intrinsic::experimental_constrained_pow: case Intrinsic::experimental_constrained_powi: @@ -5671,7 +6084,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { case Intrinsic::experimental_constrained_round: case Intrinsic::experimental_constrained_trunc: visitConstrainedFPIntrinsic(cast(I)); - return nullptr; + return; case Intrinsic::fmuladd: { EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); if (TM.Options.AllowFPOpFusion != FPOpFusion::Strict && @@ -5693,7 +6106,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { getValue(I.getArgOperand(2))); setValue(&I, Add); } - return nullptr; + return; } case Intrinsic::convert_to_fp16: setValue(&I, DAG.getNode(ISD::BITCAST, sdl, MVT::i16, @@ -5701,17 +6114,17 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { getValue(I.getArgOperand(0)), DAG.getTargetConstant(0, sdl, MVT::i32)))); - return nullptr; + return; case Intrinsic::convert_from_fp16: setValue(&I, DAG.getNode(ISD::FP_EXTEND, sdl, TLI.getValueType(DAG.getDataLayout(), I.getType()), DAG.getNode(ISD::BITCAST, sdl, MVT::f16, getValue(I.getArgOperand(0))))); - return nullptr; + return; case Intrinsic::pcmarker: { SDValue Tmp = getValue(I.getArgOperand(0)); DAG.setRoot(DAG.getNode(ISD::PCMARKER, sdl, MVT::Other, getRoot(), Tmp)); - return nullptr; + return; } case Intrinsic::readcyclecounter: { SDValue Op = getRoot(); @@ -5719,25 +6132,25 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { DAG.getVTList(MVT::i64, MVT::Other), Op); setValue(&I, Res); DAG.setRoot(Res.getValue(1)); - return nullptr; + return; } case Intrinsic::bitreverse: setValue(&I, DAG.getNode(ISD::BITREVERSE, sdl, getValue(I.getArgOperand(0)).getValueType(), getValue(I.getArgOperand(0)))); - return nullptr; + return; case Intrinsic::bswap: setValue(&I, DAG.getNode(ISD::BSWAP, sdl, getValue(I.getArgOperand(0)).getValueType(), getValue(I.getArgOperand(0)))); - return nullptr; + return; case Intrinsic::cttz: { SDValue Arg = getValue(I.getArgOperand(0)); ConstantInt *CI = cast(I.getArgOperand(1)); EVT Ty = Arg.getValueType(); setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTTZ : ISD::CTTZ_ZERO_UNDEF, sdl, Ty, Arg)); - return nullptr; + return; } case Intrinsic::ctlz: { SDValue Arg = getValue(I.getArgOperand(0)); @@ -5745,13 +6158,13 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { EVT Ty = Arg.getValueType(); setValue(&I, DAG.getNode(CI->isZero() ? ISD::CTLZ : ISD::CTLZ_ZERO_UNDEF, sdl, Ty, Arg)); - return nullptr; + return; } case Intrinsic::ctpop: { SDValue Arg = getValue(I.getArgOperand(0)); EVT Ty = Arg.getValueType(); setValue(&I, DAG.getNode(ISD::CTPOP, sdl, Ty, Arg)); - return nullptr; + return; } case Intrinsic::fshl: case Intrinsic::fshr: { @@ -5767,7 +6180,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { auto FunnelOpcode = IsFSHL ? ISD::FSHL : ISD::FSHR; if (TLI.isOperationLegalOrCustom(FunnelOpcode, VT)) { setValue(&I, DAG.getNode(FunnelOpcode, sdl, VT, X, Y, Z)); - return nullptr; + return; } // When X == Y, this is rotate. If the data type has a power-of-2 size, we @@ -5777,7 +6190,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { auto RotateOpcode = IsFSHL ? ISD::ROTL : ISD::ROTR; if (TLI.isOperationLegalOrCustom(RotateOpcode, VT)) { setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, Z)); - return nullptr; + return; } // Some targets only rotate one way. Try the opposite direction. @@ -5786,7 +6199,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { // Negate the shift amount because it is safe to ignore the high bits. SDValue NegShAmt = DAG.getNode(ISD::SUB, sdl, VT, Zero, Z); setValue(&I, DAG.getNode(RotateOpcode, sdl, VT, X, NegShAmt)); - return nullptr; + return; } // fshl (rotl): (X << (Z % BW)) | (X >> ((0 - Z) % BW)) @@ -5796,7 +6209,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { SDValue ShX = DAG.getNode(ISD::SHL, sdl, VT, X, IsFSHL ? ShAmt : NShAmt); SDValue ShY = DAG.getNode(ISD::SRL, sdl, VT, X, IsFSHL ? NShAmt : ShAmt); setValue(&I, DAG.getNode(ISD::OR, sdl, VT, ShX, ShY)); - return nullptr; + return; } // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW))) @@ -5816,39 +6229,48 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { // For fshr, 0-shift returns the 2nd arg (Y). SDValue IsZeroShift = DAG.getSetCC(sdl, CCVT, ShAmt, Zero, ISD::SETEQ); setValue(&I, DAG.getSelect(sdl, VT, IsZeroShift, IsFSHL ? X : Y, Or)); - return nullptr; + return; } case Intrinsic::sadd_sat: { SDValue Op1 = getValue(I.getArgOperand(0)); SDValue Op2 = getValue(I.getArgOperand(1)); setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2)); - return nullptr; + return; } case Intrinsic::uadd_sat: { SDValue Op1 = getValue(I.getArgOperand(0)); SDValue Op2 = getValue(I.getArgOperand(1)); setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2)); - return nullptr; + return; } case Intrinsic::ssub_sat: { SDValue Op1 = getValue(I.getArgOperand(0)); SDValue Op2 = getValue(I.getArgOperand(1)); setValue(&I, DAG.getNode(ISD::SSUBSAT, sdl, Op1.getValueType(), Op1, Op2)); - return nullptr; + return; } case Intrinsic::usub_sat: { SDValue Op1 = getValue(I.getArgOperand(0)); SDValue Op2 = getValue(I.getArgOperand(1)); setValue(&I, DAG.getNode(ISD::USUBSAT, sdl, Op1.getValueType(), Op1, Op2)); - return nullptr; + return; } - case Intrinsic::smul_fix: { + case Intrinsic::smul_fix: + case Intrinsic::umul_fix: { SDValue Op1 = getValue(I.getArgOperand(0)); SDValue Op2 = getValue(I.getArgOperand(1)); SDValue Op3 = getValue(I.getArgOperand(2)); - setValue(&I, - DAG.getNode(ISD::SMULFIX, sdl, Op1.getValueType(), Op1, Op2, Op3)); - return nullptr; + setValue(&I, DAG.getNode(FixedPointIntrinsicToOpcode(Intrinsic), sdl, + Op1.getValueType(), Op1, Op2, Op3)); + return; + } + case Intrinsic::smul_fix_sat: { + SDValue Op1 = getValue(I.getArgOperand(0)); + SDValue Op2 = getValue(I.getArgOperand(1)); + SDValue Op3 = getValue(I.getArgOperand(2)); + setValue(&I, DAG.getNode(ISD::SMULFIXSAT, sdl, Op1.getValueType(), Op1, Op2, + Op3)); + return; } case Intrinsic::stacksave: { SDValue Op = getRoot(); @@ -5857,26 +6279,26 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { DAG.getVTList(TLI.getPointerTy(DAG.getDataLayout()), MVT::Other), Op); setValue(&I, Res); DAG.setRoot(Res.getValue(1)); - return nullptr; + return; } case Intrinsic::stackrestore: Res = getValue(I.getArgOperand(0)); DAG.setRoot(DAG.getNode(ISD::STACKRESTORE, sdl, MVT::Other, getRoot(), Res)); - return nullptr; + return; case Intrinsic::get_dynamic_area_offset: { SDValue Op = getRoot(); EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout()); EVT ResTy = TLI.getValueType(DAG.getDataLayout(), I.getType()); // Result type for @llvm.get.dynamic.area.offset should match PtrTy for // target. - if (PtrTy != ResTy) + if (PtrTy.getSizeInBits() < ResTy.getSizeInBits()) report_fatal_error("Wrong result type for @llvm.get.dynamic.area.offset" " intrinsic!"); Res = DAG.getNode(ISD::GET_DYNAMIC_AREA_OFFSET, sdl, DAG.getVTList(ResTy), Op); DAG.setRoot(Op); setValue(&I, Res); - return nullptr; + return; } case Intrinsic::stackguard: { EVT PtrTy = TLI.getPointerTy(DAG.getDataLayout()); @@ -5896,7 +6318,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { Res = TLI.emitStackGuardXorFP(DAG, Res, sdl); DAG.setRoot(Chain); setValue(&I, Res); - return nullptr; + return; } case Intrinsic::stackprotector: { // Emit code into the DAG to store the stack guard onto the stack. @@ -5923,7 +6345,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { /* Alignment = */ 0, MachineMemOperand::MOVolatile); setValue(&I, Res); DAG.setRoot(Res); - return nullptr; + return; } case Intrinsic::objectsize: { // If we don't know by now, we're never going to know. @@ -5940,14 +6362,14 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { Res = DAG.getConstant(0, sdl, Ty); setValue(&I, Res); - return nullptr; + return; } case Intrinsic::is_constant: // If this wasn't constant-folded away by now, then it's not a // constant. setValue(&I, DAG.getConstant(0, sdl, MVT::i1)); - return nullptr; + return; case Intrinsic::annotation: case Intrinsic::ptr_annotation: @@ -5955,12 +6377,12 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { case Intrinsic::strip_invariant_group: // Drop the intrinsic, but forward the value setValue(&I, getValue(I.getOperand(0))); - return nullptr; + return; case Intrinsic::assume: case Intrinsic::var_annotation: case Intrinsic::sideeffect: // Discard annotate attributes, assumptions, and artificial side-effects. - return nullptr; + return; case Intrinsic::codeview_annotation: { // Emit a label associated with this metadata. @@ -5971,7 +6393,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { MF.addCodeViewAnnotation(Label, cast(MD)); Res = DAG.getLabelNode(ISD::ANNOTATION_LABEL, sdl, getRoot(), Label); DAG.setRoot(Res); - return nullptr; + return; } case Intrinsic::init_trampoline: { @@ -5988,13 +6410,13 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { Res = DAG.getNode(ISD::INIT_TRAMPOLINE, sdl, MVT::Other, Ops); DAG.setRoot(Res); - return nullptr; + return; } case Intrinsic::adjust_trampoline: setValue(&I, DAG.getNode(ISD::ADJUST_TRAMPOLINE, sdl, TLI.getPointerTy(DAG.getDataLayout()), getValue(I.getArgOperand(0)))); - return nullptr; + return; case Intrinsic::gcroot: { assert(DAG.getMachineFunction().getFunction().hasGC() && "only valid in functions with gc specified, enforced by Verifier"); @@ -6004,19 +6426,19 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { FrameIndexSDNode *FI = cast(getValue(Alloca).getNode()); GFI->addStackRoot(FI->getIndex(), TypeMap); - return nullptr; + return; } case Intrinsic::gcread: case Intrinsic::gcwrite: llvm_unreachable("GC failed to lower gcread/gcwrite intrinsics!"); case Intrinsic::flt_rounds: setValue(&I, DAG.getNode(ISD::FLT_ROUNDS_, sdl, MVT::i32)); - return nullptr; + return; case Intrinsic::expect: // Just replace __builtin_expect(exp, c) with EXP. setValue(&I, getValue(I.getArgOperand(0))); - return nullptr; + return; case Intrinsic::debugtrap: case Intrinsic::trap: { @@ -6028,7 +6450,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { ISD::NodeType Op = (Intrinsic == Intrinsic::trap) ? ISD::TRAP : ISD::DEBUGTRAP; DAG.setRoot(DAG.getNode(Op, sdl,MVT::Other, getRoot())); - return nullptr; + return; } TargetLowering::ArgListTy Args; @@ -6041,7 +6463,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { std::pair Result = TLI.LowerCallTo(CLI); DAG.setRoot(Result.second); - return nullptr; + return; } case Intrinsic::uadd_with_overflow: @@ -6063,9 +6485,15 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { SDValue Op1 = getValue(I.getArgOperand(0)); SDValue Op2 = getValue(I.getArgOperand(1)); - SDVTList VTs = DAG.getVTList(Op1.getValueType(), MVT::i1); + EVT ResultVT = Op1.getValueType(); + EVT OverflowVT = MVT::i1; + if (ResultVT.isVector()) + OverflowVT = EVT::getVectorVT( + *Context, OverflowVT, ResultVT.getVectorNumElements()); + + SDVTList VTs = DAG.getVTList(ResultVT, OverflowVT); setValue(&I, DAG.getNode(Op, sdl, VTs, Op1, Op2)); - return nullptr; + return; } case Intrinsic::prefetch: { SDValue Ops[5]; @@ -6088,21 +6516,24 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { PendingLoads.push_back(Result); Result = getRoot(); DAG.setRoot(Result); - return nullptr; + return; } case Intrinsic::lifetime_start: case Intrinsic::lifetime_end: { bool IsStart = (Intrinsic == Intrinsic::lifetime_start); // Stack coloring is not enabled in O0, discard region information. if (TM.getOptLevel() == CodeGenOpt::None) - return nullptr; + return; - SmallVector Allocas; - GetUnderlyingObjects(I.getArgOperand(1), Allocas, *DL); + const int64_t ObjectSize = + cast(I.getArgOperand(0))->getSExtValue(); + Value *const ObjectPtr = I.getArgOperand(1); + SmallVector Allocas; + GetUnderlyingObjects(ObjectPtr, Allocas, *DL); - for (SmallVectorImpl::iterator Object = Allocas.begin(), + for (SmallVectorImpl::iterator Object = Allocas.begin(), E = Allocas.end(); Object != E; ++Object) { - AllocaInst *LifetimeObject = dyn_cast_or_null(*Object); + const AllocaInst *LifetimeObject = dyn_cast_or_null(*Object); // Could not find an Alloca. if (!LifetimeObject) @@ -6112,49 +6543,50 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { // valid frame index. auto SI = FuncInfo.StaticAllocaMap.find(LifetimeObject); if (SI == FuncInfo.StaticAllocaMap.end()) - return nullptr; - - int FI = SI->second; - - SDValue Ops[2]; - Ops[0] = getRoot(); - Ops[1] = - DAG.getFrameIndex(FI, TLI.getFrameIndexTy(DAG.getDataLayout()), true); - unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END); + return; - Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops); + const int FrameIndex = SI->second; + int64_t Offset; + if (GetPointerBaseWithConstantOffset( + ObjectPtr, Offset, DAG.getDataLayout()) != LifetimeObject) + Offset = -1; // Cannot determine offset from alloca to lifetime object. + Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex, ObjectSize, + Offset); DAG.setRoot(Res); } - return nullptr; + return; } case Intrinsic::invariant_start: // Discard region information. setValue(&I, DAG.getUNDEF(TLI.getPointerTy(DAG.getDataLayout()))); - return nullptr; + return; case Intrinsic::invariant_end: // Discard region information. - return nullptr; + return; case Intrinsic::clear_cache: - return TLI.getClearCacheBuiltinName(); + /// FunctionName may be null. + if (const char *FunctionName = TLI.getClearCacheBuiltinName()) + lowerCallToExternalSymbol(I, FunctionName); + return; case Intrinsic::donothing: // ignore - return nullptr; + return; case Intrinsic::experimental_stackmap: visitStackmap(I); - return nullptr; + return; case Intrinsic::experimental_patchpoint_void: case Intrinsic::experimental_patchpoint_i64: visitPatchpoint(&I); - return nullptr; + return; case Intrinsic::experimental_gc_statepoint: LowerStatepoint(ImmutableStatepoint(&I)); - return nullptr; + return; case Intrinsic::experimental_gc_result: visitGCResult(cast(I)); - return nullptr; + return; case Intrinsic::experimental_gc_relocate: visitGCRelocate(cast(I)); - return nullptr; + return; case Intrinsic::instrprof_increment: llvm_unreachable("instrprof failed to lower an increment"); case Intrinsic::instrprof_value_profile: @@ -6182,7 +6614,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { .addFrameIndex(FI); } - return nullptr; + return; } case Intrinsic::localrecover: { @@ -6211,7 +6643,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { SDValue Add = DAG.getNode(ISD::ADD, sdl, PtrVT, FPVal, OffsetVal); setValue(&I, Add); - return nullptr; + return; } case Intrinsic::eh_exceptionpointer: @@ -6226,7 +6658,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { if (Intrinsic == Intrinsic::eh_exceptioncode) N = DAG.getZExtOrTrunc(N, getCurSDLoc(), MVT::i32); setValue(&I, N); - return nullptr; + return; } case Intrinsic::xray_customevent: { // Here we want to make sure that the intrinsic behaves as if it has a @@ -6234,7 +6666,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { // FIXME: Support other platforms later. const auto &Triple = DAG.getTarget().getTargetTriple(); if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux()) - return nullptr; + return; SDLoc DL = getCurSDLoc(); SmallVector Ops; @@ -6257,7 +6689,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { SDValue patchableNode = SDValue(MN, 0); DAG.setRoot(patchableNode); setValue(&I, patchableNode); - return nullptr; + return; } case Intrinsic::xray_typedevent: { // Here we want to make sure that the intrinsic behaves as if it has a @@ -6265,7 +6697,7 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { // FIXME: Support other platforms later. const auto &Triple = DAG.getTarget().getTargetTriple(); if (Triple.getArch() != Triple::x86_64 || !Triple.isOSLinux()) - return nullptr; + return; SDLoc DL = getCurSDLoc(); SmallVector Ops; @@ -6292,14 +6724,14 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { SDValue patchableNode = SDValue(MN, 0); DAG.setRoot(patchableNode); setValue(&I, patchableNode); - return nullptr; + return; } case Intrinsic::experimental_deoptimize: LowerDeoptimizeCall(&I); - return nullptr; + return; - case Intrinsic::experimental_vector_reduce_fadd: - case Intrinsic::experimental_vector_reduce_fmul: + case Intrinsic::experimental_vector_reduce_v2_fadd: + case Intrinsic::experimental_vector_reduce_v2_fmul: case Intrinsic::experimental_vector_reduce_add: case Intrinsic::experimental_vector_reduce_mul: case Intrinsic::experimental_vector_reduce_and: @@ -6312,11 +6744,10 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { case Intrinsic::experimental_vector_reduce_fmax: case Intrinsic::experimental_vector_reduce_fmin: visitVectorReduce(I, Intrinsic); - return nullptr; + return; case Intrinsic::icall_branch_funnel: { SmallVector Ops; - Ops.push_back(DAG.getRoot()); Ops.push_back(getValue(I.getArgOperand(0))); int64_t Offset; @@ -6359,20 +6790,34 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { Ops.push_back(T.Target); } + Ops.push_back(DAG.getRoot()); // Chain SDValue N(DAG.getMachineNode(TargetOpcode::ICALL_BRANCH_FUNNEL, getCurSDLoc(), MVT::Other, Ops), 0); DAG.setRoot(N); setValue(&I, N); HasTailCall = true; - return nullptr; + return; } case Intrinsic::wasm_landingpad_index: // Information this intrinsic contained has been transferred to // MachineFunction in SelectionDAGISel::PrepareEHLandingPad. We can safely // delete it now. - return nullptr; + return; + + case Intrinsic::aarch64_settag: + case Intrinsic::aarch64_settag_zero: { + const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); + bool ZeroMemory = Intrinsic == Intrinsic::aarch64_settag_zero; + SDValue Val = TSI.EmitTargetCodeForSetTag( + DAG, getCurSDLoc(), getRoot(), getValue(I.getArgOperand(0)), + getValue(I.getArgOperand(1)), MachinePointerInfo(I.getArgOperand(0)), + ZeroMemory); + DAG.setRoot(Val); + setValue(&I, Val); + return; + } } } @@ -6400,6 +6845,12 @@ void SelectionDAGBuilder::visitConstrainedFPIntrinsic( case Intrinsic::experimental_constrained_fma: Opcode = ISD::STRICT_FMA; break; + case Intrinsic::experimental_constrained_fptrunc: + Opcode = ISD::STRICT_FP_ROUND; + break; + case Intrinsic::experimental_constrained_fpext: + Opcode = ISD::STRICT_FP_EXTEND; + break; case Intrinsic::experimental_constrained_sqrt: Opcode = ISD::STRICT_FSQRT; break; @@ -6463,7 +6914,12 @@ void SelectionDAGBuilder::visitConstrainedFPIntrinsic( SDVTList VTs = DAG.getVTList(ValueVTs); SDValue Result; - if (FPI.isUnaryOp()) + if (Opcode == ISD::STRICT_FP_ROUND) + Result = DAG.getNode(Opcode, sdl, VTs, + { Chain, getValue(FPI.getArgOperand(0)), + DAG.getTargetConstant(0, sdl, + TLI.getPointerTy(DAG.getDataLayout())) }); + else if (FPI.isUnaryOp()) Result = DAG.getNode(Opcode, sdl, VTs, { Chain, getValue(FPI.getArgOperand(0)) }); else if (FPI.isTernaryOp()) @@ -6476,6 +6932,13 @@ void SelectionDAGBuilder::visitConstrainedFPIntrinsic( { Chain, getValue(FPI.getArgOperand(0)), getValue(FPI.getArgOperand(1)) }); + if (FPI.getExceptionBehavior() != + ConstrainedFPIntrinsic::ExceptionBehavior::ebIgnore) { + SDNodeFlags Flags; + Flags.setFPExcept(true); + Result->setFlags(Flags); + } + assert(Result.getNode()->getNumValues() == 2); SDValue OutChain = Result.getValue(1); DAG.setRoot(OutChain); @@ -6596,11 +7059,9 @@ void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee, SwiftErrorVal = V; // We find the virtual register for the actual swifterror argument. // Instead of using the Value, we use the virtual register instead. - Entry.Node = DAG.getRegister(FuncInfo - .getOrCreateSwiftErrorVRegUseAt( - CS.getInstruction(), FuncInfo.MBB, V) - .first, - EVT(TLI.getPointerTy(DL))); + Entry.Node = DAG.getRegister( + SwiftError.getOrCreateVRegUseAt(CS.getInstruction(), FuncInfo.MBB, V), + EVT(TLI.getPointerTy(DL))); } Args.push_back(Entry); @@ -6641,13 +7102,9 @@ void SelectionDAGBuilder::LowerCallTo(ImmutableCallSite CS, SDValue Callee, if (SwiftErrorVal && TLI.supportSwiftError()) { // Get the last element of InVals. SDValue Src = CLI.InVals.back(); - unsigned VReg; bool CreatedVReg; - std::tie(VReg, CreatedVReg) = - FuncInfo.getOrCreateSwiftErrorVRegDefAt(CS.getInstruction()); + unsigned VReg = SwiftError.getOrCreateVRegDefAt( + CS.getInstruction(), FuncInfo.MBB, SwiftErrorVal); SDValue CopyNode = CLI.DAG.getCopyToReg(Result.second, CLI.DL, VReg, Src); - // We update the virtual register for the actual swifterror argument. - if (CreatedVReg) - FuncInfo.setCurrentSwiftErrorVReg(FuncInfo.MBB, SwiftErrorVal, VReg); DAG.setRoot(CopyNode); } } @@ -6995,10 +7452,6 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) { return; } - MachineModuleInfo &MMI = DAG.getMachineFunction().getMMI(); - computeUsesVAFloatArgument(I, MMI); - - const char *RenameFn = nullptr; if (Function *F = I.getCalledFunction()) { if (F->isDeclaration()) { // Is this an LLVM intrinsic or a target-specific intrinsic? @@ -7008,9 +7461,8 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) { IID = II->getIntrinsicID(F); if (IID) { - RenameFn = visitIntrinsicCall(I, IID); - if (!RenameFn) - return; + visitIntrinsicCall(I, IID); + return; } } @@ -7159,20 +7611,14 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) { } } - SDValue Callee; - if (!RenameFn) - Callee = getValue(I.getCalledValue()); - else - Callee = DAG.getExternalSymbol( - RenameFn, - DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout())); - // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't // have to do anything here to lower funclet bundles. assert(!I.hasOperandBundlesOtherThan( {LLVMContext::OB_deopt, LLVMContext::OB_funclet}) && "Cannot lower calls with arbitrary operand bundles!"); + SDValue Callee = getValue(I.getCalledValue()); + if (I.countOperandBundlesOfType(LLVMContext::OB_deopt)) LowerCallSiteWithDeoptBundle(&I, Callee, nullptr); else @@ -7328,8 +7774,9 @@ static SDValue getAddressForMemoryInput(SDValue Chain, const SDLoc &Location, MachineFunction &MF = DAG.getMachineFunction(); int SSFI = MF.getFrameInfo().CreateStackObject(TySize, Align, false); SDValue StackSlot = DAG.getFrameIndex(SSFI, TLI.getFrameIndexTy(DL)); - Chain = DAG.getStore(Chain, Location, OpInfo.CallOperand, StackSlot, - MachinePointerInfo::getFixedStack(MF, SSFI)); + Chain = DAG.getTruncStore(Chain, Location, OpInfo.CallOperand, StackSlot, + MachinePointerInfo::getFixedStack(MF, SSFI), + TLI.getMemValueType(DL, Ty)); OpInfo.CallOperand = StackSlot; return Chain; @@ -7353,6 +7800,10 @@ static void GetRegistersForValue(SelectionDAG &DAG, const SDLoc &DL, SmallVector Regs; const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); + // No work to do for memory operations. + if (OpInfo.ConstraintType == TargetLowering::C_Memory) + return; + // If this is a constraint for a single physreg, or a constraint for a // register class, find it. unsigned AssignedReg; @@ -7435,7 +7886,7 @@ static void GetRegistersForValue(SelectionDAG &DAG, const SDLoc &DL, for (; NumRegs; --NumRegs, ++I) { assert(I != RC->end() && "Ran out of registers to allocate!"); - auto R = (AssignedReg) ? *I : RegInfo.createVirtualRegister(RC); + Register R = AssignedReg ? Register(*I) : RegInfo.createVirtualRegister(RC); Regs.push_back(R); } @@ -7509,9 +7960,9 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { TargetLowering::AsmOperandInfoVector TargetConstraints = TLI.ParseConstraints( DAG.getDataLayout(), DAG.getSubtarget().getRegisterInfo(), CS); - bool hasMemory = false; - - // Remember the HasSideEffect, AlignStack, AsmDialect, MayLoad and MayStore + // First Pass: Calculate HasSideEffects and ExtraFlags (AlignStack, + // AsmDialect, MayLoad, MayStore). + bool HasSideEffect = IA->hasSideEffects(); ExtraFlags ExtraInfo(CS); unsigned ArgNo = 0; // ArgNo - The argument of the CallInst. @@ -7527,7 +7978,14 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { // Process the call argument. BasicBlocks are labels, currently appearing // only in asm's. - if (const BasicBlock *BB = dyn_cast(OpInfo.CallOperandVal)) { + const Instruction *I = CS.getInstruction(); + if (isa(I) && + (ArgNo - 1) >= (cast(I)->getNumArgOperands() - + cast(I)->getNumIndirectDests())) { + const auto *BA = cast(OpInfo.CallOperandVal); + EVT VT = TLI.getValueType(DAG.getDataLayout(), BA->getType(), true); + OpInfo.CallOperand = DAG.getTargetBlockAddress(BA, VT); + } else if (const auto *BB = dyn_cast(OpInfo.CallOperandVal)) { OpInfo.CallOperand = DAG.getBasicBlock(FuncInfo.MBBMap[BB]); } else { OpInfo.CallOperand = getValue(OpInfo.CallOperandVal); @@ -7554,8 +8012,8 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { OpInfo.ConstraintVT = MVT::Other; } - if (!hasMemory) - hasMemory = OpInfo.hasMemory(TLI); + if (!HasSideEffect) + HasSideEffect = OpInfo.hasMemory(TLI); // Determine if this InlineAsm MayLoad or MayStore based on the constraints. // FIXME: Could we compute this on OpInfo rather than T? @@ -7566,17 +8024,20 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { ExtraInfo.update(T); } - SDValue Chain, Flag; // We won't need to flush pending loads if this asm doesn't touch // memory and is nonvolatile. - if (hasMemory || IA->hasSideEffects()) - Chain = getRoot(); - else - Chain = DAG.getRoot(); + SDValue Flag, Chain = (HasSideEffect) ? getRoot() : DAG.getRoot(); + + bool IsCallBr = isa(CS.getInstruction()); + if (IsCallBr) { + // If this is a callbr we need to flush pending exports since inlineasm_br + // is a terminator. We need to do this before nodes are glued to + // the inlineasm_br node. + Chain = getControlRoot(); + } - // Second pass over the constraints: compute which constraint option to use - // and assign registers to constraints that want a specific physreg. + // Second pass over the constraints: compute which constraint option to use. for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) { // If this is an output operand with a matching input operand, look up the // matching input. If their types mismatch, e.g. one is an integer, the @@ -7612,28 +8073,6 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { OpInfo.isIndirect = true; } - // If this constraint is for a specific register, allocate it before - // anything else. - SDISelAsmOperandInfo &RefOpInfo = - OpInfo.isMatchingInputConstraint() - ? ConstraintOperands[OpInfo.getMatchedOperand()] - : OpInfo; - if (RefOpInfo.ConstraintType == TargetLowering::C_Register) - GetRegistersForValue(DAG, getCurSDLoc(), OpInfo, RefOpInfo); - } - - // Third pass - Loop over all of the operands, assigning virtual or physregs - // to register class operands. - for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) { - SDISelAsmOperandInfo &RefOpInfo = - OpInfo.isMatchingInputConstraint() - ? ConstraintOperands[OpInfo.getMatchedOperand()] - : OpInfo; - - // C_Register operands have already been allocated, Other/Memory don't need - // to be. - if (RefOpInfo.ConstraintType == TargetLowering::C_RegisterClass) - GetRegistersForValue(DAG, getCurSDLoc(), OpInfo, RefOpInfo); } // AsmNodeOperands - The operands for the ISD::INLINEASM node. @@ -7653,21 +8092,21 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { AsmNodeOperands.push_back(DAG.getTargetConstant( ExtraInfo.get(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout()))); - // Loop over all of the inputs, copying the operand values into the - // appropriate registers and processing the output regs. - RegsForValue RetValRegs; - - // IndirectStoresToEmit - The set of stores to emit after the inline asm node. - std::vector> IndirectStoresToEmit; - + // Third pass: Loop over operands to prepare DAG-level operands.. As part of + // this, assign virtual and physical registers for inputs and otput. for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) { + // Assign Registers. + SDISelAsmOperandInfo &RefOpInfo = + OpInfo.isMatchingInputConstraint() + ? ConstraintOperands[OpInfo.getMatchedOperand()] + : OpInfo; + GetRegistersForValue(DAG, getCurSDLoc(), OpInfo, RefOpInfo); + switch (OpInfo.Type) { case InlineAsm::isOutput: - if (OpInfo.ConstraintType != TargetLowering::C_RegisterClass && - OpInfo.ConstraintType != TargetLowering::C_Register) { - // Memory output, or 'other' output (e.g. 'X' constraint). - assert(OpInfo.isIndirect && "Memory output must be indirect operand"); - + if (OpInfo.ConstraintType == TargetLowering::C_Memory || + (OpInfo.ConstraintType == TargetLowering::C_Other && + OpInfo.isIndirect)) { unsigned ConstraintID = TLI.getInlineAsmMemConstraint(OpInfo.ConstraintCode); assert(ConstraintID != InlineAsm::Constraint_Unknown && @@ -7680,38 +8119,27 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { MVT::i32)); AsmNodeOperands.push_back(OpInfo.CallOperand); break; - } - - // Otherwise, this is a register or register class output. - - // Copy the output from the appropriate register. Find a register that - // we can use. - if (OpInfo.AssignedRegs.Regs.empty()) { - emitInlineAsmError( - CS, "couldn't allocate output register for constraint '" + - Twine(OpInfo.ConstraintCode) + "'"); - return; - } + } else if ((OpInfo.ConstraintType == TargetLowering::C_Other && + !OpInfo.isIndirect) || + OpInfo.ConstraintType == TargetLowering::C_Register || + OpInfo.ConstraintType == TargetLowering::C_RegisterClass) { + // Otherwise, this outputs to a register (directly for C_Register / + // C_RegisterClass, and a target-defined fashion for C_Other). Find a + // register that we can use. + if (OpInfo.AssignedRegs.Regs.empty()) { + emitInlineAsmError( + CS, "couldn't allocate output register for constraint '" + + Twine(OpInfo.ConstraintCode) + "'"); + return; + } - // If this is an indirect operand, store through the pointer after the - // asm. - if (OpInfo.isIndirect) { - IndirectStoresToEmit.push_back(std::make_pair(OpInfo.AssignedRegs, - OpInfo.CallOperandVal)); - } else { - // This is the result value of the call. - assert(!CS.getType()->isVoidTy() && "Bad inline asm!"); - // Concatenate this output onto the outputs list. - RetValRegs.append(OpInfo.AssignedRegs); + // Add information to the INLINEASM node to know that this register is + // set. + OpInfo.AssignedRegs.AddInlineAsmOperands( + OpInfo.isEarlyClobber ? InlineAsm::Kind_RegDefEarlyClobber + : InlineAsm::Kind_RegDef, + false, 0, getCurSDLoc(), DAG, AsmNodeOperands); } - - // Add information to the INLINEASM node to know that this register is - // set. - OpInfo.AssignedRegs - .AddInlineAsmOperands(OpInfo.isEarlyClobber - ? InlineAsm::Kind_RegDefEarlyClobber - : InlineAsm::Kind_RegDef, - false, 0, getCurSDLoc(), DAG, AsmNodeOperands); break; case InlineAsm::isInput: { @@ -7865,98 +8293,117 @@ void SelectionDAGBuilder::visitInlineAsm(ImmutableCallSite CS) { AsmNodeOperands[InlineAsm::Op_InputChain] = Chain; if (Flag.getNode()) AsmNodeOperands.push_back(Flag); - Chain = DAG.getNode(ISD::INLINEASM, getCurSDLoc(), + unsigned ISDOpc = IsCallBr ? ISD::INLINEASM_BR : ISD::INLINEASM; + Chain = DAG.getNode(ISDOpc, getCurSDLoc(), DAG.getVTList(MVT::Other, MVT::Glue), AsmNodeOperands); Flag = Chain.getValue(1); - // If this asm returns a register value, copy the result from that register - // and set it as the value of the call. - if (!RetValRegs.Regs.empty()) { - SDValue Val = RetValRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), - Chain, &Flag, CS.getInstruction()); - - llvm::Type *CSResultType = CS.getType(); - unsigned numRet; - ArrayRef ResultTypes; - SmallVector ResultValues(1); - if (StructType *StructResult = dyn_cast(CSResultType)) { - numRet = StructResult->getNumElements(); - assert(Val->getNumOperands() == numRet && - "Mismatch in number of output operands in asm result"); - ResultTypes = StructResult->elements(); - ArrayRef ValueUses = Val->ops(); - ResultValues.resize(numRet); - std::transform(ValueUses.begin(), ValueUses.end(), ResultValues.begin(), - [](const SDUse &u) -> SDValue { return u.get(); }); - } else { - numRet = 1; - ResultValues[0] = Val; - ResultTypes = makeArrayRef(CSResultType); - } - SmallVector ResultVTs(numRet); - for (unsigned i = 0; i < numRet; i++) { - EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), ResultTypes[i]); - SDValue Val = ResultValues[i]; - assert(ResultTypes[i]->isSized() && "Unexpected unsized type"); - // If the type of the inline asm call site return value is different but - // has same size as the type of the asm output bitcast it. One example - // of this is for vectors with different width / number of elements. - // This can happen for register classes that can contain multiple - // different value types. The preg or vreg allocated may not have the - // same VT as was expected. - // - // This can also happen for a return value that disagrees with the - // register class it is put in, eg. a double in a general-purpose - // register on a 32-bit machine. - if (ResultVT != Val.getValueType() && - ResultVT.getSizeInBits() == Val.getValueSizeInBits()) - Val = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, Val); - else if (ResultVT != Val.getValueType() && ResultVT.isInteger() && - Val.getValueType().isInteger()) { - // If a result value was tied to an input value, the computed result - // may have a wider width than the expected result. Extract the - // relevant portion. - Val = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, Val); - } + // Do additional work to generate outputs. - assert(ResultVT == Val.getValueType() && "Asm result value mismatch!"); - ResultVTs[i] = ResultVT; - ResultValues[i] = Val; - } + SmallVector ResultVTs; + SmallVector ResultValues; + SmallVector OutChains; - Val = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), - DAG.getVTList(ResultVTs), ResultValues); - setValue(CS.getInstruction(), Val); - // Don't need to use this as a chain in this case. - if (!IA->hasSideEffects() && !hasMemory && IndirectStoresToEmit.empty()) - return; - } + llvm::Type *CSResultType = CS.getType(); + ArrayRef ResultTypes; + if (StructType *StructResult = dyn_cast(CSResultType)) + ResultTypes = StructResult->elements(); + else if (!CSResultType->isVoidTy()) + ResultTypes = makeArrayRef(CSResultType); + + auto CurResultType = ResultTypes.begin(); + auto handleRegAssign = [&](SDValue V) { + assert(CurResultType != ResultTypes.end() && "Unexpected value"); + assert((*CurResultType)->isSized() && "Unexpected unsized type"); + EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), *CurResultType); + ++CurResultType; + // If the type of the inline asm call site return value is different but has + // same size as the type of the asm output bitcast it. One example of this + // is for vectors with different width / number of elements. This can + // happen for register classes that can contain multiple different value + // types. The preg or vreg allocated may not have the same VT as was + // expected. + // + // This can also happen for a return value that disagrees with the register + // class it is put in, eg. a double in a general-purpose register on a + // 32-bit machine. + if (ResultVT != V.getValueType() && + ResultVT.getSizeInBits() == V.getValueSizeInBits()) + V = DAG.getNode(ISD::BITCAST, getCurSDLoc(), ResultVT, V); + else if (ResultVT != V.getValueType() && ResultVT.isInteger() && + V.getValueType().isInteger()) { + // If a result value was tied to an input value, the computed result + // may have a wider width than the expected result. Extract the + // relevant portion. + V = DAG.getNode(ISD::TRUNCATE, getCurSDLoc(), ResultVT, V); + } + assert(ResultVT == V.getValueType() && "Asm result value mismatch!"); + ResultVTs.push_back(ResultVT); + ResultValues.push_back(V); + }; - std::vector> StoresToEmit; + // Deal with output operands. + for (SDISelAsmOperandInfo &OpInfo : ConstraintOperands) { + if (OpInfo.Type == InlineAsm::isOutput) { + SDValue Val; + // Skip trivial output operands. + if (OpInfo.AssignedRegs.Regs.empty()) + continue; + + switch (OpInfo.ConstraintType) { + case TargetLowering::C_Register: + case TargetLowering::C_RegisterClass: + Val = OpInfo.AssignedRegs.getCopyFromRegs( + DAG, FuncInfo, getCurSDLoc(), Chain, &Flag, CS.getInstruction()); + break; + case TargetLowering::C_Other: + Val = TLI.LowerAsmOutputForConstraint(Chain, Flag, getCurSDLoc(), + OpInfo, DAG); + break; + case TargetLowering::C_Memory: + break; // Already handled. + case TargetLowering::C_Unknown: + assert(false && "Unexpected unknown constraint"); + } - // Process indirect outputs, first output all of the flagged copies out of - // physregs. - for (unsigned i = 0, e = IndirectStoresToEmit.size(); i != e; ++i) { - RegsForValue &OutRegs = IndirectStoresToEmit[i].first; - const Value *Ptr = IndirectStoresToEmit[i].second; - SDValue OutVal = OutRegs.getCopyFromRegs(DAG, FuncInfo, getCurSDLoc(), - Chain, &Flag, IA); - StoresToEmit.push_back(std::make_pair(OutVal, Ptr)); + // Indirect output manifest as stores. Record output chains. + if (OpInfo.isIndirect) { + const Value *Ptr = OpInfo.CallOperandVal; + assert(Ptr && "Expected value CallOperandVal for indirect asm operand"); + SDValue Store = DAG.getStore(Chain, getCurSDLoc(), Val, getValue(Ptr), + MachinePointerInfo(Ptr)); + OutChains.push_back(Store); + } else { + // generate CopyFromRegs to associated registers. + assert(!CS.getType()->isVoidTy() && "Bad inline asm!"); + if (Val.getOpcode() == ISD::MERGE_VALUES) { + for (const SDValue &V : Val->op_values()) + handleRegAssign(V); + } else + handleRegAssign(Val); + } + } } - // Emit the non-flagged stores from the physregs. - SmallVector OutChains; - for (unsigned i = 0, e = StoresToEmit.size(); i != e; ++i) { - SDValue Val = DAG.getStore(Chain, getCurSDLoc(), StoresToEmit[i].first, - getValue(StoresToEmit[i].second), - MachinePointerInfo(StoresToEmit[i].second)); - OutChains.push_back(Val); + // Set results. + if (!ResultValues.empty()) { + assert(CurResultType == ResultTypes.end() && + "Mismatch in number of ResultTypes"); + assert(ResultValues.size() == ResultTypes.size() && + "Mismatch in number of output operands in asm result"); + + SDValue V = DAG.getNode(ISD::MERGE_VALUES, getCurSDLoc(), + DAG.getVTList(ResultVTs), ResultValues); + setValue(CS.getInstruction(), V); } + // Collect store chains. if (!OutChains.empty()) Chain = DAG.getNode(ISD::TokenFactor, getCurSDLoc(), MVT::Other, OutChains); - DAG.setRoot(Chain); + // Only Update Root if inline assembly has a memory effect. + if (ResultValues.empty() || HasSideEffect || !OutChains.empty() || IsCallBr) + DAG.setRoot(Chain); } void SelectionDAGBuilder::emitInlineAsmError(ImmutableCallSite CS, @@ -7989,12 +8436,16 @@ void SelectionDAGBuilder::visitVAStart(const CallInst &I) { void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); const DataLayout &DL = DAG.getDataLayout(); - SDValue V = DAG.getVAArg(TLI.getValueType(DAG.getDataLayout(), I.getType()), - getCurSDLoc(), getRoot(), getValue(I.getOperand(0)), - DAG.getSrcValue(I.getOperand(0)), - DL.getABITypeAlignment(I.getType())); - setValue(&I, V); + SDValue V = DAG.getVAArg( + TLI.getMemValueType(DAG.getDataLayout(), I.getType()), getCurSDLoc(), + getRoot(), getValue(I.getOperand(0)), DAG.getSrcValue(I.getOperand(0)), + DL.getABITypeAlignment(I.getType())); DAG.setRoot(V.getValue(1)); + + if (I.getType()->isPointerTy()) + V = DAG.getPtrExtOrTrunc( + V, getCurSDLoc(), TLI.getValueType(DAG.getDataLayout(), I.getType())); + setValue(&I, V); } void SelectionDAGBuilder::visitVAEnd(const CallInst &I) { @@ -8021,7 +8472,7 @@ SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG, return Op; ConstantRange CR = getConstantRangeFromMetadata(*Range); - if (CR.isFullSet() || CR.isEmptySet() || CR.isWrappedSet()) + if (CR.isFullSet() || CR.isEmptySet() || CR.isUpperWrapped()) return Op; APInt Lo = CR.getUnsignedMin(); @@ -8058,7 +8509,7 @@ SDValue SelectionDAGBuilder::lowerRangeToAssertZExt(SelectionDAG &DAG, /// convention or require stack pointer adjustment. Only a subset of the /// intrinsic's operands need to participate in the calling convention. void SelectionDAGBuilder::populateCallLoweringInfo( - TargetLowering::CallLoweringInfo &CLI, ImmutableCallSite CS, + TargetLowering::CallLoweringInfo &CLI, const CallBase *Call, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, bool IsPatchPoint) { TargetLowering::ArgListTy Args; @@ -8068,21 +8519,21 @@ void SelectionDAGBuilder::populateCallLoweringInfo( // Attributes for args start at offset 1, after the return attribute. for (unsigned ArgI = ArgIdx, ArgE = ArgIdx + NumArgs; ArgI != ArgE; ++ArgI) { - const Value *V = CS->getOperand(ArgI); + const Value *V = Call->getOperand(ArgI); assert(!V->getType()->isEmptyTy() && "Empty type passed to intrinsic."); TargetLowering::ArgListEntry Entry; Entry.Node = getValue(V); Entry.Ty = V->getType(); - Entry.setAttributes(&CS, ArgI); + Entry.setAttributes(Call, ArgI); Args.push_back(Entry); } CLI.setDebugLoc(getCurSDLoc()) .setChain(getRoot()) - .setCallee(CS.getCallingConv(), ReturnTy, Callee, std::move(Args)) - .setDiscardResult(CS->use_empty()) + .setCallee(Call->getCallingConv(), ReturnTy, Callee, std::move(Args)) + .setDiscardResult(Call->use_empty()) .setIsPatchPoint(IsPatchPoint); } @@ -8093,7 +8544,7 @@ void SelectionDAGBuilder::populateCallLoweringInfo( /// avoid constant materialization and register allocation. /// /// FrameIndex operands are converted to TargetFrameIndex so that ISEL does not -/// generate addess computation nodes, and so ExpandISelPseudo can convert the +/// generate addess computation nodes, and so FinalizeISel can convert the /// TargetFrameIndex into a DirectMemRefOp StackMap location. This avoids /// address materialization and register allocation, but may also be required /// for correctness. If a StackMap (or PatchPoint) intrinsic directly uses an @@ -8226,8 +8677,8 @@ void SelectionDAGBuilder::visitPatchpoint(ImmutableCallSite CS, IsAnyRegCC ? Type::getVoidTy(*DAG.getContext()) : CS->getType(); TargetLowering::CallLoweringInfo CLI(DAG); - populateCallLoweringInfo(CLI, CS, NumMetaOpers, NumCallArgs, Callee, ReturnTy, - true); + populateCallLoweringInfo(CLI, cast(CS.getInstruction()), + NumMetaOpers, NumCallArgs, Callee, ReturnTy, true); std::pair Result = lowerInvokable(CLI, EHPadBB); SDNode *CallEnd = Result.second.getNode(); @@ -8351,15 +8802,17 @@ void SelectionDAGBuilder::visitVectorReduce(const CallInst &I, FMF = I.getFastMathFlags(); switch (Intrinsic) { - case Intrinsic::experimental_vector_reduce_fadd: - if (FMF.isFast()) - Res = DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2); + case Intrinsic::experimental_vector_reduce_v2_fadd: + if (FMF.allowReassoc()) + Res = DAG.getNode(ISD::FADD, dl, VT, Op1, + DAG.getNode(ISD::VECREDUCE_FADD, dl, VT, Op2)); else Res = DAG.getNode(ISD::VECREDUCE_STRICT_FADD, dl, VT, Op1, Op2); break; - case Intrinsic::experimental_vector_reduce_fmul: - if (FMF.isFast()) - Res = DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2); + case Intrinsic::experimental_vector_reduce_v2_fmul: + if (FMF.allowReassoc()) + Res = DAG.getNode(ISD::FMUL, dl, VT, Op1, + DAG.getNode(ISD::VECREDUCE_FMUL, dl, VT, Op2)); else Res = DAG.getNode(ISD::VECREDUCE_STRICT_FMUL, dl, VT, Op1, Op2); break; @@ -8433,8 +8886,11 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { if (CLI.IsPostTypeLegalization) { // If we are lowering a libcall after legalization, split the return type. - SmallVector OldRetTys = std::move(RetTys); - SmallVector OldOffsets = std::move(Offsets); + SmallVector OldRetTys; + SmallVector OldOffsets; + RetTys.swap(OldRetTys); + Offsets.swap(OldOffsets); + for (size_t i = 0, e = OldRetTys.size(); i != e; ++i) { EVT RetVT = OldRetTys[i]; uint64_t Offset = OldOffsets[i]; @@ -8489,7 +8945,15 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { // points into the callers stack frame. CLI.IsTailCall = false; } else { + bool NeedsRegBlock = functionArgumentNeedsConsecutiveRegisters( + CLI.RetTy, CLI.CallConv, CLI.IsVarArg); for (unsigned I = 0, E = RetTys.size(); I != E; ++I) { + ISD::ArgFlagsTy Flags; + if (NeedsRegBlock) { + Flags.setInConsecutiveRegs(); + if (I == RetTys.size() - 1) + Flags.setInConsecutiveRegsLast(); + } EVT VT = RetTys[I]; MVT RegisterVT = getRegisterTypeForCallingConv(CLI.RetTy->getContext(), CLI.CallConv, VT); @@ -8497,9 +8961,15 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { CLI.CallConv, VT); for (unsigned i = 0; i != NumRegs; ++i) { ISD::InputArg MyFlags; + MyFlags.Flags = Flags; MyFlags.VT = RegisterVT; MyFlags.ArgVT = VT; MyFlags.Used = CLI.IsReturnValueUsed; + if (CLI.RetTy->isPointerTy()) { + MyFlags.Flags.setPointer(); + MyFlags.Flags.setPointerAddrSpace( + cast(CLI.RetTy)->getAddressSpace()); + } if (CLI.RetSExt) MyFlags.Flags.setSExt(); if (CLI.RetZExt) @@ -8550,6 +9020,11 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { // specify the alignment it wants. unsigned OriginalAlignment = getABIAlignmentForCallingConv(ArgTy, DL); + if (Args[i].Ty->isPointerTy()) { + Flags.setPointer(); + Flags.setPointerAddrSpace( + cast(Args[i].Ty)->getAddressSpace()); + } if (Args[i].IsZExt) Flags.setZExt(); if (Args[i].IsSExt) @@ -8587,8 +9062,11 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { if (Args[i].IsByVal || Args[i].IsInAlloca) { PointerType *Ty = cast(Args[i].Ty); Type *ElementTy = Ty->getElementType(); - Flags.setByValSize(DL.getTypeAllocSize(ElementTy)); - // For ByVal, alignment should come from FE. BE will guess if this + + unsigned FrameSize = DL.getTypeAllocSize( + Args[i].ByValType ? Args[i].ByValType : ElementTy); + Flags.setByValSize(FrameSize); + // info is not there but there are cases it cannot get right. unsigned FrameAlign; if (Args[i].Alignment) @@ -8619,8 +9097,11 @@ TargetLowering::LowerCallTo(TargetLowering::CallLoweringInfo &CLI) const { // for now. if (Args[i].IsReturned && !Op.getValueType().isVector() && CanLowerReturn) { - assert(CLI.RetTy == Args[i].Ty && RetTys.size() == NumValues && - "unexpected use of 'returned'"); + assert((CLI.RetTy == Args[i].Ty || + (CLI.RetTy->isPointerTy() && Args[i].Ty->isPointerTy() && + CLI.RetTy->getPointerAddressSpace() == + Args[i].Ty->getPointerAddressSpace())) && + RetTys.size() == NumValues && "unexpected use of 'returned'"); // Before passing 'returned' to the target lowering code, ensure that // either the register MVT and the actual EVT are the same size or that // the return value and argument are extended in the same way; in these @@ -9023,7 +9504,7 @@ void SelectionDAGISel::LowerArguments(const Function &F) { unsigned PartBase = 0; Type *FinalType = Arg.getType(); if (Arg.hasAttribute(Attribute::ByVal)) - FinalType = cast(FinalType)->getElementType(); + FinalType = Arg.getParamByValType(); bool NeedsRegBlock = TLI->functionArgumentNeedsConsecutiveRegisters( FinalType, F.getCallingConv(), F.isVarArg()); for (unsigned Value = 0, NumValues = ValueVTs.size(); @@ -9038,6 +9519,11 @@ void SelectionDAGISel::LowerArguments(const Function &F) { unsigned OriginalAlignment = TLI->getABIAlignmentForCallingConv(ArgTy, DL); + if (Arg.getType()->isPointerTy()) { + Flags.setPointer(); + Flags.setPointerAddrSpace( + cast(Arg.getType())->getAddressSpace()); + } if (Arg.hasAttribute(Attribute::ZExt)) Flags.setZExt(); if (Arg.hasAttribute(Attribute::SExt)) @@ -9078,11 +9564,14 @@ void SelectionDAGISel::LowerArguments(const Function &F) { Flags.setByVal(); } if (Flags.isByVal() || Flags.isInAlloca()) { - PointerType *Ty = cast(Arg.getType()); - Type *ElementTy = Ty->getElementType(); - Flags.setByValSize(DL.getTypeAllocSize(ElementTy)); - // For ByVal, alignment should be passed from FE. BE will guess if - // this info is not there but there are cases it cannot get right. + Type *ElementTy = Arg.getParamByValType(); + + // For ByVal, size and alignment should be passed from FE. BE will + // guess if this info is not there but there are cases it cannot get + // right. + unsigned FrameSize = DL.getTypeAllocSize(Arg.getParamByValType()); + Flags.setByValSize(FrameSize); + unsigned FrameAlign; if (Arg.getParamAlignment()) FrameAlign = Arg.getParamAlignment(); @@ -9263,17 +9752,16 @@ void SelectionDAGISel::LowerArguments(const Function &F) { if (Res.getOpcode() == ISD::CopyFromReg && isSwiftErrorArg) { unsigned Reg = cast(Res.getOperand(1))->getReg(); if (TargetRegisterInfo::isVirtualRegister(Reg)) - FuncInfo->setCurrentSwiftErrorVReg(FuncInfo->MBB, - FuncInfo->SwiftErrorArg, Reg); + SwiftError->setCurrentVReg(FuncInfo->MBB, SwiftError->getFunctionArg(), + Reg); } // If this argument is live outside of the entry block, insert a copy from // wherever we got it to the vreg that other BB's will reference it as. - if (!TM.Options.EnableFastISel && Res.getOpcode() == ISD::CopyFromReg) { + if (Res.getOpcode() == ISD::CopyFromReg) { // If we can, though, try to skip creating an unnecessary vreg. // FIXME: This isn't very clean... it would be nice to make this more - // general. It's also subtly incompatible with the hacks FastISel - // uses with vregs. + // general. unsigned Reg = cast(Res.getOperand(1))->getReg(); if (TargetRegisterInfo::isVirtualRegister(Reg)) { FuncInfo->ValueMap[&Arg] = Reg; @@ -9354,7 +9842,7 @@ SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) { if (const Constant *C = dyn_cast(PHIOp)) { unsigned &RegOut = ConstantsOut[C]; if (RegOut == 0) { - RegOut = FuncInfo.CreateRegs(C->getType()); + RegOut = FuncInfo.CreateRegs(C); CopyValueToVirtualRegister(C, RegOut); } Reg = RegOut; @@ -9367,7 +9855,7 @@ SelectionDAGBuilder::HandlePHINodesInSuccessorBlocks(const BasicBlock *LLVMBB) { assert(isa(PHIOp) && FuncInfo.StaticAllocaMap.count(cast(PHIOp)) && "Didn't codegen value into a register!??"); - Reg = FuncInfo.CreateRegs(PHIOp->getType()); + Reg = FuncInfo.CreateRegs(PHIOp); CopyValueToVirtualRegister(PHIOp, Reg); } } @@ -9432,450 +9920,6 @@ void SelectionDAGBuilder::updateDAGForMaybeTailCall(SDValue MaybeTC) { HasTailCall = true; } -uint64_t -SelectionDAGBuilder::getJumpTableRange(const CaseClusterVector &Clusters, - unsigned First, unsigned Last) const { - assert(Last >= First); - const APInt &LowCase = Clusters[First].Low->getValue(); - const APInt &HighCase = Clusters[Last].High->getValue(); - assert(LowCase.getBitWidth() == HighCase.getBitWidth()); - - // FIXME: A range of consecutive cases has 100% density, but only requires one - // comparison to lower. We should discriminate against such consecutive ranges - // in jump tables. - - return (HighCase - LowCase).getLimitedValue((UINT64_MAX - 1) / 100) + 1; -} - -uint64_t SelectionDAGBuilder::getJumpTableNumCases( - const SmallVectorImpl &TotalCases, unsigned First, - unsigned Last) const { - assert(Last >= First); - assert(TotalCases[Last] >= TotalCases[First]); - uint64_t NumCases = - TotalCases[Last] - (First == 0 ? 0 : TotalCases[First - 1]); - return NumCases; -} - -bool SelectionDAGBuilder::buildJumpTable(const CaseClusterVector &Clusters, - unsigned First, unsigned Last, - const SwitchInst *SI, - MachineBasicBlock *DefaultMBB, - CaseCluster &JTCluster) { - assert(First <= Last); - - auto Prob = BranchProbability::getZero(); - unsigned NumCmps = 0; - std::vector Table; - DenseMap JTProbs; - - // Initialize probabilities in JTProbs. - for (unsigned I = First; I <= Last; ++I) - JTProbs[Clusters[I].MBB] = BranchProbability::getZero(); - - for (unsigned I = First; I <= Last; ++I) { - assert(Clusters[I].Kind == CC_Range); - Prob += Clusters[I].Prob; - const APInt &Low = Clusters[I].Low->getValue(); - const APInt &High = Clusters[I].High->getValue(); - NumCmps += (Low == High) ? 1 : 2; - if (I != First) { - // Fill the gap between this and the previous cluster. - const APInt &PreviousHigh = Clusters[I - 1].High->getValue(); - assert(PreviousHigh.slt(Low)); - uint64_t Gap = (Low - PreviousHigh).getLimitedValue() - 1; - for (uint64_t J = 0; J < Gap; J++) - Table.push_back(DefaultMBB); - } - uint64_t ClusterSize = (High - Low).getLimitedValue() + 1; - for (uint64_t J = 0; J < ClusterSize; ++J) - Table.push_back(Clusters[I].MBB); - JTProbs[Clusters[I].MBB] += Clusters[I].Prob; - } - - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); - unsigned NumDests = JTProbs.size(); - if (TLI.isSuitableForBitTests( - NumDests, NumCmps, Clusters[First].Low->getValue(), - Clusters[Last].High->getValue(), DAG.getDataLayout())) { - // Clusters[First..Last] should be lowered as bit tests instead. - return false; - } - - // Create the MBB that will load from and jump through the table. - // Note: We create it here, but it's not inserted into the function yet. - MachineFunction *CurMF = FuncInfo.MF; - MachineBasicBlock *JumpTableMBB = - CurMF->CreateMachineBasicBlock(SI->getParent()); - - // Add successors. Note: use table order for determinism. - SmallPtrSet Done; - for (MachineBasicBlock *Succ : Table) { - if (Done.count(Succ)) - continue; - addSuccessorWithProb(JumpTableMBB, Succ, JTProbs[Succ]); - Done.insert(Succ); - } - JumpTableMBB->normalizeSuccProbs(); - - unsigned JTI = CurMF->getOrCreateJumpTableInfo(TLI.getJumpTableEncoding()) - ->createJumpTableIndex(Table); - - // Set up the jump table info. - JumpTable JT(-1U, JTI, JumpTableMBB, nullptr); - JumpTableHeader JTH(Clusters[First].Low->getValue(), - Clusters[Last].High->getValue(), SI->getCondition(), - nullptr, false); - JTCases.emplace_back(std::move(JTH), std::move(JT)); - - JTCluster = CaseCluster::jumpTable(Clusters[First].Low, Clusters[Last].High, - JTCases.size() - 1, Prob); - return true; -} - -void SelectionDAGBuilder::findJumpTables(CaseClusterVector &Clusters, - const SwitchInst *SI, - MachineBasicBlock *DefaultMBB) { -#ifndef NDEBUG - // Clusters must be non-empty, sorted, and only contain Range clusters. - assert(!Clusters.empty()); - for (CaseCluster &C : Clusters) - assert(C.Kind == CC_Range); - for (unsigned i = 1, e = Clusters.size(); i < e; ++i) - assert(Clusters[i - 1].High->getValue().slt(Clusters[i].Low->getValue())); -#endif - - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); - if (!TLI.areJTsAllowed(SI->getParent()->getParent())) - return; - - const int64_t N = Clusters.size(); - const unsigned MinJumpTableEntries = TLI.getMinimumJumpTableEntries(); - const unsigned SmallNumberOfEntries = MinJumpTableEntries / 2; - - if (N < 2 || N < MinJumpTableEntries) - return; - - // TotalCases[i]: Total nbr of cases in Clusters[0..i]. - SmallVector TotalCases(N); - for (unsigned i = 0; i < N; ++i) { - const APInt &Hi = Clusters[i].High->getValue(); - const APInt &Lo = Clusters[i].Low->getValue(); - TotalCases[i] = (Hi - Lo).getLimitedValue() + 1; - if (i != 0) - TotalCases[i] += TotalCases[i - 1]; - } - - // Cheap case: the whole range may be suitable for jump table. - uint64_t Range = getJumpTableRange(Clusters,0, N - 1); - uint64_t NumCases = getJumpTableNumCases(TotalCases, 0, N - 1); - assert(NumCases < UINT64_MAX / 100); - assert(Range >= NumCases); - if (TLI.isSuitableForJumpTable(SI, NumCases, Range)) { - CaseCluster JTCluster; - if (buildJumpTable(Clusters, 0, N - 1, SI, DefaultMBB, JTCluster)) { - Clusters[0] = JTCluster; - Clusters.resize(1); - return; - } - } - - // The algorithm below is not suitable for -O0. - if (TM.getOptLevel() == CodeGenOpt::None) - return; - - // Split Clusters into minimum number of dense partitions. The algorithm uses - // the same idea as Kannan & Proebsting "Correction to 'Producing Good Code - // for the Case Statement'" (1994), but builds the MinPartitions array in - // reverse order to make it easier to reconstruct the partitions in ascending - // order. In the choice between two optimal partitionings, it picks the one - // which yields more jump tables. - - // MinPartitions[i] is the minimum nbr of partitions of Clusters[i..N-1]. - SmallVector MinPartitions(N); - // LastElement[i] is the last element of the partition starting at i. - SmallVector LastElement(N); - // PartitionsScore[i] is used to break ties when choosing between two - // partitionings resulting in the same number of partitions. - SmallVector PartitionsScore(N); - // For PartitionsScore, a small number of comparisons is considered as good as - // a jump table and a single comparison is considered better than a jump - // table. - enum PartitionScores : unsigned { - NoTable = 0, - Table = 1, - FewCases = 1, - SingleCase = 2 - }; - - // Base case: There is only one way to partition Clusters[N-1]. - MinPartitions[N - 1] = 1; - LastElement[N - 1] = N - 1; - PartitionsScore[N - 1] = PartitionScores::SingleCase; - - // Note: loop indexes are signed to avoid underflow. - for (int64_t i = N - 2; i >= 0; i--) { - // Find optimal partitioning of Clusters[i..N-1]. - // Baseline: Put Clusters[i] into a partition on its own. - MinPartitions[i] = MinPartitions[i + 1] + 1; - LastElement[i] = i; - PartitionsScore[i] = PartitionsScore[i + 1] + PartitionScores::SingleCase; - - // Search for a solution that results in fewer partitions. - for (int64_t j = N - 1; j > i; j--) { - // Try building a partition from Clusters[i..j]. - uint64_t Range = getJumpTableRange(Clusters, i, j); - uint64_t NumCases = getJumpTableNumCases(TotalCases, i, j); - assert(NumCases < UINT64_MAX / 100); - assert(Range >= NumCases); - if (TLI.isSuitableForJumpTable(SI, NumCases, Range)) { - unsigned NumPartitions = 1 + (j == N - 1 ? 0 : MinPartitions[j + 1]); - unsigned Score = j == N - 1 ? 0 : PartitionsScore[j + 1]; - int64_t NumEntries = j - i + 1; - - if (NumEntries == 1) - Score += PartitionScores::SingleCase; - else if (NumEntries <= SmallNumberOfEntries) - Score += PartitionScores::FewCases; - else if (NumEntries >= MinJumpTableEntries) - Score += PartitionScores::Table; - - // If this leads to fewer partitions, or to the same number of - // partitions with better score, it is a better partitioning. - if (NumPartitions < MinPartitions[i] || - (NumPartitions == MinPartitions[i] && Score > PartitionsScore[i])) { - MinPartitions[i] = NumPartitions; - LastElement[i] = j; - PartitionsScore[i] = Score; - } - } - } - } - - // Iterate over the partitions, replacing some with jump tables in-place. - unsigned DstIndex = 0; - for (unsigned First = 0, Last; First < N; First = Last + 1) { - Last = LastElement[First]; - assert(Last >= First); - assert(DstIndex <= First); - unsigned NumClusters = Last - First + 1; - - CaseCluster JTCluster; - if (NumClusters >= MinJumpTableEntries && - buildJumpTable(Clusters, First, Last, SI, DefaultMBB, JTCluster)) { - Clusters[DstIndex++] = JTCluster; - } else { - for (unsigned I = First; I <= Last; ++I) - std::memmove(&Clusters[DstIndex++], &Clusters[I], sizeof(Clusters[I])); - } - } - Clusters.resize(DstIndex); -} - -bool SelectionDAGBuilder::buildBitTests(CaseClusterVector &Clusters, - unsigned First, unsigned Last, - const SwitchInst *SI, - CaseCluster &BTCluster) { - assert(First <= Last); - if (First == Last) - return false; - - BitVector Dests(FuncInfo.MF->getNumBlockIDs()); - unsigned NumCmps = 0; - for (int64_t I = First; I <= Last; ++I) { - assert(Clusters[I].Kind == CC_Range); - Dests.set(Clusters[I].MBB->getNumber()); - NumCmps += (Clusters[I].Low == Clusters[I].High) ? 1 : 2; - } - unsigned NumDests = Dests.count(); - - APInt Low = Clusters[First].Low->getValue(); - APInt High = Clusters[Last].High->getValue(); - assert(Low.slt(High)); - - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); - const DataLayout &DL = DAG.getDataLayout(); - if (!TLI.isSuitableForBitTests(NumDests, NumCmps, Low, High, DL)) - return false; - - APInt LowBound; - APInt CmpRange; - - const int BitWidth = TLI.getPointerTy(DL).getSizeInBits(); - assert(TLI.rangeFitsInWord(Low, High, DL) && - "Case range must fit in bit mask!"); - - // Check if the clusters cover a contiguous range such that no value in the - // range will jump to the default statement. - bool ContiguousRange = true; - for (int64_t I = First + 1; I <= Last; ++I) { - if (Clusters[I].Low->getValue() != Clusters[I - 1].High->getValue() + 1) { - ContiguousRange = false; - break; - } - } - - if (Low.isStrictlyPositive() && High.slt(BitWidth)) { - // Optimize the case where all the case values fit in a word without having - // to subtract minValue. In this case, we can optimize away the subtraction. - LowBound = APInt::getNullValue(Low.getBitWidth()); - CmpRange = High; - ContiguousRange = false; - } else { - LowBound = Low; - CmpRange = High - Low; - } - - CaseBitsVector CBV; - auto TotalProb = BranchProbability::getZero(); - for (unsigned i = First; i <= Last; ++i) { - // Find the CaseBits for this destination. - unsigned j; - for (j = 0; j < CBV.size(); ++j) - if (CBV[j].BB == Clusters[i].MBB) - break; - if (j == CBV.size()) - CBV.push_back( - CaseBits(0, Clusters[i].MBB, 0, BranchProbability::getZero())); - CaseBits *CB = &CBV[j]; - - // Update Mask, Bits and ExtraProb. - uint64_t Lo = (Clusters[i].Low->getValue() - LowBound).getZExtValue(); - uint64_t Hi = (Clusters[i].High->getValue() - LowBound).getZExtValue(); - assert(Hi >= Lo && Hi < 64 && "Invalid bit case!"); - CB->Mask |= (-1ULL >> (63 - (Hi - Lo))) << Lo; - CB->Bits += Hi - Lo + 1; - CB->ExtraProb += Clusters[i].Prob; - TotalProb += Clusters[i].Prob; - } - - BitTestInfo BTI; - llvm::sort(CBV, [](const CaseBits &a, const CaseBits &b) { - // Sort by probability first, number of bits second, bit mask third. - if (a.ExtraProb != b.ExtraProb) - return a.ExtraProb > b.ExtraProb; - if (a.Bits != b.Bits) - return a.Bits > b.Bits; - return a.Mask < b.Mask; - }); - - for (auto &CB : CBV) { - MachineBasicBlock *BitTestBB = - FuncInfo.MF->CreateMachineBasicBlock(SI->getParent()); - BTI.push_back(BitTestCase(CB.Mask, BitTestBB, CB.BB, CB.ExtraProb)); - } - BitTestCases.emplace_back(std::move(LowBound), std::move(CmpRange), - SI->getCondition(), -1U, MVT::Other, false, - ContiguousRange, nullptr, nullptr, std::move(BTI), - TotalProb); - - BTCluster = CaseCluster::bitTests(Clusters[First].Low, Clusters[Last].High, - BitTestCases.size() - 1, TotalProb); - return true; -} - -void SelectionDAGBuilder::findBitTestClusters(CaseClusterVector &Clusters, - const SwitchInst *SI) { -// Partition Clusters into as few subsets as possible, where each subset has a -// range that fits in a machine word and has <= 3 unique destinations. - -#ifndef NDEBUG - // Clusters must be sorted and contain Range or JumpTable clusters. - assert(!Clusters.empty()); - assert(Clusters[0].Kind == CC_Range || Clusters[0].Kind == CC_JumpTable); - for (const CaseCluster &C : Clusters) - assert(C.Kind == CC_Range || C.Kind == CC_JumpTable); - for (unsigned i = 1; i < Clusters.size(); ++i) - assert(Clusters[i-1].High->getValue().slt(Clusters[i].Low->getValue())); -#endif - - // The algorithm below is not suitable for -O0. - if (TM.getOptLevel() == CodeGenOpt::None) - return; - - // If target does not have legal shift left, do not emit bit tests at all. - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); - const DataLayout &DL = DAG.getDataLayout(); - - EVT PTy = TLI.getPointerTy(DL); - if (!TLI.isOperationLegal(ISD::SHL, PTy)) - return; - - int BitWidth = PTy.getSizeInBits(); - const int64_t N = Clusters.size(); - - // MinPartitions[i] is the minimum nbr of partitions of Clusters[i..N-1]. - SmallVector MinPartitions(N); - // LastElement[i] is the last element of the partition starting at i. - SmallVector LastElement(N); - - // FIXME: This might not be the best algorithm for finding bit test clusters. - - // Base case: There is only one way to partition Clusters[N-1]. - MinPartitions[N - 1] = 1; - LastElement[N - 1] = N - 1; - - // Note: loop indexes are signed to avoid underflow. - for (int64_t i = N - 2; i >= 0; --i) { - // Find optimal partitioning of Clusters[i..N-1]. - // Baseline: Put Clusters[i] into a partition on its own. - MinPartitions[i] = MinPartitions[i + 1] + 1; - LastElement[i] = i; - - // Search for a solution that results in fewer partitions. - // Note: the search is limited by BitWidth, reducing time complexity. - for (int64_t j = std::min(N - 1, i + BitWidth - 1); j > i; --j) { - // Try building a partition from Clusters[i..j]. - - // Check the range. - if (!TLI.rangeFitsInWord(Clusters[i].Low->getValue(), - Clusters[j].High->getValue(), DL)) - continue; - - // Check nbr of destinations and cluster types. - // FIXME: This works, but doesn't seem very efficient. - bool RangesOnly = true; - BitVector Dests(FuncInfo.MF->getNumBlockIDs()); - for (int64_t k = i; k <= j; k++) { - if (Clusters[k].Kind != CC_Range) { - RangesOnly = false; - break; - } - Dests.set(Clusters[k].MBB->getNumber()); - } - if (!RangesOnly || Dests.count() > 3) - break; - - // Check if it's a better partition. - unsigned NumPartitions = 1 + (j == N - 1 ? 0 : MinPartitions[j + 1]); - if (NumPartitions < MinPartitions[i]) { - // Found a better partition. - MinPartitions[i] = NumPartitions; - LastElement[i] = j; - } - } - } - - // Iterate over the partitions, replacing with bit-test clusters in-place. - unsigned DstIndex = 0; - for (unsigned First = 0, Last; First < N; First = Last + 1) { - Last = LastElement[First]; - assert(First <= Last); - assert(DstIndex <= First); - - CaseCluster BitTestCluster; - if (buildBitTests(Clusters, First, Last, SI, BitTestCluster)) { - Clusters[DstIndex++] = BitTestCluster; - } else { - size_t NumClusters = Last - First + 1; - std::memmove(&Clusters[DstIndex], &Clusters[First], - sizeof(Clusters[0]) * NumClusters); - DstIndex += NumClusters; - } - } - Clusters.resize(DstIndex); -} - void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond, MachineBasicBlock *SwitchMBB, MachineBasicBlock *DefaultMBB) { @@ -9977,10 +10021,13 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond, MachineBasicBlock *CurMBB = W.MBB; for (CaseClusterIt I = W.FirstCluster, E = W.LastCluster; I <= E; ++I) { + bool FallthroughUnreachable = false; MachineBasicBlock *Fallthrough; if (I == W.LastCluster) { // For the last cluster, fall through to the default destination. Fallthrough = DefaultMBB; + FallthroughUnreachable = isa( + DefaultMBB->getBasicBlock()->getFirstNonPHIOrDbg()); } else { Fallthrough = CurMF->CreateMachineBasicBlock(CurMBB->getBasicBlock()); CurMF->insert(BBI, Fallthrough); @@ -9992,8 +10039,8 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond, switch (I->Kind) { case CC_JumpTable: { // FIXME: Optimize away range check based on pivot comparisons. - JumpTableHeader *JTH = &JTCases[I->JTCasesIndex].first; - JumpTable *JT = &JTCases[I->JTCasesIndex].second; + JumpTableHeader *JTH = &SL->JTCases[I->JTCasesIndex].first; + SwitchCG::JumpTable *JT = &SL->JTCases[I->JTCasesIndex].second; // The jump block hasn't been inserted yet; insert it here. MachineBasicBlock *JumpMBB = JT->MBB; @@ -10017,7 +10064,13 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond, } } - addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb); + if (FallthroughUnreachable) { + // Skip the range check if the fallthrough block is unreachable. + JTH->OmitRangeCheck = true; + } + + if (!JTH->OmitRangeCheck) + addSuccessorWithProb(CurMBB, Fallthrough, FallthroughProb); addSuccessorWithProb(CurMBB, JumpMBB, JumpProb); CurMBB->normalizeSuccProbs(); @@ -10034,8 +10087,10 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond, break; } case CC_BitTests: { + // FIXME: If Fallthrough is unreachable, skip the range check. + // FIXME: Optimize away range check based on pivot comparisons. - BitTestBlock *BTB = &BitTestCases[I->BTCasesIndex]; + BitTestBlock *BTB = &SL->BitTestCases[I->BTCasesIndex]; // The bit test blocks haven't been inserted yet; insert them here. for (BitTestCase &BTC : BTB->Cases) @@ -10078,6 +10133,10 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond, RHS = I->High; } + // If Fallthrough is unreachable, fold away the comparison. + if (FallthroughUnreachable) + CC = ISD::SETTRUE; + // The false probability is the sum of all unhandled cases. CaseBlock CB(CC, LHS, RHS, MHS, I->MBB, Fallthrough, CurMBB, getCurSDLoc(), I->Prob, UnhandledProbs); @@ -10085,7 +10144,7 @@ void SelectionDAGBuilder::lowerWorkItem(SwitchWorkListItem W, Value *Cond, if (CurMBB == SwitchMBB) visitSwitchCase(CB, SwitchMBB); else - SwitchCases.push_back(CB); + SL->SwitchCases.push_back(CB); break; } @@ -10236,7 +10295,7 @@ void SelectionDAGBuilder::splitWorkItem(SwitchWorkList &WorkList, if (W.MBB == SwitchMBB) visitSwitchCase(CB, SwitchMBB); else - SwitchCases.push_back(CB); + SL->SwitchCases.push_back(CB); } // Scale CaseProb after peeling a case with the probablity of PeeledCaseProb @@ -10265,7 +10324,7 @@ MachineBasicBlock *SelectionDAGBuilder::peelDominantCaseCluster( // Don't perform if there is only one cluster or optimizing for size. if (SwitchPeelThreshold > 100 || !FuncInfo.BPI || Clusters.size() < 2 || TM.getOptLevel() == CodeGenOpt::None || - SwitchMBB->getParent()->getFunction().optForMinSize()) + SwitchMBB->getParent()->getFunction().hasMinSize()) return SwitchMBB; BranchProbability TopCaseProb = BranchProbability(SwitchPeelThreshold, 100); @@ -10331,38 +10390,6 @@ void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) { // if there are many clusters. sortAndRangeify(Clusters); - if (TM.getOptLevel() != CodeGenOpt::None) { - // Replace an unreachable default with the most popular destination. - // FIXME: Exploit unreachable default more aggressively. - bool UnreachableDefault = - isa(SI.getDefaultDest()->getFirstNonPHIOrDbg()); - if (UnreachableDefault && !Clusters.empty()) { - DenseMap Popularity; - unsigned MaxPop = 0; - const BasicBlock *MaxBB = nullptr; - for (auto I : SI.cases()) { - const BasicBlock *BB = I.getCaseSuccessor(); - if (++Popularity[BB] > MaxPop) { - MaxPop = Popularity[BB]; - MaxBB = BB; - } - } - // Set new default. - assert(MaxPop > 0 && MaxBB); - DefaultMBB = FuncInfo.MBBMap[MaxBB]; - - // Remove cases that were pointing to the destination that is now the - // default. - CaseClusterVector New; - New.reserve(Clusters.size()); - for (CaseCluster &CC : Clusters) { - if (CC.MBB != DefaultMBB) - New.push_back(CC); - } - Clusters = std::move(New); - } - } - // The branch probablity of the peeled case. BranchProbability PeeledCaseProb = BranchProbability::getZero(); MachineBasicBlock *PeeledSwitchMBB = @@ -10380,8 +10407,8 @@ void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) { return; } - findJumpTables(Clusters, &SI, DefaultMBB); - findBitTestClusters(Clusters, &SI); + SL->findJumpTables(Clusters, &SI, DefaultMBB); + SL->findBitTestClusters(Clusters, &SI); LLVM_DEBUG({ dbgs() << "Case clusters: "; @@ -10420,7 +10447,7 @@ void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) { unsigned NumClusters = W.LastCluster - W.FirstCluster + 1; if (NumClusters > 3 && TM.getOptLevel() != CodeGenOpt::None && - !DefaultMBB->getParent()->getFunction().optForMinSize()) { + !DefaultMBB->getParent()->getFunction().hasMinSize()) { // For optimized builds, lower large range as a balanced binary tree. splitWorkItem(WorkList, W, SI.getCondition(), SwitchMBB); continue; diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h index 5f9cdb69daf7..0072e33f23b7 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ b/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -1,9 +1,8 @@ //===- SelectionDAGBuilder.h - Selection-DAG building -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,11 +17,13 @@ #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/MapVector.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/ISDOpcodes.h" #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/SelectionDAGNodes.h" +#include "llvm/CodeGen/SwitchLoweringUtils.h" #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/ValueTypes.h" #include "llvm/IR/CallSite.h" @@ -47,6 +48,7 @@ class AtomicRMWInst; class BasicBlock; class BranchInst; class CallInst; +class CallBrInst; class CatchPadInst; class CatchReturnInst; class CatchSwitchInst; @@ -76,6 +78,7 @@ class ResumeInst; class ReturnInst; class SDDbgValue; class StoreInst; +class SwiftErrorValueTracking; class SwitchInst; class TargetLibraryInfo; class TargetMachine; @@ -91,16 +94,16 @@ class Value; /// implementation that is parameterized by a TargetLowering object. /// class SelectionDAGBuilder { - /// CurInst - The current instruction being visited + /// The current instruction being visited. const Instruction *CurInst = nullptr; DenseMap NodeMap; - /// UnusedArgNodeMap - Maps argument value for unused arguments. This is used + /// Maps argument value for unused arguments. This is used /// to preserve debug information for incoming arguments. DenseMap UnusedArgNodeMap; - /// DanglingDebugInfo - Helper type for DanglingDebugInfoMap. + /// Helper type for DanglingDebugInfoMap. class DanglingDebugInfo { const DbgValueInst* DI = nullptr; DebugLoc dl; @@ -116,18 +119,17 @@ class SelectionDAGBuilder { unsigned getSDNodeOrder() { return SDNodeOrder; } }; - /// DanglingDebugInfoVector - Helper type for DanglingDebugInfoMap. + /// Helper type for DanglingDebugInfoMap. typedef std::vector DanglingDebugInfoVector; - /// DanglingDebugInfoMap - Keeps track of dbg_values for which we have not - /// yet seen the referent. We defer handling these until we do see it. - DenseMap DanglingDebugInfoMap; + /// Keeps track of dbg_values for which we have not yet seen the referent. + /// We defer handling these until we do see it. + MapVector DanglingDebugInfoMap; public: - /// PendingLoads - Loads are not emitted to the program immediately. We bunch - /// them up and then emit token factor nodes when possible. This allows us to - /// get simple disambiguation between loads without worrying about alias - /// analysis. + /// Loads are not emitted to the program immediately. We bunch them up and + /// then emit token factor nodes when possible. This allows us to get simple + /// disambiguation between loads without worrying about alias analysis. SmallVector PendingLoads; /// State used while lowering a statepoint sequence (gc_statepoint, @@ -135,247 +137,37 @@ public: StatepointLoweringState StatepointLowering; private: - /// PendingExports - CopyToReg nodes that copy values to virtual registers - /// for export to other blocks need to be emitted before any terminator - /// instruction, but they have no other ordering requirements. We bunch them - /// up and the emit a single tokenfactor for them just before terminator - /// instructions. + /// CopyToReg nodes that copy values to virtual registers for export to other + /// blocks need to be emitted before any terminator instruction, but they have + /// no other ordering requirements. We bunch them up and the emit a single + /// tokenfactor for them just before terminator instructions. SmallVector PendingExports; - /// SDNodeOrder - A unique monotonically increasing number used to order the - /// SDNodes we create. + /// A unique monotonically increasing number used to order the SDNodes we + /// create. unsigned SDNodeOrder; - enum CaseClusterKind { - /// A cluster of adjacent case labels with the same destination, or just one - /// case. - CC_Range, - /// A cluster of cases suitable for jump table lowering. - CC_JumpTable, - /// A cluster of cases suitable for bit test lowering. - CC_BitTests - }; - - /// A cluster of case labels. - struct CaseCluster { - CaseClusterKind Kind; - const ConstantInt *Low, *High; - union { - MachineBasicBlock *MBB; - unsigned JTCasesIndex; - unsigned BTCasesIndex; - }; - BranchProbability Prob; - - static CaseCluster range(const ConstantInt *Low, const ConstantInt *High, - MachineBasicBlock *MBB, BranchProbability Prob) { - CaseCluster C; - C.Kind = CC_Range; - C.Low = Low; - C.High = High; - C.MBB = MBB; - C.Prob = Prob; - return C; - } - - static CaseCluster jumpTable(const ConstantInt *Low, - const ConstantInt *High, unsigned JTCasesIndex, - BranchProbability Prob) { - CaseCluster C; - C.Kind = CC_JumpTable; - C.Low = Low; - C.High = High; - C.JTCasesIndex = JTCasesIndex; - C.Prob = Prob; - return C; - } - - static CaseCluster bitTests(const ConstantInt *Low, const ConstantInt *High, - unsigned BTCasesIndex, BranchProbability Prob) { - CaseCluster C; - C.Kind = CC_BitTests; - C.Low = Low; - C.High = High; - C.BTCasesIndex = BTCasesIndex; - C.Prob = Prob; - return C; - } - }; - - using CaseClusterVector = std::vector; - using CaseClusterIt = CaseClusterVector::iterator; - - struct CaseBits { - uint64_t Mask = 0; - MachineBasicBlock* BB = nullptr; - unsigned Bits = 0; - BranchProbability ExtraProb; - - CaseBits() = default; - CaseBits(uint64_t mask, MachineBasicBlock* bb, unsigned bits, - BranchProbability Prob): - Mask(mask), BB(bb), Bits(bits), ExtraProb(Prob) {} - }; - - using CaseBitsVector = std::vector; - - /// Sort Clusters and merge adjacent cases. - void sortAndRangeify(CaseClusterVector &Clusters); - - /// CaseBlock - This structure is used to communicate between - /// SelectionDAGBuilder and SDISel for the code generation of additional basic - /// blocks needed by multi-case switch statements. - struct CaseBlock { - // CC - the condition code to use for the case block's setcc node - ISD::CondCode CC; - - // CmpLHS/CmpRHS/CmpMHS - The LHS/MHS/RHS of the comparison to emit. - // Emit by default LHS op RHS. MHS is used for range comparisons: - // If MHS is not null: (LHS <= MHS) and (MHS <= RHS). - const Value *CmpLHS, *CmpMHS, *CmpRHS; - - // TrueBB/FalseBB - the block to branch to if the setcc is true/false. - MachineBasicBlock *TrueBB, *FalseBB; - - // ThisBB - the block into which to emit the code for the setcc and branches - MachineBasicBlock *ThisBB; - - /// The debug location of the instruction this CaseBlock was - /// produced from. - SDLoc DL; - - // TrueProb/FalseProb - branch weights. - BranchProbability TrueProb, FalseProb; - - CaseBlock(ISD::CondCode cc, const Value *cmplhs, const Value *cmprhs, - const Value *cmpmiddle, MachineBasicBlock *truebb, - MachineBasicBlock *falsebb, MachineBasicBlock *me, - SDLoc dl, - BranchProbability trueprob = BranchProbability::getUnknown(), - BranchProbability falseprob = BranchProbability::getUnknown()) - : CC(cc), CmpLHS(cmplhs), CmpMHS(cmpmiddle), CmpRHS(cmprhs), - TrueBB(truebb), FalseBB(falsebb), ThisBB(me), DL(dl), - TrueProb(trueprob), FalseProb(falseprob) {} - }; - - struct JumpTable { - /// Reg - the virtual register containing the index of the jump table entry - //. to jump to. - unsigned Reg; - /// JTI - the JumpTableIndex for this jump table in the function. - unsigned JTI; - /// MBB - the MBB into which to emit the code for the indirect jump. - MachineBasicBlock *MBB; - /// Default - the MBB of the default bb, which is a successor of the range - /// check MBB. This is when updating PHI nodes in successors. - MachineBasicBlock *Default; - - JumpTable(unsigned R, unsigned J, MachineBasicBlock *M, - MachineBasicBlock *D): Reg(R), JTI(J), MBB(M), Default(D) {} - }; - struct JumpTableHeader { - APInt First; - APInt Last; - const Value *SValue; - MachineBasicBlock *HeaderBB; - bool Emitted; - - JumpTableHeader(APInt F, APInt L, const Value *SV, MachineBasicBlock *H, - bool E = false) - : First(std::move(F)), Last(std::move(L)), SValue(SV), HeaderBB(H), - Emitted(E) {} - }; - using JumpTableBlock = std::pair; - - struct BitTestCase { - uint64_t Mask; - MachineBasicBlock *ThisBB; - MachineBasicBlock *TargetBB; - BranchProbability ExtraProb; - - BitTestCase(uint64_t M, MachineBasicBlock* T, MachineBasicBlock* Tr, - BranchProbability Prob): - Mask(M), ThisBB(T), TargetBB(Tr), ExtraProb(Prob) {} - }; - - using BitTestInfo = SmallVector; - - struct BitTestBlock { - APInt First; - APInt Range; - const Value *SValue; - unsigned Reg; - MVT RegVT; - bool Emitted; - bool ContiguousRange; - MachineBasicBlock *Parent; - MachineBasicBlock *Default; - BitTestInfo Cases; - BranchProbability Prob; - BranchProbability DefaultProb; - - BitTestBlock(APInt F, APInt R, const Value *SV, unsigned Rg, MVT RgVT, - bool E, bool CR, MachineBasicBlock *P, MachineBasicBlock *D, - BitTestInfo C, BranchProbability Pr) - : First(std::move(F)), Range(std::move(R)), SValue(SV), Reg(Rg), - RegVT(RgVT), Emitted(E), ContiguousRange(CR), Parent(P), Default(D), - Cases(std::move(C)), Prob(Pr) {} - }; - - /// Return the range of value in [First..Last]. - uint64_t getJumpTableRange(const CaseClusterVector &Clusters, unsigned First, - unsigned Last) const; - - /// Return the number of cases in [First..Last]. - uint64_t getJumpTableNumCases(const SmallVectorImpl &TotalCases, - unsigned First, unsigned Last) const; - - /// Build a jump table cluster from Clusters[First..Last]. Returns false if it - /// decides it's not a good idea. - bool buildJumpTable(const CaseClusterVector &Clusters, unsigned First, - unsigned Last, const SwitchInst *SI, - MachineBasicBlock *DefaultMBB, CaseCluster &JTCluster); - - /// Find clusters of cases suitable for jump table lowering. - void findJumpTables(CaseClusterVector &Clusters, const SwitchInst *SI, - MachineBasicBlock *DefaultMBB); - - /// Build a bit test cluster from Clusters[First..Last]. Returns false if it - /// decides it's not a good idea. - bool buildBitTests(CaseClusterVector &Clusters, unsigned First, unsigned Last, - const SwitchInst *SI, CaseCluster &BTCluster); - - /// Find clusters of cases suitable for bit test lowering. - void findBitTestClusters(CaseClusterVector &Clusters, const SwitchInst *SI); - - struct SwitchWorkListItem { - MachineBasicBlock *MBB; - CaseClusterIt FirstCluster; - CaseClusterIt LastCluster; - const ConstantInt *GE; - const ConstantInt *LT; - BranchProbability DefaultProb; - }; - using SwitchWorkList = SmallVector; - /// Determine the rank by weight of CC in [First,Last]. If CC has more weight /// than each cluster in the range, its rank is 0. - static unsigned caseClusterRank(const CaseCluster &CC, CaseClusterIt First, - CaseClusterIt Last); + unsigned caseClusterRank(const SwitchCG::CaseCluster &CC, + SwitchCG::CaseClusterIt First, + SwitchCG::CaseClusterIt Last); /// Emit comparison and split W into two subtrees. - void splitWorkItem(SwitchWorkList &WorkList, const SwitchWorkListItem &W, - Value *Cond, MachineBasicBlock *SwitchMBB); + void splitWorkItem(SwitchCG::SwitchWorkList &WorkList, + const SwitchCG::SwitchWorkListItem &W, Value *Cond, + MachineBasicBlock *SwitchMBB); /// Lower W. - void lowerWorkItem(SwitchWorkListItem W, Value *Cond, + void lowerWorkItem(SwitchCG::SwitchWorkListItem W, Value *Cond, MachineBasicBlock *SwitchMBB, MachineBasicBlock *DefaultMBB); /// Peel the top probability case if it exceeds the threshold - MachineBasicBlock *peelDominantCaseCluster(const SwitchInst &SI, - CaseClusterVector &Clusters, - BranchProbability &PeeledCaseProb); + MachineBasicBlock * + peelDominantCaseCluster(const SwitchInst &SI, + SwitchCG::CaseClusterVector &Clusters, + BranchProbability &PeeledCaseProb); /// A class which encapsulates all of the information needed to generate a /// stack protector check and signals to isel via its state being initialized @@ -588,17 +380,22 @@ public: AliasAnalysis *AA = nullptr; const TargetLibraryInfo *LibInfo; - /// SwitchCases - Vector of CaseBlock structures used to communicate - /// SwitchInst code generation information. - std::vector SwitchCases; + class SDAGSwitchLowering : public SwitchCG::SwitchLowering { + public: + SDAGSwitchLowering(SelectionDAGBuilder *sdb, FunctionLoweringInfo &funcinfo) + : SwitchCG::SwitchLowering(funcinfo), SDB(sdb) {} + + virtual void addSuccessorWithProb( + MachineBasicBlock *Src, MachineBasicBlock *Dst, + BranchProbability Prob = BranchProbability::getUnknown()) override { + SDB->addSuccessorWithProb(Src, Dst, Prob); + } - /// JTCases - Vector of JumpTable structures used to communicate - /// SwitchInst code generation information. - std::vector JTCases; + private: + SelectionDAGBuilder *SDB; + }; - /// BitTestCases - Vector of BitTestBlock structures used to communicate - /// SwitchInst code generation information. - std::vector BitTestCases; + std::unique_ptr SL; /// A StackProtectorDescriptor structure used to communicate stack protector /// information in between SelectBasicBlock and FinishBasicBlock. @@ -608,27 +405,29 @@ public: // PHI nodes. DenseMap ConstantsOut; - /// FuncInfo - Information about the function as a whole. - /// + /// Information about the function as a whole. FunctionLoweringInfo &FuncInfo; - /// GFI - Garbage collection metadata for the function. + /// Information about the swifterror values used throughout the function. + SwiftErrorValueTracking &SwiftError; + + /// Garbage collection metadata for the function. GCFunctionInfo *GFI; - /// LPadToCallSiteMap - Map a landing pad to the call site indexes. + /// Map a landing pad to the call site indexes. DenseMap> LPadToCallSiteMap; - /// HasTailCall - This is set to true if a call in the current - /// block has been translated as a tail call. In this case, - /// no subsequent DAG nodes should be created. + /// This is set to true if a call in the current block has been translated as + /// a tail call. In this case, no subsequent DAG nodes should be created. bool HasTailCall = false; LLVMContext *Context; SelectionDAGBuilder(SelectionDAG &dag, FunctionLoweringInfo &funcinfo, - CodeGenOpt::Level ol) - : SDNodeOrder(LowestSDNodeOrder), TM(dag.getTarget()), DAG(dag), - FuncInfo(funcinfo) {} + SwiftErrorValueTracking &swifterror, CodeGenOpt::Level ol) + : SDNodeOrder(LowestSDNodeOrder), TM(dag.getTarget()), DAG(dag), + SL(make_unique(this, funcinfo)), FuncInfo(funcinfo), + SwiftError(swifterror) {} void init(GCFunctionInfo *gfi, AliasAnalysis *AA, const TargetLibraryInfo *li); @@ -670,20 +469,34 @@ public: void visit(unsigned Opcode, const User &I); - /// getCopyFromRegs - If there was virtual register allocated for the value V - /// emit CopyFromReg of the specified type Ty. Return empty SDValue() otherwise. + /// If there was virtual register allocated for the value V emit CopyFromReg + /// of the specified type Ty. Return empty SDValue() otherwise. SDValue getCopyFromRegs(const Value *V, Type *Ty); /// If we have dangling debug info that describes \p Variable, or an /// overlapping part of variable considering the \p Expr, then this method - /// weill drop that debug info as it isn't valid any longer. + /// will drop that debug info as it isn't valid any longer. void dropDanglingDebugInfo(const DILocalVariable *Variable, const DIExpression *Expr); - // resolveDanglingDebugInfo - if we saw an earlier dbg_value referring to V, - // generate the debug data structures now that we've seen its definition. + /// If we saw an earlier dbg_value referring to V, generate the debug data + /// structures now that we've seen its definition. void resolveDanglingDebugInfo(const Value *V, SDValue Val); + /// For the given dangling debuginfo record, perform last-ditch efforts to + /// resolve the debuginfo to something that is represented in this DAG. If + /// this cannot be done, produce an Undef debug value record. + void salvageUnresolvedDbgValue(DanglingDebugInfo &DDI); + + /// For a given Value, attempt to create and record a SDDbgValue in the + /// SelectionDAG. + bool handleDebugValue(const Value *V, DILocalVariable *Var, + DIExpression *Expr, DebugLoc CurDL, + DebugLoc InstDL, unsigned Order); + + /// Evict any dangling debug information, attempting to salvage it first. + void resolveOrClearDbgInfo(); + SDValue getValue(const Value *V); bool findValue(const Value *V) const; @@ -720,7 +533,7 @@ public: MachineBasicBlock *SwitchBB, BranchProbability TProb, BranchProbability FProb, bool InvertCond); - bool ShouldEmitAsBranches(const std::vector &Cases); + bool ShouldEmitAsBranches(const std::vector &Cases); bool isExportableFromCurrentBlock(const Value *V, const BasicBlock *FromBB); void CopyToExportRegsIfNeeded(const Value *V); void ExportFromCurrentBlock(const Value *V); @@ -733,7 +546,7 @@ public: SDValue Op); void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, - ImmutableCallSite CS, unsigned ArgIdx, + const CallBase *Call, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, bool IsPatchPoint); @@ -741,7 +554,7 @@ public: lowerInvokable(TargetLowering::CallLoweringInfo &CLI, const BasicBlock *EHPadBB = nullptr); - /// UpdateSplitBlock - When an MBB was split during scheduling, update the + /// When an MBB was split during scheduling, update the /// references that need to refer to the last resulting block. void UpdateSplitBlock(MachineBasicBlock *First, MachineBasicBlock *Last); @@ -797,13 +610,13 @@ public: void LowerStatepoint(ImmutableStatepoint ISP, const BasicBlock *EHPadBB = nullptr); - void LowerCallSiteWithDeoptBundle(ImmutableCallSite CS, SDValue Callee, + void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB); void LowerDeoptimizeCall(const CallInst *CI); void LowerDeoptimizingReturn(); - void LowerCallSiteWithDeoptBundleImpl(ImmutableCallSite CS, SDValue Callee, + void LowerCallSiteWithDeoptBundleImpl(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB, bool VarArgDisallowed, bool ForceVoidReturnTy); @@ -833,25 +646,24 @@ private: BranchProbability Prob = BranchProbability::getUnknown()); public: - void visitSwitchCase(CaseBlock &CB, - MachineBasicBlock *SwitchBB); + void visitSwitchCase(SwitchCG::CaseBlock &CB, MachineBasicBlock *SwitchBB); void visitSPDescriptorParent(StackProtectorDescriptor &SPD, MachineBasicBlock *ParentBB); void visitSPDescriptorFailure(StackProtectorDescriptor &SPD); - void visitBitTestHeader(BitTestBlock &B, MachineBasicBlock *SwitchBB); - void visitBitTestCase(BitTestBlock &BB, - MachineBasicBlock* NextMBB, - BranchProbability BranchProbToNext, - unsigned Reg, - BitTestCase &B, - MachineBasicBlock *SwitchBB); - void visitJumpTable(JumpTable &JT); - void visitJumpTableHeader(JumpTable &JT, JumpTableHeader &JTH, + void visitBitTestHeader(SwitchCG::BitTestBlock &B, + MachineBasicBlock *SwitchBB); + void visitBitTestCase(SwitchCG::BitTestBlock &BB, MachineBasicBlock *NextMBB, + BranchProbability BranchProbToNext, unsigned Reg, + SwitchCG::BitTestCase &B, MachineBasicBlock *SwitchBB); + void visitJumpTable(SwitchCG::JumpTable &JT); + void visitJumpTableHeader(SwitchCG::JumpTable &JT, + SwitchCG::JumpTableHeader &JTH, MachineBasicBlock *SwitchBB); private: // These all get lowered before this pass. void visitInvoke(const InvokeInst &I); + void visitCallBr(const CallBrInst &I); void visitResume(const ResumeInst &I); void visitUnary(const User &I, unsigned Opcode); @@ -932,7 +744,7 @@ private: void visitStoreToSwiftError(const StoreInst &I); void visitInlineAsm(ImmutableCallSite CS); - const char *visitIntrinsicCall(const CallInst &I, unsigned Intrinsic); + void visitIntrinsicCall(const CallInst &I, unsigned Intrinsic); void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic); void visitConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI); @@ -982,9 +794,12 @@ private: SDDbgValue *getDbgValue(SDValue N, DILocalVariable *Variable, DIExpression *Expr, const DebugLoc &dl, unsigned DbgSDNodeOrder); + + /// Lowers CallInst to an external symbol. + void lowerCallToExternalSymbol(const CallInst &I, const char *FunctionName); }; -/// RegsForValue - This struct represents the registers (physical or virtual) +/// This struct represents the registers (physical or virtual) /// that a particular set of values is assigned, and the type information about /// the value. The most common situation is to represent one value at a time, /// but struct or array values are handled element-wise as multiple values. The diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp index 43df2abb674b..da3049881d31 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -1,9 +1,8 @@ //===- SelectionDAGDumper.cpp - Implement SelectionDAG::dump() ------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -96,6 +95,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::ATOMIC_LOAD_MAX: return "AtomicLoadMax"; case ISD::ATOMIC_LOAD_UMIN: return "AtomicLoadUMin"; case ISD::ATOMIC_LOAD_UMAX: return "AtomicLoadUMax"; + case ISD::ATOMIC_LOAD_FADD: return "AtomicLoadFAdd"; case ISD::ATOMIC_LOAD: return "AtomicLoad"; case ISD::ATOMIC_STORE: return "AtomicStore"; case ISD::PCMARKER: return "PCMarker"; @@ -145,6 +145,8 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { unsigned IID = cast(getOperand(OpNo))->getZExtValue(); if (IID < Intrinsic::num_intrinsics) return Intrinsic::getName((Intrinsic::ID)IID, None); + else if (!G) + return "Unknown intrinsic"; else if (const TargetIntrinsicInfo *TII = G->getTarget().getIntrinsicInfo()) return TII->getName(IID); llvm_unreachable("Invalid intrinsic ID"); @@ -170,7 +172,9 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::UNDEF: return "undef"; case ISD::MERGE_VALUES: return "merge_values"; case ISD::INLINEASM: return "inlineasm"; + case ISD::INLINEASM_BR: return "inlineasm_br"; case ISD::EH_LABEL: return "eh_label"; + case ISD::ANNOTATION_LABEL: return "annotation_label"; case ISD::HANDLENODE: return "handlenode"; // Unary operators @@ -297,7 +301,10 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::UADDSAT: return "uaddsat"; case ISD::SSUBSAT: return "ssubsat"; case ISD::USUBSAT: return "usubsat"; + case ISD::SMULFIX: return "smulfix"; + case ISD::SMULFIXSAT: return "smulfixsat"; + case ISD::UMULFIX: return "umulfix"; // Conversion operators. case ISD::SIGN_EXTEND: return "sign_extend"; @@ -309,9 +316,11 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::ZERO_EXTEND_VECTOR_INREG: return "zero_extend_vector_inreg"; case ISD::TRUNCATE: return "truncate"; case ISD::FP_ROUND: return "fp_round"; + case ISD::STRICT_FP_ROUND: return "strict_fp_round"; case ISD::FLT_ROUNDS_: return "flt_rounds"; case ISD::FP_ROUND_INREG: return "fp_round_inreg"; case ISD::FP_EXTEND: return "fp_extend"; + case ISD::STRICT_FP_EXTEND: return "strict_fp_extend"; case ISD::SINT_TO_FP: return "sint_to_fp"; case ISD::UINT_TO_FP: return "uint_to_fp"; @@ -321,6 +330,10 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const { case ISD::ADDRSPACECAST: return "addrspacecast"; case ISD::FP16_TO_FP: return "fp16_to_fp"; case ISD::FP_TO_FP16: return "fp_to_fp16"; + case ISD::LROUND: return "lround"; + case ISD::LLROUND: return "llround"; + case ISD::LRINT: return "lrint"; + case ISD::LLRINT: return "llrint"; // Control flow instructions case ISD::BR: return "br"; @@ -649,6 +662,36 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const { if (*AM) OS << ", " << AM; + OS << ">"; + } else if (const MaskedLoadSDNode *MLd = dyn_cast(this)) { + OS << "<"; + + printMemOperand(OS, *MLd->getMemOperand(), G); + + bool doExt = true; + switch (MLd->getExtensionType()) { + default: doExt = false; break; + case ISD::EXTLOAD: OS << ", anyext"; break; + case ISD::SEXTLOAD: OS << ", sext"; break; + case ISD::ZEXTLOAD: OS << ", zext"; break; + } + if (doExt) + OS << " from " << MLd->getMemoryVT().getEVTString(); + + if (MLd->isExpandingLoad()) + OS << ", expanding"; + + OS << ">"; + } else if (const MaskedStoreSDNode *MSt = dyn_cast(this)) { + OS << "<"; + printMemOperand(OS, *MSt->getMemOperand(), G); + + if (MSt->isTruncatingStore()) + OS << ", trunc to " << MSt->getMemoryVT().getEVTString(); + + if (MSt->isCompressingStore()) + OS << ", compressing"; + OS << ">"; } else if (const MemSDNode* M = dyn_cast(this)) { OS << "<"; @@ -675,6 +718,9 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const { << " -> " << ASC->getDestAddressSpace() << ']'; + } else if (const LifetimeSDNode *LN = dyn_cast(this)) { + if (LN->hasOffset()) + OS << "<" << LN->getOffset() << " to " << LN->getOffset() + LN->getSize() << ">"; } if (VerboseDAGDumping) { @@ -684,45 +730,63 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const { if (getNodeId() != -1) OS << " [ID=" << getNodeId() << ']'; if (!(isa(this) || (isa(this)))) - OS << "# D:" << isDivergent(); - - if (!G) - return; - - DILocation *L = getDebugLoc(); - if (!L) - return; - - if (auto *Scope = L->getScope()) - OS << Scope->getFilename(); - else - OS << ""; - OS << ':' << L->getLine(); - if (unsigned C = L->getColumn()) - OS << ':' << C; - - for (SDDbgValue *Dbg : G->GetDbgValues(this)) { - if (Dbg->getKind() != SDDbgValue::SDNODE || Dbg->isInvalidated()) - continue; - Dbg->dump(OS); - } + OS << " # D:" << isDivergent(); + + if (G && !G->GetDbgValues(this).empty()) { + OS << " [NoOfDbgValues=" << G->GetDbgValues(this).size() << ']'; + for (SDDbgValue *Dbg : G->GetDbgValues(this)) + if (!Dbg->isInvalidated()) + Dbg->print(OS); + } else if (getHasDebugValue()) + OS << " [NoOfDbgValues>0]"; } } -LLVM_DUMP_METHOD void SDDbgValue::dump(raw_ostream &OS) const { - OS << " DbgVal"; - if (kind==SDNODE) - OS << '(' << u.s.ResNo << ')'; - OS << ":\"" << Var->getName() << '"'; +LLVM_DUMP_METHOD void SDDbgValue::print(raw_ostream &OS) const { + OS << " DbgVal(Order=" << getOrder() << ')'; + if (isInvalidated()) OS << "(Invalidated)"; + if (isEmitted()) OS << "(Emitted)"; + switch (getKind()) { + case SDNODE: + if (getSDNode()) + OS << "(SDNODE=" << PrintNodeId(*getSDNode()) << ':' << getResNo() << ')'; + else + OS << "(SDNODE)"; + break; + case CONST: + OS << "(CONST)"; + break; + case FRAMEIX: + OS << "(FRAMEIX=" << getFrameIx() << ')'; + break; + case VREG: + OS << "(VREG=" << getVReg() << ')'; + break; + } + if (isIndirect()) OS << "(Indirect)"; + OS << ":\"" << Var->getName() << '"'; #ifndef NDEBUG - if (Expr->getNumElements()) - Expr->dump(); + if (Expr->getNumElements()) + Expr->dump(); #endif } +#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) +LLVM_DUMP_METHOD void SDDbgValue::dump() const { + if (isInvalidated()) + return; + print(dbgs()); + dbgs() << "\n"; +} +#endif + /// Return true if this node is so simple that we should just print it inline /// if it appears as an operand. -static bool shouldPrintInline(const SDNode &Node) { +static bool shouldPrintInline(const SDNode &Node, const SelectionDAG *G) { + // Avoid lots of cluttering when inline printing nodes with associated + // DbgValues in verbose mode. + if (VerboseDAGDumping && G && !G->GetDbgValues(&Node).empty()) + return false; if (Node.getOpcode() == ISD::EntryToken) return false; return Node.getNumOperands() == 0; @@ -731,7 +795,7 @@ static bool shouldPrintInline(const SDNode &Node) { #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) static void DumpNodes(const SDNode *N, unsigned indent, const SelectionDAG *G) { for (const SDValue &Op : N->op_values()) { - if (shouldPrintInline(*Op.getNode())) + if (shouldPrintInline(*Op.getNode(), G)) continue; if (Op.getNode()->hasOneUse()) DumpNodes(Op.getNode(), indent+2, G); @@ -748,12 +812,24 @@ LLVM_DUMP_METHOD void SelectionDAG::dump() const { I != E; ++I) { const SDNode *N = &*I; if (!N->hasOneUse() && N != getRoot().getNode() && - (!shouldPrintInline(*N) || N->use_empty())) + (!shouldPrintInline(*N, this) || N->use_empty())) DumpNodes(N, 2, this); } if (getRoot().getNode()) DumpNodes(getRoot().getNode(), 2, this); - dbgs() << "\n\n"; + dbgs() << "\n"; + + if (VerboseDAGDumping) { + if (DbgBegin() != DbgEnd()) + dbgs() << "SDDbgValues:\n"; + for (auto *Dbg : make_range(DbgBegin(), DbgEnd())) + Dbg->dump(); + if (ByvalParmDbgBegin() != ByvalParmDbgEnd()) + dbgs() << "Byval SDDbgValues:\n"; + for (auto *Dbg : make_range(ByvalParmDbgBegin(), ByvalParmDbgEnd())) + Dbg->dump(); + } + dbgs() << "\n"; } #endif @@ -769,7 +845,7 @@ static bool printOperand(raw_ostream &OS, const SelectionDAG *G, if (!Value.getNode()) { OS << ""; return false; - } else if (shouldPrintInline(*Value.getNode())) { + } else if (shouldPrintInline(*Value.getNode(), G)) { OS << Value->getOperationName(G) << ':'; Value->print_types(OS, G); Value->print_details(OS, G); diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index af5c2433fa2f..bdf9f2c166e1 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -1,9 +1,8 @@ //===- SelectionDAGISel.cpp - Implement the SelectionDAGISel class --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -42,6 +41,7 @@ #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/MachineInstrBuilder.h" #include "llvm/CodeGen/MachineMemOperand.h" +#include "llvm/CodeGen/MachineModuleInfo.h" #include "llvm/CodeGen/MachineOperand.h" #include "llvm/CodeGen/MachinePassRegistry.h" #include "llvm/CodeGen/MachineRegisterInfo.h" @@ -49,6 +49,7 @@ #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/SelectionDAGNodes.h" #include "llvm/CodeGen/StackProtector.h" +#include "llvm/CodeGen/SwiftErrorValueTracking.h" #include "llvm/CodeGen/TargetInstrInfo.h" #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetRegisterInfo.h" @@ -63,6 +64,7 @@ #include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" #include "llvm/IR/InlineAsm.h" +#include "llvm/IR/InstIterator.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instruction.h" #include "llvm/IR/Instructions.h" @@ -306,8 +308,9 @@ SelectionDAGISel::SelectionDAGISel(TargetMachine &tm, CodeGenOpt::Level OL) : MachineFunctionPass(ID), TM(tm), FuncInfo(new FunctionLoweringInfo()), + SwiftError(new SwiftErrorValueTracking()), CurDAG(new SelectionDAG(tm, OL)), - SDB(new SelectionDAGBuilder(*CurDAG, *FuncInfo, OL)), + SDB(new SelectionDAGBuilder(*CurDAG, *FuncInfo, *SwiftError, OL)), AA(), GFI(), OptLevel(OL), DAGSize(0) { @@ -323,6 +326,7 @@ SelectionDAGISel::~SelectionDAGISel() { delete SDB; delete CurDAG; delete FuncInfo; + delete SwiftError; } void SelectionDAGISel::getAnalysisUsage(AnalysisUsage &AU) const { @@ -379,6 +383,30 @@ static void SplitCriticalSideEffectEdges(Function &Fn, DominatorTree *DT, } } +static void computeUsesMSVCFloatingPoint(const Triple &TT, const Function &F, + MachineModuleInfo &MMI) { + // Only needed for MSVC + if (!TT.isWindowsMSVCEnvironment()) + return; + + // If it's already set, nothing to do. + if (MMI.usesMSVCFloatingPoint()) + return; + + for (const Instruction &I : instructions(F)) { + if (I.getType()->isFPOrFPVectorTy()) { + MMI.setUsesMSVCFloatingPoint(true); + return; + } + for (const auto &Op : I.operands()) { + if (Op->getType()->isFPOrFPVectorTy()) { + MMI.setUsesMSVCFloatingPoint(true); + return; + } + } + } +} + bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { // If we already selected that function, we do not need to run SDISel. if (mf.getProperties().hasProperty( @@ -421,6 +449,7 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { CurDAG->init(*MF, *ORE, this, LibInfo, getAnalysisIfAvailable()); FuncInfo->set(Fn, *MF, CurDAG); + SwiftError->setFunction(*MF); // Now get the optional analyzes if we want to. // This is based on the possibly changed OptLevel (after optnone is taken @@ -474,6 +503,40 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { Fn.getContext().diagnose(DiagFallback); } + // Replace forward-declared registers with the registers containing + // the desired value. + // Note: it is important that this happens **before** the call to + // EmitLiveInCopies, since implementations can skip copies of unused + // registers. If we don't apply the reg fixups before, some registers may + // appear as unused and will be skipped, resulting in bad MI. + MachineRegisterInfo &MRI = MF->getRegInfo(); + for (DenseMap::iterator I = FuncInfo->RegFixups.begin(), + E = FuncInfo->RegFixups.end(); + I != E; ++I) { + unsigned From = I->first; + unsigned To = I->second; + // If To is also scheduled to be replaced, find what its ultimate + // replacement is. + while (true) { + DenseMap::iterator J = FuncInfo->RegFixups.find(To); + if (J == E) + break; + To = J->second; + } + // Make sure the new register has a sufficiently constrained register class. + if (TargetRegisterInfo::isVirtualRegister(From) && + TargetRegisterInfo::isVirtualRegister(To)) + MRI.constrainRegClass(To, MRI.getRegClass(From)); + // Replace it. + + // Replacing one register with another won't touch the kill flags. + // We need to conservatively clear the kill flags as a kill on the old + // register might dominate existing uses of the new register. + if (!MRI.use_empty(To)) + MRI.clearKillFlags(From); + MRI.replaceRegWith(From, To); + } + // If the first basic block in the function has live ins that need to be // copied into vregs, emit the copies into the top of the block before // emitting the code for the block. @@ -507,7 +570,7 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { for (unsigned i = 0, e = FuncInfo->ArgDbgValues.size(); i != e; ++i) { MachineInstr *MI = FuncInfo->ArgDbgValues[e-i-1]; bool hasFI = MI->getOperand(0).isFI(); - unsigned Reg = + Register Reg = hasFI ? TRI.getFrameRegister(*MF) : MI->getOperand(0).getReg(); if (TargetRegisterInfo::isPhysicalRegister(Reg)) EntryMBB->insert(EntryMBB->begin(), MI); @@ -590,9 +653,11 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { // Determine if there is a call to setjmp in the machine function. MF->setExposesReturnsTwice(Fn.callsFunctionThatReturnsTwice()); + // Determine if floating point is used for msvc + computeUsesMSVCFloatingPoint(TM.getTargetTriple(), Fn, MF->getMMI()); + // Replace forward-declared registers with the registers containing // the desired value. - MachineRegisterInfo &MRI = MF->getRegInfo(); for (DenseMap::iterator I = FuncInfo->RegFixups.begin(), E = FuncInfo->RegFixups.end(); I != E; ++I) { @@ -663,6 +728,7 @@ void SelectionDAGISel::SelectBasicBlock(BasicBlock::const_iterator Begin, // Make sure the root of the DAG is up-to-date. CurDAG->setRoot(SDB->getControlRoot()); HadTailCall = SDB->HasTailCall; + SDB->resolveOrClearDbgInfo(); SDB->clear(); // Final step, emit the lowered DAG as machine code. @@ -713,8 +779,6 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { StringRef GroupName = "sdag"; StringRef GroupDescription = "Instruction Selection and Scheduling"; std::string BlockName; - int BlockNumber = -1; - (void)BlockNumber; bool MatchFilterBB = false; (void)MatchFilterBB; #ifndef NDEBUG TargetTransformInfo &TTI = @@ -735,7 +799,6 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { ViewSUnitDAGs) #endif { - BlockNumber = FuncInfo->MBB->getNumber(); BlockName = (MF->getName() + ":" + FuncInfo->MBB->getBasicBlock()->getName()).str(); } @@ -1092,16 +1155,14 @@ void SelectionDAGISel::DoInstructionSelection() { #endif // When we are using non-default rounding modes or FP exception behavior - // FP operations are represented by StrictFP pseudo-operations. They - // need to be simplified here so that the target-specific instruction - // selectors know how to handle them. - // - // If the current node is a strict FP pseudo-op, the isStrictFPOp() - // function will provide the corresponding normal FP opcode to which the - // node should be mutated. - // - // FIXME: The backends need a way to handle FP constraints. - if (Node->isStrictFPOpcode()) + // FP operations are represented by StrictFP pseudo-operations. For + // targets that do not (yet) understand strict FP operations directly, + // we convert them to normal FP opcodes instead at this point. This + // will allow them to be handled by existing target-specific instruction + // selectors. + if (Node->isStrictFPOpcode() && + (TLI->getOperationAction(Node->getOpcode(), Node->getValueType(0)) + != TargetLowering::Legal)) Node = CurDAG->mutateStrictFPToFP(Node); LLVM_DEBUG(dbgs() << "\nISEL: Starting selection on root node: "; @@ -1228,77 +1289,6 @@ static bool isFoldedOrDeadInstruction(const Instruction *I, !FuncInfo->isExportedInst(I); // Exported instrs must be computed. } -/// Set up SwiftErrorVals by going through the function. If the function has -/// swifterror argument, it will be the first entry. -static void setupSwiftErrorVals(const Function &Fn, const TargetLowering *TLI, - FunctionLoweringInfo *FuncInfo) { - if (!TLI->supportSwiftError()) - return; - - FuncInfo->SwiftErrorVals.clear(); - FuncInfo->SwiftErrorVRegDefMap.clear(); - FuncInfo->SwiftErrorVRegUpwardsUse.clear(); - FuncInfo->SwiftErrorVRegDefUses.clear(); - FuncInfo->SwiftErrorArg = nullptr; - - // Check if function has a swifterror argument. - bool HaveSeenSwiftErrorArg = false; - for (Function::const_arg_iterator AI = Fn.arg_begin(), AE = Fn.arg_end(); - AI != AE; ++AI) - if (AI->hasSwiftErrorAttr()) { - assert(!HaveSeenSwiftErrorArg && - "Must have only one swifterror parameter"); - (void)HaveSeenSwiftErrorArg; // silence warning. - HaveSeenSwiftErrorArg = true; - FuncInfo->SwiftErrorArg = &*AI; - FuncInfo->SwiftErrorVals.push_back(&*AI); - } - - for (const auto &LLVMBB : Fn) - for (const auto &Inst : LLVMBB) { - if (const AllocaInst *Alloca = dyn_cast(&Inst)) - if (Alloca->isSwiftError()) - FuncInfo->SwiftErrorVals.push_back(Alloca); - } -} - -static void createSwiftErrorEntriesInEntryBlock(FunctionLoweringInfo *FuncInfo, - FastISel *FastIS, - const TargetLowering *TLI, - const TargetInstrInfo *TII, - SelectionDAGBuilder *SDB) { - if (!TLI->supportSwiftError()) - return; - - // We only need to do this when we have swifterror parameter or swifterror - // alloc. - if (FuncInfo->SwiftErrorVals.empty()) - return; - - assert(FuncInfo->MBB == &*FuncInfo->MF->begin() && - "expected to insert into entry block"); - auto &DL = FuncInfo->MF->getDataLayout(); - auto const *RC = TLI->getRegClassFor(TLI->getPointerTy(DL)); - for (const auto *SwiftErrorVal : FuncInfo->SwiftErrorVals) { - // We will always generate a copy from the argument. It is always used at - // least by the 'return' of the swifterror. - if (FuncInfo->SwiftErrorArg && FuncInfo->SwiftErrorArg == SwiftErrorVal) - continue; - unsigned VReg = FuncInfo->MF->getRegInfo().createVirtualRegister(RC); - // Assign Undef to Vreg. We construct MI directly to make sure it works - // with FastISel. - BuildMI(*FuncInfo->MBB, FuncInfo->MBB->getFirstNonPHI(), - SDB->getCurDebugLoc(), TII->get(TargetOpcode::IMPLICIT_DEF), - VReg); - - // Keep FastIS informed about the value we just inserted. - if (FastIS) - FastIS->setLastLocalValue(&*std::prev(FuncInfo->InsertPt)); - - FuncInfo->setCurrentSwiftErrorVReg(FuncInfo->MBB, SwiftErrorVal, VReg); - } -} - /// Collect llvm.dbg.declare information. This is done after argument lowering /// in case the declarations refer to arguments. static void processDbgDeclares(FunctionLoweringInfo *FuncInfo) { @@ -1337,202 +1327,13 @@ static void processDbgDeclares(FunctionLoweringInfo *FuncInfo) { DIExpression *Expr = DI->getExpression(); if (Offset.getBoolValue()) - Expr = DIExpression::prepend(Expr, DIExpression::NoDeref, + Expr = DIExpression::prepend(Expr, DIExpression::ApplyOffset, Offset.getZExtValue()); MF->setVariableDbgInfo(DI->getVariable(), Expr, FI, DI->getDebugLoc()); } } } -/// Propagate swifterror values through the machine function CFG. -static void propagateSwiftErrorVRegs(FunctionLoweringInfo *FuncInfo) { - auto *TLI = FuncInfo->TLI; - if (!TLI->supportSwiftError()) - return; - - // We only need to do this when we have swifterror parameter or swifterror - // alloc. - if (FuncInfo->SwiftErrorVals.empty()) - return; - - // For each machine basic block in reverse post order. - ReversePostOrderTraversal RPOT(FuncInfo->MF); - for (MachineBasicBlock *MBB : RPOT) { - // For each swifterror value in the function. - for(const auto *SwiftErrorVal : FuncInfo->SwiftErrorVals) { - auto Key = std::make_pair(MBB, SwiftErrorVal); - auto UUseIt = FuncInfo->SwiftErrorVRegUpwardsUse.find(Key); - auto VRegDefIt = FuncInfo->SwiftErrorVRegDefMap.find(Key); - bool UpwardsUse = UUseIt != FuncInfo->SwiftErrorVRegUpwardsUse.end(); - unsigned UUseVReg = UpwardsUse ? UUseIt->second : 0; - bool DownwardDef = VRegDefIt != FuncInfo->SwiftErrorVRegDefMap.end(); - assert(!(UpwardsUse && !DownwardDef) && - "We can't have an upwards use but no downwards def"); - - // If there is no upwards exposed use and an entry for the swifterror in - // the def map for this value we don't need to do anything: We already - // have a downward def for this basic block. - if (!UpwardsUse && DownwardDef) - continue; - - // Otherwise we either have an upwards exposed use vreg that we need to - // materialize or need to forward the downward def from predecessors. - - // Check whether we have a single vreg def from all predecessors. - // Otherwise we need a phi. - SmallVector, 4> VRegs; - SmallSet Visited; - for (auto *Pred : MBB->predecessors()) { - if (!Visited.insert(Pred).second) - continue; - VRegs.push_back(std::make_pair( - Pred, FuncInfo->getOrCreateSwiftErrorVReg(Pred, SwiftErrorVal))); - if (Pred != MBB) - continue; - // We have a self-edge. - // If there was no upwards use in this basic block there is now one: the - // phi needs to use it self. - if (!UpwardsUse) { - UpwardsUse = true; - UUseIt = FuncInfo->SwiftErrorVRegUpwardsUse.find(Key); - assert(UUseIt != FuncInfo->SwiftErrorVRegUpwardsUse.end()); - UUseVReg = UUseIt->second; - } - } - - // We need a phi node if we have more than one predecessor with different - // downward defs. - bool needPHI = - VRegs.size() >= 1 && - std::find_if( - VRegs.begin(), VRegs.end(), - [&](const std::pair &V) - -> bool { return V.second != VRegs[0].second; }) != - VRegs.end(); - - // If there is no upwards exposed used and we don't need a phi just - // forward the swifterror vreg from the predecessor(s). - if (!UpwardsUse && !needPHI) { - assert(!VRegs.empty() && - "No predecessors? The entry block should bail out earlier"); - // Just forward the swifterror vreg from the predecessor(s). - FuncInfo->setCurrentSwiftErrorVReg(MBB, SwiftErrorVal, VRegs[0].second); - continue; - } - - auto DLoc = isa(SwiftErrorVal) - ? cast(SwiftErrorVal)->getDebugLoc() - : DebugLoc(); - const auto *TII = FuncInfo->MF->getSubtarget().getInstrInfo(); - - // If we don't need a phi create a copy to the upward exposed vreg. - if (!needPHI) { - assert(UpwardsUse); - assert(!VRegs.empty() && - "No predecessors? Is the Calling Convention correct?"); - unsigned DestReg = UUseVReg; - BuildMI(*MBB, MBB->getFirstNonPHI(), DLoc, TII->get(TargetOpcode::COPY), - DestReg) - .addReg(VRegs[0].second); - continue; - } - - // We need a phi: if there is an upwards exposed use we already have a - // destination virtual register number otherwise we generate a new one. - auto &DL = FuncInfo->MF->getDataLayout(); - auto const *RC = TLI->getRegClassFor(TLI->getPointerTy(DL)); - unsigned PHIVReg = - UpwardsUse ? UUseVReg - : FuncInfo->MF->getRegInfo().createVirtualRegister(RC); - MachineInstrBuilder SwiftErrorPHI = - BuildMI(*MBB, MBB->getFirstNonPHI(), DLoc, - TII->get(TargetOpcode::PHI), PHIVReg); - for (auto BBRegPair : VRegs) { - SwiftErrorPHI.addReg(BBRegPair.second).addMBB(BBRegPair.first); - } - - // We did not have a definition in this block before: store the phi's vreg - // as this block downward exposed def. - if (!UpwardsUse) - FuncInfo->setCurrentSwiftErrorVReg(MBB, SwiftErrorVal, PHIVReg); - } - } -} - -static void preassignSwiftErrorRegs(const TargetLowering *TLI, - FunctionLoweringInfo *FuncInfo, - BasicBlock::const_iterator Begin, - BasicBlock::const_iterator End) { - if (!TLI->supportSwiftError() || FuncInfo->SwiftErrorVals.empty()) - return; - - // Iterator over instructions and assign vregs to swifterror defs and uses. - for (auto It = Begin; It != End; ++It) { - ImmutableCallSite CS(&*It); - if (CS) { - // A call-site with a swifterror argument is both use and def. - const Value *SwiftErrorAddr = nullptr; - for (auto &Arg : CS.args()) { - if (!Arg->isSwiftError()) - continue; - // Use of swifterror. - assert(!SwiftErrorAddr && "Cannot have multiple swifterror arguments"); - SwiftErrorAddr = &*Arg; - assert(SwiftErrorAddr->isSwiftError() && - "Must have a swifterror value argument"); - unsigned VReg; bool CreatedReg; - std::tie(VReg, CreatedReg) = FuncInfo->getOrCreateSwiftErrorVRegUseAt( - &*It, FuncInfo->MBB, SwiftErrorAddr); - assert(CreatedReg); - } - if (!SwiftErrorAddr) - continue; - - // Def of swifterror. - unsigned VReg; bool CreatedReg; - std::tie(VReg, CreatedReg) = - FuncInfo->getOrCreateSwiftErrorVRegDefAt(&*It); - assert(CreatedReg); - FuncInfo->setCurrentSwiftErrorVReg(FuncInfo->MBB, SwiftErrorAddr, VReg); - - // A load is a use. - } else if (const LoadInst *LI = dyn_cast(&*It)) { - const Value *V = LI->getOperand(0); - if (!V->isSwiftError()) - continue; - - unsigned VReg; bool CreatedReg; - std::tie(VReg, CreatedReg) = - FuncInfo->getOrCreateSwiftErrorVRegUseAt(LI, FuncInfo->MBB, V); - assert(CreatedReg); - - // A store is a def. - } else if (const StoreInst *SI = dyn_cast(&*It)) { - const Value *SwiftErrorAddr = SI->getOperand(1); - if (!SwiftErrorAddr->isSwiftError()) - continue; - - // Def of swifterror. - unsigned VReg; bool CreatedReg; - std::tie(VReg, CreatedReg) = - FuncInfo->getOrCreateSwiftErrorVRegDefAt(&*It); - assert(CreatedReg); - FuncInfo->setCurrentSwiftErrorVReg(FuncInfo->MBB, SwiftErrorAddr, VReg); - - // A return in a swiferror returning function is a use. - } else if (const ReturnInst *R = dyn_cast(&*It)) { - const Function *F = R->getParent()->getParent(); - if(!F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) - continue; - - unsigned VReg; bool CreatedReg; - std::tie(VReg, CreatedReg) = FuncInfo->getOrCreateSwiftErrorVRegUseAt( - R, FuncInfo->MBB, FuncInfo->SwiftErrorArg); - assert(CreatedReg); - } - } -} - void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { FastISelFailed = false; // Initialize the Fast-ISel state, if needed. @@ -1542,8 +1343,6 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { FastIS = TLI->createFastISel(*FuncInfo, LibInfo); } - setupSwiftErrorVals(Fn, TLI, FuncInfo); - ReversePostOrderTraversal RPOT(&Fn); // Lower arguments up front. An RPO iteration always visits the entry block @@ -1589,7 +1388,11 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { else FastIS->setLastLocalValue(nullptr); } - createSwiftErrorEntriesInEntryBlock(FuncInfo, FastIS, TLI, TII, SDB); + + bool Inserted = SwiftError->createEntriesInEntryBlock(SDB->getCurDebugLoc()); + + if (FastIS && Inserted) + FastIS->setLastLocalValue(&*std::prev(FuncInfo->InsertPt)); processDbgDeclares(FuncInfo); @@ -1644,7 +1447,7 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { unsigned NumFastIselRemaining = std::distance(Begin, End); // Pre-assign swifterror vregs. - preassignSwiftErrorRegs(TLI, FuncInfo, Begin, End); + SwiftError->preassignVRegs(FuncInfo->MBB, Begin, End); // Do FastISel on as many instructions as possible. for (; BI != Begin; --BI) { @@ -1692,7 +1495,8 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { // to keep track of gc-relocates for a particular gc-statepoint. This is // done by SelectionDAGBuilder::LowerAsSTATEPOINT, called before // visitGCRelocate. - if (isa(Inst) && !isStatepoint(Inst) && !isGCRelocate(Inst)) { + if (isa(Inst) && !isStatepoint(Inst) && !isGCRelocate(Inst) && + !isGCResult(Inst)) { OptimizationRemarkMissed R("sdagisel", "FastISelFailure", Inst->getDebugLoc(), LLVMBB); @@ -1712,7 +1516,7 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { !Inst->use_empty()) { unsigned &R = FuncInfo->ValueMap[Inst]; if (!R) - R = FuncInfo->CreateRegs(Inst->getType()); + R = FuncInfo->CreateRegs(Inst); } bool HadTailCall = false; @@ -1799,7 +1603,7 @@ void SelectionDAGISel::SelectAllBasicBlocks(const Function &Fn) { SP.copyToMachineFrameInfo(MF->getFrameInfo()); - propagateSwiftErrorVRegs(FuncInfo); + SwiftError->propagateVRegs(); delete FastIS; SDB->clearDanglingDebugInfo(); @@ -1969,7 +1773,7 @@ SelectionDAGISel::FinishBasicBlock() { } // Lower each BitTestBlock. - for (auto &BTB : SDB->BitTestCases) { + for (auto &BTB : SDB->SL->BitTestCases) { // Lower header first, if it wasn't already lowered if (!BTB.Emitted) { // Set the current basic block to the mbb we wish to insert the code into @@ -2050,30 +1854,30 @@ SelectionDAGISel::FinishBasicBlock() { } } } - SDB->BitTestCases.clear(); + SDB->SL->BitTestCases.clear(); // If the JumpTable record is filled in, then we need to emit a jump table. // Updating the PHI nodes is tricky in this case, since we need to determine // whether the PHI is a successor of the range check MBB or the jump table MBB - for (unsigned i = 0, e = SDB->JTCases.size(); i != e; ++i) { + for (unsigned i = 0, e = SDB->SL->JTCases.size(); i != e; ++i) { // Lower header first, if it wasn't already lowered - if (!SDB->JTCases[i].first.Emitted) { + if (!SDB->SL->JTCases[i].first.Emitted) { // Set the current basic block to the mbb we wish to insert the code into - FuncInfo->MBB = SDB->JTCases[i].first.HeaderBB; + FuncInfo->MBB = SDB->SL->JTCases[i].first.HeaderBB; FuncInfo->InsertPt = FuncInfo->MBB->end(); // Emit the code - SDB->visitJumpTableHeader(SDB->JTCases[i].second, SDB->JTCases[i].first, - FuncInfo->MBB); + SDB->visitJumpTableHeader(SDB->SL->JTCases[i].second, + SDB->SL->JTCases[i].first, FuncInfo->MBB); CurDAG->setRoot(SDB->getRoot()); SDB->clear(); CodeGenAndEmitDAG(); } // Set the current basic block to the mbb we wish to insert the code into - FuncInfo->MBB = SDB->JTCases[i].second.MBB; + FuncInfo->MBB = SDB->SL->JTCases[i].second.MBB; FuncInfo->InsertPt = FuncInfo->MBB->end(); // Emit the code - SDB->visitJumpTable(SDB->JTCases[i].second); + SDB->visitJumpTable(SDB->SL->JTCases[i].second); CurDAG->setRoot(SDB->getRoot()); SDB->clear(); CodeGenAndEmitDAG(); @@ -2086,31 +1890,31 @@ SelectionDAGISel::FinishBasicBlock() { assert(PHI->isPHI() && "This is not a machine PHI node that we are updating!"); // "default" BB. We can go there only from header BB. - if (PHIBB == SDB->JTCases[i].second.Default) + if (PHIBB == SDB->SL->JTCases[i].second.Default) PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second) - .addMBB(SDB->JTCases[i].first.HeaderBB); + .addMBB(SDB->SL->JTCases[i].first.HeaderBB); // JT BB. Just iterate over successors here if (FuncInfo->MBB->isSuccessor(PHIBB)) PHI.addReg(FuncInfo->PHINodesToUpdate[pi].second).addMBB(FuncInfo->MBB); } } - SDB->JTCases.clear(); + SDB->SL->JTCases.clear(); // If we generated any switch lowering information, build and codegen any // additional DAGs necessary. - for (unsigned i = 0, e = SDB->SwitchCases.size(); i != e; ++i) { + for (unsigned i = 0, e = SDB->SL->SwitchCases.size(); i != e; ++i) { // Set the current basic block to the mbb we wish to insert the code into - FuncInfo->MBB = SDB->SwitchCases[i].ThisBB; + FuncInfo->MBB = SDB->SL->SwitchCases[i].ThisBB; FuncInfo->InsertPt = FuncInfo->MBB->end(); // Determine the unique successors. SmallVector Succs; - Succs.push_back(SDB->SwitchCases[i].TrueBB); - if (SDB->SwitchCases[i].TrueBB != SDB->SwitchCases[i].FalseBB) - Succs.push_back(SDB->SwitchCases[i].FalseBB); + Succs.push_back(SDB->SL->SwitchCases[i].TrueBB); + if (SDB->SL->SwitchCases[i].TrueBB != SDB->SL->SwitchCases[i].FalseBB) + Succs.push_back(SDB->SL->SwitchCases[i].FalseBB); // Emit the code. Note that this could result in FuncInfo->MBB being split. - SDB->visitSwitchCase(SDB->SwitchCases[i], FuncInfo->MBB); + SDB->visitSwitchCase(SDB->SL->SwitchCases[i], FuncInfo->MBB); CurDAG->setRoot(SDB->getRoot()); SDB->clear(); CodeGenAndEmitDAG(); @@ -2146,7 +1950,7 @@ SelectionDAGISel::FinishBasicBlock() { } } } - SDB->SwitchCases.clear(); + SDB->SL->SwitchCases.clear(); } /// Create the scheduler. If a specific scheduler was specified @@ -2413,14 +2217,14 @@ bool SelectionDAGISel::IsLegalToFold(SDValue N, SDNode *U, SDNode *Root, return !findNonImmUse(Root, N.getNode(), U, IgnoreChains); } -void SelectionDAGISel::Select_INLINEASM(SDNode *N) { +void SelectionDAGISel::Select_INLINEASM(SDNode *N, bool Branch) { SDLoc DL(N); std::vector Ops(N->op_begin(), N->op_end()); SelectInlineAsmMemoryOperands(Ops, DL); const EVT VTs[] = {MVT::Other, MVT::Glue}; - SDValue New = CurDAG->getNode(ISD::INLINEASM, DL, VTs, Ops); + SDValue New = CurDAG->getNode(Branch ? ISD::INLINEASM_BR : ISD::INLINEASM, DL, VTs, Ops); New->setNodeId(-1); ReplaceUses(N, New.getNode()); CurDAG->RemoveDeadNode(N); @@ -2727,6 +2531,14 @@ CheckCondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, (ISD::CondCode)MatcherTable[MatcherIndex++]; } +LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool +CheckChild2CondCode(const unsigned char *MatcherTable, unsigned &MatcherIndex, + SDValue N) { + if (2 >= N.getNumOperands()) + return false; + return ::CheckCondCode(MatcherTable, MatcherIndex, N.getOperand(2)); +} + LLVM_ATTRIBUTE_ALWAYS_INLINE static inline bool CheckValueType(const unsigned char *MatcherTable, unsigned &MatcherIndex, SDValue N, const TargetLowering *TLI, const DataLayout &DL) { @@ -2842,6 +2654,9 @@ static unsigned IsPredicateKnownToFail(const unsigned char *Table, case SelectionDAGISel::OPC_CheckCondCode: Result = !::CheckCondCode(Table, Index, N); return Index; + case SelectionDAGISel::OPC_CheckChild2CondCode: + Result = !::CheckChild2CondCode(Table, Index, N); + return Index; case SelectionDAGISel::OPC_CheckValueType: Result = !::CheckValueType(Table, Index, N, SDISel.TLI, SDISel.CurDAG->getDataLayout()); @@ -2970,7 +2785,9 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, CurDAG->RemoveDeadNode(NodeToMatch); return; case ISD::INLINEASM: - Select_INLINEASM(NodeToMatch); + case ISD::INLINEASM_BR: + Select_INLINEASM(NodeToMatch, + NodeToMatch->getOpcode() == ISD::INLINEASM_BR); return; case ISD::READ_REGISTER: Select_READ_REGISTER(NodeToMatch); @@ -3328,6 +3145,9 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, case OPC_CheckCondCode: if (!::CheckCondCode(MatcherTable, MatcherIndex, N)) break; continue; + case OPC_CheckChild2CondCode: + if (!::CheckChild2CondCode(MatcherTable, MatcherIndex, N)) break; + continue; case OPC_CheckValueType: if (!::CheckValueType(MatcherTable, MatcherIndex, N, TLI, CurDAG->getDataLayout())) @@ -3348,6 +3168,12 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, case OPC_CheckOrImm: if (!::CheckOrImm(MatcherTable, MatcherIndex, N, *this)) break; continue; + case OPC_CheckImmAllOnesV: + if (!ISD::isBuildVectorAllOnes(N.getNode())) break; + continue; + case OPC_CheckImmAllZerosV: + if (!ISD::isBuildVectorAllZeros(N.getNode())) break; + continue; case OPC_CheckFoldableChainNode: { assert(NodeStack.size() != 1 && "No parent node"); diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp index 3b19bff4743d..cdc09d59f6a4 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGPrinter.cpp @@ -1,9 +1,8 @@ //===-- SelectionDAGPrinter.cpp - Implement SelectionDAG::viewGraph() -----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/SelectionDAG/SelectionDAGTargetInfo.cpp b/lib/CodeGen/SelectionDAG/SelectionDAGTargetInfo.cpp index 3a283bc5fdc0..3a2df6f60593 100644 --- a/lib/CodeGen/SelectionDAG/SelectionDAGTargetInfo.cpp +++ b/lib/CodeGen/SelectionDAG/SelectionDAGTargetInfo.cpp @@ -1,9 +1,8 @@ //===- SelectionDAGTargetInfo.cpp - SelectionDAG Info ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/lib/CodeGen/SelectionDAG/StatepointLowering.cpp index 90a1b350fc94..395e9a8a4fc5 100644 --- a/lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ b/lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -1,9 +1,8 @@ //===- StatepointLowering.cpp - SDAGBuilder's statepoint code -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -348,16 +347,28 @@ static std::pair lowerCallFromStatepointLoweringInfo( return std::make_pair(ReturnValue, CallEnd->getOperand(0).getNode()); } +static MachineMemOperand* getMachineMemOperand(MachineFunction &MF, + FrameIndexSDNode &FI) { + auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI.getIndex()); + auto MMOFlags = MachineMemOperand::MOStore | + MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile; + auto &MFI = MF.getFrameInfo(); + return MF.getMachineMemOperand(PtrInfo, MMOFlags, + MFI.getObjectSize(FI.getIndex()), + MFI.getObjectAlignment(FI.getIndex())); +} + /// Spill a value incoming to the statepoint. It might be either part of /// vmstate /// or gcstate. In both cases unconditionally spill it on the stack unless it /// is a null constant. Return pair with first element being frame index /// containing saved value and second element with outgoing chain from the /// emitted store -static std::pair +static std::tuple spillIncomingStatepointValue(SDValue Incoming, SDValue Chain, SelectionDAGBuilder &Builder) { SDValue Loc = Builder.StatepointLowering.getLocation(Incoming); + MachineMemOperand* MMO = nullptr; // Emit new store if we didn't do it for this ptr before if (!Loc.getNode()) { @@ -367,10 +378,6 @@ spillIncomingStatepointValue(SDValue Incoming, SDValue Chain, // We use TargetFrameIndex so that isel will not select it into LEA Loc = Builder.DAG.getTargetFrameIndex(Index, Builder.getFrameIndexTy()); - // TODO: We can create TokenFactor node instead of - // chaining stores one after another, this may allow - // a bit more optimal scheduling for them - #ifndef NDEBUG // Right now we always allocate spill slots that are of the same // size as the value we're about to spill (the size of spillee can @@ -382,15 +389,18 @@ spillIncomingStatepointValue(SDValue Incoming, SDValue Chain, "Bad spill: stack slot does not match!"); #endif + auto &MF = Builder.DAG.getMachineFunction(); + auto PtrInfo = MachinePointerInfo::getFixedStack(MF, Index); Chain = Builder.DAG.getStore(Chain, Builder.getCurSDLoc(), Incoming, Loc, - MachinePointerInfo::getFixedStack( - Builder.DAG.getMachineFunction(), Index)); + PtrInfo); + MMO = getMachineMemOperand(MF, *cast(Loc)); + Builder.StatepointLowering.setLocation(Incoming, Loc); } assert(Loc.getNode()); - return std::make_pair(Loc, Chain); + return std::make_tuple(Loc, Chain, MMO); } /// Lower a single value incoming to a statepoint node. This value can be @@ -398,7 +408,11 @@ spillIncomingStatepointValue(SDValue Incoming, SDValue Chain, /// case constants and allocas, then fall back to spilling if required. static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly, SmallVectorImpl &Ops, + SmallVectorImpl &MemRefs, SelectionDAGBuilder &Builder) { + // Note: We know all of these spills are independent, but don't bother to + // exploit that chain wise. DAGCombine will happily do so as needed, so + // doing it here would be a small compile time win at most. SDValue Chain = Builder.getRoot(); if (ConstantSDNode *C = dyn_cast(Incoming)) { @@ -417,6 +431,11 @@ static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly, "Incoming value is a frame index!"); Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(), Builder.getFrameIndexTy())); + + auto &MF = Builder.DAG.getMachineFunction(); + auto *MMO = getMachineMemOperand(MF, *FI); + MemRefs.push_back(MMO); + } else if (LiveInOnly) { // If this value is live in (not live-on-return, or live-through), we can // treat it the same way patchpoint treats it's "live in" values. We'll @@ -433,8 +452,10 @@ static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly, // need to be optional since it requires a lot of complexity on the // runtime side which not all would support. auto Res = spillIncomingStatepointValue(Incoming, Chain, Builder); - Ops.push_back(Res.first); - Chain = Res.second; + Ops.push_back(std::get<0>(Res)); + if (auto *MMO = std::get<2>(Res)) + MemRefs.push_back(MMO); + Chain = std::get<1>(Res);; } Builder.DAG.setRoot(Chain); @@ -449,7 +470,7 @@ static void lowerIncomingStatepointValue(SDValue Incoming, bool LiveInOnly, /// will be set to the last value spilled (if any were). static void lowerStatepointMetaArgs(SmallVectorImpl &Ops, - SelectionDAGBuilder::StatepointLoweringInfo &SI, + SmallVectorImpl &MemRefs, SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder) { // Lower the deopt and gc arguments for this statepoint. Layout will be: // deopt argument length, deopt arguments.., gc arguments... @@ -533,7 +554,7 @@ lowerStatepointMetaArgs(SmallVectorImpl &Ops, if (!Incoming.getNode()) Incoming = Builder.getValue(V); const bool LiveInValue = LiveInDeopt && !isGCValue(V); - lowerIncomingStatepointValue(Incoming, LiveInValue, Ops, Builder); + lowerIncomingStatepointValue(Incoming, LiveInValue, Ops, MemRefs, Builder); } // Finally, go ahead and lower all the gc arguments. There's no prefixed @@ -544,11 +565,11 @@ lowerStatepointMetaArgs(SmallVectorImpl &Ops, for (unsigned i = 0; i < SI.Bases.size(); ++i) { const Value *Base = SI.Bases[i]; lowerIncomingStatepointValue(Builder.getValue(Base), /*LiveInOnly*/ false, - Ops, Builder); + Ops, MemRefs, Builder); const Value *Ptr = SI.Ptrs[i]; lowerIncomingStatepointValue(Builder.getValue(Ptr), /*LiveInOnly*/ false, - Ops, Builder); + Ops, MemRefs, Builder); } // If there are any explicit spill slots passed to the statepoint, record @@ -564,6 +585,10 @@ lowerStatepointMetaArgs(SmallVectorImpl &Ops, "Incoming value is a frame index!"); Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(), Builder.getFrameIndexTy())); + + auto &MF = Builder.DAG.getMachineFunction(); + auto *MMO = getMachineMemOperand(MF, *FI); + MemRefs.push_back(MMO); } } @@ -630,7 +655,8 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT( // Lower statepoint vmstate and gcstate arguments SmallVector LoweredMetaArgs; - lowerStatepointMetaArgs(LoweredMetaArgs, SI, *this); + SmallVector MemRefs; + lowerStatepointMetaArgs(LoweredMetaArgs, MemRefs, SI, *this); // Now that we've emitted the spills, we need to update the root so that the // call sequence is ordered correctly. @@ -746,8 +772,9 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT( // input. This allows someone else to chain off us as needed. SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); - SDNode *StatepointMCNode = - DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops); + MachineSDNode *StatepointMCNode = + DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops); + DAG.setNodeMemRefs(StatepointMCNode, MemRefs); SDNode *SinkNode = StatepointMCNode; @@ -799,7 +826,7 @@ SDValue SelectionDAGBuilder::LowerAsSTATEPOINT( void SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP, const BasicBlock *EHPadBB /*= nullptr*/) { - assert(ISP.getCallSite().getCallingConv() != CallingConv::AnyReg && + assert(ISP.getCall()->getCallingConv() != CallingConv::AnyReg && "anyregcc is not supported on statepoints!"); #ifndef NDEBUG @@ -832,7 +859,7 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP, } StatepointLoweringInfo SI(DAG); - populateCallLoweringInfo(SI.CLI, ISP.getCallSite(), + populateCallLoweringInfo(SI.CLI, ISP.getCall(), ImmutableStatepoint::CallArgsBeginPos, ISP.getNumCallArgs(), ActualCallee, ISP.getActualReturnType(), false /* IsPatchPoint */); @@ -859,7 +886,7 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP, const GCResultInst *GCResult = ISP.getGCResult(); Type *RetTy = ISP.getActualReturnType(); if (!RetTy->isVoidTy() && GCResult) { - if (GCResult->getParent() != ISP.getCallSite().getParent()) { + if (GCResult->getParent() != ISP.getCall()->getParent()) { // Result value will be used in a different basic block so we need to // export it now. Default exporting mechanism will not work here because // statepoint call has a different type than the actual call. It means @@ -871,7 +898,7 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP, unsigned Reg = FuncInfo.CreateRegs(RetTy); RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(), DAG.getDataLayout(), Reg, RetTy, - ISP.getCallSite().getCallingConv()); + ISP.getCall()->getCallingConv()); SDValue Chain = DAG.getEntryNode(); RFV.getCopyToRegs(ReturnValue, DAG, getCurSDLoc(), Chain, nullptr); @@ -891,22 +918,22 @@ SelectionDAGBuilder::LowerStatepoint(ImmutableStatepoint ISP, } void SelectionDAGBuilder::LowerCallSiteWithDeoptBundleImpl( - ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB, + const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB, bool VarArgDisallowed, bool ForceVoidReturnTy) { StatepointLoweringInfo SI(DAG); - unsigned ArgBeginIndex = CS.arg_begin() - CS.getInstruction()->op_begin(); + unsigned ArgBeginIndex = Call->arg_begin() - Call->op_begin(); populateCallLoweringInfo( - SI.CLI, CS, ArgBeginIndex, CS.getNumArgOperands(), Callee, - ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : CS.getType(), + SI.CLI, Call, ArgBeginIndex, Call->getNumArgOperands(), Callee, + ForceVoidReturnTy ? Type::getVoidTy(*DAG.getContext()) : Call->getType(), false); if (!VarArgDisallowed) - SI.CLI.IsVarArg = CS.getFunctionType()->isVarArg(); + SI.CLI.IsVarArg = Call->getFunctionType()->isVarArg(); - auto DeoptBundle = *CS.getOperandBundle(LLVMContext::OB_deopt); + auto DeoptBundle = *Call->getOperandBundle(LLVMContext::OB_deopt); unsigned DefaultID = StatepointDirectives::DeoptBundleStatepointID; - auto SD = parseStatepointDirectivesFromAttrs(CS.getAttributes()); + auto SD = parseStatepointDirectivesFromAttrs(Call->getAttributes()); SI.ID = SD.StatepointID.getValueOr(DefaultID); SI.NumPatchBytes = SD.NumPatchBytes.getValueOr(0); @@ -918,15 +945,14 @@ void SelectionDAGBuilder::LowerCallSiteWithDeoptBundleImpl( // NB! The GC arguments are deliberately left empty. if (SDValue ReturnVal = LowerAsSTATEPOINT(SI)) { - const Instruction *Inst = CS.getInstruction(); - ReturnVal = lowerRangeToAssertZExt(DAG, *Inst, ReturnVal); - setValue(Inst, ReturnVal); + ReturnVal = lowerRangeToAssertZExt(DAG, *Call, ReturnVal); + setValue(Call, ReturnVal); } } void SelectionDAGBuilder::LowerCallSiteWithDeoptBundle( - ImmutableCallSite CS, SDValue Callee, const BasicBlock *EHPadBB) { - LowerCallSiteWithDeoptBundleImpl(CS, Callee, EHPadBB, + const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB) { + LowerCallSiteWithDeoptBundleImpl(Call, Callee, EHPadBB, /* VarArgDisallowed = */ false, /* ForceVoidReturnTy = */ false); } @@ -986,11 +1012,11 @@ void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) { } SDValue SpillSlot = - DAG.getTargetFrameIndex(*DerivedPtrLocation, getFrameIndexTy()); + DAG.getTargetFrameIndex(*DerivedPtrLocation, getFrameIndexTy()); - // Be conservative: flush all pending loads - // TODO: Probably we can be less restrictive on this, - // it may allow more scheduling opportunities. + // Note: We know all of these reloads are independent, but don't bother to + // exploit that chain wise. DAGCombine will happily do so as needed, so + // doing it here would be a small compile time win at most. SDValue Chain = getRoot(); SDValue SpillLoad = @@ -1000,7 +1026,6 @@ void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) { MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), *DerivedPtrLocation)); - // Again, be conservative, don't emit pending loads DAG.setRoot(SpillLoad.getValue(1)); assert(SpillLoad.getNode()); diff --git a/lib/CodeGen/SelectionDAG/StatepointLowering.h b/lib/CodeGen/SelectionDAG/StatepointLowering.h index 372c82a359f6..70507932681d 100644 --- a/lib/CodeGen/SelectionDAG/StatepointLowering.h +++ b/lib/CodeGen/SelectionDAG/StatepointLowering.h @@ -1,9 +1,8 @@ //===- StatepointLowering.h - SDAGBuilder's statepoint code ---*- C++ -*---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -67,13 +66,18 @@ public: /// before the next statepoint. If we don't see it, we'll report /// an assertion. void scheduleRelocCall(const CallInst &RelocCall) { - PendingGCRelocateCalls.push_back(&RelocCall); + // We are not interested in lowering dead instructions. + if (!RelocCall.use_empty()) + PendingGCRelocateCalls.push_back(&RelocCall); } /// Remove this gc_relocate from the list we're expecting to see /// before the next statepoint. If we weren't expecting to see /// it, we'll report an assertion. void relocCallVisited(const CallInst &RelocCall) { + // We are not interested in lowering dead instructions. + if (RelocCall.use_empty()) + return; auto I = llvm::find(PendingGCRelocateCalls, &RelocCall); assert(I != PendingGCRelocateCalls.end() && "Visited unexpected gcrelocate call"); diff --git a/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/lib/CodeGen/SelectionDAG/TargetLowering.cpp index a2f05c1e3cef..b260cd91d468 100644 --- a/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -1,9 +1,8 @@ //===-- TargetLowering.cpp - Implement the TargetLowering class -----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -100,19 +99,22 @@ bool TargetLowering::parametersInCSRMatch(const MachineRegisterInfo &MRI, /// Set CallLoweringInfo attribute flags based on a call instruction /// and called function attributes. -void TargetLoweringBase::ArgListEntry::setAttributes(ImmutableCallSite *CS, +void TargetLoweringBase::ArgListEntry::setAttributes(const CallBase *Call, unsigned ArgIdx) { - IsSExt = CS->paramHasAttr(ArgIdx, Attribute::SExt); - IsZExt = CS->paramHasAttr(ArgIdx, Attribute::ZExt); - IsInReg = CS->paramHasAttr(ArgIdx, Attribute::InReg); - IsSRet = CS->paramHasAttr(ArgIdx, Attribute::StructRet); - IsNest = CS->paramHasAttr(ArgIdx, Attribute::Nest); - IsByVal = CS->paramHasAttr(ArgIdx, Attribute::ByVal); - IsInAlloca = CS->paramHasAttr(ArgIdx, Attribute::InAlloca); - IsReturned = CS->paramHasAttr(ArgIdx, Attribute::Returned); - IsSwiftSelf = CS->paramHasAttr(ArgIdx, Attribute::SwiftSelf); - IsSwiftError = CS->paramHasAttr(ArgIdx, Attribute::SwiftError); - Alignment = CS->getParamAlignment(ArgIdx); + IsSExt = Call->paramHasAttr(ArgIdx, Attribute::SExt); + IsZExt = Call->paramHasAttr(ArgIdx, Attribute::ZExt); + IsInReg = Call->paramHasAttr(ArgIdx, Attribute::InReg); + IsSRet = Call->paramHasAttr(ArgIdx, Attribute::StructRet); + IsNest = Call->paramHasAttr(ArgIdx, Attribute::Nest); + IsByVal = Call->paramHasAttr(ArgIdx, Attribute::ByVal); + IsInAlloca = Call->paramHasAttr(ArgIdx, Attribute::InAlloca); + IsReturned = Call->paramHasAttr(ArgIdx, Attribute::Returned); + IsSwiftSelf = Call->paramHasAttr(ArgIdx, Attribute::SwiftSelf); + IsSwiftError = Call->paramHasAttr(ArgIdx, Attribute::SwiftError); + Alignment = Call->getParamAlignment(ArgIdx); + ByValType = nullptr; + if (Call->paramHasAttr(ArgIdx, Attribute::ByVal)) + ByValType = Call->getParamByValType(ArgIdx); } /// Generate a libcall taking the given operands as arguments and returning a @@ -121,7 +123,8 @@ std::pair TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef Ops, bool isSigned, const SDLoc &dl, bool doesNotReturn, - bool isReturnValueUsed) const { + bool isReturnValueUsed, + bool isPostTypeLegalization) const { TargetLowering::ArgListTy Args; Args.reserve(Ops.size()); @@ -147,11 +150,114 @@ TargetLowering::makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, .setLibCallee(getLibcallCallingConv(LC), RetTy, Callee, std::move(Args)) .setNoReturn(doesNotReturn) .setDiscardResult(!isReturnValueUsed) + .setIsPostTypeLegalization(isPostTypeLegalization) .setSExtResult(signExtend) .setZExtResult(!signExtend); return LowerCallTo(CLI); } +bool +TargetLowering::findOptimalMemOpLowering(std::vector &MemOps, + unsigned Limit, uint64_t Size, + unsigned DstAlign, unsigned SrcAlign, + bool IsMemset, + bool ZeroMemset, + bool MemcpyStrSrc, + bool AllowOverlap, + unsigned DstAS, unsigned SrcAS, + const AttributeList &FuncAttributes) const { + // If 'SrcAlign' is zero, that means the memory operation does not need to + // load the value, i.e. memset or memcpy from constant string. Otherwise, + // it's the inferred alignment of the source. 'DstAlign', on the other hand, + // is the specified alignment of the memory operation. If it is zero, that + // means it's possible to change the alignment of the destination. + // 'MemcpyStrSrc' indicates whether the memcpy source is constant so it does + // not need to be loaded. + if (!(SrcAlign == 0 || SrcAlign >= DstAlign)) + return false; + + EVT VT = getOptimalMemOpType(Size, DstAlign, SrcAlign, + IsMemset, ZeroMemset, MemcpyStrSrc, + FuncAttributes); + + if (VT == MVT::Other) { + // Use the largest integer type whose alignment constraints are satisfied. + // We only need to check DstAlign here as SrcAlign is always greater or + // equal to DstAlign (or zero). + VT = MVT::i64; + while (DstAlign && DstAlign < VT.getSizeInBits() / 8 && + !allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign)) + VT = (MVT::SimpleValueType)(VT.getSimpleVT().SimpleTy - 1); + assert(VT.isInteger()); + + // Find the largest legal integer type. + MVT LVT = MVT::i64; + while (!isTypeLegal(LVT)) + LVT = (MVT::SimpleValueType)(LVT.SimpleTy - 1); + assert(LVT.isInteger()); + + // If the type we've chosen is larger than the largest legal integer type + // then use that instead. + if (VT.bitsGT(LVT)) + VT = LVT; + } + + unsigned NumMemOps = 0; + while (Size != 0) { + unsigned VTSize = VT.getSizeInBits() / 8; + while (VTSize > Size) { + // For now, only use non-vector load / store's for the left-over pieces. + EVT NewVT = VT; + unsigned NewVTSize; + + bool Found = false; + if (VT.isVector() || VT.isFloatingPoint()) { + NewVT = (VT.getSizeInBits() > 64) ? MVT::i64 : MVT::i32; + if (isOperationLegalOrCustom(ISD::STORE, NewVT) && + isSafeMemOpType(NewVT.getSimpleVT())) + Found = true; + else if (NewVT == MVT::i64 && + isOperationLegalOrCustom(ISD::STORE, MVT::f64) && + isSafeMemOpType(MVT::f64)) { + // i64 is usually not legal on 32-bit targets, but f64 may be. + NewVT = MVT::f64; + Found = true; + } + } + + if (!Found) { + do { + NewVT = (MVT::SimpleValueType)(NewVT.getSimpleVT().SimpleTy - 1); + if (NewVT == MVT::i8) + break; + } while (!isSafeMemOpType(NewVT.getSimpleVT())); + } + NewVTSize = NewVT.getSizeInBits() / 8; + + // If the new VT cannot cover all of the remaining bits, then consider + // issuing a (or a pair of) unaligned and overlapping load / store. + bool Fast; + if (NumMemOps && AllowOverlap && NewVTSize < Size && + allowsMisalignedMemoryAccesses(VT, DstAS, DstAlign, + MachineMemOperand::MONone, &Fast) && + Fast) + VTSize = Size; + else { + VT = NewVT; + VTSize = NewVTSize; + } + } + + if (++NumMemOps > Limit) + return false; + + MemOps.push_back(VT); + Size -= VTSize; + } + + return true; +} + /// Soften the operands of a comparison. This code is shared among BR_CC, /// SELECT_CC, and SETCC handlers. void TargetLowering::softenSetCCOperands(SelectionDAG &DAG, EVT VT, @@ -346,7 +452,6 @@ TargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const { /// return true. bool TargetLowering::ShrinkDemandedConstant(SDValue Op, const APInt &Demanded, TargetLoweringOpt &TLO) const { - SelectionDAG &DAG = TLO.DAG; SDLoc DL(Op); unsigned Opcode = Op.getOpcode(); @@ -372,8 +477,8 @@ bool TargetLowering::ShrinkDemandedConstant(SDValue Op, const APInt &Demanded, if (!C.isSubsetOf(Demanded)) { EVT VT = Op.getValueType(); - SDValue NewC = DAG.getConstant(Demanded & C, DL, VT); - SDValue NewOp = DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC); + SDValue NewC = TLO.DAG.getConstant(Demanded & C, DL, VT); + SDValue NewOp = TLO.DAG.getNode(Opcode, DL, VT, Op.getOperand(0), NewC); return TLO.CombineTo(Op, NewOp); } @@ -487,6 +592,10 @@ bool TargetLowering::SimplifyDemandedBits( // Don't know anything. Known = KnownBits(BitWidth); + // Undef operand. + if (Op.isUndef()) + return false; + if (Op.getOpcode() == ISD::Constant) { // We know all of the bits for a constant! Known.One = cast(Op)->getAPIntValue(); @@ -509,40 +618,116 @@ bool TargetLowering::SimplifyDemandedBits( DemandedElts = APInt::getAllOnesValue(NumElts); } else if (OriginalDemandedBits == 0 || OriginalDemandedElts == 0) { // Not demanding any bits/elts from Op. - if (!Op.isUndef()) - return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT)); - return false; + return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT)); } else if (Depth == 6) { // Limit search depth. return false; } KnownBits Known2, KnownOut; switch (Op.getOpcode()) { + case ISD::SCALAR_TO_VECTOR: { + if (!DemandedElts[0]) + return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT)); + + KnownBits SrcKnown; + SDValue Src = Op.getOperand(0); + unsigned SrcBitWidth = Src.getScalarValueSizeInBits(); + APInt SrcDemandedBits = DemandedBits.zextOrSelf(SrcBitWidth); + if (SimplifyDemandedBits(Src, SrcDemandedBits, SrcKnown, TLO, Depth + 1)) + return true; + Known = SrcKnown.zextOrTrunc(BitWidth, false); + break; + } case ISD::BUILD_VECTOR: - // Collect the known bits that are shared by every constant vector element. - Known.Zero.setAllBits(); Known.One.setAllBits(); - for (SDValue SrcOp : Op->ops()) { - if (!isa(SrcOp)) { - // We can only handle all constant values - bail out with no known bits. - Known = KnownBits(BitWidth); - return false; - } - Known2.One = cast(SrcOp)->getAPIntValue(); - Known2.Zero = ~Known2.One; - - // BUILD_VECTOR can implicitly truncate sources, we must handle this. - if (Known2.One.getBitWidth() != BitWidth) { - assert(Known2.getBitWidth() > BitWidth && - "Expected BUILD_VECTOR implicit truncation"); - Known2 = Known2.trunc(BitWidth); + // Collect the known bits that are shared by every demanded element. + // TODO: Call SimplifyDemandedBits for non-constant demanded elements. + Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth); + return false; // Don't fall through, will infinitely loop. + case ISD::LOAD: { + LoadSDNode *LD = cast(Op); + if (getTargetConstantFromLoad(LD)) { + Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth); + return false; // Don't fall through, will infinitely loop. + } + break; + } + case ISD::INSERT_VECTOR_ELT: { + SDValue Vec = Op.getOperand(0); + SDValue Scl = Op.getOperand(1); + auto *CIdx = dyn_cast(Op.getOperand(2)); + EVT VecVT = Vec.getValueType(); + + // If index isn't constant, assume we need all vector elements AND the + // inserted element. + APInt DemandedVecElts(DemandedElts); + if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) { + unsigned Idx = CIdx->getZExtValue(); + DemandedVecElts.clearBit(Idx); + + // Inserted element is not required. + if (!DemandedElts[Idx]) + return TLO.CombineTo(Op, Vec); + } + + KnownBits KnownScl; + unsigned NumSclBits = Scl.getScalarValueSizeInBits(); + APInt DemandedSclBits = DemandedBits.zextOrTrunc(NumSclBits); + if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1)) + return true; + + Known = KnownScl.zextOrTrunc(BitWidth, false); + + KnownBits KnownVec; + if (SimplifyDemandedBits(Vec, DemandedBits, DemandedVecElts, KnownVec, TLO, + Depth + 1)) + return true; + + if (!!DemandedVecElts) { + Known.One &= KnownVec.One; + Known.Zero &= KnownVec.Zero; + } + + return false; + } + case ISD::INSERT_SUBVECTOR: { + SDValue Base = Op.getOperand(0); + SDValue Sub = Op.getOperand(1); + EVT SubVT = Sub.getValueType(); + unsigned NumSubElts = SubVT.getVectorNumElements(); + + // If index isn't constant, assume we need the original demanded base + // elements and ALL the inserted subvector elements. + APInt BaseElts = DemandedElts; + APInt SubElts = APInt::getAllOnesValue(NumSubElts); + if (isa(Op.getOperand(2))) { + const APInt &Idx = Op.getConstantOperandAPInt(2); + if (Idx.ule(NumElts - NumSubElts)) { + unsigned SubIdx = Idx.getZExtValue(); + SubElts = DemandedElts.extractBits(NumSubElts, SubIdx); + BaseElts.insertBits(APInt::getNullValue(NumSubElts), SubIdx); } + } - // Known bits are the values that are shared by every element. - // TODO: support per-element known bits. - Known.One &= Known2.One; - Known.Zero &= Known2.Zero; + KnownBits KnownSub, KnownBase; + if (SimplifyDemandedBits(Sub, DemandedBits, SubElts, KnownSub, TLO, + Depth + 1)) + return true; + if (SimplifyDemandedBits(Base, DemandedBits, BaseElts, KnownBase, TLO, + Depth + 1)) + return true; + + Known.Zero.setAllBits(); + Known.One.setAllBits(); + if (!!SubElts) { + Known.One &= KnownSub.One; + Known.Zero &= KnownSub.Zero; } - return false; // Don't fall through, will infinitely loop. + if (!!BaseElts) { + Known.One &= KnownBase.One; + Known.Zero &= KnownBase.Zero; + } + break; + } case ISD::CONCAT_VECTORS: { Known.Zero.setAllBits(); Known.One.setAllBits(); @@ -640,11 +825,12 @@ bool TargetLowering::SimplifyDemandedBits( } } - if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO, Depth + 1)) + if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO, + Depth + 1)) return true; assert(!Known.hasConflict() && "Bits known to be one AND zero?"); - if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts, Known2, TLO, - Depth + 1)) + if (SimplifyDemandedBits(Op0, ~Known.Zero & DemandedBits, DemandedElts, + Known2, TLO, Depth + 1)) return true; assert(!Known2.hasConflict() && "Bits known to be one AND zero?"); @@ -674,11 +860,12 @@ bool TargetLowering::SimplifyDemandedBits( SDValue Op0 = Op.getOperand(0); SDValue Op1 = Op.getOperand(1); - if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO, Depth + 1)) + if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO, + Depth + 1)) return true; assert(!Known.hasConflict() && "Bits known to be one AND zero?"); - if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts, Known2, TLO, - Depth + 1)) + if (SimplifyDemandedBits(Op0, ~Known.One & DemandedBits, DemandedElts, + Known2, TLO, Depth + 1)) return true; assert(!Known2.hasConflict() && "Bits known to be one AND zero?"); @@ -705,10 +892,12 @@ bool TargetLowering::SimplifyDemandedBits( SDValue Op0 = Op.getOperand(0); SDValue Op1 = Op.getOperand(1); - if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO, Depth + 1)) + if (SimplifyDemandedBits(Op1, DemandedBits, DemandedElts, Known, TLO, + Depth + 1)) return true; assert(!Known.hasConflict() && "Bits known to be one AND zero?"); - if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO, Depth + 1)) + if (SimplifyDemandedBits(Op0, DemandedBits, DemandedElts, Known2, TLO, + Depth + 1)) return true; assert(!Known2.hasConflict() && "Bits known to be one AND zero?"); @@ -831,20 +1020,23 @@ bool TargetLowering::SimplifyDemandedBits( SDValue Op0 = Op.getOperand(0); SDValue Op1 = Op.getOperand(1); - if (ConstantSDNode *SA = isConstOrConstSplat(Op1)) { + if (ConstantSDNode *SA = isConstOrConstSplat(Op1, DemandedElts)) { // If the shift count is an invalid immediate, don't do anything. if (SA->getAPIntValue().uge(BitWidth)) break; unsigned ShAmt = SA->getZExtValue(); + if (ShAmt == 0) + return TLO.CombineTo(Op, Op0); // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a // single shift. We can do this if the bottom bits (which are shifted // out) are never demanded. + // TODO - support non-uniform vector amounts. if (Op0.getOpcode() == ISD::SRL) { - if (ShAmt && - (DemandedBits & APInt::getLowBitsSet(BitWidth, ShAmt)) == 0) { - if (ConstantSDNode *SA2 = isConstOrConstSplat(Op0.getOperand(1))) { + if ((DemandedBits & APInt::getLowBitsSet(BitWidth, ShAmt)) == 0) { + if (ConstantSDNode *SA2 = + isConstOrConstSplat(Op0.getOperand(1), DemandedElts)) { if (SA2->getAPIntValue().ult(BitWidth)) { unsigned C1 = SA2->getZExtValue(); unsigned Opc = ISD::SHL; @@ -862,8 +1054,14 @@ bool TargetLowering::SimplifyDemandedBits( } } - if (SimplifyDemandedBits(Op0, DemandedBits.lshr(ShAmt), DemandedElts, Known, TLO, - Depth + 1)) + if (SimplifyDemandedBits(Op0, DemandedBits.lshr(ShAmt), DemandedElts, + Known, TLO, Depth + 1)) + return true; + + // Try shrinking the operation as long as the shift amount will still be + // in range. + if ((ShAmt < DemandedBits.getActiveBits()) && + ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) return true; // Convert (shl (anyext x, c)) to (anyext (shl x, c)) if the high bits @@ -919,12 +1117,16 @@ bool TargetLowering::SimplifyDemandedBits( SDValue Op0 = Op.getOperand(0); SDValue Op1 = Op.getOperand(1); - if (ConstantSDNode *SA = isConstOrConstSplat(Op1)) { + if (ConstantSDNode *SA = isConstOrConstSplat(Op1, DemandedElts)) { // If the shift count is an invalid immediate, don't do anything. if (SA->getAPIntValue().uge(BitWidth)) break; unsigned ShAmt = SA->getZExtValue(); + if (ShAmt == 0) + return TLO.CombineTo(Op, Op0); + + EVT ShiftVT = Op1.getValueType(); APInt InDemandedMask = (DemandedBits << ShAmt); // If the shift is exact, then it does demand the low bits (and knows that @@ -935,10 +1137,11 @@ bool TargetLowering::SimplifyDemandedBits( // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a // single shift. We can do this if the top bits (which are shifted out) // are never demanded. + // TODO - support non-uniform vector amounts. if (Op0.getOpcode() == ISD::SHL) { - if (ConstantSDNode *SA2 = isConstOrConstSplat(Op0.getOperand(1))) { - if (ShAmt && - (DemandedBits & APInt::getHighBitsSet(BitWidth, ShAmt)) == 0) { + if (ConstantSDNode *SA2 = + isConstOrConstSplat(Op0.getOperand(1), DemandedElts)) { + if ((DemandedBits & APInt::getHighBitsSet(BitWidth, ShAmt)) == 0) { if (SA2->getAPIntValue().ult(BitWidth)) { unsigned C1 = SA2->getZExtValue(); unsigned Opc = ISD::SRL; @@ -948,7 +1151,7 @@ bool TargetLowering::SimplifyDemandedBits( Opc = ISD::SHL; } - SDValue NewSA = TLO.DAG.getConstant(Diff, dl, Op1.getValueType()); + SDValue NewSA = TLO.DAG.getConstant(Diff, dl, ShiftVT); return TLO.CombineTo( Op, TLO.DAG.getNode(Opc, dl, VT, Op0.getOperand(0), NewSA)); } @@ -957,7 +1160,8 @@ bool TargetLowering::SimplifyDemandedBits( } // Compute the new bits that are at the top now. - if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO, Depth + 1)) + if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO, + Depth + 1)) return true; assert(!Known.hasConflict() && "Bits known to be one AND zero?"); Known.Zero.lshrInPlace(ShAmt); @@ -978,12 +1182,15 @@ bool TargetLowering::SimplifyDemandedBits( if (DemandedBits.isOneValue()) return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::SRL, dl, VT, Op0, Op1)); - if (ConstantSDNode *SA = isConstOrConstSplat(Op1)) { + if (ConstantSDNode *SA = isConstOrConstSplat(Op1, DemandedElts)) { // If the shift count is an invalid immediate, don't do anything. if (SA->getAPIntValue().uge(BitWidth)) break; unsigned ShAmt = SA->getZExtValue(); + if (ShAmt == 0) + return TLO.CombineTo(Op, Op0); + APInt InDemandedMask = (DemandedBits << ShAmt); // If the shift is exact, then it does demand the low bits (and knows that @@ -996,7 +1203,8 @@ bool TargetLowering::SimplifyDemandedBits( if (DemandedBits.countLeadingZeros() < ShAmt) InDemandedMask.setSignBit(); - if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO, Depth + 1)) + if (SimplifyDemandedBits(Op0, InDemandedMask, DemandedElts, Known, TLO, + Depth + 1)) return true; assert(!Known.hasConflict() && "Bits known to be one AND zero?"); Known.Zero.lshrInPlace(ShAmt); @@ -1026,6 +1234,55 @@ bool TargetLowering::SimplifyDemandedBits( } break; } + case ISD::FSHL: + case ISD::FSHR: { + SDValue Op0 = Op.getOperand(0); + SDValue Op1 = Op.getOperand(1); + SDValue Op2 = Op.getOperand(2); + bool IsFSHL = (Op.getOpcode() == ISD::FSHL); + + if (ConstantSDNode *SA = isConstOrConstSplat(Op2, DemandedElts)) { + unsigned Amt = SA->getAPIntValue().urem(BitWidth); + + // For fshl, 0-shift returns the 1st arg. + // For fshr, 0-shift returns the 2nd arg. + if (Amt == 0) { + if (SimplifyDemandedBits(IsFSHL ? Op0 : Op1, DemandedBits, DemandedElts, + Known, TLO, Depth + 1)) + return true; + break; + } + + // fshl: (Op0 << Amt) | (Op1 >> (BW - Amt)) + // fshr: (Op0 << (BW - Amt)) | (Op1 >> Amt) + APInt Demanded0 = DemandedBits.lshr(IsFSHL ? Amt : (BitWidth - Amt)); + APInt Demanded1 = DemandedBits << (IsFSHL ? (BitWidth - Amt) : Amt); + if (SimplifyDemandedBits(Op0, Demanded0, DemandedElts, Known2, TLO, + Depth + 1)) + return true; + if (SimplifyDemandedBits(Op1, Demanded1, DemandedElts, Known, TLO, + Depth + 1)) + return true; + + Known2.One <<= (IsFSHL ? Amt : (BitWidth - Amt)); + Known2.Zero <<= (IsFSHL ? Amt : (BitWidth - Amt)); + Known.One.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt); + Known.Zero.lshrInPlace(IsFSHL ? (BitWidth - Amt) : Amt); + Known.One |= Known2.One; + Known.Zero |= Known2.Zero; + } + break; + } + case ISD::BITREVERSE: { + SDValue Src = Op.getOperand(0); + APInt DemandedSrcBits = DemandedBits.reverseBits(); + if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, Known2, TLO, + Depth + 1)) + return true; + Known.One = Known2.One.reverseBits(); + Known.Zero = Known2.Zero.reverseBits(); + break; + } case ISD::SIGN_EXTEND_INREG: { SDValue Op0 = Op.getOperand(0); EVT ExVT = cast(Op.getOperand(1))->getVT(); @@ -1033,8 +1290,8 @@ bool TargetLowering::SimplifyDemandedBits( // If we only care about the highest bit, don't bother shifting right. if (DemandedBits.isSignMask()) { - bool AlreadySignExtended = - TLO.DAG.ComputeNumSignBits(Op0) >= BitWidth - ExVTBits + 1; + unsigned NumSignBits = TLO.DAG.ComputeNumSignBits(Op0); + bool AlreadySignExtended = NumSignBits >= BitWidth - ExVTBits + 1; // However if the input is already sign extended we expect the sign // extension to be dropped altogether later and do not simplify. if (!AlreadySignExtended) { @@ -1099,79 +1356,116 @@ bool TargetLowering::SimplifyDemandedBits( return true; Known.Zero = KnownLo.Zero.zext(BitWidth) | - KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth); + KnownHi.Zero.zext(BitWidth).shl(HalfBitWidth); Known.One = KnownLo.One.zext(BitWidth) | - KnownHi.One.zext(BitWidth).shl(HalfBitWidth); + KnownHi.One.zext(BitWidth).shl(HalfBitWidth); break; } - case ISD::ZERO_EXTEND: { + case ISD::ZERO_EXTEND: + case ISD::ZERO_EXTEND_VECTOR_INREG: { SDValue Src = Op.getOperand(0); - unsigned InBits = Src.getScalarValueSizeInBits(); + EVT SrcVT = Src.getValueType(); + unsigned InBits = SrcVT.getScalarSizeInBits(); + unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1; + bool IsVecInReg = Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG; // If none of the top bits are demanded, convert this into an any_extend. - if (DemandedBits.getActiveBits() <= InBits) - return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, Src)); + if (DemandedBits.getActiveBits() <= InBits) { + // If we only need the non-extended bits of the bottom element + // then we can just bitcast to the result. + if (IsVecInReg && DemandedElts == 1 && + VT.getSizeInBits() == SrcVT.getSizeInBits() && + TLO.DAG.getDataLayout().isLittleEndian()) + return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src)); + + unsigned Opc = + IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND; + if (!TLO.LegalOperations() || isOperationLegal(Opc, VT)) + return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src)); + } APInt InDemandedBits = DemandedBits.trunc(InBits); - if (SimplifyDemandedBits(Src, InDemandedBits, Known, TLO, Depth+1)) + APInt InDemandedElts = DemandedElts.zextOrSelf(InElts); + if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO, + Depth + 1)) return true; assert(!Known.hasConflict() && "Bits known to be one AND zero?"); - Known = Known.zext(BitWidth); - Known.Zero.setBitsFrom(InBits); + assert(Known.getBitWidth() == InBits && "Src width has changed?"); + Known = Known.zext(BitWidth, true /* ExtendedBitsAreKnownZero */); break; } - case ISD::SIGN_EXTEND: { + case ISD::SIGN_EXTEND: + case ISD::SIGN_EXTEND_VECTOR_INREG: { SDValue Src = Op.getOperand(0); - unsigned InBits = Src.getScalarValueSizeInBits(); + EVT SrcVT = Src.getValueType(); + unsigned InBits = SrcVT.getScalarSizeInBits(); + unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1; + bool IsVecInReg = Op.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG; // If none of the top bits are demanded, convert this into an any_extend. - if (DemandedBits.getActiveBits() <= InBits) - return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::ANY_EXTEND, dl, VT, Src)); + if (DemandedBits.getActiveBits() <= InBits) { + // If we only need the non-extended bits of the bottom element + // then we can just bitcast to the result. + if (IsVecInReg && DemandedElts == 1 && + VT.getSizeInBits() == SrcVT.getSizeInBits() && + TLO.DAG.getDataLayout().isLittleEndian()) + return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src)); + + unsigned Opc = + IsVecInReg ? ISD::ANY_EXTEND_VECTOR_INREG : ISD::ANY_EXTEND; + if (!TLO.LegalOperations() || isOperationLegal(Opc, VT)) + return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src)); + } + + APInt InDemandedBits = DemandedBits.trunc(InBits); + APInt InDemandedElts = DemandedElts.zextOrSelf(InElts); // Since some of the sign extended bits are demanded, we know that the sign // bit is demanded. - APInt InDemandedBits = DemandedBits.trunc(InBits); InDemandedBits.setBit(InBits - 1); - if (SimplifyDemandedBits(Src, InDemandedBits, Known, TLO, Depth + 1)) + if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO, + Depth + 1)) return true; assert(!Known.hasConflict() && "Bits known to be one AND zero?"); + assert(Known.getBitWidth() == InBits && "Src width has changed?"); + // If the sign bit is known one, the top bits match. Known = Known.sext(BitWidth); // If the sign bit is known zero, convert this to a zero extend. - if (Known.isNonNegative()) - return TLO.CombineTo(Op, TLO.DAG.getNode(ISD::ZERO_EXTEND, dl, VT, Src)); + if (Known.isNonNegative()) { + unsigned Opc = + IsVecInReg ? ISD::ZERO_EXTEND_VECTOR_INREG : ISD::ZERO_EXTEND; + if (!TLO.LegalOperations() || isOperationLegal(Opc, VT)) + return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, dl, VT, Src)); + } break; } - case ISD::SIGN_EXTEND_VECTOR_INREG: { - // TODO - merge this with SIGN_EXTEND above? + case ISD::ANY_EXTEND: + case ISD::ANY_EXTEND_VECTOR_INREG: { SDValue Src = Op.getOperand(0); - unsigned InBits = Src.getScalarValueSizeInBits(); - - APInt InDemandedBits = DemandedBits.trunc(InBits); + EVT SrcVT = Src.getValueType(); + unsigned InBits = SrcVT.getScalarSizeInBits(); + unsigned InElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1; + bool IsVecInReg = Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG; - // If some of the sign extended bits are demanded, we know that the sign - // bit is demanded. - if (InBits < DemandedBits.getActiveBits()) - InDemandedBits.setBit(InBits - 1); + // If we only need the bottom element then we can just bitcast. + // TODO: Handle ANY_EXTEND? + if (IsVecInReg && DemandedElts == 1 && + VT.getSizeInBits() == SrcVT.getSizeInBits() && + TLO.DAG.getDataLayout().isLittleEndian()) + return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src)); - if (SimplifyDemandedBits(Src, InDemandedBits, Known, TLO, Depth + 1)) - return true; - assert(!Known.hasConflict() && "Bits known to be one AND zero?"); - // If the sign bit is known one, the top bits match. - Known = Known.sext(BitWidth); - break; - } - case ISD::ANY_EXTEND: { - SDValue Src = Op.getOperand(0); - unsigned InBits = Src.getScalarValueSizeInBits(); APInt InDemandedBits = DemandedBits.trunc(InBits); - if (SimplifyDemandedBits(Src, InDemandedBits, Known, TLO, Depth+1)) + APInt InDemandedElts = DemandedElts.zextOrSelf(InElts); + if (SimplifyDemandedBits(Src, InDemandedBits, InDemandedElts, Known, TLO, + Depth + 1)) return true; assert(!Known.hasConflict() && "Bits known to be one AND zero?"); - Known = Known.zext(BitWidth); + assert(Known.getBitWidth() == InBits && "Src width has changed?"); + Known = Known.zext(BitWidth, false /* => any extend */); break; } case ISD::TRUNCATE: { @@ -1198,29 +1492,29 @@ bool TargetLowering::SimplifyDemandedBits( // Do not turn (vt1 truncate (vt2 srl)) into (vt1 srl) if vt1 is // undesirable. break; - ConstantSDNode *ShAmt = dyn_cast(Src.getOperand(1)); - if (!ShAmt) + + auto *ShAmt = dyn_cast(Src.getOperand(1)); + if (!ShAmt || ShAmt->getAPIntValue().uge(BitWidth)) break; + SDValue Shift = Src.getOperand(1); - if (TLO.LegalTypes()) { - uint64_t ShVal = ShAmt->getZExtValue(); + uint64_t ShVal = ShAmt->getZExtValue(); + + if (TLO.LegalTypes()) Shift = TLO.DAG.getConstant(ShVal, dl, getShiftAmountTy(VT, DL)); - } - if (ShAmt->getZExtValue() < BitWidth) { - APInt HighBits = APInt::getHighBitsSet(OperandBitWidth, - OperandBitWidth - BitWidth); - HighBits.lshrInPlace(ShAmt->getZExtValue()); - HighBits = HighBits.trunc(BitWidth); - - if (!(HighBits & DemandedBits)) { - // None of the shifted in bits are needed. Add a truncate of the - // shift input, then shift it. - SDValue NewTrunc = - TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0)); - return TLO.CombineTo( - Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, Shift)); - } + APInt HighBits = + APInt::getHighBitsSet(OperandBitWidth, OperandBitWidth - BitWidth); + HighBits.lshrInPlace(ShVal); + HighBits = HighBits.trunc(BitWidth); + + if (!(HighBits & DemandedBits)) { + // None of the shifted in bits are needed. Add a truncate of the + // shift input, then shift it. + SDValue NewTrunc = + TLO.DAG.getNode(ISD::TRUNCATE, dl, VT, Src.getOperand(0)); + return TLO.CombineTo( + Op, TLO.DAG.getNode(ISD::SRL, dl, VT, NewTrunc, Shift)); } break; } @@ -1234,8 +1528,8 @@ bool TargetLowering::SimplifyDemandedBits( // demanded by its users. EVT ZVT = cast(Op.getOperand(1))->getVT(); APInt InMask = APInt::getLowBitsSet(BitWidth, ZVT.getSizeInBits()); - if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, - Known, TLO, Depth+1)) + if (SimplifyDemandedBits(Op.getOperand(0), ~InMask | DemandedBits, Known, + TLO, Depth + 1)) return true; assert(!Known.hasConflict() && "Bits known to be one AND zero?"); @@ -1266,7 +1560,7 @@ bool TargetLowering::SimplifyDemandedBits( Known = Known2; if (BitWidth > EltBitWidth) - Known = Known.zext(BitWidth); + Known = Known.zext(BitWidth, false /* => any extend */); break; } case ISD::BITCAST: { @@ -1297,40 +1591,68 @@ bool TargetLowering::SimplifyDemandedBits( TLO.DAG.getNode(ISD::SHL, dl, VT, Sign, ShAmt)); } } - // If bitcast from a vector, see if we can use SimplifyDemandedVectorElts by - // demanding the element if any bits from it are demanded. + + // Bitcast from a vector using SimplifyDemanded Bits/VectorElts. + // Demand the elt/bit if any of the original elts/bits are demanded. // TODO - bigendian once we have test coverage. // TODO - bool vectors once SimplifyDemandedVectorElts has SETCC support. if (SrcVT.isVector() && NumSrcEltBits > 1 && (BitWidth % NumSrcEltBits) == 0 && TLO.DAG.getDataLayout().isLittleEndian()) { unsigned Scale = BitWidth / NumSrcEltBits; - auto GetDemandedSubMask = [&](APInt &DemandedSubElts) -> bool { - DemandedSubElts = APInt::getNullValue(Scale); - for (unsigned i = 0; i != Scale; ++i) { - unsigned Offset = i * NumSrcEltBits; - APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset); - if (!Sub.isNullValue()) - DemandedSubElts.setBit(i); + unsigned NumSrcElts = SrcVT.getVectorNumElements(); + APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits); + APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts); + for (unsigned i = 0; i != Scale; ++i) { + unsigned Offset = i * NumSrcEltBits; + APInt Sub = DemandedBits.extractBits(NumSrcEltBits, Offset); + if (!Sub.isNullValue()) { + DemandedSrcBits |= Sub; + for (unsigned j = 0; j != NumElts; ++j) + if (DemandedElts[j]) + DemandedSrcElts.setBit((j * Scale) + i); } + } + + APInt KnownSrcUndef, KnownSrcZero; + if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef, + KnownSrcZero, TLO, Depth + 1)) return true; - }; - APInt DemandedSubElts; - if (GetDemandedSubMask(DemandedSubElts)) { - unsigned NumSrcElts = SrcVT.getVectorNumElements(); - APInt DemandedElts = APInt::getSplat(NumSrcElts, DemandedSubElts); + KnownBits KnownSrcBits; + if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, + KnownSrcBits, TLO, Depth + 1)) + return true; + } else if ((NumSrcEltBits % BitWidth) == 0 && + TLO.DAG.getDataLayout().isLittleEndian()) { + unsigned Scale = NumSrcEltBits / BitWidth; + unsigned NumSrcElts = SrcVT.isVector() ? SrcVT.getVectorNumElements() : 1; + APInt DemandedSrcBits = APInt::getNullValue(NumSrcEltBits); + APInt DemandedSrcElts = APInt::getNullValue(NumSrcElts); + for (unsigned i = 0; i != NumElts; ++i) + if (DemandedElts[i]) { + unsigned Offset = (i % Scale) * BitWidth; + DemandedSrcBits.insertBits(DemandedBits, Offset); + DemandedSrcElts.setBit(i / Scale); + } - APInt KnownUndef, KnownZero; - if (SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef, KnownZero, - TLO, Depth + 1)) + if (SrcVT.isVector()) { + APInt KnownSrcUndef, KnownSrcZero; + if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, KnownSrcUndef, + KnownSrcZero, TLO, Depth + 1)) return true; } + + KnownBits KnownSrcBits; + if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedSrcElts, + KnownSrcBits, TLO, Depth + 1)) + return true; } + // If this is a bitcast, let computeKnownBits handle it. Only do this on a // recursive call where Known may be useful to the caller. if (Depth > 0) { - Known = TLO.DAG.computeKnownBits(Op, Depth); + Known = TLO.DAG.computeKnownBits(Op, DemandedElts, Depth); return false; } break; @@ -1343,8 +1665,10 @@ bool TargetLowering::SimplifyDemandedBits( SDValue Op0 = Op.getOperand(0), Op1 = Op.getOperand(1); unsigned DemandedBitsLZ = DemandedBits.countLeadingZeros(); APInt LoMask = APInt::getLowBitsSet(BitWidth, BitWidth - DemandedBitsLZ); - if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO, Depth + 1) || - SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO, Depth + 1) || + if (SimplifyDemandedBits(Op0, LoMask, DemandedElts, Known2, TLO, + Depth + 1) || + SimplifyDemandedBits(Op1, LoMask, DemandedElts, Known2, TLO, + Depth + 1) || // See if the operation should be performed at a smaller bit width. ShrinkDemandedOp(Op, BitWidth, DemandedBits, TLO)) { SDNodeFlags Flags = Op.getNode()->getFlags(); @@ -1353,8 +1677,8 @@ bool TargetLowering::SimplifyDemandedBits( // won't wrap after simplification. Flags.setNoSignedWrap(false); Flags.setNoUnsignedWrap(false); - SDValue NewOp = TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, - Flags); + SDValue NewOp = + TLO.DAG.getNode(Op.getOpcode(), dl, VT, Op0, Op1, Flags); return TLO.CombineTo(Op, NewOp); } return true; @@ -1431,15 +1755,64 @@ bool TargetLowering::SimplifyDemandedVectorElts(SDValue Op, DCI.AddToWorklist(Op.getNode()); DCI.CommitTargetLoweringOpt(TLO); } + return Simplified; } +/// Given a vector binary operation and known undefined elements for each input +/// operand, compute whether each element of the output is undefined. +static APInt getKnownUndefForVectorBinop(SDValue BO, SelectionDAG &DAG, + const APInt &UndefOp0, + const APInt &UndefOp1) { + EVT VT = BO.getValueType(); + assert(DAG.getTargetLoweringInfo().isBinOp(BO.getOpcode()) && VT.isVector() && + "Vector binop only"); + + EVT EltVT = VT.getVectorElementType(); + unsigned NumElts = VT.getVectorNumElements(); + assert(UndefOp0.getBitWidth() == NumElts && + UndefOp1.getBitWidth() == NumElts && "Bad type for undef analysis"); + + auto getUndefOrConstantElt = [&](SDValue V, unsigned Index, + const APInt &UndefVals) { + if (UndefVals[Index]) + return DAG.getUNDEF(EltVT); + + if (auto *BV = dyn_cast(V)) { + // Try hard to make sure that the getNode() call is not creating temporary + // nodes. Ignore opaque integers because they do not constant fold. + SDValue Elt = BV->getOperand(Index); + auto *C = dyn_cast(Elt); + if (isa(Elt) || Elt.isUndef() || (C && !C->isOpaque())) + return Elt; + } + + return SDValue(); + }; + + APInt KnownUndef = APInt::getNullValue(NumElts); + for (unsigned i = 0; i != NumElts; ++i) { + // If both inputs for this element are either constant or undef and match + // the element type, compute the constant/undef result for this element of + // the vector. + // TODO: Ideally we would use FoldConstantArithmetic() here, but that does + // not handle FP constants. The code within getNode() should be refactored + // to avoid the danger of creating a bogus temporary node here. + SDValue C0 = getUndefOrConstantElt(BO.getOperand(0), i, UndefOp0); + SDValue C1 = getUndefOrConstantElt(BO.getOperand(1), i, UndefOp1); + if (C0 && C1 && C0.getValueType() == EltVT && C1.getValueType() == EltVT) + if (DAG.getNode(BO.getOpcode(), SDLoc(BO), EltVT, C0, C1).isUndef()) + KnownUndef.setBit(i); + } + return KnownUndef; +} + bool TargetLowering::SimplifyDemandedVectorElts( - SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, + SDValue Op, const APInt &OriginalDemandedElts, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth, bool AssumeSingleUse) const { EVT VT = Op.getValueType(); - APInt DemandedElts = DemandedEltMask; + APInt DemandedElts = OriginalDemandedElts; unsigned NumElts = DemandedElts.getBitWidth(); assert(VT.isVector() && "Expected vector op"); assert(VT.getVectorNumElements() == NumElts && @@ -1617,7 +1990,7 @@ bool TargetLowering::SimplifyDemandedVectorElts( SDValue Sub = Op.getOperand(1); EVT SubVT = Sub.getValueType(); unsigned NumSubElts = SubVT.getVectorNumElements(); - const APInt& Idx = cast(Op.getOperand(2))->getAPIntValue(); + const APInt &Idx = Op.getConstantOperandAPInt(2); if (Idx.ugt(NumElts - NumSubElts)) break; unsigned SubIdx = Idx.getZExtValue(); @@ -1786,18 +2159,26 @@ bool TargetLowering::SimplifyDemandedVectorElts( } break; } + case ISD::ANY_EXTEND_VECTOR_INREG: case ISD::SIGN_EXTEND_VECTOR_INREG: case ISD::ZERO_EXTEND_VECTOR_INREG: { APInt SrcUndef, SrcZero; SDValue Src = Op.getOperand(0); unsigned NumSrcElts = Src.getValueType().getVectorNumElements(); APInt DemandedSrcElts = DemandedElts.zextOrSelf(NumSrcElts); - if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, - SrcZero, TLO, Depth + 1)) + if (SimplifyDemandedVectorElts(Src, DemandedSrcElts, SrcUndef, SrcZero, TLO, + Depth + 1)) return true; KnownZero = SrcZero.zextOrTrunc(NumElts); KnownUndef = SrcUndef.zextOrTrunc(NumElts); + if (Op.getOpcode() == ISD::ANY_EXTEND_VECTOR_INREG && + Op.getValueSizeInBits() == Src.getValueSizeInBits() && + DemandedSrcElts == 1 && TLO.DAG.getDataLayout().isLittleEndian()) { + // aext - if we just need the bottom element then we can bitcast. + return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, Src)); + } + if (Op.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG) { // zext(undef) upper bits are guaranteed to be zero. if (DemandedElts.isSubsetOf(KnownUndef)) @@ -1806,6 +2187,9 @@ bool TargetLowering::SimplifyDemandedVectorElts( } break; } + + // TODO: There are more binop opcodes that could be handled here - MUL, MIN, + // MAX, saturated math, etc. case ISD::OR: case ISD::XOR: case ISD::ADD: @@ -1815,17 +2199,38 @@ bool TargetLowering::SimplifyDemandedVectorElts( case ISD::FMUL: case ISD::FDIV: case ISD::FREM: { - APInt SrcUndef, SrcZero; - if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, SrcUndef, - SrcZero, TLO, Depth + 1)) + APInt UndefRHS, ZeroRHS; + if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, UndefRHS, + ZeroRHS, TLO, Depth + 1)) return true; - if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, KnownUndef, - KnownZero, TLO, Depth + 1)) + APInt UndefLHS, ZeroLHS; + if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UndefLHS, + ZeroLHS, TLO, Depth + 1)) return true; - KnownZero &= SrcZero; - KnownUndef &= SrcUndef; + + KnownZero = ZeroLHS & ZeroRHS; + KnownUndef = getKnownUndefForVectorBinop(Op, TLO.DAG, UndefLHS, UndefRHS); + break; + } + case ISD::SHL: + case ISD::SRL: + case ISD::SRA: + case ISD::ROTL: + case ISD::ROTR: { + APInt UndefRHS, ZeroRHS; + if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, UndefRHS, + ZeroRHS, TLO, Depth + 1)) + return true; + APInt UndefLHS, ZeroLHS; + if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, UndefLHS, + ZeroLHS, TLO, Depth + 1)) + return true; + + KnownZero = ZeroLHS; + KnownUndef = UndefLHS & UndefRHS; // TODO: use getKnownUndefForVectorBinop? break; } + case ISD::MUL: case ISD::AND: { APInt SrcUndef, SrcZero; if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, SrcUndef, @@ -1837,6 +2242,8 @@ bool TargetLowering::SimplifyDemandedVectorElts( // If either side has a zero element, then the result element is zero, even // if the other is an UNDEF. + // TODO: Extend getKnownUndefForVectorBinop to also deal with known zeros + // and then handle 'and' nodes with the rest of the binop opcodes. KnownZero |= SrcZero; KnownUndef &= SrcUndef; KnownUndef &= ~KnownZero; @@ -1864,8 +2271,8 @@ bool TargetLowering::SimplifyDemandedVectorElts( } else { KnownBits Known; APInt DemandedBits = APInt::getAllOnesValue(EltSizeInBits); - if (SimplifyDemandedBits(Op, DemandedBits, DemandedEltMask, Known, TLO, - Depth, AssumeSingleUse)) + if (SimplifyDemandedBits(Op, DemandedBits, OriginalDemandedElts, Known, + TLO, Depth, AssumeSingleUse)) return true; } break; @@ -1950,6 +2357,10 @@ bool TargetLowering::SimplifyDemandedBitsForTargetNode( return false; } +const Constant *TargetLowering::getTargetConstantFromLoad(LoadSDNode*) const { + return nullptr; +} + bool TargetLowering::isKnownNeverNaNForTargetNode(SDValue Op, const SelectionDAG &DAG, bool SNaN, @@ -2044,10 +2455,9 @@ bool TargetLowering::isExtendedTrueVal(const ConstantSDNode *N, EVT VT, /// This helper function of SimplifySetCC tries to optimize the comparison when /// either operand of the SetCC node is a bitwise-and instruction. -SDValue TargetLowering::simplifySetCCWithAnd(EVT VT, SDValue N0, SDValue N1, - ISD::CondCode Cond, - DAGCombinerInfo &DCI, - const SDLoc &DL) const { +SDValue TargetLowering::foldSetCCWithAnd(EVT VT, SDValue N0, SDValue N1, + ISD::CondCode Cond, const SDLoc &DL, + DAGCombinerInfo &DCI) const { // Match these patterns in any of their permutations: // (X & Y) == Y // (X & Y) != Y @@ -2200,6 +2610,49 @@ SDValue TargetLowering::optimizeSetCCOfSignedTruncationCheck( return T2; } +/// Try to fold an equality comparison with a {add/sub/xor} binary operation as +/// the 1st operand (N0). Callers are expected to swap the N0/N1 parameters to +/// handle the commuted versions of these patterns. +SDValue TargetLowering::foldSetCCWithBinOp(EVT VT, SDValue N0, SDValue N1, + ISD::CondCode Cond, const SDLoc &DL, + DAGCombinerInfo &DCI) const { + unsigned BOpcode = N0.getOpcode(); + assert((BOpcode == ISD::ADD || BOpcode == ISD::SUB || BOpcode == ISD::XOR) && + "Unexpected binop"); + assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && "Unexpected condcode"); + + // (X + Y) == X --> Y == 0 + // (X - Y) == X --> Y == 0 + // (X ^ Y) == X --> Y == 0 + SelectionDAG &DAG = DCI.DAG; + EVT OpVT = N0.getValueType(); + SDValue X = N0.getOperand(0); + SDValue Y = N0.getOperand(1); + if (X == N1) + return DAG.getSetCC(DL, VT, Y, DAG.getConstant(0, DL, OpVT), Cond); + + if (Y != N1) + return SDValue(); + + // (X + Y) == Y --> X == 0 + // (X ^ Y) == Y --> X == 0 + if (BOpcode == ISD::ADD || BOpcode == ISD::XOR) + return DAG.getSetCC(DL, VT, X, DAG.getConstant(0, DL, OpVT), Cond); + + // The shift would not be valid if the operands are boolean (i1). + if (!N0.hasOneUse() || OpVT.getScalarSizeInBits() == 1) + return SDValue(); + + // (X - Y) == Y --> X == Y << 1 + EVT ShiftVT = getShiftAmountTy(OpVT, DAG.getDataLayout(), + !DCI.isBeforeLegalize()); + SDValue One = DAG.getConstant(1, DL, ShiftVT); + SDValue YShl1 = DAG.getNode(ISD::SHL, DL, N1.getValueType(), Y, One); + if (!DCI.isCalledByLegalizer()) + DCI.AddToWorklist(YShl1.getNode()); + return DAG.getSetCC(DL, VT, X, YShl1, Cond); +} + /// Try to simplify a setcc built with the specified operands and cc. If it is /// unable to simplify it, return a null SDValue. SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, @@ -2209,14 +2662,9 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, SelectionDAG &DAG = DCI.DAG; EVT OpVT = N0.getValueType(); - // These setcc operations always fold. - switch (Cond) { - default: break; - case ISD::SETFALSE: - case ISD::SETFALSE2: return DAG.getBoolConstant(false, dl, VT, OpVT); - case ISD::SETTRUE: - case ISD::SETTRUE2: return DAG.getBoolConstant(true, dl, VT, OpVT); - } + // Constant fold or commute setcc. + if (SDValue Fold = DAG.FoldSetCC(VT, N0, N1, Cond, dl)) + return Fold; // Ensure that the constant occurs on the RHS and fold constant comparisons. // TODO: Handle non-splat vector constants. All undef causes trouble. @@ -2226,6 +2674,17 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, isCondCodeLegal(SwappedCC, N0.getSimpleValueType()))) return DAG.getSetCC(dl, VT, N1, N0, SwappedCC); + // If we have a subtract with the same 2 non-constant operands as this setcc + // -- but in reverse order -- then try to commute the operands of this setcc + // to match. A matching pair of setcc (cmp) and sub may be combined into 1 + // instruction on some targets. + if (!isConstOrConstSplat(N0) && !isConstOrConstSplat(N1) && + (DCI.isBeforeLegalizeOps() || + isCondCodeLegal(SwappedCC, N0.getSimpleValueType())) && + DAG.getNodeIfExists(ISD::SUB, DAG.getVTList(OpVT), { N1, N0 } ) && + !DAG.getNodeIfExists(ISD::SUB, DAG.getVTList(OpVT), { N0, N1 } )) + return DAG.getSetCC(dl, VT, N1, N0, SwappedCC); + if (auto *N1C = dyn_cast(N1.getNode())) { const APInt &C1 = N1C->getAPIntValue(); @@ -2235,8 +2694,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, if (N0.getOpcode() == ISD::SRL && (C1.isNullValue() || C1.isOneValue()) && N0.getOperand(0).getOpcode() == ISD::CTLZ && N0.getOperand(1).getOpcode() == ISD::Constant) { - const APInt &ShAmt - = cast(N0.getOperand(1))->getAPIntValue(); + const APInt &ShAmt = N0.getConstantOperandAPInt(1); if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && ShAmt == Log2_32(N0.getValueSizeInBits())) { if ((C1 == 0) == (Cond == ISD::SETEQ)) { @@ -2275,7 +2733,21 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, return DAG.getSetCC(dl, VT, And, DAG.getConstant(0, dl, CTVT), CC); } - // TODO: (ctpop x) == 1 -> x && (x & x-1) == 0 iff ctpop is illegal. + // If ctpop is not supported, expand a power-of-2 comparison based on it. + if (C1 == 1 && !isOperationLegalOrCustom(ISD::CTPOP, CTVT) && + (Cond == ISD::SETEQ || Cond == ISD::SETNE)) { + // (ctpop x) == 1 --> (x != 0) && ((x & x-1) == 0) + // (ctpop x) != 1 --> (x == 0) || ((x & x-1) != 0) + SDValue Zero = DAG.getConstant(0, dl, CTVT); + SDValue NegOne = DAG.getAllOnesConstant(dl, CTVT); + ISD::CondCode InvCond = ISD::getSetCCInverse(Cond, true); + SDValue Add = DAG.getNode(ISD::ADD, dl, CTVT, CTOp, NegOne); + SDValue And = DAG.getNode(ISD::AND, dl, CTVT, CTOp, Add); + SDValue LHS = DAG.getSetCC(dl, VT, CTOp, Zero, InvCond); + SDValue RHS = DAG.getSetCC(dl, VT, And, Zero, Cond); + unsigned LogicOpcode = Cond == ISD::SETEQ ? ISD::AND : ISD::OR; + return DAG.getNode(LogicOpcode, dl, VT, LHS, RHS); + } } // (zext x) == C --> x == (trunc C) @@ -2387,8 +2859,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, // 8 bits, but have to be careful... if (Lod->getExtensionType() != ISD::NON_EXTLOAD) origWidth = Lod->getMemoryVT().getSizeInBits(); - const APInt &Mask = - cast(N0.getOperand(1))->getAPIntValue(); + const APInt &Mask = N0.getConstantOperandAPInt(1); for (unsigned width = origWidth / 2; width>=8; width /= 2) { APInt newMask = APInt::getLowBitsSet(maskWidth, width); for (unsigned offset=0; offsetisNullValue() && + (Cond == ISD::SETEQ || Cond == ISD::SETNE)) { + KnownBits XKnown = DAG.computeKnownBits(N0.getOperand(0)); + KnownBits YKnown = DAG.computeKnownBits(N0.getOperand(1)); + if (XKnown.countMaxPopulation() == 1 && YKnown.countMinPopulation() >= 2) + return DAG.getSetCC(dl, VT, N0.getOperand(0), N1, Cond); + } + if (SDValue V = optimizeSetCCOfSignedTruncationCheck(VT, N0, N1, Cond, DCI, dl)) return V; @@ -2805,25 +3288,9 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, } } - if (isa(N0.getNode())) { - // Constant fold or commute setcc. - SDValue O = DAG.FoldSetCC(VT, N0, N1, Cond, dl); - if (O.getNode()) return O; - } else if (auto *CFP = dyn_cast(N1.getNode())) { - // If the RHS of an FP comparison is a constant, simplify it away in - // some cases. - if (CFP->getValueAPF().isNaN()) { - // If an operand is known to be a nan, we can fold it. - switch (ISD::getUnorderedFlavor(Cond)) { - default: llvm_unreachable("Unknown flavor!"); - case 0: // Known false. - return DAG.getBoolConstant(false, dl, VT, OpVT); - case 1: // Known true. - return DAG.getBoolConstant(true, dl, VT, OpVT); - case 2: // Undefined. - return DAG.getUNDEF(VT); - } - } + if (!isa(N0) && isa(N1)) { + auto *CFP = cast(N1); + assert(!CFP->getValueAPF().isNaN() && "Unexpected NaN value"); // Otherwise, we know the RHS is not a NaN. Simplify the node to drop the // constant if knowing that the operand is non-nan is enough. We prefer to @@ -2883,15 +3350,12 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, if (N0 == N1) { // The sext(setcc()) => setcc() optimization relies on the appropriate // constant being emitted. + assert(!N0.getValueType().isInteger() && + "Integer types should be handled by FoldSetCC"); bool EqTrue = ISD::isTrueWhenEqual(Cond); - - // We can always fold X == X for integer setcc's. - if (N0.getValueType().isInteger()) - return DAG.getBoolConstant(EqTrue, dl, VT, OpVT); - unsigned UOF = ISD::getUnorderedFlavor(Cond); - if (UOF == 2) // FP operators that are undefined on NaNs. + if (UOF == 2) // FP operators that are undefined on NaNs. return DAG.getBoolConstant(EqTrue, dl, VT, OpVT); if (UOF == unsigned(EqTrue)) return DAG.getBoolConstant(EqTrue, dl, VT, OpVT); @@ -2900,7 +3364,7 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, ISD::CondCode NewCond = UOF == 0 ? ISD::SETO : ISD::SETUO; if (NewCond != Cond && (DCI.isBeforeLegalizeOps() || - isCondCodeLegal(NewCond, N0.getSimpleValueType()))) + isCondCodeLegal(NewCond, N0.getSimpleValueType()))) return DAG.getSetCC(dl, VT, N0, N1, NewCond); } @@ -2969,69 +3433,39 @@ SDValue TargetLowering::SimplifySetCC(EVT VT, SDValue N0, SDValue N1, LegalRHSImm = isLegalICmpImmediate(RHSC->getSExtValue()); } - // Simplify (X+Z) == X --> Z == 0 + // (X+Y) == X --> Y == 0 and similar folds. // Don't do this if X is an immediate that can fold into a cmp - // instruction and X+Z has other uses. It could be an induction variable + // instruction and X+Y has other uses. It could be an induction variable // chain, and the transform would increase register pressure. - if (!LegalRHSImm || N0.getNode()->hasOneUse()) { - if (N0.getOperand(0) == N1) - return DAG.getSetCC(dl, VT, N0.getOperand(1), - DAG.getConstant(0, dl, N0.getValueType()), Cond); - if (N0.getOperand(1) == N1) { - if (isCommutativeBinOp(N0.getOpcode())) - return DAG.getSetCC(dl, VT, N0.getOperand(0), - DAG.getConstant(0, dl, N0.getValueType()), - Cond); - if (N0.getNode()->hasOneUse()) { - assert(N0.getOpcode() == ISD::SUB && "Unexpected operation!"); - auto &DL = DAG.getDataLayout(); - // (Z-X) == X --> Z == X<<1 - SDValue SH = DAG.getNode( - ISD::SHL, dl, N1.getValueType(), N1, - DAG.getConstant(1, dl, - getShiftAmountTy(N1.getValueType(), DL, - !DCI.isBeforeLegalize()))); - if (!DCI.isCalledByLegalizer()) - DCI.AddToWorklist(SH.getNode()); - return DAG.getSetCC(dl, VT, N0.getOperand(0), SH, Cond); - } - } - } + if (!LegalRHSImm || N0.hasOneUse()) + if (SDValue V = foldSetCCWithBinOp(VT, N0, N1, Cond, dl, DCI)) + return V; } if (N1.getOpcode() == ISD::ADD || N1.getOpcode() == ISD::SUB || - N1.getOpcode() == ISD::XOR) { - // Simplify X == (X+Z) --> Z == 0 - if (N1.getOperand(0) == N0) - return DAG.getSetCC(dl, VT, N1.getOperand(1), - DAG.getConstant(0, dl, N1.getValueType()), Cond); - if (N1.getOperand(1) == N0) { - if (isCommutativeBinOp(N1.getOpcode())) - return DAG.getSetCC(dl, VT, N1.getOperand(0), - DAG.getConstant(0, dl, N1.getValueType()), Cond); - if (N1.getNode()->hasOneUse()) { - assert(N1.getOpcode() == ISD::SUB && "Unexpected operation!"); - auto &DL = DAG.getDataLayout(); - // X == (Z-X) --> X<<1 == Z - SDValue SH = DAG.getNode( - ISD::SHL, dl, N1.getValueType(), N0, - DAG.getConstant(1, dl, getShiftAmountTy(N0.getValueType(), DL, - !DCI.isBeforeLegalize()))); - if (!DCI.isCalledByLegalizer()) - DCI.AddToWorklist(SH.getNode()); - return DAG.getSetCC(dl, VT, SH, N1.getOperand(0), Cond); - } - } - } + N1.getOpcode() == ISD::XOR) + if (SDValue V = foldSetCCWithBinOp(VT, N1, N0, Cond, dl, DCI)) + return V; - if (SDValue V = simplifySetCCWithAnd(VT, N0, N1, Cond, DCI, dl)) + if (SDValue V = foldSetCCWithAnd(VT, N0, N1, Cond, dl, DCI)) return V; } + // Fold remainder of division by a constant. + if (N0.getOpcode() == ISD::UREM && N0.hasOneUse() && + (Cond == ISD::SETEQ || Cond == ISD::SETNE)) { + AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes(); + + // When division is cheap or optimizing for minimum size, + // fall through to DIVREM creation by skipping this fold. + if (!isIntDivCheap(VT, Attr) && !Attr.hasFnAttribute(Attribute::MinSize)) + if (SDValue Folded = buildUREMEqFold(VT, N0, N1, Cond, DCI, dl)) + return Folded; + } + // Fold away ALL boolean setcc's. - SDValue Temp; if (N0.getValueType().getScalarType() == MVT::i1 && foldBooleans) { - EVT OpVT = N0.getValueType(); + SDValue Temp; switch (Cond) { default: llvm_unreachable("Unknown integer setcc!"); case ISD::SETEQ: // X == Y -> ~(X^Y) @@ -3134,18 +3568,18 @@ TargetLowering::getConstraintType(StringRef Constraint) const { switch (Constraint[0]) { default: break; case 'r': return C_RegisterClass; - case 'm': // memory - case 'o': // offsetable - case 'V': // not offsetable + case 'm': // memory + case 'o': // offsetable + case 'V': // not offsetable return C_Memory; - case 'i': // Simple Integer or Relocatable Constant - case 'n': // Simple Integer - case 'E': // Floating Point Constant - case 'F': // Floating Point Constant - case 's': // Relocatable Constant - case 'p': // Address. - case 'X': // Allow ANY value. - case 'I': // Target registers. + case 'i': // Simple Integer or Relocatable Constant + case 'n': // Simple Integer + case 'E': // Floating Point Constant + case 'F': // Floating Point Constant + case 's': // Relocatable Constant + case 'p': // Address. + case 'X': // Allow ANY value. + case 'I': // Target registers. case 'J': case 'K': case 'L': @@ -3159,7 +3593,7 @@ TargetLowering::getConstraintType(StringRef Constraint) const { } } - if (S > 1 && Constraint[0] == '{' && Constraint[S-1] == '}') { + if (S > 1 && Constraint[0] == '{' && Constraint[S - 1] == '}') { if (S == 8 && Constraint.substr(1, 6) == "memory") // "{memory}" return C_Memory; return C_Register; @@ -3170,14 +3604,20 @@ TargetLowering::getConstraintType(StringRef Constraint) const { /// Try to replace an X constraint, which matches anything, with another that /// has more specific requirements based on the type of the corresponding /// operand. -const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const{ +const char *TargetLowering::LowerXConstraint(EVT ConstraintVT) const { if (ConstraintVT.isInteger()) return "r"; if (ConstraintVT.isFloatingPoint()) - return "f"; // works for many targets + return "f"; // works for many targets return nullptr; } +SDValue TargetLowering::LowerAsmOutputForConstraint( + SDValue &Chain, SDValue &Flag, SDLoc DL, const AsmOperandInfo &OpInfo, + SelectionDAG &DAG) const { + return SDValue(); +} + /// Lower the specified operand into the Ops vector. /// If it is invalid, don't add anything to Ops. void TargetLowering::LowerAsmOperandForConstraint(SDValue Op, @@ -3191,7 +3631,8 @@ void TargetLowering::LowerAsmOperandForConstraint(SDValue Op, switch (ConstraintLetter) { default: break; case 'X': // Allows any operand; labels (basic block) use this. - if (Op.getOpcode() == ISD::BasicBlock) { + if (Op.getOpcode() == ISD::BasicBlock || + Op.getOpcode() == ISD::TargetBlockAddress) { Ops.push_back(Op); return; } @@ -3199,46 +3640,57 @@ void TargetLowering::LowerAsmOperandForConstraint(SDValue Op, case 'i': // Simple Integer or Relocatable Constant case 'n': // Simple Integer case 's': { // Relocatable Constant - // These operands are interested in values of the form (GV+C), where C may - // be folded in as an offset of GV, or it may be explicitly added. Also, it - // is possible and fine if either GV or C are missing. - ConstantSDNode *C = dyn_cast(Op); - GlobalAddressSDNode *GA = dyn_cast(Op); - - // If we have "(add GV, C)", pull out GV/C - if (Op.getOpcode() == ISD::ADD) { - C = dyn_cast(Op.getOperand(1)); - GA = dyn_cast(Op.getOperand(0)); - if (!C || !GA) { - C = dyn_cast(Op.getOperand(0)); - GA = dyn_cast(Op.getOperand(1)); - } - if (!C || !GA) { - C = nullptr; - GA = nullptr; - } - } - // If we find a valid operand, map to the TargetXXX version so that the - // value itself doesn't get selected. - if (GA) { // Either &GV or &GV+C - if (ConstraintLetter != 'n') { - int64_t Offs = GA->getOffset(); - if (C) Offs += C->getZExtValue(); - Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), - C ? SDLoc(C) : SDLoc(), - Op.getValueType(), Offs)); - } - return; - } - if (C) { // just C, no GV. - // Simple constants are not allowed for 's'. - if (ConstraintLetter != 's') { + GlobalAddressSDNode *GA; + ConstantSDNode *C; + BlockAddressSDNode *BA; + uint64_t Offset = 0; + + // Match (GA) or (C) or (GA+C) or (GA-C) or ((GA+C)+C) or (((GA+C)+C)+C), + // etc., since getelementpointer is variadic. We can't use + // SelectionDAG::FoldSymbolOffset because it expects the GA to be accessible + // while in this case the GA may be furthest from the root node which is + // likely an ISD::ADD. + while (1) { + if ((GA = dyn_cast(Op)) && ConstraintLetter != 'n') { + Ops.push_back(DAG.getTargetGlobalAddress(GA->getGlobal(), SDLoc(Op), + GA->getValueType(0), + Offset + GA->getOffset())); + return; + } else if ((C = dyn_cast(Op)) && + ConstraintLetter != 's') { // gcc prints these as sign extended. Sign extend value to 64 bits // now; without this it would get ZExt'd later in // ScheduleDAGSDNodes::EmitNode, which is very generic. - Ops.push_back(DAG.getTargetConstant(C->getSExtValue(), + bool IsBool = C->getConstantIntValue()->getBitWidth() == 1; + BooleanContent BCont = getBooleanContents(MVT::i64); + ISD::NodeType ExtOpc = IsBool ? getExtendForContent(BCont) + : ISD::SIGN_EXTEND; + int64_t ExtVal = ExtOpc == ISD::ZERO_EXTEND ? C->getZExtValue() + : C->getSExtValue(); + Ops.push_back(DAG.getTargetConstant(Offset + ExtVal, SDLoc(C), MVT::i64)); + return; + } else if ((BA = dyn_cast(Op)) && + ConstraintLetter != 'n') { + Ops.push_back(DAG.getTargetBlockAddress( + BA->getBlockAddress(), BA->getValueType(0), + Offset + BA->getOffset(), BA->getTargetFlags())); + return; + } else { + const unsigned OpCode = Op.getOpcode(); + if (OpCode == ISD::ADD || OpCode == ISD::SUB) { + if ((C = dyn_cast(Op.getOperand(0)))) + Op = Op.getOperand(1); + // Subtraction is not commutative. + else if (OpCode == ISD::ADD && + (C = dyn_cast(Op.getOperand(1)))) + Op = Op.getOperand(0); + else + return; + Offset += (OpCode == ISD::ADD ? 1 : -1) * C->getSExtValue(); + continue; + } } return; } @@ -3252,14 +3704,14 @@ TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI, StringRef Constraint, MVT VT) const { if (Constraint.empty() || Constraint[0] != '{') - return std::make_pair(0u, static_cast(nullptr)); - assert(*(Constraint.end()-1) == '}' && "Not a brace enclosed constraint?"); + return std::make_pair(0u, static_cast(nullptr)); + assert(*(Constraint.end() - 1) == '}' && "Not a brace enclosed constraint?"); // Remove the braces from around the name. - StringRef RegName(Constraint.data()+1, Constraint.size()-2); + StringRef RegName(Constraint.data() + 1, Constraint.size() - 2); - std::pair R = - std::make_pair(0u, static_cast(nullptr)); + std::pair R = + std::make_pair(0u, static_cast(nullptr)); // Figure out which register class contains this reg. for (const TargetRegisterClass *RC : RI->regclasses()) { @@ -3271,8 +3723,8 @@ TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *RI, for (TargetRegisterClass::iterator I = RC->begin(), E = RC->end(); I != E; ++I) { if (RegName.equals_lower(RI->getRegAsmName(*I))) { - std::pair S = - std::make_pair(*I, RC); + std::pair S = + std::make_pair(*I, RC); // If this register class has the requested value type, return it, // otherwise keep searching and return the first class found @@ -3321,8 +3773,8 @@ TargetLowering::ParseConstraints(const DataLayout &DL, // Do a prepass over the constraints, canonicalizing them, and building up the // ConstraintOperands list. - unsigned ArgNo = 0; // ArgNo - The argument of the CallInst. - unsigned ResNo = 0; // ResNo - The result number of the next output. + unsigned ArgNo = 0; // ArgNo - The argument of the CallInst. + unsigned ResNo = 0; // ResNo - The result number of the next output. for (InlineAsm::ConstraintInfo &CI : IA->ParseConstraints()) { ConstraintOperands.emplace_back(std::move(CI)); @@ -3391,7 +3843,7 @@ TargetLowering::ParseConstraints(const DataLayout &DL, case 64: case 128: OpInfo.ConstraintVT = - MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true); + MVT::getVT(IntegerType::get(OpTy->getContext(), BitSize), true); break; } } else if (PointerType *PT = dyn_cast(OpTy)) { @@ -3416,8 +3868,8 @@ TargetLowering::ParseConstraints(const DataLayout &DL, for (maIndex = 0; maIndex < maCount; ++maIndex) { int weightSum = 0; for (unsigned cIndex = 0, eIndex = ConstraintOperands.size(); - cIndex != eIndex; ++cIndex) { - AsmOperandInfo& OpInfo = ConstraintOperands[cIndex]; + cIndex != eIndex; ++cIndex) { + AsmOperandInfo &OpInfo = ConstraintOperands[cIndex]; if (OpInfo.Type == InlineAsm::isClobber) continue; @@ -3432,7 +3884,7 @@ TargetLowering::ParseConstraints(const DataLayout &DL, Input.ConstraintVT.isInteger()) || (OpInfo.ConstraintVT.getSizeInBits() != Input.ConstraintVT.getSizeInBits())) { - weightSum = -1; // Can't match. + weightSum = -1; // Can't match. break; } } @@ -3453,8 +3905,8 @@ TargetLowering::ParseConstraints(const DataLayout &DL, // Now select chosen alternative in each constraint. for (unsigned cIndex = 0, eIndex = ConstraintOperands.size(); - cIndex != eIndex; ++cIndex) { - AsmOperandInfo& cInfo = ConstraintOperands[cIndex]; + cIndex != eIndex; ++cIndex) { + AsmOperandInfo &cInfo = ConstraintOperands[cIndex]; if (cInfo.Type == InlineAsm::isClobber) continue; cInfo.selectAlternative(bestMAIndex); @@ -3464,8 +3916,8 @@ TargetLowering::ParseConstraints(const DataLayout &DL, // Check and hook up tied operands, choose constraint code to use. for (unsigned cIndex = 0, eIndex = ConstraintOperands.size(); - cIndex != eIndex; ++cIndex) { - AsmOperandInfo& OpInfo = ConstraintOperands[cIndex]; + cIndex != eIndex; ++cIndex) { + AsmOperandInfo &OpInfo = ConstraintOperands[cIndex]; // If this is an output operand with a matching input operand, look up the // matching input. If their types mismatch, e.g. one is an integer, the @@ -3577,9 +4029,9 @@ TargetLowering::ConstraintWeight weight = CW_Register; break; case 'X': // any operand. - default: - weight = CW_Default; - break; + default: + weight = CW_Default; + break; } return weight; } @@ -3678,6 +4130,9 @@ void TargetLowering::ComputeConstraintToUse(AsmOperandInfo &OpInfo, return; } + if (Op.getNode() && Op.getOpcode() == ISD::TargetBlockAddress) + return; + // Otherwise, try to resolve it to something we know about by looking at // the actual operand type. if (const char *Repl = LowerXConstraint(OpInfo.ConstraintVT)) { @@ -3749,12 +4204,12 @@ static SDValue BuildExactSDIV(const TargetLowering &TLI, SDNode *N, } SDValue TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor, - SelectionDAG &DAG, - SmallVectorImpl &Created) const { + SelectionDAG &DAG, + SmallVectorImpl &Created) const { AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes(); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); if (TLI.isIntDivCheap(N->getValueType(0), Attr)) - return SDValue(N,0); // Lower SDIV as SDIV + return SDValue(N, 0); // Lower SDIV as SDIV return SDValue(); } @@ -4000,6 +4455,104 @@ SDValue TargetLowering::BuildUDIV(SDNode *N, SelectionDAG &DAG, return DAG.getSelect(dl, VT, IsOne, N0, Q); } +/// Given an ISD::UREM used only by an ISD::SETEQ or ISD::SETNE +/// where the divisor is constant and the comparison target is zero, +/// return a DAG expression that will generate the same comparison result +/// using only multiplications, additions and shifts/rotations. +/// Ref: "Hacker's Delight" 10-17. +SDValue TargetLowering::buildUREMEqFold(EVT SETCCVT, SDValue REMNode, + SDValue CompTargetNode, + ISD::CondCode Cond, + DAGCombinerInfo &DCI, + const SDLoc &DL) const { + SmallVector Built; + if (SDValue Folded = prepareUREMEqFold(SETCCVT, REMNode, CompTargetNode, Cond, + DCI, DL, Built)) { + for (SDNode *N : Built) + DCI.AddToWorklist(N); + return Folded; + } + + return SDValue(); +} + +SDValue +TargetLowering::prepareUREMEqFold(EVT SETCCVT, SDValue REMNode, + SDValue CompTargetNode, ISD::CondCode Cond, + DAGCombinerInfo &DCI, const SDLoc &DL, + SmallVectorImpl &Created) const { + // fold (seteq/ne (urem N, D), 0) -> (setule/ugt (rotr (mul N, P), K), Q) + // - D must be constant with D = D0 * 2^K where D0 is odd and D0 != 1 + // - P is the multiplicative inverse of D0 modulo 2^W + // - Q = floor((2^W - 1) / D0) + // where W is the width of the common type of N and D. + assert((Cond == ISD::SETEQ || Cond == ISD::SETNE) && + "Only applicable for (in)equality comparisons."); + + EVT VT = REMNode.getValueType(); + + // If MUL is unavailable, we cannot proceed in any case. + if (!isOperationLegalOrCustom(ISD::MUL, VT)) + return SDValue(); + + // TODO: Add non-uniform constant support. + ConstantSDNode *Divisor = isConstOrConstSplat(REMNode->getOperand(1)); + ConstantSDNode *CompTarget = isConstOrConstSplat(CompTargetNode); + if (!Divisor || !CompTarget || Divisor->isNullValue() || + !CompTarget->isNullValue()) + return SDValue(); + + const APInt &D = Divisor->getAPIntValue(); + + // Decompose D into D0 * 2^K + unsigned K = D.countTrailingZeros(); + bool DivisorIsEven = (K != 0); + APInt D0 = D.lshr(K); + + // The fold is invalid when D0 == 1. + // This is reachable because visitSetCC happens before visitREM. + if (D0.isOneValue()) + return SDValue(); + + // P = inv(D0, 2^W) + // 2^W requires W + 1 bits, so we have to extend and then truncate. + unsigned W = D.getBitWidth(); + APInt P = D0.zext(W + 1) + .multiplicativeInverse(APInt::getSignedMinValue(W + 1)) + .trunc(W); + assert(!P.isNullValue() && "No multiplicative inverse!"); // unreachable + assert((D0 * P).isOneValue() && "Multiplicative inverse sanity check."); + + // Q = floor((2^W - 1) / D) + APInt Q = APInt::getAllOnesValue(W).udiv(D); + + SelectionDAG &DAG = DCI.DAG; + + SDValue PVal = DAG.getConstant(P, DL, VT); + SDValue QVal = DAG.getConstant(Q, DL, VT); + // (mul N, P) + SDValue Op1 = DAG.getNode(ISD::MUL, DL, VT, REMNode->getOperand(0), PVal); + Created.push_back(Op1.getNode()); + + // Rotate right only if D was even. + if (DivisorIsEven) { + // We need ROTR to do this. + if (!isOperationLegalOrCustom(ISD::ROTR, VT)) + return SDValue(); + SDValue ShAmt = + DAG.getConstant(K, DL, getShiftAmountTy(VT, DAG.getDataLayout())); + SDNodeFlags Flags; + Flags.setExact(true); + // UREM: (rotr (mul N, P), K) + Op1 = DAG.getNode(ISD::ROTR, DL, VT, Op1, ShAmt, Flags); + Created.push_back(Op1.getNode()); + } + + // UREM: (setule/setugt (rotr (mul N, P), K), Q) + return DAG.getSetCC(DL, SETCCVT, Op1, QVal, + ((Cond == ISD::SETEQ) ? ISD::SETULE : ISD::SETUGT)); +} + bool TargetLowering:: verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const { if (!isa(Op.getOperand(0))) { @@ -4308,7 +4861,7 @@ bool TargetLowering::expandROT(SDNode *Node, SDValue &Result, } bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result, - SelectionDAG &DAG) const { + SelectionDAG &DAG) const { SDValue Src = Node->getOperand(0); EVT SrcVT = Src.getValueType(); EVT DstVT = Node->getValueType(0); @@ -4320,7 +4873,7 @@ bool TargetLowering::expandFP_TO_SINT(SDNode *Node, SDValue &Result, // Expand f32 -> i64 conversion // This algorithm comes from compiler-rt's implementation of fixsfdi: - // https://github.com/llvm-mirror/compiler-rt/blob/master/lib/builtins/fixsfdi.c + // https://github.com/llvm/llvm-project/blob/master/compiler-rt/lib/builtins/fixsfdi.c unsigned SrcEltBits = SrcVT.getScalarSizeInBits(); EVT IntVT = SrcVT.changeTypeToInteger(); EVT IntShVT = getShiftAmountTy(IntVT, DAG.getDataLayout()); @@ -4544,6 +5097,17 @@ SDValue TargetLowering::expandFMINNUM_FMAXNUM(SDNode *Node, return DAG.getNode(NewOp, dl, VT, Quiet0, Quiet1, Node->getFlags()); } + // If the target has FMINIMUM/FMAXIMUM but not FMINNUM/FMAXNUM use that + // instead if there are no NaNs. + if (Node->getFlags().hasNoNaNs()) { + unsigned IEEE2018Op = + Node->getOpcode() == ISD::FMINNUM ? ISD::FMINIMUM : ISD::FMAXIMUM; + if (isOperationLegalOrCustom(IEEE2018Op, VT)) { + return DAG.getNode(IEEE2018Op, dl, VT, Node->getOperand(0), + Node->getOperand(1), Node->getFlags()); + } + } + return SDValue(); } @@ -4771,7 +5335,7 @@ SDValue TargetLowering::scalarizeVectorLoad(LoadSDNode *LD, SDValue NewChain = DAG.getNode(ISD::TokenFactor, SL, MVT::Other, LoadChains); SDValue Value = DAG.getBuildVector(LD->getValueType(0), SL, Vals); - return DAG.getMergeValues({ Value, NewChain }, SL); + return DAG.getMergeValues({Value, NewChain}, SL); } SDValue TargetLowering::scalarizeVectorStore(StoreSDNode *ST, @@ -4826,7 +5390,7 @@ SDValue TargetLowering::scalarizeVectorStore(StoreSDNode *ST, // Store Stride in bytes unsigned Stride = MemSclVT.getSizeInBits() / 8; - assert (Stride && "Zero stride!"); + assert(Stride && "Zero stride!"); // Extract each of the elements from the original vector and save them into // memory individually. SmallVector Stores; @@ -5013,17 +5577,16 @@ SDValue TargetLowering::expandUnalignedStore(StoreSDNode *ST, EVT VT = Val.getValueType(); int Alignment = ST->getAlignment(); auto &MF = DAG.getMachineFunction(); - EVT MemVT = ST->getMemoryVT(); + EVT StoreMemVT = ST->getMemoryVT(); SDLoc dl(ST); - if (MemVT.isFloatingPoint() || MemVT.isVector()) { + if (StoreMemVT.isFloatingPoint() || StoreMemVT.isVector()) { EVT intVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); if (isTypeLegal(intVT)) { if (!isOperationLegalOrCustom(ISD::STORE, intVT) && - MemVT.isVector()) { + StoreMemVT.isVector()) { // Scalarize the store and let the individual components be handled. SDValue Result = scalarizeVectorStore(ST, DAG); - return Result; } // Expand to a bitconvert of the value to the integer type of the @@ -5036,24 +5599,22 @@ SDValue TargetLowering::expandUnalignedStore(StoreSDNode *ST, } // Do a (aligned) store to a stack slot, then copy from the stack slot // to the final destination using (unaligned) integer loads and stores. - EVT StoredVT = ST->getMemoryVT(); - MVT RegVT = - getRegisterType(*DAG.getContext(), - EVT::getIntegerVT(*DAG.getContext(), - StoredVT.getSizeInBits())); + MVT RegVT = getRegisterType( + *DAG.getContext(), + EVT::getIntegerVT(*DAG.getContext(), StoreMemVT.getSizeInBits())); EVT PtrVT = Ptr.getValueType(); - unsigned StoredBytes = StoredVT.getStoreSize(); + unsigned StoredBytes = StoreMemVT.getStoreSize(); unsigned RegBytes = RegVT.getSizeInBits() / 8; unsigned NumRegs = (StoredBytes + RegBytes - 1) / RegBytes; // Make sure the stack slot is also aligned for the register type. - SDValue StackPtr = DAG.CreateStackTemporary(StoredVT, RegVT); + SDValue StackPtr = DAG.CreateStackTemporary(StoreMemVT, RegVT); auto FrameIndex = cast(StackPtr.getNode())->getIndex(); // Perform the original store, only redirected to the stack slot. SDValue Store = DAG.getTruncStore( Chain, dl, Val, StackPtr, - MachinePointerInfo::getFixedStack(MF, FrameIndex, 0), StoredVT); + MachinePointerInfo::getFixedStack(MF, FrameIndex, 0), StoreMemVT); EVT StackPtrVT = StackPtr.getValueType(); @@ -5082,17 +5643,17 @@ SDValue TargetLowering::expandUnalignedStore(StoreSDNode *ST, // The last store may be partial. Do a truncating store. On big-endian // machines this requires an extending load from the stack slot to ensure // that the bits are in the right place. - EVT MemVT = EVT::getIntegerVT(*DAG.getContext(), - 8 * (StoredBytes - Offset)); + EVT LoadMemVT = + EVT::getIntegerVT(*DAG.getContext(), 8 * (StoredBytes - Offset)); // Load from the stack slot. SDValue Load = DAG.getExtLoad( ISD::EXTLOAD, dl, RegVT, Store, StackPtr, - MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), MemVT); + MachinePointerInfo::getFixedStack(MF, FrameIndex, Offset), LoadMemVT); Stores.push_back( DAG.getTruncStore(Load.getValue(1), dl, Load, Ptr, - ST->getPointerInfo().getWithOffset(Offset), MemVT, + ST->getPointerInfo().getWithOffset(Offset), LoadMemVT, MinAlign(ST->getAlignment(), Offset), ST->getMemOperand()->getFlags(), ST->getAAInfo())); // The order of the stores doesn't matter - say it with a TokenFactor. @@ -5100,18 +5661,16 @@ SDValue TargetLowering::expandUnalignedStore(StoreSDNode *ST, return Result; } - assert(ST->getMemoryVT().isInteger() && - !ST->getMemoryVT().isVector() && + assert(StoreMemVT.isInteger() && !StoreMemVT.isVector() && "Unaligned store of unknown type."); // Get the half-size VT - EVT NewStoredVT = ST->getMemoryVT().getHalfSizedIntegerVT(*DAG.getContext()); + EVT NewStoredVT = StoreMemVT.getHalfSizedIntegerVT(*DAG.getContext()); int NumBits = NewStoredVT.getSizeInBits(); int IncrementSize = NumBits / 8; // Divide the stored value in two parts. - SDValue ShiftAmount = - DAG.getConstant(NumBits, dl, getShiftAmountTy(Val.getValueType(), - DAG.getDataLayout())); + SDValue ShiftAmount = DAG.getConstant( + NumBits, dl, getShiftAmountTy(Val.getValueType(), DAG.getDataLayout())); SDValue Lo = Val; SDValue Hi = DAG.getNode(ISD::SRL, dl, VT, Val, ShiftAmount); @@ -5130,7 +5689,7 @@ SDValue TargetLowering::expandUnalignedStore(StoreSDNode *ST, ST->getMemOperand()->getFlags(), ST->getAAInfo()); SDValue Result = - DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2); + DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Store1, Store2); return Result; } @@ -5242,7 +5801,7 @@ SDValue TargetLowering::LowerToTLSEmulatedModel(const GlobalAddressSDNode *GA, // TLSADDR will be codegen'ed as call. Inform MFI that function has calls. // At last for X86 targets, maybe good for other targets too? MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); - MFI.setAdjustsStack(true); // Is this only for X86 target? + MFI.setAdjustsStack(true); // Is this only for X86 target? MFI.setHasCalls(true); assert((GA->getOffset() == 0) && @@ -5282,15 +5841,19 @@ SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const { EVT VT = LHS.getValueType(); SDLoc dl(Node); + assert(VT == RHS.getValueType() && "Expected operands to be the same type"); + assert(VT.isInteger() && "Expected operands to be integers"); + // usub.sat(a, b) -> umax(a, b) - b if (Opcode == ISD::USUBSAT && isOperationLegalOrCustom(ISD::UMAX, VT)) { SDValue Max = DAG.getNode(ISD::UMAX, dl, VT, LHS, RHS); return DAG.getNode(ISD::SUB, dl, VT, Max, RHS); } - if (VT.isVector()) { - // TODO: Consider not scalarizing here. - return SDValue(); + if (Opcode == ISD::UADDSAT && isOperationLegalOrCustom(ISD::UMIN, VT)) { + SDValue InvRHS = DAG.getNOT(dl, RHS, VT); + SDValue Min = DAG.getNode(ISD::UMIN, dl, VT, LHS, InvRHS); + return DAG.getNode(ISD::ADD, dl, VT, Min, RHS); } unsigned OverflowOp; @@ -5312,96 +5875,410 @@ SDValue TargetLowering::expandAddSubSat(SDNode *Node, SelectionDAG &DAG) const { "addition or subtraction node."); } - assert(LHS.getValueType().isScalarInteger() && - "Expected operands to be integers. Vector of int arguments should " - "already be unrolled."); - assert(RHS.getValueType().isScalarInteger() && - "Expected operands to be integers. Vector of int arguments should " - "already be unrolled."); - assert(LHS.getValueType() == RHS.getValueType() && - "Expected both operands to be the same type"); - - unsigned BitWidth = LHS.getValueSizeInBits(); - EVT ResultType = LHS.getValueType(); - EVT BoolVT = - getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ResultType); - SDValue Result = - DAG.getNode(OverflowOp, dl, DAG.getVTList(ResultType, BoolVT), LHS, RHS); + unsigned BitWidth = LHS.getScalarValueSizeInBits(); + EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); + SDValue Result = DAG.getNode(OverflowOp, dl, DAG.getVTList(VT, BoolVT), + LHS, RHS); SDValue SumDiff = Result.getValue(0); SDValue Overflow = Result.getValue(1); - SDValue Zero = DAG.getConstant(0, dl, ResultType); + SDValue Zero = DAG.getConstant(0, dl, VT); + SDValue AllOnes = DAG.getAllOnesConstant(dl, VT); if (Opcode == ISD::UADDSAT) { - // Just need to check overflow for SatMax. - APInt MaxVal = APInt::getMaxValue(BitWidth); - SDValue SatMax = DAG.getConstant(MaxVal, dl, ResultType); - return DAG.getSelect(dl, ResultType, Overflow, SatMax, SumDiff); + if (getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) { + // (LHS + RHS) | OverflowMask + SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT); + return DAG.getNode(ISD::OR, dl, VT, SumDiff, OverflowMask); + } + // Overflow ? 0xffff.... : (LHS + RHS) + return DAG.getSelect(dl, VT, Overflow, AllOnes, SumDiff); } else if (Opcode == ISD::USUBSAT) { - // Just need to check overflow for SatMin. - APInt MinVal = APInt::getMinValue(BitWidth); - SDValue SatMin = DAG.getConstant(MinVal, dl, ResultType); - return DAG.getSelect(dl, ResultType, Overflow, SatMin, SumDiff); + if (getBooleanContents(VT) == ZeroOrNegativeOneBooleanContent) { + // (LHS - RHS) & ~OverflowMask + SDValue OverflowMask = DAG.getSExtOrTrunc(Overflow, dl, VT); + SDValue Not = DAG.getNOT(dl, OverflowMask, VT); + return DAG.getNode(ISD::AND, dl, VT, SumDiff, Not); + } + // Overflow ? 0 : (LHS - RHS) + return DAG.getSelect(dl, VT, Overflow, Zero, SumDiff); } else { // SatMax -> Overflow && SumDiff < 0 // SatMin -> Overflow && SumDiff >= 0 APInt MinVal = APInt::getSignedMinValue(BitWidth); APInt MaxVal = APInt::getSignedMaxValue(BitWidth); - SDValue SatMin = DAG.getConstant(MinVal, dl, ResultType); - SDValue SatMax = DAG.getConstant(MaxVal, dl, ResultType); + SDValue SatMin = DAG.getConstant(MinVal, dl, VT); + SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); SDValue SumNeg = DAG.getSetCC(dl, BoolVT, SumDiff, Zero, ISD::SETLT); - Result = DAG.getSelect(dl, ResultType, SumNeg, SatMax, SatMin); - return DAG.getSelect(dl, ResultType, Overflow, Result, SumDiff); + Result = DAG.getSelect(dl, VT, SumNeg, SatMax, SatMin); + return DAG.getSelect(dl, VT, Overflow, Result, SumDiff); } } SDValue -TargetLowering::getExpandedFixedPointMultiplication(SDNode *Node, - SelectionDAG &DAG) const { - assert(Node->getOpcode() == ISD::SMULFIX && "Expected opcode to be SMULFIX."); - assert(Node->getNumOperands() == 3 && - "Expected signed fixed point multiplication to have 3 operands."); +TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const { + assert((Node->getOpcode() == ISD::SMULFIX || + Node->getOpcode() == ISD::UMULFIX || + Node->getOpcode() == ISD::SMULFIXSAT) && + "Expected a fixed point multiplication opcode"); SDLoc dl(Node); SDValue LHS = Node->getOperand(0); SDValue RHS = Node->getOperand(1); - assert(LHS.getValueType().isScalarInteger() && - "Expected operands to be integers. Vector of int arguments should " - "already be unrolled."); - assert(RHS.getValueType().isScalarInteger() && - "Expected operands to be integers. Vector of int arguments should " - "already be unrolled."); - assert(LHS.getValueType() == RHS.getValueType() && - "Expected both operands to be the same type"); - - unsigned Scale = Node->getConstantOperandVal(2); EVT VT = LHS.getValueType(); - assert(Scale < VT.getScalarSizeInBits() && - "Expected scale to be less than the number of bits."); + unsigned Scale = Node->getConstantOperandVal(2); + bool Saturating = Node->getOpcode() == ISD::SMULFIXSAT; + EVT BoolVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); + unsigned VTSize = VT.getScalarSizeInBits(); + + if (!Scale) { + // [us]mul.fix(a, b, 0) -> mul(a, b) + if (!Saturating && isOperationLegalOrCustom(ISD::MUL, VT)) { + return DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); + } else if (Saturating && isOperationLegalOrCustom(ISD::SMULO, VT)) { + SDValue Result = + DAG.getNode(ISD::SMULO, dl, DAG.getVTList(VT, BoolVT), LHS, RHS); + SDValue Product = Result.getValue(0); + SDValue Overflow = Result.getValue(1); + SDValue Zero = DAG.getConstant(0, dl, VT); + + APInt MinVal = APInt::getSignedMinValue(VTSize); + APInt MaxVal = APInt::getSignedMaxValue(VTSize); + SDValue SatMin = DAG.getConstant(MinVal, dl, VT); + SDValue SatMax = DAG.getConstant(MaxVal, dl, VT); + SDValue ProdNeg = DAG.getSetCC(dl, BoolVT, Product, Zero, ISD::SETLT); + Result = DAG.getSelect(dl, VT, ProdNeg, SatMax, SatMin); + return DAG.getSelect(dl, VT, Overflow, Result, Product); + } + } - if (!Scale) - return DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); + bool Signed = + Node->getOpcode() == ISD::SMULFIX || Node->getOpcode() == ISD::SMULFIXSAT; + assert(((Signed && Scale < VTSize) || (!Signed && Scale <= VTSize)) && + "Expected scale to be less than the number of bits if signed or at " + "most the number of bits if unsigned."); + assert(LHS.getValueType() == RHS.getValueType() && + "Expected both operands to be the same type"); // Get the upper and lower bits of the result. SDValue Lo, Hi; - if (isOperationLegalOrCustom(ISD::SMUL_LOHI, VT)) { - SDValue Result = - DAG.getNode(ISD::SMUL_LOHI, dl, DAG.getVTList(VT, VT), LHS, RHS); + unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI; + unsigned HiOp = Signed ? ISD::MULHS : ISD::MULHU; + if (isOperationLegalOrCustom(LoHiOp, VT)) { + SDValue Result = DAG.getNode(LoHiOp, dl, DAG.getVTList(VT, VT), LHS, RHS); Lo = Result.getValue(0); Hi = Result.getValue(1); - } else if (isOperationLegalOrCustom(ISD::MULHS, VT)) { + } else if (isOperationLegalOrCustom(HiOp, VT)) { Lo = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); - Hi = DAG.getNode(ISD::MULHS, dl, VT, LHS, RHS); + Hi = DAG.getNode(HiOp, dl, VT, LHS, RHS); + } else if (VT.isVector()) { + return SDValue(); } else { - report_fatal_error("Unable to expand signed fixed point multiplication."); + report_fatal_error("Unable to expand fixed point multiplication."); } + if (Scale == VTSize) + // Result is just the top half since we'd be shifting by the width of the + // operand. + return Hi; + // The result will need to be shifted right by the scale since both operands // are scaled. The result is given to us in 2 halves, so we only want part of // both in the result. EVT ShiftTy = getShiftAmountTy(VT, DAG.getDataLayout()); - Lo = DAG.getNode(ISD::SRL, dl, VT, Lo, DAG.getConstant(Scale, dl, ShiftTy)); - Hi = DAG.getNode( - ISD::SHL, dl, VT, Hi, - DAG.getConstant(VT.getScalarSizeInBits() - Scale, dl, ShiftTy)); - return DAG.getNode(ISD::OR, dl, VT, Lo, Hi); + SDValue Result = DAG.getNode(ISD::FSHR, dl, VT, Hi, Lo, + DAG.getConstant(Scale, dl, ShiftTy)); + if (!Saturating) + return Result; + + unsigned OverflowBits = VTSize - Scale + 1; // +1 for the sign + SDValue HiMask = + DAG.getConstant(APInt::getHighBitsSet(VTSize, OverflowBits), dl, VT); + SDValue LoMask = DAG.getConstant( + APInt::getLowBitsSet(VTSize, VTSize - OverflowBits), dl, VT); + APInt MaxVal = APInt::getSignedMaxValue(VTSize); + APInt MinVal = APInt::getSignedMinValue(VTSize); + + Result = DAG.getSelectCC(dl, Hi, LoMask, + DAG.getConstant(MaxVal, dl, VT), Result, + ISD::SETGT); + return DAG.getSelectCC(dl, Hi, HiMask, + DAG.getConstant(MinVal, dl, VT), Result, + ISD::SETLT); +} + +void TargetLowering::expandUADDSUBO( + SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const { + SDLoc dl(Node); + SDValue LHS = Node->getOperand(0); + SDValue RHS = Node->getOperand(1); + bool IsAdd = Node->getOpcode() == ISD::UADDO; + + // If ADD/SUBCARRY is legal, use that instead. + unsigned OpcCarry = IsAdd ? ISD::ADDCARRY : ISD::SUBCARRY; + if (isOperationLegalOrCustom(OpcCarry, Node->getValueType(0))) { + SDValue CarryIn = DAG.getConstant(0, dl, Node->getValueType(1)); + SDValue NodeCarry = DAG.getNode(OpcCarry, dl, Node->getVTList(), + { LHS, RHS, CarryIn }); + Result = SDValue(NodeCarry.getNode(), 0); + Overflow = SDValue(NodeCarry.getNode(), 1); + return; + } + + Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl, + LHS.getValueType(), LHS, RHS); + + EVT ResultType = Node->getValueType(1); + EVT SetCCType = getSetCCResultType( + DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0)); + ISD::CondCode CC = IsAdd ? ISD::SETULT : ISD::SETUGT; + SDValue SetCC = DAG.getSetCC(dl, SetCCType, Result, LHS, CC); + Overflow = DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType); +} + +void TargetLowering::expandSADDSUBO( + SDNode *Node, SDValue &Result, SDValue &Overflow, SelectionDAG &DAG) const { + SDLoc dl(Node); + SDValue LHS = Node->getOperand(0); + SDValue RHS = Node->getOperand(1); + bool IsAdd = Node->getOpcode() == ISD::SADDO; + + Result = DAG.getNode(IsAdd ? ISD::ADD : ISD::SUB, dl, + LHS.getValueType(), LHS, RHS); + + EVT ResultType = Node->getValueType(1); + EVT OType = getSetCCResultType( + DAG.getDataLayout(), *DAG.getContext(), Node->getValueType(0)); + + // If SADDSAT/SSUBSAT is legal, compare results to detect overflow. + unsigned OpcSat = IsAdd ? ISD::SADDSAT : ISD::SSUBSAT; + if (isOperationLegalOrCustom(OpcSat, LHS.getValueType())) { + SDValue Sat = DAG.getNode(OpcSat, dl, LHS.getValueType(), LHS, RHS); + SDValue SetCC = DAG.getSetCC(dl, OType, Result, Sat, ISD::SETNE); + Overflow = DAG.getBoolExtOrTrunc(SetCC, dl, ResultType, ResultType); + return; + } + + SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType()); + + // LHSSign -> LHS >= 0 + // RHSSign -> RHS >= 0 + // SumSign -> Result >= 0 + // + // Add: + // Overflow -> (LHSSign == RHSSign) && (LHSSign != SumSign) + // Sub: + // Overflow -> (LHSSign != RHSSign) && (LHSSign != SumSign) + SDValue LHSSign = DAG.getSetCC(dl, OType, LHS, Zero, ISD::SETGE); + SDValue RHSSign = DAG.getSetCC(dl, OType, RHS, Zero, ISD::SETGE); + SDValue SignsMatch = DAG.getSetCC(dl, OType, LHSSign, RHSSign, + IsAdd ? ISD::SETEQ : ISD::SETNE); + + SDValue SumSign = DAG.getSetCC(dl, OType, Result, Zero, ISD::SETGE); + SDValue SumSignNE = DAG.getSetCC(dl, OType, LHSSign, SumSign, ISD::SETNE); + + SDValue Cmp = DAG.getNode(ISD::AND, dl, OType, SignsMatch, SumSignNE); + Overflow = DAG.getBoolExtOrTrunc(Cmp, dl, ResultType, ResultType); +} + +bool TargetLowering::expandMULO(SDNode *Node, SDValue &Result, + SDValue &Overflow, SelectionDAG &DAG) const { + SDLoc dl(Node); + EVT VT = Node->getValueType(0); + EVT SetCCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT); + SDValue LHS = Node->getOperand(0); + SDValue RHS = Node->getOperand(1); + bool isSigned = Node->getOpcode() == ISD::SMULO; + + // For power-of-two multiplications we can use a simpler shift expansion. + if (ConstantSDNode *RHSC = isConstOrConstSplat(RHS)) { + const APInt &C = RHSC->getAPIntValue(); + // mulo(X, 1 << S) -> { X << S, (X << S) >> S != X } + if (C.isPowerOf2()) { + // smulo(x, signed_min) is same as umulo(x, signed_min). + bool UseArithShift = isSigned && !C.isMinSignedValue(); + EVT ShiftAmtTy = getShiftAmountTy(VT, DAG.getDataLayout()); + SDValue ShiftAmt = DAG.getConstant(C.logBase2(), dl, ShiftAmtTy); + Result = DAG.getNode(ISD::SHL, dl, VT, LHS, ShiftAmt); + Overflow = DAG.getSetCC(dl, SetCCVT, + DAG.getNode(UseArithShift ? ISD::SRA : ISD::SRL, + dl, VT, Result, ShiftAmt), + LHS, ISD::SETNE); + return true; + } + } + + EVT WideVT = EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits() * 2); + if (VT.isVector()) + WideVT = EVT::getVectorVT(*DAG.getContext(), WideVT, + VT.getVectorNumElements()); + + SDValue BottomHalf; + SDValue TopHalf; + static const unsigned Ops[2][3] = + { { ISD::MULHU, ISD::UMUL_LOHI, ISD::ZERO_EXTEND }, + { ISD::MULHS, ISD::SMUL_LOHI, ISD::SIGN_EXTEND }}; + if (isOperationLegalOrCustom(Ops[isSigned][0], VT)) { + BottomHalf = DAG.getNode(ISD::MUL, dl, VT, LHS, RHS); + TopHalf = DAG.getNode(Ops[isSigned][0], dl, VT, LHS, RHS); + } else if (isOperationLegalOrCustom(Ops[isSigned][1], VT)) { + BottomHalf = DAG.getNode(Ops[isSigned][1], dl, DAG.getVTList(VT, VT), LHS, + RHS); + TopHalf = BottomHalf.getValue(1); + } else if (isTypeLegal(WideVT)) { + LHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, LHS); + RHS = DAG.getNode(Ops[isSigned][2], dl, WideVT, RHS); + SDValue Mul = DAG.getNode(ISD::MUL, dl, WideVT, LHS, RHS); + BottomHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, Mul); + SDValue ShiftAmt = DAG.getConstant(VT.getScalarSizeInBits(), dl, + getShiftAmountTy(WideVT, DAG.getDataLayout())); + TopHalf = DAG.getNode(ISD::TRUNCATE, dl, VT, + DAG.getNode(ISD::SRL, dl, WideVT, Mul, ShiftAmt)); + } else { + if (VT.isVector()) + return false; + + // We can fall back to a libcall with an illegal type for the MUL if we + // have a libcall big enough. + // Also, we can fall back to a division in some cases, but that's a big + // performance hit in the general case. + RTLIB::Libcall LC = RTLIB::UNKNOWN_LIBCALL; + if (WideVT == MVT::i16) + LC = RTLIB::MUL_I16; + else if (WideVT == MVT::i32) + LC = RTLIB::MUL_I32; + else if (WideVT == MVT::i64) + LC = RTLIB::MUL_I64; + else if (WideVT == MVT::i128) + LC = RTLIB::MUL_I128; + assert(LC != RTLIB::UNKNOWN_LIBCALL && "Cannot expand this operation!"); + + SDValue HiLHS; + SDValue HiRHS; + if (isSigned) { + // The high part is obtained by SRA'ing all but one of the bits of low + // part. + unsigned LoSize = VT.getSizeInBits(); + HiLHS = + DAG.getNode(ISD::SRA, dl, VT, LHS, + DAG.getConstant(LoSize - 1, dl, + getPointerTy(DAG.getDataLayout()))); + HiRHS = + DAG.getNode(ISD::SRA, dl, VT, RHS, + DAG.getConstant(LoSize - 1, dl, + getPointerTy(DAG.getDataLayout()))); + } else { + HiLHS = DAG.getConstant(0, dl, VT); + HiRHS = DAG.getConstant(0, dl, VT); + } + + // Here we're passing the 2 arguments explicitly as 4 arguments that are + // pre-lowered to the correct types. This all depends upon WideVT not + // being a legal type for the architecture and thus has to be split to + // two arguments. + SDValue Ret; + if (shouldSplitFunctionArgumentsAsLittleEndian(DAG.getDataLayout())) { + // Halves of WideVT are packed into registers in different order + // depending on platform endianness. This is usually handled by + // the C calling convention, but we can't defer to it in + // the legalizer. + SDValue Args[] = { LHS, HiLHS, RHS, HiRHS }; + Ret = makeLibCall(DAG, LC, WideVT, Args, isSigned, dl, + /* doesNotReturn */ false, /* isReturnValueUsed */ true, + /* isPostTypeLegalization */ true).first; + } else { + SDValue Args[] = { HiLHS, LHS, HiRHS, RHS }; + Ret = makeLibCall(DAG, LC, WideVT, Args, isSigned, dl, + /* doesNotReturn */ false, /* isReturnValueUsed */ true, + /* isPostTypeLegalization */ true).first; + } + assert(Ret.getOpcode() == ISD::MERGE_VALUES && + "Ret value is a collection of constituent nodes holding result."); + if (DAG.getDataLayout().isLittleEndian()) { + // Same as above. + BottomHalf = Ret.getOperand(0); + TopHalf = Ret.getOperand(1); + } else { + BottomHalf = Ret.getOperand(1); + TopHalf = Ret.getOperand(0); + } + } + + Result = BottomHalf; + if (isSigned) { + SDValue ShiftAmt = DAG.getConstant( + VT.getScalarSizeInBits() - 1, dl, + getShiftAmountTy(BottomHalf.getValueType(), DAG.getDataLayout())); + SDValue Sign = DAG.getNode(ISD::SRA, dl, VT, BottomHalf, ShiftAmt); + Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf, Sign, ISD::SETNE); + } else { + Overflow = DAG.getSetCC(dl, SetCCVT, TopHalf, + DAG.getConstant(0, dl, VT), ISD::SETNE); + } + + // Truncate the result if SetCC returns a larger type than needed. + EVT RType = Node->getValueType(1); + if (RType.getSizeInBits() < Overflow.getValueSizeInBits()) + Overflow = DAG.getNode(ISD::TRUNCATE, dl, RType, Overflow); + + assert(RType.getSizeInBits() == Overflow.getValueSizeInBits() && + "Unexpected result type for S/UMULO legalization"); + return true; +} + +SDValue TargetLowering::expandVecReduce(SDNode *Node, SelectionDAG &DAG) const { + SDLoc dl(Node); + bool NoNaN = Node->getFlags().hasNoNaNs(); + unsigned BaseOpcode = 0; + switch (Node->getOpcode()) { + default: llvm_unreachable("Expected VECREDUCE opcode"); + case ISD::VECREDUCE_FADD: BaseOpcode = ISD::FADD; break; + case ISD::VECREDUCE_FMUL: BaseOpcode = ISD::FMUL; break; + case ISD::VECREDUCE_ADD: BaseOpcode = ISD::ADD; break; + case ISD::VECREDUCE_MUL: BaseOpcode = ISD::MUL; break; + case ISD::VECREDUCE_AND: BaseOpcode = ISD::AND; break; + case ISD::VECREDUCE_OR: BaseOpcode = ISD::OR; break; + case ISD::VECREDUCE_XOR: BaseOpcode = ISD::XOR; break; + case ISD::VECREDUCE_SMAX: BaseOpcode = ISD::SMAX; break; + case ISD::VECREDUCE_SMIN: BaseOpcode = ISD::SMIN; break; + case ISD::VECREDUCE_UMAX: BaseOpcode = ISD::UMAX; break; + case ISD::VECREDUCE_UMIN: BaseOpcode = ISD::UMIN; break; + case ISD::VECREDUCE_FMAX: + BaseOpcode = NoNaN ? ISD::FMAXNUM : ISD::FMAXIMUM; + break; + case ISD::VECREDUCE_FMIN: + BaseOpcode = NoNaN ? ISD::FMINNUM : ISD::FMINIMUM; + break; + } + + SDValue Op = Node->getOperand(0); + EVT VT = Op.getValueType(); + + // Try to use a shuffle reduction for power of two vectors. + if (VT.isPow2VectorType()) { + while (VT.getVectorNumElements() > 1) { + EVT HalfVT = VT.getHalfNumVectorElementsVT(*DAG.getContext()); + if (!isOperationLegalOrCustom(BaseOpcode, HalfVT)) + break; + + SDValue Lo, Hi; + std::tie(Lo, Hi) = DAG.SplitVector(Op, dl); + Op = DAG.getNode(BaseOpcode, dl, HalfVT, Lo, Hi); + VT = HalfVT; + } + } + + EVT EltVT = VT.getVectorElementType(); + unsigned NumElts = VT.getVectorNumElements(); + + SmallVector Ops; + DAG.ExtractVectorElements(Op, Ops, 0, NumElts); + + SDValue Res = Ops[0]; + for (unsigned i = 1; i < NumElts; i++) + Res = DAG.getNode(BaseOpcode, dl, EltVT, Res, Ops[i], Node->getFlags()); + + // Result type may be wider than element type. + if (EltVT != Node->getValueType(0)) + Res = DAG.getNode(ISD::ANY_EXTEND, dl, Node->getValueType(0), Res); + return Res; } diff --git a/lib/CodeGen/ShadowStackGCLowering.cpp b/lib/CodeGen/ShadowStackGCLowering.cpp index 3e12b32b12d4..17a4d76c4c80 100644 --- a/lib/CodeGen/ShadowStackGCLowering.cpp +++ b/lib/CodeGen/ShadowStackGCLowering.cpp @@ -1,9 +1,8 @@ //===- ShadowStackGCLowering.cpp - Custom lowering for shadow-stack gc ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -313,7 +312,8 @@ bool ShadowStackGCLowering::runOnFunction(Function &F) { AtEntry.SetInsertPoint(IP->getParent(), IP); // Initialize the map pointer and load the current head of the shadow stack. - Instruction *CurrentHead = AtEntry.CreateLoad(Head, "gc_currhead"); + Instruction *CurrentHead = + AtEntry.CreateLoad(StackEntryTy->getPointerTo(), Head, "gc_currhead"); Instruction *EntryMapPtr = CreateGEP(Context, AtEntry, ConcreteStackEntryTy, StackEntry, 0, 1, "gc_frame.map"); AtEntry.CreateStore(FrameMap, EntryMapPtr); @@ -354,7 +354,8 @@ bool ShadowStackGCLowering::runOnFunction(Function &F) { Instruction *EntryNextPtr2 = CreateGEP(Context, *AtExit, ConcreteStackEntryTy, StackEntry, 0, 0, "gc_frame.next"); - Value *SavedHead = AtExit->CreateLoad(EntryNextPtr2, "gc_savedhead"); + Value *SavedHead = AtExit->CreateLoad(StackEntryTy->getPointerTo(), + EntryNextPtr2, "gc_savedhead"); AtExit->CreateStore(SavedHead, Head); } diff --git a/lib/CodeGen/ShrinkWrap.cpp b/lib/CodeGen/ShrinkWrap.cpp index d3454ca6ba6a..2db0ea570598 100644 --- a/lib/CodeGen/ShrinkWrap.cpp +++ b/lib/CodeGen/ShrinkWrap.cpp @@ -1,9 +1,8 @@ //===- ShrinkWrap.cpp - Compute safe point for prolog/epilog insertion ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -259,6 +258,15 @@ INITIALIZE_PASS_END(ShrinkWrap, DEBUG_TYPE, "Shrink Wrap Pass", false, false) bool ShrinkWrap::useOrDefCSROrFI(const MachineInstr &MI, RegScavenger *RS) const { + // This prevents premature stack popping when occurs a indirect stack + // access. It is overly aggressive for the moment. + // TODO: - Obvious non-stack loads and store, such as global values, + // are known to not access the stack. + // - Further, data dependency and alias analysis can validate + // that load and stores never derive from the stack pointer. + if (MI.mayLoadOrStore()) + return true; + if (MI.getOpcode() == FrameSetupOpcode || MI.getOpcode() == FrameDestroyOpcode) { LLVM_DEBUG(dbgs() << "Frame instruction: " << MI << '\n'); diff --git a/lib/CodeGen/SjLjEHPrepare.cpp b/lib/CodeGen/SjLjEHPrepare.cpp index 5d2669f5ae92..23e5ce0acae8 100644 --- a/lib/CodeGen/SjLjEHPrepare.cpp +++ b/lib/CodeGen/SjLjEHPrepare.cpp @@ -1,9 +1,8 @@ //===- SjLjEHPrepare.cpp - Eliminate Invoke & Unwind instructions ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -40,15 +39,15 @@ class SjLjEHPrepare : public FunctionPass { Type *doubleUnderDataTy; Type *doubleUnderJBufTy; Type *FunctionContextTy; - Constant *RegisterFn; - Constant *UnregisterFn; - Constant *BuiltinSetupDispatchFn; - Constant *FrameAddrFn; - Constant *StackAddrFn; - Constant *StackRestoreFn; - Constant *LSDAAddrFn; - Constant *CallSiteFn; - Constant *FuncCtxFn; + FunctionCallee RegisterFn; + FunctionCallee UnregisterFn; + Function *BuiltinSetupDispatchFn; + Function *FrameAddrFn; + Function *StackAddrFn; + Function *StackRestoreFn; + Function *LSDAAddrFn; + Function *CallSiteFn; + Function *FuncCtxFn; AllocaInst *FuncCtx; public: @@ -190,14 +189,16 @@ Value *SjLjEHPrepare::setupFunctionContext(Function &F, Builder.CreateConstGEP2_32(FunctionContextTy, FuncCtx, 0, 2, "__data"); // The exception values come back in context->__data[0]. + Type *Int32Ty = Type::getInt32Ty(F.getContext()); Value *ExceptionAddr = Builder.CreateConstGEP2_32(doubleUnderDataTy, FCData, 0, 0, "exception_gep"); - Value *ExnVal = Builder.CreateLoad(ExceptionAddr, true, "exn_val"); + Value *ExnVal = Builder.CreateLoad(Int32Ty, ExceptionAddr, true, "exn_val"); ExnVal = Builder.CreateIntToPtr(ExnVal, Builder.getInt8PtrTy()); Value *SelectorAddr = Builder.CreateConstGEP2_32(doubleUnderDataTy, FCData, 0, 1, "exn_selector_gep"); - Value *SelVal = Builder.CreateLoad(SelectorAddr, true, "exn_selector_val"); + Value *SelVal = + Builder.CreateLoad(Int32Ty, SelectorAddr, true, "exn_selector_val"); substituteLPadValues(LPI, ExnVal, SelVal); } diff --git a/lib/CodeGen/SlotIndexes.cpp b/lib/CodeGen/SlotIndexes.cpp index fccbb8ec91cb..9fff873324d0 100644 --- a/lib/CodeGen/SlotIndexes.cpp +++ b/lib/CodeGen/SlotIndexes.cpp @@ -1,9 +1,8 @@ //===-- SlotIndexes.cpp - Slot Indexes Pass ------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -23,7 +22,6 @@ INITIALIZE_PASS(SlotIndexes, DEBUG_TYPE, "Slot index numbering", false, false) STATISTIC(NumLocalRenum, "Number of local renumberings"); -STATISTIC(NumGlobalRenum, "Number of global renumberings"); void SlotIndexes::getAnalysisUsage(AnalysisUsage &au) const { au.setPreservesAll(); @@ -95,7 +93,7 @@ bool SlotIndexes::runOnMachineFunction(MachineFunction &fn) { } // Sort the Idx2MBBMap - llvm::sort(idx2MBBMap, Idx2MBBCompare()); + llvm::sort(idx2MBBMap, less_first()); LLVM_DEBUG(mf->print(dbgs(), this)); @@ -145,20 +143,6 @@ void SlotIndexes::removeSingleMachineInstrFromMaps(MachineInstr &MI) { } } -void SlotIndexes::renumberIndexes() { - // Renumber updates the index of every element of the index list. - LLVM_DEBUG(dbgs() << "\n*** Renumbering SlotIndexes ***\n"); - ++NumGlobalRenum; - - unsigned index = 0; - - for (IndexList::iterator I = indexList.begin(), E = indexList.end(); - I != E; ++I) { - I->setIndex(index); - index += SlotIndex::InstrDist; - } -} - // Renumber indexes locally after curItr was inserted, but failed to get a new // index. void SlotIndexes::renumberIndexes(IndexList::iterator curItr) { diff --git a/lib/CodeGen/SpillPlacement.cpp b/lib/CodeGen/SpillPlacement.cpp index f6786b30b21c..11452fdb747a 100644 --- a/lib/CodeGen/SpillPlacement.cpp +++ b/lib/CodeGen/SpillPlacement.cpp @@ -1,9 +1,8 @@ //===- SpillPlacement.cpp - Optimal Spill Code Placement ------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/SpillPlacement.h b/lib/CodeGen/SpillPlacement.h index aa3ac444e0da..aa0e07ef92e3 100644 --- a/lib/CodeGen/SpillPlacement.h +++ b/lib/CodeGen/SpillPlacement.h @@ -1,9 +1,8 @@ //===- SpillPlacement.h - Optimal Spill Code Placement ---------*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/Spiller.h b/lib/CodeGen/Spiller.h index 330ee81342b6..66dabf78f873 100644 --- a/lib/CodeGen/Spiller.h +++ b/lib/CodeGen/Spiller.h @@ -1,9 +1,8 @@ //===- llvm/CodeGen/Spiller.h - Spiller -------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/SplitKit.cpp b/lib/CodeGen/SplitKit.cpp index d639f4475301..5c944fe3f6b3 100644 --- a/lib/CodeGen/SplitKit.cpp +++ b/lib/CodeGen/SplitKit.cpp @@ -1,9 +1,8 @@ //===- SplitKit.cpp - Toolkit for splitting live ranges -------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -521,17 +520,18 @@ SlotIndex SplitEditor::buildSingleSubRegCopy(unsigned FromReg, unsigned ToReg, .addReg(FromReg, 0, SubIdx); BumpPtrAllocator &Allocator = LIS.getVNInfoAllocator(); + SlotIndexes &Indexes = *LIS.getSlotIndexes(); if (FirstCopy) { - SlotIndexes &Indexes = *LIS.getSlotIndexes(); Def = Indexes.insertMachineInstrInMaps(*CopyMI, Late).getRegSlot(); } else { CopyMI->bundleWithPred(); } LaneBitmask LaneMask = TRI.getSubRegIndexLaneMask(SubIdx); DestLI.refineSubRanges(Allocator, LaneMask, - [Def, &Allocator](LiveInterval::SubRange& SR) { - SR.createDeadDef(Def, Allocator); - }); + [Def, &Allocator](LiveInterval::SubRange &SR) { + SR.createDeadDef(Def, Allocator); + }, + Indexes, TRI); return Def; } diff --git a/lib/CodeGen/SplitKit.h b/lib/CodeGen/SplitKit.h index bcc8f8cf18bc..86ad3811e3ad 100644 --- a/lib/CodeGen/SplitKit.h +++ b/lib/CodeGen/SplitKit.h @@ -1,9 +1,8 @@ //===- SplitKit.h - Toolkit for splitting live ranges -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/StackColoring.cpp b/lib/CodeGen/StackColoring.cpp index eb8552915e2a..641b54205d62 100644 --- a/lib/CodeGen/StackColoring.cpp +++ b/lib/CodeGen/StackColoring.cpp @@ -1,9 +1,8 @@ //===- StackColoring.cpp --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -1221,11 +1220,12 @@ bool StackColoring::runOnMachineFunction(MachineFunction &Func) { // Sort the slots according to their size. Place unused slots at the end. // Use stable sort to guarantee deterministic code generation. - std::stable_sort(SortedSlots.begin(), SortedSlots.end(), - [this](int LHS, int RHS) { + llvm::stable_sort(SortedSlots, [this](int LHS, int RHS) { // We use -1 to denote a uninteresting slot. Place these slots at the end. - if (LHS == -1) return false; - if (RHS == -1) return true; + if (LHS == -1) + return false; + if (RHS == -1) + return true; // Sort according to size. return MFI->getObjectSize(LHS) > MFI->getObjectSize(RHS); }); diff --git a/lib/CodeGen/StackMapLivenessAnalysis.cpp b/lib/CodeGen/StackMapLivenessAnalysis.cpp index 00cf8070be5e..fb2abf3daa7f 100644 --- a/lib/CodeGen/StackMapLivenessAnalysis.cpp +++ b/lib/CodeGen/StackMapLivenessAnalysis.cpp @@ -1,9 +1,8 @@ //===-- StackMapLivenessAnalysis.cpp - StackMap live Out Analysis ----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/StackMaps.cpp b/lib/CodeGen/StackMaps.cpp index 0676fa2421e8..ae9401b89700 100644 --- a/lib/CodeGen/StackMaps.cpp +++ b/lib/CodeGen/StackMaps.cpp @@ -1,9 +1,8 @@ //===- StackMaps.cpp ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/CodeGen/StackProtector.cpp b/lib/CodeGen/StackProtector.cpp index 3b578c7391da..809960c7fdf9 100644 --- a/lib/CodeGen/StackProtector.cpp +++ b/lib/CodeGen/StackProtector.cpp @@ -1,9 +1,8 @@ //===- StackProtector.cpp - Stack Protector Insertion ---------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,6 +17,7 @@ #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/Statistic.h" #include "llvm/Analysis/BranchProbabilityInfo.h" +#include "llvm/Analysis/CaptureTracking.h" #include "llvm/Analysis/EHPersonalities.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/CodeGen/Passes.h" @@ -157,40 +157,6 @@ bool StackProtector::ContainsProtectableArray(Type *Ty, bool &IsLarge, return NeedsProtector; } -bool StackProtector::HasAddressTaken(const Instruction *AI) { - for (const User *U : AI->users()) { - if (const StoreInst *SI = dyn_cast(U)) { - if (AI == SI->getValueOperand()) - return true; - } else if (const PtrToIntInst *SI = dyn_cast(U)) { - if (AI == SI->getOperand(0)) - return true; - } else if (const CallInst *CI = dyn_cast(U)) { - // Ignore intrinsics that are not calls. TODO: Use isLoweredToCall(). - if (!isa(CI) && !CI->isLifetimeStartOrEnd()) - return true; - } else if (isa(U)) { - return true; - } else if (const SelectInst *SI = dyn_cast(U)) { - if (HasAddressTaken(SI)) - return true; - } else if (const PHINode *PN = dyn_cast(U)) { - // Keep track of what PHI nodes we have already visited to ensure - // they are only visited once. - if (VisitedPHIs.insert(PN).second) - if (HasAddressTaken(PN)) - return true; - } else if (const GetElementPtrInst *GEP = dyn_cast(U)) { - if (HasAddressTaken(GEP)) - return true; - } else if (const BitCastInst *BI = dyn_cast(U)) { - if (HasAddressTaken(BI)) - return true; - } - } - return false; -} - /// Search for the first call to the llvm.stackprotector intrinsic and return it /// if present. static const CallInst *findStackProtectorIntrinsic(Function &F) { @@ -298,7 +264,9 @@ bool StackProtector::RequiresStackProtector() { continue; } - if (Strong && HasAddressTaken(AI)) { + if (Strong && PointerMayBeCaptured(AI, + /* ReturnCaptures */ false, + /* StoreCaptures */ true)) { ++NumAddrTaken; Layout.insert(std::make_pair(AI, MachineFrameInfo::SSPLK_AddrOf)); ORE.emit([&]() { @@ -323,7 +291,7 @@ static Value *getStackGuard(const TargetLoweringBase *TLI, Module *M, IRBuilder<> &B, bool *SupportsSelectionDAGSP = nullptr) { if (Value *Guard = TLI->getIRStackGuard(B)) - return B.CreateLoad(Guard, true, "StackGuard"); + return B.CreateLoad(B.getInt8PtrTy(), Guard, true, "StackGuard"); // Use SelectionDAG SSP handling, since there isn't an IR guard. // @@ -414,15 +382,14 @@ bool StackProtector::InsertStackProtectors() { // Generate epilogue instrumentation. The epilogue intrumentation can be // function-based or inlined depending on which mechanism the target is // providing. - if (Value* GuardCheck = TLI->getSSPStackGuardCheck(*M)) { + if (Function *GuardCheck = TLI->getSSPStackGuardCheck(*M)) { // Generate the function-based epilogue instrumentation. // The target provides a guard check function, generate a call to it. IRBuilder<> B(RI); - LoadInst *Guard = B.CreateLoad(AI, true, "Guard"); + LoadInst *Guard = B.CreateLoad(B.getInt8PtrTy(), AI, true, "Guard"); CallInst *Call = B.CreateCall(GuardCheck, {Guard}); - llvm::Function *Function = cast(GuardCheck); - Call->setAttributes(Function->getAttributes()); - Call->setCallingConv(Function->getCallingConv()); + Call->setAttributes(GuardCheck->getAttributes()); + Call->setCallingConv(GuardCheck->getCallingConv()); } else { // Generate the epilogue with inline instrumentation. // If we do not support SelectionDAG based tail calls, generate IR level @@ -474,7 +441,7 @@ bool StackProtector::InsertStackProtectors() { // Generate the stack protector instructions in the old basic block. IRBuilder<> B(BB); Value *Guard = getStackGuard(TLI, M, B); - LoadInst *LI2 = B.CreateLoad(AI, true); + LoadInst *LI2 = B.CreateLoad(B.getInt8PtrTy(), AI, true); Value *Cmp = B.CreateICmpEQ(Guard, LI2); auto SuccessProb = BranchProbabilityInfo::getBranchProbStackProtector(true); @@ -500,14 +467,13 @@ BasicBlock *StackProtector::CreateFailBB() { IRBuilder<> B(FailBB); B.SetCurrentDebugLocation(DebugLoc::get(0, 0, F->getSubprogram())); if (Trip.isOSOpenBSD()) { - Constant *StackChkFail = - M->getOrInsertFunction("__stack_smash_handler", - Type::getVoidTy(Context), - Type::getInt8PtrTy(Context)); + FunctionCallee StackChkFail = M->getOrInsertFunction( + "__stack_smash_handler", Type::getVoidTy(Context), + Type::getInt8PtrTy(Context)); B.CreateCall(StackChkFail, B.CreateGlobalStringPtr(F->getName(), "SSH")); } else { - Constant *StackChkFail = + FunctionCallee StackChkFail = M->getOrInsertFunction("__stack_chk_fail", Type::getVoidTy(Context)); B.CreateCall(StackChkFail, {}); @@ -517,7 +483,7 @@ BasicBlock *StackProtector::CreateFailBB() { } bool StackProtector::shouldEmitSDCheck(const BasicBlock &BB) const { - return HasPrologue && !HasIRCheck && dyn_cast(BB.getTerminator()); + return HasPrologue && !HasIRCheck && isa(BB.getTerminator()); } void StackProtector::copyToMachineFrameInfo(MachineFrameInfo &MFI) const { diff --git a/lib/CodeGen/StackSlotColoring.cpp b/lib/CodeGen/StackSlotColoring.cpp index d8c6a249e4da..99b533e10b87 100644 --- a/lib/CodeGen/StackSlotColoring.cpp +++ b/lib/CodeGen/StackSlotColoring.cpp @@ -1,9 +1,8 @@ //===- StackSlotColoring.cpp - Stack slot coloring pass. ------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -243,7 +242,7 @@ void StackSlotColoring::InitializeSlots() { LLVM_DEBUG(dbgs() << '\n'); // Sort them by weight. - std::stable_sort(SSIntervals.begin(), SSIntervals.end(), IntervalSorter()); + llvm::stable_sort(SSIntervals, IntervalSorter()); NextColors.resize(AllColors.size()); @@ -348,7 +347,7 @@ bool StackSlotColoring::ColorSlots(MachineFunction &MF) { li->weight = SlotWeights[SS]; } // Sort them by new weight. - std::stable_sort(SSIntervals.begin(), SSIntervals.end(), IntervalSorter()); + llvm::stable_sort(SSIntervals, IntervalSorter()); #ifndef NDEBUG for (unsigned i = 0, e = SSIntervals.size(); i != e; ++i) diff --git a/lib/CodeGen/SwiftErrorValueTracking.cpp b/lib/CodeGen/SwiftErrorValueTracking.cpp new file mode 100644 index 000000000000..96821cadb1b6 --- /dev/null +++ b/lib/CodeGen/SwiftErrorValueTracking.cpp @@ -0,0 +1,312 @@ +//===-- SwiftErrorValueTracking.cpp --------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This implements a limited mem2reg-like analysis to promote uses of function +// arguments and allocas marked with swiftalloc from memory into virtual +// registers tracked by this class. +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/SwiftErrorValueTracking.h" +#include "llvm/ADT/SmallSet.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/CodeGen/TargetLowering.h" +#include "llvm/IR/Value.h" + +using namespace llvm; + +Register SwiftErrorValueTracking::getOrCreateVReg(const MachineBasicBlock *MBB, + const Value *Val) { + auto Key = std::make_pair(MBB, Val); + auto It = VRegDefMap.find(Key); + // If this is the first use of this swifterror value in this basic block, + // create a new virtual register. + // After we processed all basic blocks we will satisfy this "upwards exposed + // use" by inserting a copy or phi at the beginning of this block. + if (It == VRegDefMap.end()) { + auto &DL = MF->getDataLayout(); + const TargetRegisterClass *RC = TLI->getRegClassFor(TLI->getPointerTy(DL)); + auto VReg = MF->getRegInfo().createVirtualRegister(RC); + VRegDefMap[Key] = VReg; + VRegUpwardsUse[Key] = VReg; + return VReg; + } else + return It->second; +} + +void SwiftErrorValueTracking::setCurrentVReg(const MachineBasicBlock *MBB, + const Value *Val, Register VReg) { + VRegDefMap[std::make_pair(MBB, Val)] = VReg; +} + +Register SwiftErrorValueTracking::getOrCreateVRegDefAt( + const Instruction *I, const MachineBasicBlock *MBB, const Value *Val) { + auto Key = PointerIntPair(I, true); + auto It = VRegDefUses.find(Key); + if (It != VRegDefUses.end()) + return It->second; + + auto &DL = MF->getDataLayout(); + const TargetRegisterClass *RC = TLI->getRegClassFor(TLI->getPointerTy(DL)); + Register VReg = MF->getRegInfo().createVirtualRegister(RC); + VRegDefUses[Key] = VReg; + setCurrentVReg(MBB, Val, VReg); + return VReg; +} + +Register SwiftErrorValueTracking::getOrCreateVRegUseAt( + const Instruction *I, const MachineBasicBlock *MBB, const Value *Val) { + auto Key = PointerIntPair(I, false); + auto It = VRegDefUses.find(Key); + if (It != VRegDefUses.end()) + return It->second; + + Register VReg = getOrCreateVReg(MBB, Val); + VRegDefUses[Key] = VReg; + return VReg; +} + +/// Set up SwiftErrorVals by going through the function. If the function has +/// swifterror argument, it will be the first entry. +void SwiftErrorValueTracking::setFunction(MachineFunction &mf) { + MF = &mf; + Fn = &MF->getFunction(); + TLI = MF->getSubtarget().getTargetLowering(); + TII = MF->getSubtarget().getInstrInfo(); + + if (!TLI->supportSwiftError()) + return; + + SwiftErrorVals.clear(); + VRegDefMap.clear(); + VRegUpwardsUse.clear(); + VRegDefUses.clear(); + SwiftErrorArg = nullptr; + + // Check if function has a swifterror argument. + bool HaveSeenSwiftErrorArg = false; + for (Function::const_arg_iterator AI = Fn->arg_begin(), AE = Fn->arg_end(); + AI != AE; ++AI) + if (AI->hasSwiftErrorAttr()) { + assert(!HaveSeenSwiftErrorArg && + "Must have only one swifterror parameter"); + (void)HaveSeenSwiftErrorArg; // silence warning. + HaveSeenSwiftErrorArg = true; + SwiftErrorArg = &*AI; + SwiftErrorVals.push_back(&*AI); + } + + for (const auto &LLVMBB : *Fn) + for (const auto &Inst : LLVMBB) { + if (const AllocaInst *Alloca = dyn_cast(&Inst)) + if (Alloca->isSwiftError()) + SwiftErrorVals.push_back(Alloca); + } +} + +bool SwiftErrorValueTracking::createEntriesInEntryBlock(DebugLoc DbgLoc) { + if (!TLI->supportSwiftError()) + return false; + + // We only need to do this when we have swifterror parameter or swifterror + // alloc. + if (SwiftErrorVals.empty()) + return false; + + MachineBasicBlock *MBB = &*MF->begin(); + auto &DL = MF->getDataLayout(); + auto const *RC = TLI->getRegClassFor(TLI->getPointerTy(DL)); + bool Inserted = false; + for (const auto *SwiftErrorVal : SwiftErrorVals) { + // We will always generate a copy from the argument. It is always used at + // least by the 'return' of the swifterror. + if (SwiftErrorArg && SwiftErrorArg == SwiftErrorVal) + continue; + Register VReg = MF->getRegInfo().createVirtualRegister(RC); + // Assign Undef to Vreg. We construct MI directly to make sure it works + // with FastISel. + BuildMI(*MBB, MBB->getFirstNonPHI(), DbgLoc, + TII->get(TargetOpcode::IMPLICIT_DEF), VReg); + + setCurrentVReg(MBB, SwiftErrorVal, VReg); + Inserted = true; + } + + return Inserted; +} + +/// Propagate swifterror values through the machine function CFG. +void SwiftErrorValueTracking::propagateVRegs() { + if (!TLI->supportSwiftError()) + return; + + // We only need to do this when we have swifterror parameter or swifterror + // alloc. + if (SwiftErrorVals.empty()) + return; + + // For each machine basic block in reverse post order. + ReversePostOrderTraversal RPOT(MF); + for (MachineBasicBlock *MBB : RPOT) { + // For each swifterror value in the function. + for (const auto *SwiftErrorVal : SwiftErrorVals) { + auto Key = std::make_pair(MBB, SwiftErrorVal); + auto UUseIt = VRegUpwardsUse.find(Key); + auto VRegDefIt = VRegDefMap.find(Key); + bool UpwardsUse = UUseIt != VRegUpwardsUse.end(); + Register UUseVReg = UpwardsUse ? UUseIt->second : Register(); + bool DownwardDef = VRegDefIt != VRegDefMap.end(); + assert(!(UpwardsUse && !DownwardDef) && + "We can't have an upwards use but no downwards def"); + + // If there is no upwards exposed use and an entry for the swifterror in + // the def map for this value we don't need to do anything: We already + // have a downward def for this basic block. + if (!UpwardsUse && DownwardDef) + continue; + + // Otherwise we either have an upwards exposed use vreg that we need to + // materialize or need to forward the downward def from predecessors. + + // Check whether we have a single vreg def from all predecessors. + // Otherwise we need a phi. + SmallVector, 4> VRegs; + SmallSet Visited; + for (auto *Pred : MBB->predecessors()) { + if (!Visited.insert(Pred).second) + continue; + VRegs.push_back(std::make_pair( + Pred, getOrCreateVReg(Pred, SwiftErrorVal))); + if (Pred != MBB) + continue; + // We have a self-edge. + // If there was no upwards use in this basic block there is now one: the + // phi needs to use it self. + if (!UpwardsUse) { + UpwardsUse = true; + UUseIt = VRegUpwardsUse.find(Key); + assert(UUseIt != VRegUpwardsUse.end()); + UUseVReg = UUseIt->second; + } + } + + // We need a phi node if we have more than one predecessor with different + // downward defs. + bool needPHI = + VRegs.size() >= 1 && + std::find_if( + VRegs.begin(), VRegs.end(), + [&](const std::pair &V) + -> bool { return V.second != VRegs[0].second; }) != + VRegs.end(); + + // If there is no upwards exposed used and we don't need a phi just + // forward the swifterror vreg from the predecessor(s). + if (!UpwardsUse && !needPHI) { + assert(!VRegs.empty() && + "No predecessors? The entry block should bail out earlier"); + // Just forward the swifterror vreg from the predecessor(s). + setCurrentVReg(MBB, SwiftErrorVal, VRegs[0].second); + continue; + } + + auto DLoc = isa(SwiftErrorVal) + ? cast(SwiftErrorVal)->getDebugLoc() + : DebugLoc(); + const auto *TII = MF->getSubtarget().getInstrInfo(); + + // If we don't need a phi create a copy to the upward exposed vreg. + if (!needPHI) { + assert(UpwardsUse); + assert(!VRegs.empty() && + "No predecessors? Is the Calling Convention correct?"); + Register DestReg = UUseVReg; + BuildMI(*MBB, MBB->getFirstNonPHI(), DLoc, TII->get(TargetOpcode::COPY), + DestReg) + .addReg(VRegs[0].second); + continue; + } + + // We need a phi: if there is an upwards exposed use we already have a + // destination virtual register number otherwise we generate a new one. + auto &DL = MF->getDataLayout(); + auto const *RC = TLI->getRegClassFor(TLI->getPointerTy(DL)); + Register PHIVReg = + UpwardsUse ? UUseVReg : MF->getRegInfo().createVirtualRegister(RC); + MachineInstrBuilder PHI = + BuildMI(*MBB, MBB->getFirstNonPHI(), DLoc, + TII->get(TargetOpcode::PHI), PHIVReg); + for (auto BBRegPair : VRegs) { + PHI.addReg(BBRegPair.second).addMBB(BBRegPair.first); + } + + // We did not have a definition in this block before: store the phi's vreg + // as this block downward exposed def. + if (!UpwardsUse) + setCurrentVReg(MBB, SwiftErrorVal, PHIVReg); + } + } +} + +void SwiftErrorValueTracking::preassignVRegs( + MachineBasicBlock *MBB, BasicBlock::const_iterator Begin, + BasicBlock::const_iterator End) { + if (!TLI->supportSwiftError() || SwiftErrorVals.empty()) + return; + + // Iterator over instructions and assign vregs to swifterror defs and uses. + for (auto It = Begin; It != End; ++It) { + ImmutableCallSite CS(&*It); + if (CS) { + // A call-site with a swifterror argument is both use and def. + const Value *SwiftErrorAddr = nullptr; + for (auto &Arg : CS.args()) { + if (!Arg->isSwiftError()) + continue; + // Use of swifterror. + assert(!SwiftErrorAddr && "Cannot have multiple swifterror arguments"); + SwiftErrorAddr = &*Arg; + assert(SwiftErrorAddr->isSwiftError() && + "Must have a swifterror value argument"); + getOrCreateVRegUseAt(&*It, MBB, SwiftErrorAddr); + } + if (!SwiftErrorAddr) + continue; + + // Def of swifterror. + getOrCreateVRegDefAt(&*It, MBB, SwiftErrorAddr); + + // A load is a use. + } else if (const LoadInst *LI = dyn_cast(&*It)) { + const Value *V = LI->getOperand(0); + if (!V->isSwiftError()) + continue; + + getOrCreateVRegUseAt(LI, MBB, V); + + // A store is a def. + } else if (const StoreInst *SI = dyn_cast(&*It)) { + const Value *SwiftErrorAddr = SI->getOperand(1); + if (!SwiftErrorAddr->isSwiftError()) + continue; + + // Def of swifterror. + getOrCreateVRegDefAt(&*It, MBB, SwiftErrorAddr); + + // A return in a swiferror returning function is a use. + } else if (const ReturnInst *R = dyn_cast(&*It)) { + const Function *F = R->getParent()->getParent(); + if (!F->getAttributes().hasAttrSomewhere(Attribute::SwiftError)) + continue; + + getOrCreateVRegUseAt(R, MBB, SwiftErrorArg); + } + } +} diff --git a/lib/CodeGen/SwitchLoweringUtils.cpp b/lib/CodeGen/SwitchLoweringUtils.cpp new file mode 100644 index 000000000000..83acf7f80715 --- /dev/null +++ b/lib/CodeGen/SwitchLoweringUtils.cpp @@ -0,0 +1,489 @@ +//===- SwitchLoweringUtils.cpp - Switch Lowering --------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file contains switch inst lowering optimizations and utilities for +// codegen, so that it can be used for both SelectionDAG and GlobalISel. +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/MachineJumpTableInfo.h" +#include "llvm/CodeGen/SwitchLoweringUtils.h" + +using namespace llvm; +using namespace SwitchCG; + +uint64_t SwitchCG::getJumpTableRange(const CaseClusterVector &Clusters, + unsigned First, unsigned Last) { + assert(Last >= First); + const APInt &LowCase = Clusters[First].Low->getValue(); + const APInt &HighCase = Clusters[Last].High->getValue(); + assert(LowCase.getBitWidth() == HighCase.getBitWidth()); + + // FIXME: A range of consecutive cases has 100% density, but only requires one + // comparison to lower. We should discriminate against such consecutive ranges + // in jump tables. + return (HighCase - LowCase).getLimitedValue((UINT64_MAX - 1) / 100) + 1; +} + +uint64_t +SwitchCG::getJumpTableNumCases(const SmallVectorImpl &TotalCases, + unsigned First, unsigned Last) { + assert(Last >= First); + assert(TotalCases[Last] >= TotalCases[First]); + uint64_t NumCases = + TotalCases[Last] - (First == 0 ? 0 : TotalCases[First - 1]); + return NumCases; +} + +void SwitchCG::SwitchLowering::findJumpTables(CaseClusterVector &Clusters, + const SwitchInst *SI, + MachineBasicBlock *DefaultMBB) { +#ifndef NDEBUG + // Clusters must be non-empty, sorted, and only contain Range clusters. + assert(!Clusters.empty()); + for (CaseCluster &C : Clusters) + assert(C.Kind == CC_Range); + for (unsigned i = 1, e = Clusters.size(); i < e; ++i) + assert(Clusters[i - 1].High->getValue().slt(Clusters[i].Low->getValue())); +#endif + + assert(TLI && "TLI not set!"); + if (!TLI->areJTsAllowed(SI->getParent()->getParent())) + return; + + const unsigned MinJumpTableEntries = TLI->getMinimumJumpTableEntries(); + const unsigned SmallNumberOfEntries = MinJumpTableEntries / 2; + + // Bail if not enough cases. + const int64_t N = Clusters.size(); + if (N < 2 || N < MinJumpTableEntries) + return; + + // Accumulated number of cases in each cluster and those prior to it. + SmallVector TotalCases(N); + for (unsigned i = 0; i < N; ++i) { + const APInt &Hi = Clusters[i].High->getValue(); + const APInt &Lo = Clusters[i].Low->getValue(); + TotalCases[i] = (Hi - Lo).getLimitedValue() + 1; + if (i != 0) + TotalCases[i] += TotalCases[i - 1]; + } + + uint64_t Range = getJumpTableRange(Clusters,0, N - 1); + uint64_t NumCases = getJumpTableNumCases(TotalCases, 0, N - 1); + assert(NumCases < UINT64_MAX / 100); + assert(Range >= NumCases); + + // Cheap case: the whole range may be suitable for jump table. + if (TLI->isSuitableForJumpTable(SI, NumCases, Range)) { + CaseCluster JTCluster; + if (buildJumpTable(Clusters, 0, N - 1, SI, DefaultMBB, JTCluster)) { + Clusters[0] = JTCluster; + Clusters.resize(1); + return; + } + } + + // The algorithm below is not suitable for -O0. + if (TM->getOptLevel() == CodeGenOpt::None) + return; + + // Split Clusters into minimum number of dense partitions. The algorithm uses + // the same idea as Kannan & Proebsting "Correction to 'Producing Good Code + // for the Case Statement'" (1994), but builds the MinPartitions array in + // reverse order to make it easier to reconstruct the partitions in ascending + // order. In the choice between two optimal partitionings, it picks the one + // which yields more jump tables. + + // MinPartitions[i] is the minimum nbr of partitions of Clusters[i..N-1]. + SmallVector MinPartitions(N); + // LastElement[i] is the last element of the partition starting at i. + SmallVector LastElement(N); + // PartitionsScore[i] is used to break ties when choosing between two + // partitionings resulting in the same number of partitions. + SmallVector PartitionsScore(N); + // For PartitionsScore, a small number of comparisons is considered as good as + // a jump table and a single comparison is considered better than a jump + // table. + enum PartitionScores : unsigned { + NoTable = 0, + Table = 1, + FewCases = 1, + SingleCase = 2 + }; + + // Base case: There is only one way to partition Clusters[N-1]. + MinPartitions[N - 1] = 1; + LastElement[N - 1] = N - 1; + PartitionsScore[N - 1] = PartitionScores::SingleCase; + + // Note: loop indexes are signed to avoid underflow. + for (int64_t i = N - 2; i >= 0; i--) { + // Find optimal partitioning of Clusters[i..N-1]. + // Baseline: Put Clusters[i] into a partition on its own. + MinPartitions[i] = MinPartitions[i + 1] + 1; + LastElement[i] = i; + PartitionsScore[i] = PartitionsScore[i + 1] + PartitionScores::SingleCase; + + // Search for a solution that results in fewer partitions. + for (int64_t j = N - 1; j > i; j--) { + // Try building a partition from Clusters[i..j]. + Range = getJumpTableRange(Clusters, i, j); + NumCases = getJumpTableNumCases(TotalCases, i, j); + assert(NumCases < UINT64_MAX / 100); + assert(Range >= NumCases); + + if (TLI->isSuitableForJumpTable(SI, NumCases, Range)) { + unsigned NumPartitions = 1 + (j == N - 1 ? 0 : MinPartitions[j + 1]); + unsigned Score = j == N - 1 ? 0 : PartitionsScore[j + 1]; + int64_t NumEntries = j - i + 1; + + if (NumEntries == 1) + Score += PartitionScores::SingleCase; + else if (NumEntries <= SmallNumberOfEntries) + Score += PartitionScores::FewCases; + else if (NumEntries >= MinJumpTableEntries) + Score += PartitionScores::Table; + + // If this leads to fewer partitions, or to the same number of + // partitions with better score, it is a better partitioning. + if (NumPartitions < MinPartitions[i] || + (NumPartitions == MinPartitions[i] && Score > PartitionsScore[i])) { + MinPartitions[i] = NumPartitions; + LastElement[i] = j; + PartitionsScore[i] = Score; + } + } + } + } + + // Iterate over the partitions, replacing some with jump tables in-place. + unsigned DstIndex = 0; + for (unsigned First = 0, Last; First < N; First = Last + 1) { + Last = LastElement[First]; + assert(Last >= First); + assert(DstIndex <= First); + unsigned NumClusters = Last - First + 1; + + CaseCluster JTCluster; + if (NumClusters >= MinJumpTableEntries && + buildJumpTable(Clusters, First, Last, SI, DefaultMBB, JTCluster)) { + Clusters[DstIndex++] = JTCluster; + } else { + for (unsigned I = First; I <= Last; ++I) + std::memmove(&Clusters[DstIndex++], &Clusters[I], sizeof(Clusters[I])); + } + } + Clusters.resize(DstIndex); +} + +bool SwitchCG::SwitchLowering::buildJumpTable(const CaseClusterVector &Clusters, + unsigned First, unsigned Last, + const SwitchInst *SI, + MachineBasicBlock *DefaultMBB, + CaseCluster &JTCluster) { + assert(First <= Last); + + auto Prob = BranchProbability::getZero(); + unsigned NumCmps = 0; + std::vector Table; + DenseMap JTProbs; + + // Initialize probabilities in JTProbs. + for (unsigned I = First; I <= Last; ++I) + JTProbs[Clusters[I].MBB] = BranchProbability::getZero(); + + for (unsigned I = First; I <= Last; ++I) { + assert(Clusters[I].Kind == CC_Range); + Prob += Clusters[I].Prob; + const APInt &Low = Clusters[I].Low->getValue(); + const APInt &High = Clusters[I].High->getValue(); + NumCmps += (Low == High) ? 1 : 2; + if (I != First) { + // Fill the gap between this and the previous cluster. + const APInt &PreviousHigh = Clusters[I - 1].High->getValue(); + assert(PreviousHigh.slt(Low)); + uint64_t Gap = (Low - PreviousHigh).getLimitedValue() - 1; + for (uint64_t J = 0; J < Gap; J++) + Table.push_back(DefaultMBB); + } + uint64_t ClusterSize = (High - Low).getLimitedValue() + 1; + for (uint64_t J = 0; J < ClusterSize; ++J) + Table.push_back(Clusters[I].MBB); + JTProbs[Clusters[I].MBB] += Clusters[I].Prob; + } + + unsigned NumDests = JTProbs.size(); + if (TLI->isSuitableForBitTests(NumDests, NumCmps, + Clusters[First].Low->getValue(), + Clusters[Last].High->getValue(), *DL)) { + // Clusters[First..Last] should be lowered as bit tests instead. + return false; + } + + // Create the MBB that will load from and jump through the table. + // Note: We create it here, but it's not inserted into the function yet. + MachineFunction *CurMF = FuncInfo.MF; + MachineBasicBlock *JumpTableMBB = + CurMF->CreateMachineBasicBlock(SI->getParent()); + + // Add successors. Note: use table order for determinism. + SmallPtrSet Done; + for (MachineBasicBlock *Succ : Table) { + if (Done.count(Succ)) + continue; + addSuccessorWithProb(JumpTableMBB, Succ, JTProbs[Succ]); + Done.insert(Succ); + } + JumpTableMBB->normalizeSuccProbs(); + + unsigned JTI = CurMF->getOrCreateJumpTableInfo(TLI->getJumpTableEncoding()) + ->createJumpTableIndex(Table); + + // Set up the jump table info. + JumpTable JT(-1U, JTI, JumpTableMBB, nullptr); + JumpTableHeader JTH(Clusters[First].Low->getValue(), + Clusters[Last].High->getValue(), SI->getCondition(), + nullptr, false); + JTCases.emplace_back(std::move(JTH), std::move(JT)); + + JTCluster = CaseCluster::jumpTable(Clusters[First].Low, Clusters[Last].High, + JTCases.size() - 1, Prob); + return true; +} + +void SwitchCG::SwitchLowering::findBitTestClusters(CaseClusterVector &Clusters, + const SwitchInst *SI) { + // Partition Clusters into as few subsets as possible, where each subset has a + // range that fits in a machine word and has <= 3 unique destinations. + +#ifndef NDEBUG + // Clusters must be sorted and contain Range or JumpTable clusters. + assert(!Clusters.empty()); + assert(Clusters[0].Kind == CC_Range || Clusters[0].Kind == CC_JumpTable); + for (const CaseCluster &C : Clusters) + assert(C.Kind == CC_Range || C.Kind == CC_JumpTable); + for (unsigned i = 1; i < Clusters.size(); ++i) + assert(Clusters[i-1].High->getValue().slt(Clusters[i].Low->getValue())); +#endif + + // The algorithm below is not suitable for -O0. + if (TM->getOptLevel() == CodeGenOpt::None) + return; + + // If target does not have legal shift left, do not emit bit tests at all. + EVT PTy = TLI->getPointerTy(*DL); + if (!TLI->isOperationLegal(ISD::SHL, PTy)) + return; + + int BitWidth = PTy.getSizeInBits(); + const int64_t N = Clusters.size(); + + // MinPartitions[i] is the minimum nbr of partitions of Clusters[i..N-1]. + SmallVector MinPartitions(N); + // LastElement[i] is the last element of the partition starting at i. + SmallVector LastElement(N); + + // FIXME: This might not be the best algorithm for finding bit test clusters. + + // Base case: There is only one way to partition Clusters[N-1]. + MinPartitions[N - 1] = 1; + LastElement[N - 1] = N - 1; + + // Note: loop indexes are signed to avoid underflow. + for (int64_t i = N - 2; i >= 0; --i) { + // Find optimal partitioning of Clusters[i..N-1]. + // Baseline: Put Clusters[i] into a partition on its own. + MinPartitions[i] = MinPartitions[i + 1] + 1; + LastElement[i] = i; + + // Search for a solution that results in fewer partitions. + // Note: the search is limited by BitWidth, reducing time complexity. + for (int64_t j = std::min(N - 1, i + BitWidth - 1); j > i; --j) { + // Try building a partition from Clusters[i..j]. + + // Check the range. + if (!TLI->rangeFitsInWord(Clusters[i].Low->getValue(), + Clusters[j].High->getValue(), *DL)) + continue; + + // Check nbr of destinations and cluster types. + // FIXME: This works, but doesn't seem very efficient. + bool RangesOnly = true; + BitVector Dests(FuncInfo.MF->getNumBlockIDs()); + for (int64_t k = i; k <= j; k++) { + if (Clusters[k].Kind != CC_Range) { + RangesOnly = false; + break; + } + Dests.set(Clusters[k].MBB->getNumber()); + } + if (!RangesOnly || Dests.count() > 3) + break; + + // Check if it's a better partition. + unsigned NumPartitions = 1 + (j == N - 1 ? 0 : MinPartitions[j + 1]); + if (NumPartitions < MinPartitions[i]) { + // Found a better partition. + MinPartitions[i] = NumPartitions; + LastElement[i] = j; + } + } + } + + // Iterate over the partitions, replacing with bit-test clusters in-place. + unsigned DstIndex = 0; + for (unsigned First = 0, Last; First < N; First = Last + 1) { + Last = LastElement[First]; + assert(First <= Last); + assert(DstIndex <= First); + + CaseCluster BitTestCluster; + if (buildBitTests(Clusters, First, Last, SI, BitTestCluster)) { + Clusters[DstIndex++] = BitTestCluster; + } else { + size_t NumClusters = Last - First + 1; + std::memmove(&Clusters[DstIndex], &Clusters[First], + sizeof(Clusters[0]) * NumClusters); + DstIndex += NumClusters; + } + } + Clusters.resize(DstIndex); +} + +bool SwitchCG::SwitchLowering::buildBitTests(CaseClusterVector &Clusters, + unsigned First, unsigned Last, + const SwitchInst *SI, + CaseCluster &BTCluster) { + assert(First <= Last); + if (First == Last) + return false; + + BitVector Dests(FuncInfo.MF->getNumBlockIDs()); + unsigned NumCmps = 0; + for (int64_t I = First; I <= Last; ++I) { + assert(Clusters[I].Kind == CC_Range); + Dests.set(Clusters[I].MBB->getNumber()); + NumCmps += (Clusters[I].Low == Clusters[I].High) ? 1 : 2; + } + unsigned NumDests = Dests.count(); + + APInt Low = Clusters[First].Low->getValue(); + APInt High = Clusters[Last].High->getValue(); + assert(Low.slt(High)); + + if (!TLI->isSuitableForBitTests(NumDests, NumCmps, Low, High, *DL)) + return false; + + APInt LowBound; + APInt CmpRange; + + const int BitWidth = TLI->getPointerTy(*DL).getSizeInBits(); + assert(TLI->rangeFitsInWord(Low, High, *DL) && + "Case range must fit in bit mask!"); + + // Check if the clusters cover a contiguous range such that no value in the + // range will jump to the default statement. + bool ContiguousRange = true; + for (int64_t I = First + 1; I <= Last; ++I) { + if (Clusters[I].Low->getValue() != Clusters[I - 1].High->getValue() + 1) { + ContiguousRange = false; + break; + } + } + + if (Low.isStrictlyPositive() && High.slt(BitWidth)) { + // Optimize the case where all the case values fit in a word without having + // to subtract minValue. In this case, we can optimize away the subtraction. + LowBound = APInt::getNullValue(Low.getBitWidth()); + CmpRange = High; + ContiguousRange = false; + } else { + LowBound = Low; + CmpRange = High - Low; + } + + CaseBitsVector CBV; + auto TotalProb = BranchProbability::getZero(); + for (unsigned i = First; i <= Last; ++i) { + // Find the CaseBits for this destination. + unsigned j; + for (j = 0; j < CBV.size(); ++j) + if (CBV[j].BB == Clusters[i].MBB) + break; + if (j == CBV.size()) + CBV.push_back( + CaseBits(0, Clusters[i].MBB, 0, BranchProbability::getZero())); + CaseBits *CB = &CBV[j]; + + // Update Mask, Bits and ExtraProb. + uint64_t Lo = (Clusters[i].Low->getValue() - LowBound).getZExtValue(); + uint64_t Hi = (Clusters[i].High->getValue() - LowBound).getZExtValue(); + assert(Hi >= Lo && Hi < 64 && "Invalid bit case!"); + CB->Mask |= (-1ULL >> (63 - (Hi - Lo))) << Lo; + CB->Bits += Hi - Lo + 1; + CB->ExtraProb += Clusters[i].Prob; + TotalProb += Clusters[i].Prob; + } + + BitTestInfo BTI; + llvm::sort(CBV, [](const CaseBits &a, const CaseBits &b) { + // Sort by probability first, number of bits second, bit mask third. + if (a.ExtraProb != b.ExtraProb) + return a.ExtraProb > b.ExtraProb; + if (a.Bits != b.Bits) + return a.Bits > b.Bits; + return a.Mask < b.Mask; + }); + + for (auto &CB : CBV) { + MachineBasicBlock *BitTestBB = + FuncInfo.MF->CreateMachineBasicBlock(SI->getParent()); + BTI.push_back(BitTestCase(CB.Mask, BitTestBB, CB.BB, CB.ExtraProb)); + } + BitTestCases.emplace_back(std::move(LowBound), std::move(CmpRange), + SI->getCondition(), -1U, MVT::Other, false, + ContiguousRange, nullptr, nullptr, std::move(BTI), + TotalProb); + + BTCluster = CaseCluster::bitTests(Clusters[First].Low, Clusters[Last].High, + BitTestCases.size() - 1, TotalProb); + return true; +} + +void SwitchCG::sortAndRangeify(CaseClusterVector &Clusters) { +#ifndef NDEBUG + for (const CaseCluster &CC : Clusters) + assert(CC.Low == CC.High && "Input clusters must be single-case"); +#endif + + llvm::sort(Clusters, [](const CaseCluster &a, const CaseCluster &b) { + return a.Low->getValue().slt(b.Low->getValue()); + }); + + // Merge adjacent clusters with the same destination. + const unsigned N = Clusters.size(); + unsigned DstIndex = 0; + for (unsigned SrcIndex = 0; SrcIndex < N; ++SrcIndex) { + CaseCluster &CC = Clusters[SrcIndex]; + const ConstantInt *CaseVal = CC.Low; + MachineBasicBlock *Succ = CC.MBB; + + if (DstIndex != 0 && Clusters[DstIndex - 1].MBB == Succ && + (CaseVal->getValue() - Clusters[DstIndex - 1].High->getValue()) == 1) { + // If this case has the same successor and is a neighbour, merge it into + // the previous cluster. + Clusters[DstIndex - 1].High = CaseVal; + Clusters[DstIndex - 1].Prob += CC.Prob; + } else { + std::memmove(&Clusters[DstIndex++], &Clusters[SrcIndex], + sizeof(Clusters[SrcIndex])); + } + } + Clusters.resize(DstIndex); +} diff --git a/lib/CodeGen/TailDuplication.cpp b/lib/CodeGen/TailDuplication.cpp index 25cd7802264e..ba348b4a9d41 100644 --- a/lib/CodeGen/TailDuplication.cpp +++ b/lib/CodeGen/TailDuplication.cpp @@ -1,9 +1,8 @@ //===- TailDuplication.cpp - Duplicate blocks into predecessors' tails ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/TailDuplicator.cpp b/lib/CodeGen/TailDuplicator.cpp index b118c176a897..a0590a8a6cc6 100644 --- a/lib/CodeGen/TailDuplicator.cpp +++ b/lib/CodeGen/TailDuplicator.cpp @@ -1,9 +1,8 @@ //===- TailDuplicator.cpp - Duplicate blocks into predecessors' tails -----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -435,7 +434,7 @@ void TailDuplicator::duplicateInstruction( if (NewRC == nullptr) NewRC = OrigRC; unsigned NewReg = MRI->createVirtualRegister(NewRC); - BuildMI(*PredBB, MI, MI->getDebugLoc(), + BuildMI(*PredBB, NewMI, NewMI.getDebugLoc(), TII->get(TargetOpcode::COPY), NewReg) .addReg(VI->second.Reg, 0, VI->second.SubReg); LocalVRMap.erase(VI); @@ -558,7 +557,7 @@ bool TailDuplicator::shouldTailDuplicate(bool IsSimple, unsigned MaxDuplicateCount; if (TailDupSize == 0 && TailDuplicateSize.getNumOccurrences() == 0 && - MF->getFunction().optForSize()) + MF->getFunction().hasOptSize()) MaxDuplicateCount = 1; else if (TailDupSize == 0) MaxDuplicateCount = TailDuplicateSize; @@ -857,11 +856,6 @@ bool TailDuplicator::tailDuplicate(bool IsSimple, MachineBasicBlock *TailBB, } appendCopies(PredBB, CopyInfos, Copies); - // Simplify - MachineBasicBlock *PredTBB = nullptr, *PredFBB = nullptr; - SmallVector PredCond; - TII->analyzeBranch(*PredBB, PredTBB, PredFBB, PredCond); - NumTailDupAdded += TailBB->size() - 1; // subtract one for removed branch // Update the CFG. diff --git a/lib/CodeGen/TargetFrameLoweringImpl.cpp b/lib/CodeGen/TargetFrameLoweringImpl.cpp index cf78fb5a1f12..9c4483cb240d 100644 --- a/lib/CodeGen/TargetFrameLoweringImpl.cpp +++ b/lib/CodeGen/TargetFrameLoweringImpl.cpp @@ -1,9 +1,8 @@ //===- TargetFrameLoweringImpl.cpp - Implement target frame interface ------==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/TargetInstrInfo.cpp b/lib/CodeGen/TargetInstrInfo.cpp index 2a17af391105..868617ffe14d 100644 --- a/lib/CodeGen/TargetInstrInfo.cpp +++ b/lib/CodeGen/TargetInstrInfo.cpp @@ -1,9 +1,8 @@ //===-- TargetInstrInfo.cpp - Target Instruction Information --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -86,11 +85,13 @@ static bool isAsmComment(const char *Str, const MCAsmInfo &MAI) { /// simple--i.e. not a logical or arithmetic expression--size values without /// the optional fill value. This is primarily used for creating arbitrary /// sized inline asm blocks for testing purposes. -unsigned TargetInstrInfo::getInlineAsmLength(const char *Str, - const MCAsmInfo &MAI) const { +unsigned TargetInstrInfo::getInlineAsmLength( + const char *Str, + const MCAsmInfo &MAI, const TargetSubtargetInfo *STI) const { // Count the number of instructions in the asm. bool AtInsnStart = true; unsigned Length = 0; + const unsigned MaxInstLength = MAI.getMaxInstLength(STI); for (; *Str; ++Str) { if (*Str == '\n' || strncmp(Str, MAI.getSeparatorString(), strlen(MAI.getSeparatorString())) == 0) { @@ -102,7 +103,7 @@ unsigned TargetInstrInfo::getInlineAsmLength(const char *Str, } if (AtInsnStart && !std::isspace(static_cast(*Str))) { - unsigned AddLength = MAI.getMaxInstLength(); + unsigned AddLength = MaxInstLength; if (strncmp(Str, ".space", 6) == 0) { char *EStr; int SpaceSize; @@ -136,8 +137,14 @@ TargetInstrInfo::ReplaceTailWithBranchTo(MachineBasicBlock::iterator Tail, // Save off the debug loc before erasing the instruction. DebugLoc DL = Tail->getDebugLoc(); - // Remove all the dead instructions from the end of MBB. - MBB->erase(Tail, MBB->end()); + // Update call site info and remove all the dead instructions + // from the end of MBB. + while (Tail != MBB->end()) { + auto MI = Tail++; + if (MI->isCall()) + MBB->getParent()->updateCallSiteInfo(&*MI); + MBB->erase(MI); + } // If MBB isn't immediately before MBB, insert a branch to it. if (++MachineFunction::iterator(MBB) != MachineFunction::iterator(NewDest)) @@ -162,9 +169,9 @@ MachineInstr *TargetInstrInfo::commuteInstructionImpl(MachineInstr &MI, assert(MI.getOperand(Idx1).isReg() && MI.getOperand(Idx2).isReg() && "This only knows how to commute register operands so far"); - unsigned Reg0 = HasDef ? MI.getOperand(0).getReg() : 0; - unsigned Reg1 = MI.getOperand(Idx1).getReg(); - unsigned Reg2 = MI.getOperand(Idx2).getReg(); + Register Reg0 = HasDef ? MI.getOperand(0).getReg() : Register(); + Register Reg1 = MI.getOperand(Idx1).getReg(); + Register Reg2 = MI.getOperand(Idx2).getReg(); unsigned SubReg0 = HasDef ? MI.getOperand(0).getSubReg() : 0; unsigned SubReg1 = MI.getOperand(Idx1).getSubReg(); unsigned SubReg2 = MI.getOperand(Idx2).getSubReg(); @@ -523,7 +530,8 @@ static MachineInstr *foldPatchpoint(MachineFunction &MF, MachineInstr &MI, MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI, ArrayRef Ops, int FI, - LiveIntervals *LIS) const { + LiveIntervals *LIS, + VirtRegMap *VRM) const { auto Flags = MachineMemOperand::MONone; for (unsigned OpIdx : Ops) Flags |= MI.getOperand(OpIdx).isDef() ? MachineMemOperand::MOStore @@ -569,7 +577,7 @@ MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI, MBB->insert(MI, NewMI); } else { // Ask the target to do the actual folding. - NewMI = foldMemoryOperandImpl(MF, MI, Ops, MI, FI, LIS); + NewMI = foldMemoryOperandImpl(MF, MI, Ops, MI, FI, LIS, VRM); } if (NewMI) { @@ -898,7 +906,8 @@ bool TargetInstrInfo::isReallyTriviallyReMaterializableGeneric( return true; // Avoid instructions obviously unsafe for remat. - if (MI.isNotDuplicable() || MI.mayStore() || MI.hasUnmodeledSideEffects()) + if (MI.isNotDuplicable() || MI.mayStore() || MI.mayRaiseFPException() || + MI.hasUnmodeledSideEffects()) return false; // Don't remat inline asm. We have no idea how expensive it is @@ -1010,7 +1019,7 @@ ScheduleHazardRecognizer *TargetInstrInfo:: CreateTargetMIHazardRecognizer(const InstrItineraryData *II, const ScheduleDAG *DAG) const { return (ScheduleHazardRecognizer *) - new ScoreboardHazardRecognizer(II, DAG, "misched"); + new ScoreboardHazardRecognizer(II, DAG, "machine-scheduler"); } // Default implementation of CreateTargetPostRAHazardRecognizer. diff --git a/lib/CodeGen/TargetLoweringBase.cpp b/lib/CodeGen/TargetLoweringBase.cpp index e86190375642..9b28c1a6c450 100644 --- a/lib/CodeGen/TargetLoweringBase.cpp +++ b/lib/CodeGen/TargetLoweringBase.cpp @@ -1,9 +1,8 @@ //===- TargetLoweringBase.cpp - Implement the TargetLoweringBase class ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -74,8 +73,8 @@ static cl::opt MinimumJumpTableEntries cl::desc("Set minimum number of entries to use a jump table.")); static cl::opt MaximumJumpTableSize - ("max-jump-table-size", cl::init(0), cl::Hidden, - cl::desc("Set maximum size of jump tables; zero for no limit.")); + ("max-jump-table-size", cl::init(UINT_MAX), cl::Hidden, + cl::desc("Set maximum size of jump tables.")); /// Minimum jump table density for normal functions. static cl::opt @@ -124,6 +123,34 @@ void TargetLoweringBase::InitLibcalls(const Triple &TT) { for (int LC = 0; LC < RTLIB::UNKNOWN_LIBCALL; ++LC) setLibcallCallingConv((RTLIB::Libcall)LC, CallingConv::C); + // For IEEE quad-precision libcall names, PPC uses "kf" instead of "tf". + if (TT.getArch() == Triple::ppc || TT.isPPC64()) { + setLibcallName(RTLIB::ADD_F128, "__addkf3"); + setLibcallName(RTLIB::SUB_F128, "__subkf3"); + setLibcallName(RTLIB::MUL_F128, "__mulkf3"); + setLibcallName(RTLIB::DIV_F128, "__divkf3"); + setLibcallName(RTLIB::FPEXT_F32_F128, "__extendsfkf2"); + setLibcallName(RTLIB::FPEXT_F64_F128, "__extenddfkf2"); + setLibcallName(RTLIB::FPROUND_F128_F32, "__trunckfsf2"); + setLibcallName(RTLIB::FPROUND_F128_F64, "__trunckfdf2"); + setLibcallName(RTLIB::FPTOSINT_F128_I32, "__fixkfsi"); + setLibcallName(RTLIB::FPTOSINT_F128_I64, "__fixkfdi"); + setLibcallName(RTLIB::FPTOUINT_F128_I32, "__fixunskfsi"); + setLibcallName(RTLIB::FPTOUINT_F128_I64, "__fixunskfdi"); + setLibcallName(RTLIB::SINTTOFP_I32_F128, "__floatsikf"); + setLibcallName(RTLIB::SINTTOFP_I64_F128, "__floatdikf"); + setLibcallName(RTLIB::UINTTOFP_I32_F128, "__floatunsikf"); + setLibcallName(RTLIB::UINTTOFP_I64_F128, "__floatundikf"); + setLibcallName(RTLIB::OEQ_F128, "__eqkf2"); + setLibcallName(RTLIB::UNE_F128, "__nekf2"); + setLibcallName(RTLIB::OGE_F128, "__gekf2"); + setLibcallName(RTLIB::OLT_F128, "__ltkf2"); + setLibcallName(RTLIB::OLE_F128, "__lekf2"); + setLibcallName(RTLIB::OGT_F128, "__gtkf2"); + setLibcallName(RTLIB::UO_F128, "__unordkf2"); + setLibcallName(RTLIB::O_F128, "__unordkf2"); + } + // A few names are different on particular architectures or environments. if (TT.isOSDarwin()) { // For f16/f32 conversions, Darwin uses the standard naming scheme, instead @@ -546,7 +573,6 @@ TargetLoweringBase::TargetLoweringBase(const TargetMachine &tm) : TM(tm) { JumpIsExpensive = JumpIsExpensiveOverride; PredictableSelectIsExpensive = false; EnableExtLdPromotion = false; - HasFloatingPointExceptions = true; StackPointerRegisterToSaveRestore = 0; BooleanContents = UndefinedBooleanContent; BooleanFloatContents = UndefinedBooleanContent; @@ -583,6 +609,14 @@ void TargetLoweringBase::initActions() { std::fill(std::begin(TargetDAGCombineArray), std::end(TargetDAGCombineArray), 0); + for (MVT VT : MVT::fp_valuetypes()) { + MVT IntVT = MVT::getIntegerVT(VT.getSizeInBits()); + if (IntVT.isValid()) { + setOperationAction(ISD::ATOMIC_SWAP, VT, Promote); + AddPromotedToType(ISD::ATOMIC_SWAP, VT, IntVT); + } + } + // Set default actions for various operations. for (MVT VT : MVT::all_valuetypes()) { // Default all indexed load / store to expand. @@ -617,6 +651,8 @@ void TargetLoweringBase::initActions() { setOperationAction(ISD::SSUBSAT, VT, Expand); setOperationAction(ISD::USUBSAT, VT, Expand); setOperationAction(ISD::SMULFIX, VT, Expand); + setOperationAction(ISD::SMULFIXSAT, VT, Expand); + setOperationAction(ISD::UMULFIX, VT, Expand); // Overflow operations default to expand setOperationAction(ISD::SADDO, VT, Expand); @@ -655,8 +691,51 @@ void TargetLoweringBase::initActions() { setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Expand); } + // Constrained floating-point operations default to expand. + setOperationAction(ISD::STRICT_FADD, VT, Expand); + setOperationAction(ISD::STRICT_FSUB, VT, Expand); + setOperationAction(ISD::STRICT_FMUL, VT, Expand); + setOperationAction(ISD::STRICT_FDIV, VT, Expand); + setOperationAction(ISD::STRICT_FREM, VT, Expand); + setOperationAction(ISD::STRICT_FMA, VT, Expand); + setOperationAction(ISD::STRICT_FSQRT, VT, Expand); + setOperationAction(ISD::STRICT_FPOW, VT, Expand); + setOperationAction(ISD::STRICT_FPOWI, VT, Expand); + setOperationAction(ISD::STRICT_FSIN, VT, Expand); + setOperationAction(ISD::STRICT_FCOS, VT, Expand); + setOperationAction(ISD::STRICT_FEXP, VT, Expand); + setOperationAction(ISD::STRICT_FEXP2, VT, Expand); + setOperationAction(ISD::STRICT_FLOG, VT, Expand); + setOperationAction(ISD::STRICT_FLOG10, VT, Expand); + setOperationAction(ISD::STRICT_FLOG2, VT, Expand); + setOperationAction(ISD::STRICT_FRINT, VT, Expand); + setOperationAction(ISD::STRICT_FNEARBYINT, VT, Expand); + setOperationAction(ISD::STRICT_FCEIL, VT, Expand); + setOperationAction(ISD::STRICT_FFLOOR, VT, Expand); + setOperationAction(ISD::STRICT_FROUND, VT, Expand); + setOperationAction(ISD::STRICT_FTRUNC, VT, Expand); + setOperationAction(ISD::STRICT_FMAXNUM, VT, Expand); + setOperationAction(ISD::STRICT_FMINNUM, VT, Expand); + setOperationAction(ISD::STRICT_FP_ROUND, VT, Expand); + setOperationAction(ISD::STRICT_FP_EXTEND, VT, Expand); + // For most targets @llvm.get.dynamic.area.offset just returns 0. setOperationAction(ISD::GET_DYNAMIC_AREA_OFFSET, VT, Expand); + + // Vector reduction default to expand. + setOperationAction(ISD::VECREDUCE_FADD, VT, Expand); + setOperationAction(ISD::VECREDUCE_FMUL, VT, Expand); + setOperationAction(ISD::VECREDUCE_ADD, VT, Expand); + setOperationAction(ISD::VECREDUCE_MUL, VT, Expand); + setOperationAction(ISD::VECREDUCE_AND, VT, Expand); + setOperationAction(ISD::VECREDUCE_OR, VT, Expand); + setOperationAction(ISD::VECREDUCE_XOR, VT, Expand); + setOperationAction(ISD::VECREDUCE_SMAX, VT, Expand); + setOperationAction(ISD::VECREDUCE_SMIN, VT, Expand); + setOperationAction(ISD::VECREDUCE_UMAX, VT, Expand); + setOperationAction(ISD::VECREDUCE_UMIN, VT, Expand); + setOperationAction(ISD::VECREDUCE_FMAX, VT, Expand); + setOperationAction(ISD::VECREDUCE_FMIN, VT, Expand); } // Most targets ignore the @llvm.prefetch intrinsic. @@ -688,6 +767,10 @@ void TargetLoweringBase::initActions() { setOperationAction(ISD::FRINT, VT, Expand); setOperationAction(ISD::FTRUNC, VT, Expand); setOperationAction(ISD::FROUND, VT, Expand); + setOperationAction(ISD::LROUND, VT, Expand); + setOperationAction(ISD::LLROUND, VT, Expand); + setOperationAction(ISD::LRINT, VT, Expand); + setOperationAction(ISD::LLRINT, VT, Expand); } // Default ISD::TRAP to expand (which turns it into abort). @@ -700,7 +783,7 @@ void TargetLoweringBase::initActions() { MVT TargetLoweringBase::getScalarShiftAmountTy(const DataLayout &DL, EVT) const { - return MVT::getIntegerVT(8 * DL.getPointerSize(0)); + return MVT::getIntegerVT(DL.getPointerSizeInBits(0)); } EVT TargetLoweringBase::getShiftAmountTy(EVT LHSTy, const DataLayout &DL, @@ -985,16 +1068,16 @@ TargetLoweringBase::emitPatchPoint(MachineInstr &InitialMI, // Add a new memory operand for this FI. assert(MFI.getObjectOffset(FI) != -1); - auto Flags = MachineMemOperand::MOLoad; - if (MI->getOpcode() == TargetOpcode::STATEPOINT) { - Flags |= MachineMemOperand::MOStore; - Flags |= MachineMemOperand::MOVolatile; + // Note: STATEPOINT MMOs are added during SelectionDAG. STACKMAP, and + // PATCHPOINT should be updated to do the same. (TODO) + if (MI->getOpcode() != TargetOpcode::STATEPOINT) { + auto Flags = MachineMemOperand::MOLoad; + MachineMemOperand *MMO = MF.getMachineMemOperand( + MachinePointerInfo::getFixedStack(MF, FI), Flags, + MF.getDataLayout().getPointerSize(), MFI.getObjectAlignment(FI)); + MIB->addMemOperand(MF, MMO); } - MachineMemOperand *MMO = MF.getMachineMemOperand( - MachinePointerInfo::getFixedStack(MF, FI), Flags, - MF.getDataLayout().getPointerSize(), MFI.getObjectAlignment(FI)); - MIB->addMemOperand(MF, MMO); - + // Replace the instruction and update the operand index. MBB->insert(MachineBasicBlock::iterator(MI), MIB); OperIdx += (MIB->getNumOperands() - MI->getNumOperands()) - 1; @@ -1393,7 +1476,7 @@ void llvm::GetReturnInfo(CallingConv::ID CC, Type *ReturnType, Flags.setZExt(); for (unsigned i = 0; i < NumParts; ++i) - Outs.push_back(ISD::OutputArg(Flags, PartVT, VT, /*isFixed=*/true, 0, 0)); + Outs.push_back(ISD::OutputArg(Flags, PartVT, VT, /*isfixed=*/true, 0, 0)); } } @@ -1409,6 +1492,7 @@ bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace, unsigned Alignment, + MachineMemOperand::Flags Flags, bool *Fast) const { // Check if the specified alignment is sufficient based on the data layout. // TODO: While using the data layout works in practice, a better solution @@ -1424,7 +1508,15 @@ bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context, } // This is a misaligned access. - return allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment, Fast); + return allowsMisalignedMemoryAccesses(VT, AddrSpace, Alignment, Flags, Fast); +} + +bool TargetLoweringBase::allowsMemoryAccess(LLVMContext &Context, + const DataLayout &DL, EVT VT, + const MachineMemOperand &MMO, + bool *Fast) const { + return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(), + MMO.getAlignment(), MMO.getFlags(), Fast); } BranchProbability TargetLoweringBase::getPredictableBranchThreshold() const { @@ -1447,6 +1539,7 @@ int TargetLoweringBase::InstructionOpcodeToISD(unsigned Opcode) const { case Switch: return 0; case IndirectBr: return 0; case Invoke: return 0; + case CallBr: return 0; case Resume: return 0; case Unreachable: return 0; case CleanupRet: return 0; @@ -1580,8 +1673,8 @@ Value *TargetLoweringBase::getSafeStackPointerLocation(IRBuilder<> &IRB) const { // thread's unsafe stack pointer. Module *M = IRB.GetInsertBlock()->getParent()->getParent(); Type *StackPtrTy = Type::getInt8PtrTy(M->getContext()); - Value *Fn = M->getOrInsertFunction("__safestack_pointer_address", - StackPtrTy->getPointerTo(0)); + FunctionCallee Fn = M->getOrInsertFunction("__safestack_pointer_address", + StackPtrTy->getPointerTo(0)); return IRB.CreateCall(Fn); } @@ -1656,7 +1749,7 @@ Value *TargetLoweringBase::getSDagStackGuard(const Module &M) const { return M.getNamedValue("__stack_chk_guard"); } -Value *TargetLoweringBase::getSSPStackGuardCheck(const Module &M) const { +Function *TargetLoweringBase::getSSPStackGuardCheck(const Module &M) const { return nullptr; } diff --git a/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/lib/CodeGen/TargetLoweringObjectFileImpl.cpp index cb2fe691d702..4c8f75b237aa 100644 --- a/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -1,9 +1,8 @@ //===- llvm/CodeGen/TargetLoweringObjectFileImpl.cpp - Object File Info ---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -219,6 +218,16 @@ void TargetLoweringObjectFileELF::Initialize(MCContext &Ctx, PersonalityEncoding = dwarf::DW_EH_PE_absptr; TTypeEncoding = dwarf::DW_EH_PE_absptr; } + CallSiteEncoding = dwarf::DW_EH_PE_udata4; + break; + case Triple::riscv32: + case Triple::riscv64: + LSDAEncoding = dwarf::DW_EH_PE_pcrel | dwarf::DW_EH_PE_sdata4; + PersonalityEncoding = dwarf::DW_EH_PE_indirect | dwarf::DW_EH_PE_pcrel | + dwarf::DW_EH_PE_sdata4; + TTypeEncoding = dwarf::DW_EH_PE_indirect | dwarf::DW_EH_PE_pcrel | + dwarf::DW_EH_PE_sdata4; + CallSiteEncoding = dwarf::DW_EH_PE_udata4; break; case Triple::sparcv9: LSDAEncoding = dwarf::DW_EH_PE_pcrel | dwarf::DW_EH_PE_sdata4; @@ -272,6 +281,19 @@ void TargetLoweringObjectFileELF::emitModuleMetadata(MCStreamer &Streamer, } } + if (NamedMDNode *DependentLibraries = M.getNamedMetadata("llvm.dependent-libraries")) { + auto *S = C.getELFSection(".deplibs", ELF::SHT_LLVM_DEPENDENT_LIBRARIES, + ELF::SHF_MERGE | ELF::SHF_STRINGS, 1, ""); + + Streamer.SwitchSection(S); + + for (const auto &Operand : DependentLibraries->operands()) { + Streamer.EmitBytes( + cast(cast(Operand)->getOperand(0))->getString()); + Streamer.EmitIntValue(0, 1); + } + } + unsigned Version = 0; unsigned Flags = 0; StringRef Section; @@ -1458,7 +1480,7 @@ void TargetLoweringObjectFileCOFF::Initialize(MCContext &Ctx, const TargetMachine &TM) { TargetLoweringObjectFile::Initialize(Ctx, TM); const Triple &T = TM.getTargetTriple(); - if (T.isKnownWindowsMSVCEnvironment() || T.isWindowsItaniumEnvironment()) { + if (T.isWindowsMSVCEnvironment() || T.isWindowsItaniumEnvironment()) { StaticCtorSection = Ctx.getCOFFSection(".CRT$XCU", COFF::IMAGE_SCN_CNT_INITIALIZED_DATA | COFF::IMAGE_SCN_MEM_READ, @@ -1484,7 +1506,7 @@ static MCSectionCOFF *getCOFFStaticStructorSection(MCContext &Ctx, unsigned Priority, const MCSymbol *KeySym, MCSectionCOFF *Default) { - if (T.isKnownWindowsMSVCEnvironment() || T.isWindowsItaniumEnvironment()) { + if (T.isWindowsMSVCEnvironment() || T.isWindowsItaniumEnvironment()) { // If the priority is the default, use .CRT$XCU, possibly associative. if (Priority == 65535) return Ctx.getAssociativeCOFFSection(Default, KeySym, 0); @@ -1544,9 +1566,7 @@ const MCExpr *TargetLoweringObjectFileCOFF::lowerRelativeReference( const GlobalValue *LHS, const GlobalValue *RHS, const TargetMachine &TM) const { const Triple &T = TM.getTargetTriple(); - if (!T.isKnownWindowsMSVCEnvironment() && - !T.isWindowsItaniumEnvironment() && - !T.isWindowsCoreCLREnvironment()) + if (T.isOSCygMing()) return nullptr; // Our symbols should exist in address space zero, cowardly no-op if @@ -1694,8 +1714,11 @@ MCSection *TargetLoweringObjectFileWasm::getExplicitSectionGlobal( Group = C->getName(); } - return getContext().getWasmSection(Name, Kind, Group, - MCContext::GenericSectionID); + MCSectionWasm* Section = + getContext().getWasmSection(Name, Kind, Group, + MCContext::GenericSectionID); + + return Section; } static MCSectionWasm *selectWasmSectionForGlobal( @@ -1724,6 +1747,7 @@ static MCSectionWasm *selectWasmSectionForGlobal( UniqueID = *NextUniqueID; (*NextUniqueID)++; } + return Ctx.getWasmSection(Name, Kind, Group, UniqueID); } diff --git a/lib/CodeGen/TargetOptionsImpl.cpp b/lib/CodeGen/TargetOptionsImpl.cpp index 3c133fb8594e..039748d817ca 100644 --- a/lib/CodeGen/TargetOptionsImpl.cpp +++ b/lib/CodeGen/TargetOptionsImpl.cpp @@ -1,9 +1,8 @@ //===-- TargetOptionsImpl.cpp - Options that apply to all targets ----------==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/TargetPassConfig.cpp b/lib/CodeGen/TargetPassConfig.cpp index 28126fcf766d..36df02692f86 100644 --- a/lib/CodeGen/TargetPassConfig.cpp +++ b/lib/CodeGen/TargetPassConfig.cpp @@ -1,9 +1,8 @@ //===- TargetPassConfig.cpp - Target independent code generation passes ---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -23,6 +22,7 @@ #include "llvm/Analysis/ScopedNoAliasAA.h" #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/TypeBasedAliasAnalysis.h" +#include "llvm/CodeGen/CSEConfigBase.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachinePassRegistry.h" #include "llvm/CodeGen/Passes.h" @@ -408,7 +408,7 @@ TargetPassConfig::TargetPassConfig(LLVMTargetMachine &TM, PassManagerBase &pm) TM.Options.EnableIPRA = EnableIPRA; else { // If not explicitly specified, use target default. - TM.Options.EnableIPRA = TM.useIPRA(); + TM.Options.EnableIPRA |= TM.useIPRA(); } if (TM.Options.EnableIPRA) @@ -646,7 +646,7 @@ void TargetPassConfig::addIRPasses() { // into optimally-sized loads and compares. The transforms are enabled by a // target lowering hook. if (!DisableMergeICmps) - addPass(createMergeICmpsPass()); + addPass(createMergeICmpsLegacyPass()); addPass(createExpandMemCmpPass()); } @@ -815,6 +815,13 @@ bool TargetPassConfig::addCoreISelPasses() { } else if (addInstSelector()) return true; + // Expand pseudo-instructions emitted by ISel. Don't run the verifier before + // FinalizeISel. + addPass(&FinalizeISelID); + + // Print the instruction selected machine code... + printAndVerify("After Instruction Selection"); + return false; } @@ -874,12 +881,6 @@ void TargetPassConfig::addMachinePasses() { } } - // Print the instruction selected machine code... - printAndVerify("After Instruction Selection"); - - // Expand pseudo-instructions emitted by ISel. - addPass(&ExpandISelPseudosID); - // Add passes that optimize machine instructions in SSA form. if (getOptLevel() != CodeGenOpt::None) { addMachineSSAOptimization(); @@ -898,13 +899,9 @@ void TargetPassConfig::addMachinePasses() { // Run register allocation and passes that are tightly coupled with it, // including phi elimination and scheduling. if (getOptimizeRegAlloc()) - addOptimizedRegAlloc(createRegAllocPass(true)); - else { - if (RegAlloc != &useDefaultRegisterAllocator && - RegAlloc != &createFastRegisterAllocator) - report_fatal_error("Must use fast (default) register allocator for unoptimized regalloc."); - addFastRegAlloc(createRegAllocPass(false)); - } + addOptimizedRegAlloc(); + else + addFastRegAlloc(); // Run post-ra passes. addPostRegAlloc(); @@ -1039,10 +1036,6 @@ bool TargetPassConfig::getOptimizeRegAlloc() const { llvm_unreachable("Invalid optimize-regalloc state"); } -/// RegisterRegAlloc's global Registry tracks allocator registration. -MachinePassRegistry - RegisterRegAlloc::Registry; - /// A dummy default pass factory indicates whether the register allocator is /// overridden on the command line. static llvm::once_flag InitializeDefaultRegisterAllocatorFlag; @@ -1053,12 +1046,8 @@ defaultRegAlloc("default", useDefaultRegisterAllocator); static void initializeDefaultRegisterAllocatorOnce() { - RegisterRegAlloc::FunctionPassCtor Ctor = RegisterRegAlloc::getDefault(); - - if (!Ctor) { - Ctor = RegAlloc; + if (!RegisterRegAlloc::getDefault()) RegisterRegAlloc::setDefault(RegAlloc); - } } /// Instantiate the default register allocator pass for this target for either @@ -1098,6 +1087,33 @@ FunctionPass *TargetPassConfig::createRegAllocPass(bool Optimized) { return createTargetRegisterAllocator(Optimized); } +bool TargetPassConfig::addRegAssignmentFast() { + if (RegAlloc != &useDefaultRegisterAllocator && + RegAlloc != &createFastRegisterAllocator) + report_fatal_error("Must use fast (default) register allocator for unoptimized regalloc."); + + addPass(createRegAllocPass(false)); + return true; +} + +bool TargetPassConfig::addRegAssignmentOptimized() { + // Add the selected register allocation pass. + addPass(createRegAllocPass(true)); + + // Allow targets to change the register assignments before rewriting. + addPreRewrite(); + + // Finally rewrite virtual registers. + addPass(&VirtRegRewriterID); + // Perform stack slot coloring and post-ra machine LICM. + // + // FIXME: Re-enable coloring with register when it's capable of adding + // kill markers. + addPass(&StackSlotColoringID); + + return true; +} + /// Return true if the default global register allocator is in use and /// has not be overriden on the command line with '-regalloc=...' bool TargetPassConfig::usingDefaultRegAlloc() const { @@ -1106,18 +1122,17 @@ bool TargetPassConfig::usingDefaultRegAlloc() const { /// Add the minimum set of target-independent passes that are required for /// register allocation. No coalescing or scheduling. -void TargetPassConfig::addFastRegAlloc(FunctionPass *RegAllocPass) { +void TargetPassConfig::addFastRegAlloc() { addPass(&PHIEliminationID, false); addPass(&TwoAddressInstructionPassID, false); - if (RegAllocPass) - addPass(RegAllocPass); + addRegAssignmentFast(); } /// Add standard target-independent passes that are tightly coupled with /// optimized register allocation, including coalescing, machine instruction /// scheduling, and register allocation itself. -void TargetPassConfig::addOptimizedRegAlloc(FunctionPass *RegAllocPass) { +void TargetPassConfig::addOptimizedRegAlloc() { addPass(&DetectDeadLanesID, false); addPass(&ProcessImplicitDefsID, false); @@ -1149,21 +1164,10 @@ void TargetPassConfig::addOptimizedRegAlloc(FunctionPass *RegAllocPass) { // PreRA instruction scheduling. addPass(&MachineSchedulerID); - if (RegAllocPass) { - // Add the selected register allocation pass. - addPass(RegAllocPass); - - // Allow targets to change the register assignments before rewriting. - addPreRewrite(); - - // Finally rewrite virtual registers. - addPass(&VirtRegRewriterID); - - // Perform stack slot coloring and post-ra machine LICM. - // - // FIXME: Re-enable coloring with register when it's capable of adding - // kill markers. - addPass(&StackSlotColoringID); + if (addRegAssignmentOptimized()) { + // Allow targets to expand pseudo instructions depending on the choice of + // registers before MachineCopyPropagation. + addPostRewrite(); // Copy propagate to forward register uses and try to eliminate COPYs that // were not coalesced. @@ -1221,3 +1225,11 @@ bool TargetPassConfig::isGlobalISelAbortEnabled() const { bool TargetPassConfig::reportDiagnosticWhenGlobalISelFallback() const { return TM->Options.GlobalISelAbort == GlobalISelAbortMode::DisableWithDiag; } + +bool TargetPassConfig::isGISelCSEEnabled() const { + return true; +} + +std::unique_ptr TargetPassConfig::getCSEConfig() const { + return make_unique(); +} diff --git a/lib/CodeGen/TargetRegisterInfo.cpp b/lib/CodeGen/TargetRegisterInfo.cpp index 661dc18f7a85..f1b2ecf3243b 100644 --- a/lib/CodeGen/TargetRegisterInfo.cpp +++ b/lib/CodeGen/TargetRegisterInfo.cpp @@ -1,9 +1,8 @@ //==- TargetRegisterInfo.cpp - Target Register Information Implementation --==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -14,6 +13,7 @@ #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/BitVector.h" +#include "llvm/ADT/SmallSet.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/StringExtras.h" #include "llvm/CodeGen/MachineFrameInfo.h" @@ -398,6 +398,7 @@ TargetRegisterInfo::getRegAllocationHints(unsigned VirtReg, const std::pair> &Hints_MRI = MRI.getRegAllocationHints(VirtReg); + SmallSet HintedRegs; // First hint may be a target hint. bool Skip = (Hints_MRI.first != 0); for (auto Reg : Hints_MRI.second) { @@ -411,6 +412,10 @@ TargetRegisterInfo::getRegAllocationHints(unsigned VirtReg, if (VRM && isVirtualRegister(Phys)) Phys = VRM->getPhys(Phys); + // Don't add the same reg twice (Hints_MRI may contain multiple virtual + // registers allocated to the same physreg). + if (!HintedRegs.insert(Phys).second) + continue; // Check that Phys is a valid hint in VirtReg's register class. if (!isPhysicalRegister(Phys)) continue; diff --git a/lib/CodeGen/TargetSchedule.cpp b/lib/CodeGen/TargetSchedule.cpp index 3cff31ad4933..195279719ad4 100644 --- a/lib/CodeGen/TargetSchedule.cpp +++ b/lib/CodeGen/TargetSchedule.cpp @@ -1,9 +1,8 @@ //===- llvm/Target/TargetSchedule.cpp - Sched Machine Model ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/CodeGen/TargetSubtargetInfo.cpp b/lib/CodeGen/TargetSubtargetInfo.cpp index fa29c05fd6c2..59eb2f9c88cb 100644 --- a/lib/CodeGen/TargetSubtargetInfo.cpp +++ b/lib/CodeGen/TargetSubtargetInfo.cpp @@ -1,9 +1,8 @@ //===- TargetSubtargetInfo.cpp - General Target Information ----------------==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -12,24 +11,16 @@ //===----------------------------------------------------------------------===// #include "llvm/CodeGen/TargetSubtargetInfo.h" -#include "llvm/ADT/Optional.h" -#include "llvm/CodeGen/MachineInstr.h" -#include "llvm/CodeGen/TargetInstrInfo.h" -#include "llvm/CodeGen/TargetSchedule.h" -#include "llvm/MC/MCInst.h" -#include "llvm/Support/Format.h" -#include "llvm/Support/raw_ostream.h" -#include using namespace llvm; TargetSubtargetInfo::TargetSubtargetInfo( const Triple &TT, StringRef CPU, StringRef FS, - ArrayRef PF, ArrayRef PD, - const SubtargetInfoKV *ProcSched, const MCWriteProcResEntry *WPR, + ArrayRef PF, ArrayRef PD, + const MCWriteProcResEntry *WPR, const MCWriteLatencyEntry *WL, const MCReadAdvanceEntry *RA, const InstrStage *IS, const unsigned *OC, const unsigned *FP) - : MCSubtargetInfo(TT, CPU, FS, PF, PD, ProcSched, WPR, WL, RA, IS, OC, FP) { + : MCSubtargetInfo(TT, CPU, FS, PF, PD, WPR, WL, RA, IS, OC, FP) { } TargetSubtargetInfo::~TargetSubtargetInfo() = default; @@ -67,50 +58,4 @@ bool TargetSubtargetInfo::useAA() const { return false; } -static std::string createSchedInfoStr(unsigned Latency, double RThroughput) { - static const char *SchedPrefix = " sched: ["; - std::string Comment; - raw_string_ostream CS(Comment); - if (RThroughput != 0.0) - CS << SchedPrefix << Latency << format(":%2.2f", RThroughput) - << "]"; - else - CS << SchedPrefix << Latency << ":?]"; - CS.flush(); - return Comment; -} - -/// Returns string representation of scheduler comment -std::string TargetSubtargetInfo::getSchedInfoStr(const MachineInstr &MI) const { - if (MI.isPseudo() || MI.isTerminator()) - return std::string(); - // We don't cache TSchedModel because it depends on TargetInstrInfo - // that could be changed during the compilation - TargetSchedModel TSchedModel; - TSchedModel.init(this); - unsigned Latency = TSchedModel.computeInstrLatency(&MI); - double RThroughput = TSchedModel.computeReciprocalThroughput(&MI); - return createSchedInfoStr(Latency, RThroughput); -} - -/// Returns string representation of scheduler comment -std::string TargetSubtargetInfo::getSchedInfoStr(MCInst const &MCI) const { - // We don't cache TSchedModel because it depends on TargetInstrInfo - // that could be changed during the compilation - TargetSchedModel TSchedModel; - TSchedModel.init(this); - unsigned Latency; - if (TSchedModel.hasInstrSchedModel()) - Latency = TSchedModel.computeInstrLatency(MCI); - else if (TSchedModel.hasInstrItineraries()) { - auto *ItinData = TSchedModel.getInstrItineraries(); - Latency = ItinData->getStageLatency( - getInstrInfo()->get(MCI.getOpcode()).getSchedClass()); - } else - return std::string(); - double RThroughput = TSchedModel.computeReciprocalThroughput(MCI); - return createSchedInfoStr(Latency, RThroughput); -} - -void TargetSubtargetInfo::mirFileLoaded(MachineFunction &MF) const { -} +void TargetSubtargetInfo::mirFileLoaded(MachineFunction &MF) const { } diff --git a/lib/CodeGen/TwoAddressInstructionPass.cpp b/lib/CodeGen/TwoAddressInstructionPass.cpp index 4b72f6a84ca1..43d876646967 100644 --- a/lib/CodeGen/TwoAddressInstructionPass.cpp +++ b/lib/CodeGen/TwoAddressInstructionPass.cpp @@ -1,9 +1,8 @@ //===- TwoAddressInstructionPass.cpp - Two-Address instruction pass -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -1245,8 +1244,13 @@ bool TwoAddressInstructionPass::tryInstructionCommute(MachineInstr *MI, ++NumAggrCommuted; // There might be more than two commutable operands, update BaseOp and // continue scanning. + // FIXME: This assumes that the new instruction's operands are in the + // same positions and were simply swapped. BaseOpReg = OtherOpReg; BaseOpKilled = OtherOpKilled; + // Resamples OpsNum in case the number of operands was reduced. This + // happens with X86. + OpsNum = MI->getDesc().getNumOperands(); continue; } // If this was a commute based on kill, we won't do better continuing. diff --git a/lib/CodeGen/UnreachableBlockElim.cpp b/lib/CodeGen/UnreachableBlockElim.cpp index 5288ca672774..177bab32bccc 100644 --- a/lib/CodeGen/UnreachableBlockElim.cpp +++ b/lib/CodeGen/UnreachableBlockElim.cpp @@ -1,9 +1,8 @@ //===-- UnreachableBlockElim.cpp - Remove unreachable blocks for codegen --===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -38,43 +37,13 @@ #include "llvm/IR/Instructions.h" #include "llvm/IR/Type.h" #include "llvm/Pass.h" +#include "llvm/Transforms/Utils/BasicBlockUtils.h" using namespace llvm; -static bool eliminateUnreachableBlock(Function &F) { - df_iterator_default_set Reachable; - - // Mark all reachable blocks. - for (BasicBlock *BB : depth_first_ext(&F, Reachable)) - (void)BB/* Mark all reachable blocks */; - - // Loop over all dead blocks, remembering them and deleting all instructions - // in them. - std::vector DeadBlocks; - for (Function::iterator I = F.begin(), E = F.end(); I != E; ++I) - if (!Reachable.count(&*I)) { - BasicBlock *BB = &*I; - DeadBlocks.push_back(BB); - while (PHINode *PN = dyn_cast(BB->begin())) { - PN->replaceAllUsesWith(Constant::getNullValue(PN->getType())); - BB->getInstList().pop_front(); - } - for (succ_iterator SI = succ_begin(BB), E = succ_end(BB); SI != E; ++SI) - (*SI)->removePredecessor(BB); - BB->dropAllReferences(); - } - - // Actually remove the blocks now. - for (unsigned i = 0, e = DeadBlocks.size(); i != e; ++i) { - DeadBlocks[i]->eraseFromParent(); - } - - return !DeadBlocks.empty(); -} - namespace { class UnreachableBlockElimLegacyPass : public FunctionPass { bool runOnFunction(Function &F) override { - return eliminateUnreachableBlock(F); + return llvm::EliminateUnreachableBlocks(F); } public: @@ -99,7 +68,7 @@ FunctionPass *llvm::createUnreachableBlockEliminationPass() { PreservedAnalyses UnreachableBlockElimPass::run(Function &F, FunctionAnalysisManager &AM) { - bool Changed = eliminateUnreachableBlock(F); + bool Changed = llvm::EliminateUnreachableBlocks(F); if (!Changed) return PreservedAnalyses::all(); PreservedAnalyses PA; diff --git a/lib/CodeGen/ValueTypes.cpp b/lib/CodeGen/ValueTypes.cpp index adb7075de651..a911cdcbec9d 100644 --- a/lib/CodeGen/ValueTypes.cpp +++ b/lib/CodeGen/ValueTypes.cpp @@ -1,9 +1,8 @@ //===----------- ValueTypes.cpp - Implementation of EVT methods -----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -166,11 +165,18 @@ std::string EVT::getEVTString() const { case MVT::v128i16: return "v128i16"; case MVT::v1i32: return "v1i32"; case MVT::v2i32: return "v2i32"; + case MVT::v3i32: return "v3i32"; case MVT::v4i32: return "v4i32"; + case MVT::v5i32: return "v5i32"; case MVT::v8i32: return "v8i32"; case MVT::v16i32: return "v16i32"; case MVT::v32i32: return "v32i32"; case MVT::v64i32: return "v64i32"; + case MVT::v128i32: return "v128i32"; + case MVT::v256i32: return "v256i32"; + case MVT::v512i32: return "v512i32"; + case MVT::v1024i32:return "v1024i32"; + case MVT::v2048i32:return "v2048i32"; case MVT::v1i64: return "v1i64"; case MVT::v2i64: return "v2i64"; case MVT::v4i64: return "v4i64"; @@ -183,16 +189,25 @@ std::string EVT::getEVTString() const { case MVT::v2f16: return "v2f16"; case MVT::v4f16: return "v4f16"; case MVT::v8f16: return "v8f16"; + case MVT::v3f32: return "v3f32"; case MVT::v4f32: return "v4f32"; + case MVT::v5f32: return "v5f32"; case MVT::v8f32: return "v8f32"; case MVT::v16f32: return "v16f32"; + case MVT::v32f32: return "v32f32"; + case MVT::v64f32: return "v64f32"; + case MVT::v128f32: return "v128f32"; + case MVT::v256f32: return "v256f32"; + case MVT::v512f32: return "v512f32"; + case MVT::v1024f32:return "v1024f32"; + case MVT::v2048f32:return "v2048f32"; case MVT::v1f64: return "v1f64"; case MVT::v2f64: return "v2f64"; case MVT::v4f64: return "v4f64"; case MVT::v8f64: return "v8f64"; case MVT::Metadata:return "Metadata"; case MVT::Untyped: return "Untyped"; - case MVT::ExceptRef: return "ExceptRef"; + case MVT::exnref : return "exnref"; } } @@ -247,11 +262,18 @@ Type *EVT::getTypeForEVT(LLVMContext &Context) const { case MVT::v128i16: return VectorType::get(Type::getInt16Ty(Context), 128); case MVT::v1i32: return VectorType::get(Type::getInt32Ty(Context), 1); case MVT::v2i32: return VectorType::get(Type::getInt32Ty(Context), 2); + case MVT::v3i32: return VectorType::get(Type::getInt32Ty(Context), 3); case MVT::v4i32: return VectorType::get(Type::getInt32Ty(Context), 4); + case MVT::v5i32: return VectorType::get(Type::getInt32Ty(Context), 5); case MVT::v8i32: return VectorType::get(Type::getInt32Ty(Context), 8); case MVT::v16i32: return VectorType::get(Type::getInt32Ty(Context), 16); case MVT::v32i32: return VectorType::get(Type::getInt32Ty(Context), 32); case MVT::v64i32: return VectorType::get(Type::getInt32Ty(Context), 64); + case MVT::v128i32: return VectorType::get(Type::getInt32Ty(Context), 128); + case MVT::v256i32: return VectorType::get(Type::getInt32Ty(Context), 256); + case MVT::v512i32: return VectorType::get(Type::getInt32Ty(Context), 512); + case MVT::v1024i32:return VectorType::get(Type::getInt32Ty(Context), 1024); + case MVT::v2048i32:return VectorType::get(Type::getInt32Ty(Context), 2048); case MVT::v1i64: return VectorType::get(Type::getInt64Ty(Context), 1); case MVT::v2i64: return VectorType::get(Type::getInt64Ty(Context), 2); case MVT::v4i64: return VectorType::get(Type::getInt64Ty(Context), 4); @@ -264,9 +286,18 @@ Type *EVT::getTypeForEVT(LLVMContext &Context) const { case MVT::v8f16: return VectorType::get(Type::getHalfTy(Context), 8); case MVT::v1f32: return VectorType::get(Type::getFloatTy(Context), 1); case MVT::v2f32: return VectorType::get(Type::getFloatTy(Context), 2); + case MVT::v3f32: return VectorType::get(Type::getFloatTy(Context), 3); case MVT::v4f32: return VectorType::get(Type::getFloatTy(Context), 4); + case MVT::v5f32: return VectorType::get(Type::getFloatTy(Context), 5); case MVT::v8f32: return VectorType::get(Type::getFloatTy(Context), 8); - case MVT::v16f32: return VectorType::get(Type::getFloatTy(Context), 16); + case MVT::v16f32: return VectorType::get(Type::getFloatTy(Context), 16); + case MVT::v32f32: return VectorType::get(Type::getFloatTy(Context), 32); + case MVT::v64f32: return VectorType::get(Type::getFloatTy(Context), 64); + case MVT::v128f32: return VectorType::get(Type::getFloatTy(Context), 128); + case MVT::v256f32: return VectorType::get(Type::getFloatTy(Context), 256); + case MVT::v512f32: return VectorType::get(Type::getFloatTy(Context), 512); + case MVT::v1024f32:return VectorType::get(Type::getFloatTy(Context), 1024); + case MVT::v2048f32:return VectorType::get(Type::getFloatTy(Context), 2048); case MVT::v1f64: return VectorType::get(Type::getDoubleTy(Context), 1); case MVT::v2f64: return VectorType::get(Type::getDoubleTy(Context), 2); case MVT::v4f64: return VectorType::get(Type::getDoubleTy(Context), 4); diff --git a/lib/CodeGen/VirtRegMap.cpp b/lib/CodeGen/VirtRegMap.cpp index ed7bef667e77..4a06704a8876 100644 --- a/lib/CodeGen/VirtRegMap.cpp +++ b/lib/CodeGen/VirtRegMap.cpp @@ -1,9 +1,8 @@ //===- llvm/CodeGen/VirtRegMap.cpp - Virtual Register Map -----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -385,7 +384,7 @@ void VirtRegRewriter::handleIdentityCopy(MachineInstr &MI) const { // give us additional liveness information: The target (super-)register // must not be valid before this point. Replace the COPY with a KILL // instruction to maintain this information. - if (MI.getOperand(0).isUndef() || MI.getNumOperands() > 2) { + if (MI.getOperand(1).isUndef() || MI.getNumOperands() > 2) { MI.setDesc(TII->get(TargetOpcode::KILL)); LLVM_DEBUG(dbgs() << " replace by: " << MI); return; diff --git a/lib/CodeGen/WasmEHPrepare.cpp b/lib/CodeGen/WasmEHPrepare.cpp index e5002eb95346..865a1cfbf43a 100644 --- a/lib/CodeGen/WasmEHPrepare.cpp +++ b/lib/CodeGen/WasmEHPrepare.cpp @@ -1,14 +1,14 @@ //===-- WasmEHPrepare - Prepare excepton handling for WebAssembly --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // // This transformation is designed for use by code generators which use -// WebAssembly exception handling scheme. +// WebAssembly exception handling scheme. This currently supports C++ +// exceptions. // // WebAssembly exception handling uses Windows exception IR for the middle level // representation. This pass does the following transformation for every @@ -23,53 +23,20 @@ // // - After: // catchpad ... -// exn = wasm.catch(0); // 0 is a tag for C++ -// wasm.landingpad.index(index); +// exn = wasm.extract.exception(); // // Only add below in case it's not a single catch (...) +// wasm.landingpad.index(index); // __wasm_lpad_context.lpad_index = index; // __wasm_lpad_context.lsda = wasm.lsda(); // _Unwind_CallPersonality(exn); -// int selector = __wasm.landingpad_context.selector; +// selector = __wasm.landingpad_context.selector; // ... // -// Also, does the following for a cleanuppad block with a call to -// __clang_call_terminate(): -// - Before: -// cleanuppad ... -// exn = wasm.get.exception(); -// __clang_call_terminate(exn); -// -// - After: -// cleanuppad ... -// exn = wasm.catch(0); // 0 is a tag for C++ -// __clang_call_terminate(exn); -// -// -// * Background: WebAssembly EH instructions -// WebAssembly's try and catch instructions are structured as follows: -// try -// instruction* -// catch (C++ tag) -// instruction* -// ... -// catch_all -// instruction* -// try_end -// -// A catch instruction in WebAssembly does not correspond to a C++ catch clause. -// In WebAssembly, there is a single catch instruction for all C++ exceptions. -// There can be more catch instructions for exceptions in other languages, but -// they are not generated for now. catch_all catches all exceptions including -// foreign exceptions (e.g. JavaScript). We turn catchpads into catch (C++ tag) -// and cleanuppads into catch_all, with one exception: cleanuppad with a call to -// __clang_call_terminate should be both in catch (C++ tag) and catch_all. -// // // * Background: Direct personality function call // In WebAssembly EH, the VM is responsible for unwinding the stack once an // exception is thrown. After the stack is unwound, the control flow is -// transfered to WebAssembly 'catch' instruction, which returns a caught -// exception object. +// transfered to WebAssembly 'catch' instruction. // // Unwinding the stack is not done by libunwind but the VM, so the personality // function in libcxxabi cannot be called from libunwind during the unwinding @@ -137,19 +104,19 @@ class WasmEHPrepare : public FunctionPass { Value *LSDAField = nullptr; // lsda field Value *SelectorField = nullptr; // selector - Function *ThrowF = nullptr; // wasm.throw() intrinsic - Function *CatchF = nullptr; // wasm.catch.extract() intrinsic - Function *LPadIndexF = nullptr; // wasm.landingpad.index() intrinsic - Function *LSDAF = nullptr; // wasm.lsda() intrinsic - Function *GetExnF = nullptr; // wasm.get.exception() intrinsic - Function *GetSelectorF = nullptr; // wasm.get.ehselector() intrinsic - Function *CallPersonalityF = nullptr; // _Unwind_CallPersonality() wrapper - Function *ClangCallTermF = nullptr; // __clang_call_terminate() function + Function *ThrowF = nullptr; // wasm.throw() intrinsic + Function *LPadIndexF = nullptr; // wasm.landingpad.index() intrinsic + Function *LSDAF = nullptr; // wasm.lsda() intrinsic + Function *GetExnF = nullptr; // wasm.get.exception() intrinsic + Function *ExtractExnF = nullptr; // wasm.extract.exception() intrinsic + Function *GetSelectorF = nullptr; // wasm.get.ehselector() intrinsic + FunctionCallee CallPersonalityF = + nullptr; // _Unwind_CallPersonality() wrapper bool prepareEHPads(Function &F); bool prepareThrows(Function &F); - void prepareEHPad(BasicBlock *BB, unsigned Index); + void prepareEHPad(BasicBlock *BB, bool NeedLSDA, unsigned Index = 0); void prepareTerminateCleanupPad(BasicBlock *BB); public: @@ -209,14 +176,12 @@ bool WasmEHPrepare::prepareThrows(Function &F) { // wasm.throw() intinsic, which will be lowered to wasm 'throw' instruction. ThrowF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_throw); - // Insert an unreachable instruction after a call to @llvm.wasm.throw and // delete all following instructions within the BB, and delete all the dead // children of the BB as well. for (User *U : ThrowF->users()) { - // A call to @llvm.wasm.throw() is only generated from - // __builtin_wasm_throw() builtin call within libcxxabi, and cannot be an - // InvokeInst. + // A call to @llvm.wasm.throw() is only generated from __cxa_throw() + // builtin call within libcxxabi, and cannot be an InvokeInst. auto *ThrowI = cast(U); if (ThrowI->getFunction() != &F) continue; @@ -263,8 +228,6 @@ bool WasmEHPrepare::prepareEHPads(Function &F) { SelectorField = IRB.CreateConstGEP2_32(LPadContextTy, LPadContextGV, 0, 2, "selector_gep"); - // wasm.catch() intinsic, which will be lowered to wasm 'catch' instruction. - CatchF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_catch); // wasm.landingpad.index() intrinsic, which is to specify landingpad index LPadIndexF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_landingpad_index); // wasm.lsda() intrinsic. Returns the address of LSDA table for the current @@ -275,14 +238,18 @@ bool WasmEHPrepare::prepareEHPads(Function &F) { GetExnF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_get_exception); GetSelectorF = Intrinsic::getDeclaration(&M, Intrinsic::wasm_get_ehselector); - // _Unwind_CallPersonality() wrapper function, which calls the personality - CallPersonalityF = cast(M.getOrInsertFunction( - "_Unwind_CallPersonality", IRB.getInt32Ty(), IRB.getInt8PtrTy())); - CallPersonalityF->setDoesNotThrow(); + // wasm.extract.exception() is the same as wasm.get.exception() but it does + // not take a token argument. This will be lowered down to EXTRACT_EXCEPTION + // pseudo instruction in instruction selection, which will be expanded using + // 'br_on_exn' instruction later. + ExtractExnF = + Intrinsic::getDeclaration(&M, Intrinsic::wasm_extract_exception); - // __clang_call_terminate() function, which is inserted by clang in case a - // cleanup throws - ClangCallTermF = M.getFunction("__clang_call_terminate"); + // _Unwind_CallPersonality() wrapper function, which calls the personality + CallPersonalityF = M.getOrInsertFunction( + "_Unwind_CallPersonality", IRB.getInt32Ty(), IRB.getInt8PtrTy()); + if (Function *F = dyn_cast(CallPersonalityF.getCallee())) + F->setDoesNotThrow(); unsigned Index = 0; for (auto *BB : CatchPads) { @@ -290,60 +257,52 @@ bool WasmEHPrepare::prepareEHPads(Function &F) { // In case of a single catch (...), we don't need to emit LSDA if (CPI->getNumArgOperands() == 1 && cast(CPI->getArgOperand(0))->isNullValue()) - prepareEHPad(BB, -1); + prepareEHPad(BB, false); else - prepareEHPad(BB, Index++); + prepareEHPad(BB, true, Index++); } - if (!ClangCallTermF) - return !CatchPads.empty(); - - // Cleanuppads will turn into catch_all later, but cleanuppads with a call to - // __clang_call_terminate() is a special case. __clang_call_terminate() takes - // an exception object, so we have to duplicate call in both 'catch ' - // and 'catch_all' clauses. Here we only insert a call to catch; the - // duplication will be done later. In catch_all, the exception object will be - // set to null. + // Cleanup pads don't need LSDA. for (auto *BB : CleanupPads) - for (auto &I : *BB) - if (auto *CI = dyn_cast(&I)) - if (CI->getCalledValue() == ClangCallTermF) - prepareEHPad(BB, -1); + prepareEHPad(BB, false); return true; } -void WasmEHPrepare::prepareEHPad(BasicBlock *BB, unsigned Index) { +// Prepare an EH pad for Wasm EH handling. If NeedLSDA is false, Index is +// ignored. +void WasmEHPrepare::prepareEHPad(BasicBlock *BB, bool NeedLSDA, + unsigned Index) { assert(BB->isEHPad() && "BB is not an EHPad!"); IRBuilder<> IRB(BB->getContext()); - IRB.SetInsertPoint(&*BB->getFirstInsertionPt()); - // The argument to wasm.catch() is the tag for C++ exceptions, which we set to - // 0 for this module. - // Pseudocode: void *exn = wasm.catch(0); - Instruction *Exn = IRB.CreateCall(CatchF, IRB.getInt32(0), "exn"); - // Replace the return value of wasm.get.exception() with the return value from - // wasm.catch(). + auto *FPI = cast(BB->getFirstNonPHI()); Instruction *GetExnCI = nullptr, *GetSelectorCI = nullptr; for (auto &U : FPI->uses()) { if (auto *CI = dyn_cast(U.getUser())) { if (CI->getCalledValue() == GetExnF) GetExnCI = CI; - else if (CI->getCalledValue() == GetSelectorF) + if (CI->getCalledValue() == GetSelectorF) GetSelectorCI = CI; } } - assert(GetExnCI && "wasm.get.exception() call does not exist"); - GetExnCI->replaceAllUsesWith(Exn); + // Cleanup pads w/o __clang_call_terminate call do not have any of + // wasm.get.exception() or wasm.get.ehselector() calls. We need to do nothing. + if (!GetExnCI) { + assert(!GetSelectorCI && + "wasm.get.ehselector() cannot exist w/o wasm.get.exception()"); + return; + } + + Instruction *ExtractExnCI = IRB.CreateCall(ExtractExnF, {}, "exn"); + GetExnCI->replaceAllUsesWith(ExtractExnCI); GetExnCI->eraseFromParent(); // In case it is a catchpad with single catch (...) or a cleanuppad, we don't // need to call personality function because we don't need a selector. - if (FPI->getNumArgOperands() == 0 || - (FPI->getNumArgOperands() == 1 && - cast(FPI->getArgOperand(0))->isNullValue())) { + if (!NeedLSDA) { if (GetSelectorCI) { assert(GetSelectorCI->use_empty() && "wasm.get.ehselector() still has uses!"); @@ -351,7 +310,7 @@ void WasmEHPrepare::prepareEHPad(BasicBlock *BB, unsigned Index) { } return; } - IRB.SetInsertPoint(Exn->getNextNode()); + IRB.SetInsertPoint(ExtractExnCI->getNextNode()); // This is to create a map of in // SelectionDAGISel, which is to be used in EHStreamer to emit LSDA tables. @@ -373,12 +332,13 @@ void WasmEHPrepare::prepareEHPad(BasicBlock *BB, unsigned Index) { IRB.CreateStore(IRB.CreateCall(LSDAF), LSDAField); // Pseudocode: _Unwind_CallPersonality(exn); - CallInst *PersCI = - IRB.CreateCall(CallPersonalityF, Exn, OperandBundleDef("funclet", CPI)); + CallInst *PersCI = IRB.CreateCall(CallPersonalityF, ExtractExnCI, + OperandBundleDef("funclet", CPI)); PersCI->setDoesNotThrow(); // Pseudocode: int selector = __wasm.landingpad_context.selector; - Instruction *Selector = IRB.CreateLoad(SelectorField, "selector"); + Instruction *Selector = + IRB.CreateLoad(IRB.getInt32Ty(), SelectorField, "selector"); // Replace the return value from wasm.get.ehselector() with the selector value // loaded from __wasm_lpad_context.selector. @@ -388,15 +348,15 @@ void WasmEHPrepare::prepareEHPad(BasicBlock *BB, unsigned Index) { } void llvm::calculateWasmEHInfo(const Function *F, WasmEHFuncInfo &EHInfo) { + // If an exception is not caught by a catchpad (i.e., it is a foreign + // exception), it will unwind to its parent catchswitch's unwind destination. + // We don't record an unwind destination for cleanuppads because every + // exception should be caught by it. for (const auto &BB : *F) { if (!BB.isEHPad()) continue; const Instruction *Pad = BB.getFirstNonPHI(); - // If an exception is not caught by a catchpad (i.e., it is a foreign - // exception), it will unwind to its parent catchswitch's unwind - // destination. We don't record an unwind destination for cleanuppads - // because every exception should be caught by it. if (const auto *CatchPad = dyn_cast(Pad)) { const auto *UnwindBB = CatchPad->getCatchSwitch()->getUnwindDest(); if (!UnwindBB) @@ -409,22 +369,4 @@ void llvm::calculateWasmEHInfo(const Function *F, WasmEHFuncInfo &EHInfo) { EHInfo.setEHPadUnwindDest(&BB, UnwindBB); } } - - // Record the unwind destination for invoke and cleanupret instructions. - for (const auto &BB : *F) { - const Instruction *TI = BB.getTerminator(); - BasicBlock *UnwindBB = nullptr; - if (const auto *Invoke = dyn_cast(TI)) - UnwindBB = Invoke->getUnwindDest(); - else if (const auto *CleanupRet = dyn_cast(TI)) - UnwindBB = CleanupRet->getUnwindDest(); - if (!UnwindBB) - continue; - const Instruction *UnwindPad = UnwindBB->getFirstNonPHI(); - if (const auto *CatchSwitch = dyn_cast(UnwindPad)) - // Currently there should be only one handler per a catchswitch. - EHInfo.setThrowUnwindDest(&BB, *CatchSwitch->handlers().begin()); - else // cleanuppad - EHInfo.setThrowUnwindDest(&BB, UnwindBB); - } } diff --git a/lib/CodeGen/WinEHPrepare.cpp b/lib/CodeGen/WinEHPrepare.cpp index 6a15240fa6e0..cdf79374e974 100644 --- a/lib/CodeGen/WinEHPrepare.cpp +++ b/lib/CodeGen/WinEHPrepare.cpp @@ -1,9 +1,8 @@ //===-- WinEHPrepare - Prepare exception handling for code generation ---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -1080,7 +1079,8 @@ AllocaInst *WinEHPrepare::insertPHILoads(PHINode *PN, Function &F) { SpillSlot = new AllocaInst(PN->getType(), DL->getAllocaAddrSpace(), nullptr, Twine(PN->getName(), ".wineh.spillslot"), &F.getEntryBlock().front()); - Value *V = new LoadInst(SpillSlot, Twine(PN->getName(), ".wineh.reload"), + Value *V = new LoadInst(PN->getType(), SpillSlot, + Twine(PN->getName(), ".wineh.reload"), &*PHIBlock->getFirstInsertionPt()); PN->replaceAllUsesWith(V); return SpillSlot; @@ -1222,14 +1222,16 @@ void WinEHPrepare::replaceUseWithLoad(Value *V, Use &U, AllocaInst *&SpillSlot, Value *&Load = Loads[IncomingBlock]; // Insert the load into the predecessor block if (!Load) - Load = new LoadInst(SpillSlot, Twine(V->getName(), ".wineh.reload"), - /*Volatile=*/false, IncomingBlock->getTerminator()); + Load = new LoadInst(V->getType(), SpillSlot, + Twine(V->getName(), ".wineh.reload"), + /*isVolatile=*/false, IncomingBlock->getTerminator()); U.set(Load); } else { // Reload right before the old use. - auto *Load = new LoadInst(SpillSlot, Twine(V->getName(), ".wineh.reload"), - /*Volatile=*/false, UsingInst); + auto *Load = new LoadInst(V->getType(), SpillSlot, + Twine(V->getName(), ".wineh.reload"), + /*isVolatile=*/false, UsingInst); U.set(Load); } } diff --git a/lib/CodeGen/XRayInstrumentation.cpp b/lib/CodeGen/XRayInstrumentation.cpp index 32a7457c2060..19c59e9542b4 100644 --- a/lib/CodeGen/XRayInstrumentation.cpp +++ b/lib/CodeGen/XRayInstrumentation.cpp @@ -1,9 +1,8 @@ //===- XRayInstrumentation.cpp - Adds XRay instrumentation to functions. --===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -111,6 +110,8 @@ void XRayInstrumentation::replaceRetWithPatchableRet( for (auto &MO : T.operands()) MIB.add(MO); Terminators.push_back(&T); + if (T.isCall()) + MF.updateCallSiteInfo(&T); } } } diff --git a/lib/DebugInfo/CodeView/AppendingTypeTableBuilder.cpp b/lib/DebugInfo/CodeView/AppendingTypeTableBuilder.cpp index 8828671d9be9..86a6f9eebfa2 100644 --- a/lib/DebugInfo/CodeView/AppendingTypeTableBuilder.cpp +++ b/lib/DebugInfo/CodeView/AppendingTypeTableBuilder.cpp @@ -1,9 +1,8 @@ //===- AppendingTypeTableBuilder.cpp --------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -50,13 +49,8 @@ Optional AppendingTypeTableBuilder::getNext(TypeIndex Prev) { return Prev; } -CVType AppendingTypeTableBuilder::getType(TypeIndex Index) { - CVType Type; - Type.RecordData = SeenRecords[Index.toArrayIndex()]; - const RecordPrefix *P = - reinterpret_cast(Type.RecordData.data()); - Type.Type = static_cast(uint16_t(P->RecordKind)); - return Type; +CVType AppendingTypeTableBuilder::getType(TypeIndex Index){ + return CVType(SeenRecords[Index.toArrayIndex()]); } StringRef AppendingTypeTableBuilder::getTypeName(TypeIndex Index) { diff --git a/lib/DebugInfo/CodeView/CVSymbolVisitor.cpp b/lib/DebugInfo/CodeView/CVSymbolVisitor.cpp index cbcaa5692828..48b9b0496ffe 100644 --- a/lib/DebugInfo/CodeView/CVSymbolVisitor.cpp +++ b/lib/DebugInfo/CodeView/CVSymbolVisitor.cpp @@ -1,9 +1,8 @@ //===- CVSymbolVisitor.cpp --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -21,7 +20,7 @@ CVSymbolVisitor::CVSymbolVisitor(SymbolVisitorCallbacks &Callbacks) template static Error visitKnownRecord(CVSymbol &Record, SymbolVisitorCallbacks &Callbacks) { - SymbolRecordKind RK = static_cast(Record.Type); + SymbolRecordKind RK = static_cast(Record.kind()); T KnownRecord(RK); if (auto EC = Callbacks.visitKnownRecord(Record, KnownRecord)) return EC; @@ -30,7 +29,7 @@ static Error visitKnownRecord(CVSymbol &Record, static Error finishVisitation(CVSymbol &Record, SymbolVisitorCallbacks &Callbacks) { - switch (Record.Type) { + switch (Record.kind()) { default: if (auto EC = Callbacks.visitUnknownSymbol(Record)) return EC; diff --git a/lib/DebugInfo/CodeView/CVTypeVisitor.cpp b/lib/DebugInfo/CodeView/CVTypeVisitor.cpp index a4182a3b2fa1..ec4773d571c8 100644 --- a/lib/DebugInfo/CodeView/CVTypeVisitor.cpp +++ b/lib/DebugInfo/CodeView/CVTypeVisitor.cpp @@ -1,9 +1,8 @@ //===- CVTypeVisitor.cpp ----------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -23,7 +22,7 @@ using namespace llvm::codeview; template static Error visitKnownRecord(CVType &Record, TypeVisitorCallbacks &Callbacks) { - TypeRecordKind RK = static_cast(Record.Type); + TypeRecordKind RK = static_cast(Record.kind()); T KnownRecord(RK); if (auto EC = Callbacks.visitKnownRecord(Record, KnownRecord)) return EC; @@ -97,7 +96,7 @@ CVTypeVisitor::CVTypeVisitor(TypeVisitorCallbacks &Callbacks) : Callbacks(Callbacks) {} Error CVTypeVisitor::finishVisitation(CVType &Record) { - switch (Record.Type) { + switch (Record.kind()) { default: if (auto EC = Callbacks.visitUnknownType(Record)) return EC; @@ -210,6 +209,14 @@ struct VisitHelper { } } + VisitHelper(TypeVisitorCallbackPipeline &Callbacks, VisitorDataSource Source) + : Visitor((Source == VDS_BytesPresent) ? Pipeline : Callbacks) { + if (Source == VDS_BytesPresent) { + Pipeline = Callbacks; + Pipeline.addCallbackToPipelineFront(Deserializer); + } + } + TypeDeserializer Deserializer; TypeVisitorCallbackPipeline Pipeline; CVTypeVisitor Visitor; @@ -223,6 +230,13 @@ Error llvm::codeview::visitTypeRecord(CVType &Record, TypeIndex Index, return V.Visitor.visitTypeRecord(Record, Index); } +Error llvm::codeview::visitTypeRecord(CVType &Record, TypeIndex Index, + TypeVisitorCallbackPipeline &Callbacks, + VisitorDataSource Source) { + VisitHelper V(Callbacks, Source); + return V.Visitor.visitTypeRecord(Record, Index); +} + Error llvm::codeview::visitTypeRecord(CVType &Record, TypeVisitorCallbacks &Callbacks, VisitorDataSource Source) { diff --git a/lib/DebugInfo/CodeView/CodeViewError.cpp b/lib/DebugInfo/CodeView/CodeViewError.cpp index 2a9753add311..69390c708f59 100644 --- a/lib/DebugInfo/CodeView/CodeViewError.cpp +++ b/lib/DebugInfo/CodeView/CodeViewError.cpp @@ -1,9 +1,8 @@ //===- CodeViewError.cpp - Error extensions for CodeView --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -14,6 +13,7 @@ using namespace llvm; using namespace llvm::codeview; +namespace { // FIXME: This class is only here to support the transition to llvm::Error. It // will be removed once this transition is complete. Clients should prefer to // deal with the Error value directly, rather than converting to error_code. @@ -39,6 +39,7 @@ public: llvm_unreachable("Unrecognized cv_error_code"); } }; +} // namespace static llvm::ManagedStatic CodeViewErrCategory; const std::error_category &llvm::codeview::CVErrorCategory() { diff --git a/lib/DebugInfo/CodeView/CodeViewRecordIO.cpp b/lib/DebugInfo/CodeView/CodeViewRecordIO.cpp index 4fc14480578e..2f49474115a1 100644 --- a/lib/DebugInfo/CodeView/CodeViewRecordIO.cpp +++ b/lib/DebugInfo/CodeView/CodeViewRecordIO.cpp @@ -1,9 +1,8 @@ //===- CodeViewRecordIO.cpp -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -21,6 +20,7 @@ Error CodeViewRecordIO::beginRecord(Optional MaxLength) { Limit.MaxLength = MaxLength; Limit.BeginOffset = getCurrentOffset(); Limits.push_back(Limit); + resetStreamedLen(); return Error::success(); } @@ -35,10 +35,29 @@ Error CodeViewRecordIO::endRecord() { // we don't know how big the record is until we're finished writing it, so // even though we don't commit the extraneous data, we still can't guarantee // we're at the end of the allocated data. + + if (isStreaming()) { + // For streaming mode, add padding to align with 4 byte boundaries for each + // record + uint32_t Align = getStreamedLen() % 4; + if (Align == 0) + return Error::success(); + + int PaddingBytes = 4 - Align; + while (PaddingBytes > 0) { + char Pad = static_cast(LF_PAD0 + PaddingBytes); + StringRef BytesSR = StringRef(&Pad, sizeof(Pad)); + Streamer->EmitBytes(BytesSR); + --PaddingBytes; + } + } return Error::success(); } uint32_t CodeViewRecordIO::maxFieldLength() const { + if (isStreaming()) + return 0; + assert(!Limits.empty() && "Not in a record!"); // The max length of the next field is the minimum of all lengths that would @@ -78,8 +97,13 @@ Error CodeViewRecordIO::skipPadding() { return Reader->skip(BytesToAdvance); } -Error CodeViewRecordIO::mapByteVectorTail(ArrayRef &Bytes) { - if (isWriting()) { +Error CodeViewRecordIO::mapByteVectorTail(ArrayRef &Bytes, + const Twine &Comment) { + if (isStreaming()) { + emitComment(Comment); + Streamer->EmitBinaryData(toStringRef(Bytes)); + incrStreamedLen(Bytes.size()); + } else if (isWriting()) { if (auto EC = Writer->writeBytes(Bytes)) return EC; } else { @@ -89,9 +113,10 @@ Error CodeViewRecordIO::mapByteVectorTail(ArrayRef &Bytes) { return Error::success(); } -Error CodeViewRecordIO::mapByteVectorTail(std::vector &Bytes) { +Error CodeViewRecordIO::mapByteVectorTail(std::vector &Bytes, + const Twine &Comment) { ArrayRef BytesRef(Bytes); - if (auto EC = mapByteVectorTail(BytesRef)) + if (auto EC = mapByteVectorTail(BytesRef, Comment)) return EC; if (!isWriting()) Bytes.assign(BytesRef.begin(), BytesRef.end()); @@ -99,22 +124,31 @@ Error CodeViewRecordIO::mapByteVectorTail(std::vector &Bytes) { return Error::success(); } -Error CodeViewRecordIO::mapInteger(TypeIndex &TypeInd) { - if (isWriting()) { +Error CodeViewRecordIO::mapInteger(TypeIndex &TypeInd, const Twine &Comment) { + if (isStreaming()) { + emitComment(Comment); + Streamer->EmitIntValue(TypeInd.getIndex(), sizeof(TypeInd.getIndex())); + incrStreamedLen(sizeof(TypeInd.getIndex())); + } else if (isWriting()) { if (auto EC = Writer->writeInteger(TypeInd.getIndex())) return EC; - return Error::success(); + } else { + uint32_t I; + if (auto EC = Reader->readInteger(I)) + return EC; + TypeInd.setIndex(I); } - - uint32_t I; - if (auto EC = Reader->readInteger(I)) - return EC; - TypeInd.setIndex(I); return Error::success(); } -Error CodeViewRecordIO::mapEncodedInteger(int64_t &Value) { - if (isWriting()) { +Error CodeViewRecordIO::mapEncodedInteger(int64_t &Value, + const Twine &Comment) { + if (isStreaming()) { + if (Value >= 0) + emitEncodedUnsignedInteger(static_cast(Value), Comment); + else + emitEncodedSignedInteger(Value, Comment); + } else if (isWriting()) { if (Value >= 0) { if (auto EC = writeEncodedUnsignedInteger(static_cast(Value))) return EC; @@ -132,8 +166,11 @@ Error CodeViewRecordIO::mapEncodedInteger(int64_t &Value) { return Error::success(); } -Error CodeViewRecordIO::mapEncodedInteger(uint64_t &Value) { - if (isWriting()) { +Error CodeViewRecordIO::mapEncodedInteger(uint64_t &Value, + const Twine &Comment) { + if (isStreaming()) + emitEncodedUnsignedInteger(Value, Comment); + else if (isWriting()) { if (auto EC = writeEncodedUnsignedInteger(Value)) return EC; } else { @@ -145,18 +182,28 @@ Error CodeViewRecordIO::mapEncodedInteger(uint64_t &Value) { return Error::success(); } -Error CodeViewRecordIO::mapEncodedInteger(APSInt &Value) { - if (isWriting()) { +Error CodeViewRecordIO::mapEncodedInteger(APSInt &Value, const Twine &Comment) { + if (isStreaming()) { + if (Value.isSigned()) + emitEncodedSignedInteger(Value.getSExtValue(), Comment); + else + emitEncodedUnsignedInteger(Value.getZExtValue(), Comment); + } else if (isWriting()) { if (Value.isSigned()) return writeEncodedSignedInteger(Value.getSExtValue()); return writeEncodedUnsignedInteger(Value.getZExtValue()); - } - - return consume(*Reader, Value); + } else + return consume(*Reader, Value); + return Error::success(); } -Error CodeViewRecordIO::mapStringZ(StringRef &Value) { - if (isWriting()) { +Error CodeViewRecordIO::mapStringZ(StringRef &Value, const Twine &Comment) { + if (isStreaming()) { + auto NullTerminatedString = StringRef(Value.data(), Value.size() + 1); + emitComment(Comment); + Streamer->EmitBytes(NullTerminatedString); + incrStreamedLen(NullTerminatedString.size()); + } else if (isWriting()) { // Truncate if we attempt to write too much. StringRef S = Value.take_front(maxFieldLength() - 1); if (auto EC = Writer->writeCString(S)) @@ -168,8 +215,18 @@ Error CodeViewRecordIO::mapStringZ(StringRef &Value) { return Error::success(); } -Error CodeViewRecordIO::mapGuid(GUID &Guid) { +Error CodeViewRecordIO::mapGuid(GUID &Guid, const Twine &Comment) { constexpr uint32_t GuidSize = 16; + + if (isStreaming()) { + StringRef GuidSR = + StringRef((reinterpret_cast(&Guid)), GuidSize); + emitComment(Comment); + Streamer->EmitBytes(GuidSR); + incrStreamedLen(GuidSize); + return Error::success(); + } + if (maxFieldLength() < GuidSize) return make_error(cv_error_code::insufficient_buffer); @@ -185,13 +242,17 @@ Error CodeViewRecordIO::mapGuid(GUID &Guid) { return Error::success(); } -Error CodeViewRecordIO::mapStringZVectorZ(std::vector &Value) { - if (isWriting()) { +Error CodeViewRecordIO::mapStringZVectorZ(std::vector &Value, + const Twine &Comment) { + + if (!isReading()) { + emitComment(Comment); for (auto V : Value) { if (auto EC = mapStringZ(V)) return EC; } - if (auto EC = Writer->writeInteger(0)) + uint8_t FinalZero = 0; + if (auto EC = mapInteger(FinalZero)) return EC; } else { StringRef S; @@ -206,6 +267,56 @@ Error CodeViewRecordIO::mapStringZVectorZ(std::vector &Value) { return Error::success(); } +void CodeViewRecordIO::emitEncodedSignedInteger(const int64_t &Value, + const Twine &Comment) { + assert(Value < 0 && "Encoded integer is not signed!"); + if (Value >= std::numeric_limits::min()) { + Streamer->EmitIntValue(LF_CHAR, 2); + emitComment(Comment); + Streamer->EmitIntValue(Value, 1); + incrStreamedLen(3); + } else if (Value >= std::numeric_limits::min()) { + Streamer->EmitIntValue(LF_SHORT, 2); + emitComment(Comment); + Streamer->EmitIntValue(Value, 2); + incrStreamedLen(4); + } else if (Value >= std::numeric_limits::min()) { + Streamer->EmitIntValue(LF_LONG, 2); + emitComment(Comment); + Streamer->EmitIntValue(Value, 4); + incrStreamedLen(6); + } else { + Streamer->EmitIntValue(LF_QUADWORD, 2); + emitComment(Comment); + Streamer->EmitIntValue(Value, 4); + incrStreamedLen(6); + } +} + +void CodeViewRecordIO::emitEncodedUnsignedInteger(const uint64_t &Value, + const Twine &Comment) { + if (Value < LF_NUMERIC) { + emitComment(Comment); + Streamer->EmitIntValue(Value, 2); + incrStreamedLen(2); + } else if (Value <= std::numeric_limits::max()) { + Streamer->EmitIntValue(LF_USHORT, 2); + emitComment(Comment); + Streamer->EmitIntValue(Value, 2); + incrStreamedLen(4); + } else if (Value <= std::numeric_limits::max()) { + Streamer->EmitIntValue(LF_ULONG, 2); + emitComment(Comment); + Streamer->EmitIntValue(Value, 4); + incrStreamedLen(6); + } else { + Streamer->EmitIntValue(LF_UQUADWORD, 2); + emitComment(Comment); + Streamer->EmitIntValue(Value, 8); + incrStreamedLen(6); + } +} + Error CodeViewRecordIO::writeEncodedSignedInteger(const int64_t &Value) { assert(Value < 0 && "Encoded integer is not signed!"); if (Value >= std::numeric_limits::min()) { diff --git a/lib/DebugInfo/CodeView/ContinuationRecordBuilder.cpp b/lib/DebugInfo/CodeView/ContinuationRecordBuilder.cpp index f180fc6990fc..799cffb7116e 100644 --- a/lib/DebugInfo/CodeView/ContinuationRecordBuilder.cpp +++ b/lib/DebugInfo/CodeView/ContinuationRecordBuilder.cpp @@ -66,14 +66,11 @@ void ContinuationRecordBuilder::begin(ContinuationRecordKind RecordKind) { InjectedSegmentBytes = ArrayRef(FLIB, FLIB + sizeof(SegmentInjection)); - CVType Type; - Type.Type = getTypeLeafKind(RecordKind); + // Seed the first record with an appropriate record prefix. + RecordPrefix Prefix(getTypeLeafKind(RecordKind)); + CVType Type(&Prefix, sizeof(Prefix)); cantFail(Mapping.visitTypeBegin(Type)); - // Seed the first trecord with an appropriate record prefix. - RecordPrefix Prefix; - Prefix.RecordLen = 0; - Prefix.RecordKind = Type.Type; cantFail(SegmentWriter.writeObject(Prefix)); } @@ -156,14 +153,9 @@ CVType ContinuationRecordBuilder::createSegmentRecord( MutableArrayRef Data = Buffer.data(); Data = Data.slice(OffBegin, OffEnd - OffBegin); - CVType Type; - Type.Type = getTypeLeafKind(*Kind); - Type.RecordData = Data; - // Write the length to the RecordPrefix, making sure it does not include // sizeof(RecordPrefix.Length) RecordPrefix *Prefix = reinterpret_cast(Data.data()); - assert(Prefix->RecordKind == Type.Type); Prefix->RecordLen = Data.size() - sizeof(RecordPrefix::RecordLen); if (RefersTo.hasValue()) { @@ -175,12 +167,12 @@ CVType ContinuationRecordBuilder::createSegmentRecord( CR->IndexRef = RefersTo->getIndex(); } - return Type; + return CVType(Data); } std::vector ContinuationRecordBuilder::end(TypeIndex Index) { - CVType Type; - Type.Type = getTypeLeafKind(*Kind); + RecordPrefix Prefix(getTypeLeafKind(*Kind)); + CVType Type(&Prefix, sizeof(Prefix)); cantFail(Mapping.visitTypeEnd(Type)); // We're now done, and we have a series of segments each beginning at an diff --git a/lib/DebugInfo/CodeView/DebugChecksumsSubsection.cpp b/lib/DebugInfo/CodeView/DebugChecksumsSubsection.cpp index 0f155a95d607..3d28bac00c44 100644 --- a/lib/DebugInfo/CodeView/DebugChecksumsSubsection.cpp +++ b/lib/DebugInfo/CodeView/DebugChecksumsSubsection.cpp @@ -1,9 +1,8 @@ //===- DebugChecksumsSubsection.cpp ---------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/CodeView/DebugCrossExSubsection.cpp b/lib/DebugInfo/CodeView/DebugCrossExSubsection.cpp index cef27787cfd1..b23410409f88 100644 --- a/lib/DebugInfo/CodeView/DebugCrossExSubsection.cpp +++ b/lib/DebugInfo/CodeView/DebugCrossExSubsection.cpp @@ -1,9 +1,8 @@ //===- DebugCrossExSubsection.cpp -----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/CodeView/DebugCrossImpSubsection.cpp b/lib/DebugInfo/CodeView/DebugCrossImpSubsection.cpp index 4001741f560a..dbadafd3aaf3 100644 --- a/lib/DebugInfo/CodeView/DebugCrossImpSubsection.cpp +++ b/lib/DebugInfo/CodeView/DebugCrossImpSubsection.cpp @@ -1,9 +1,8 @@ //===- DebugCrossImpSubsection.cpp ----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/CodeView/DebugFrameDataSubsection.cpp b/lib/DebugInfo/CodeView/DebugFrameDataSubsection.cpp index 5881bf177a55..be8c32d5b294 100644 --- a/lib/DebugInfo/CodeView/DebugFrameDataSubsection.cpp +++ b/lib/DebugInfo/CodeView/DebugFrameDataSubsection.cpp @@ -1,9 +1,8 @@ //===- DebugFrameDataSubsection.cpp -----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/CodeView/DebugInlineeLinesSubsection.cpp b/lib/DebugInfo/CodeView/DebugInlineeLinesSubsection.cpp index 077c103a615b..48ec7e4ecdd6 100644 --- a/lib/DebugInfo/CodeView/DebugInlineeLinesSubsection.cpp +++ b/lib/DebugInfo/CodeView/DebugInlineeLinesSubsection.cpp @@ -1,9 +1,8 @@ //===- DebugInlineeLinesSubsection.cpp ------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/CodeView/DebugLinesSubsection.cpp b/lib/DebugInfo/CodeView/DebugLinesSubsection.cpp index 57ad40819fbc..ea16c0a6c671 100644 --- a/lib/DebugInfo/CodeView/DebugLinesSubsection.cpp +++ b/lib/DebugInfo/CodeView/DebugLinesSubsection.cpp @@ -1,9 +1,8 @@ //===- DebugLinesSubsection.cpp -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/CodeView/DebugStringTableSubsection.cpp b/lib/DebugInfo/CodeView/DebugStringTableSubsection.cpp index 9b251f5931b3..63342749918d 100644 --- a/lib/DebugInfo/CodeView/DebugStringTableSubsection.cpp +++ b/lib/DebugInfo/CodeView/DebugStringTableSubsection.cpp @@ -1,9 +1,8 @@ //===- DebugStringTableSubsection.cpp - CodeView String Table -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/CodeView/DebugSubsection.cpp b/lib/DebugInfo/CodeView/DebugSubsection.cpp index 67b428bfa713..3f93463fe6d6 100644 --- a/lib/DebugInfo/CodeView/DebugSubsection.cpp +++ b/lib/DebugInfo/CodeView/DebugSubsection.cpp @@ -1,9 +1,8 @@ //===- DebugSubsection.cpp -----------------------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/CodeView/DebugSubsectionRecord.cpp b/lib/DebugInfo/CodeView/DebugSubsectionRecord.cpp index 55f343c11e7f..0f704f286ee9 100644 --- a/lib/DebugInfo/CodeView/DebugSubsectionRecord.cpp +++ b/lib/DebugInfo/CodeView/DebugSubsectionRecord.cpp @@ -1,9 +1,8 @@ //===- DebugSubsectionRecord.cpp ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/CodeView/DebugSubsectionVisitor.cpp b/lib/DebugInfo/CodeView/DebugSubsectionVisitor.cpp index 9b824333369b..7968b6a2d757 100644 --- a/lib/DebugInfo/CodeView/DebugSubsectionVisitor.cpp +++ b/lib/DebugInfo/CodeView/DebugSubsectionVisitor.cpp @@ -1,9 +1,8 @@ //===- DebugSubsectionVisitor.cpp -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/CodeView/DebugSymbolRVASubsection.cpp b/lib/DebugInfo/CodeView/DebugSymbolRVASubsection.cpp index 60fbf9d747b2..52328967357b 100644 --- a/lib/DebugInfo/CodeView/DebugSymbolRVASubsection.cpp +++ b/lib/DebugInfo/CodeView/DebugSymbolRVASubsection.cpp @@ -1,9 +1,8 @@ //===- DebugSymbolRVASubsection.cpp ---------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/CodeView/DebugSymbolsSubsection.cpp b/lib/DebugInfo/CodeView/DebugSymbolsSubsection.cpp index dc8ba8c929ae..c833103663e4 100644 --- a/lib/DebugInfo/CodeView/DebugSymbolsSubsection.cpp +++ b/lib/DebugInfo/CodeView/DebugSymbolsSubsection.cpp @@ -1,9 +1,8 @@ //===- DebugSymbolsSubsection.cpp -------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -31,4 +30,4 @@ Error DebugSymbolsSubsection::commit(BinaryStreamWriter &Writer) const { void DebugSymbolsSubsection::addSymbol(CVSymbol Symbol) { Records.push_back(Symbol); Length += Symbol.length(); -} \ No newline at end of file +} diff --git a/lib/DebugInfo/CodeView/EnumTables.cpp b/lib/DebugInfo/CodeView/EnumTables.cpp index ef4e42f79ebc..54e68ae4ea9f 100644 --- a/lib/DebugInfo/CodeView/EnumTables.cpp +++ b/lib/DebugInfo/CodeView/EnumTables.cpp @@ -1,9 +1,8 @@ //===- EnumTables.cpp - Enum to string conversion tables ------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -32,10 +31,20 @@ static const EnumEntry TypeLeafNames[] = { #undef CV_TYPE }; -static const EnumEntry RegisterNames[] = { +static const EnumEntry RegisterNames_X86[] = { +#define CV_REGISTERS_X86 +#define CV_REGISTER(name, val) CV_ENUM_CLASS_ENT(RegisterId, name), +#include "llvm/DebugInfo/CodeView/CodeViewRegisters.def" +#undef CV_REGISTER +#undef CV_REGISTERS_X86 +}; + +static const EnumEntry RegisterNames_ARM64[] = { +#define CV_REGISTERS_ARM64 #define CV_REGISTER(name, val) CV_ENUM_CLASS_ENT(RegisterId, name), #include "llvm/DebugInfo/CodeView/CodeViewRegisters.def" #undef CV_REGISTER +#undef CV_REGISTERS_ARM64 }; static const EnumEntry PublicSymFlagNames[] = { @@ -87,6 +96,7 @@ static const EnumEntry SourceLanguages[] = { CV_ENUM_ENT(SourceLanguage, ILAsm), CV_ENUM_ENT(SourceLanguage, Java), CV_ENUM_ENT(SourceLanguage, JScript), CV_ENUM_ENT(SourceLanguage, MSIL), CV_ENUM_ENT(SourceLanguage, HLSL), CV_ENUM_ENT(SourceLanguage, D), + CV_ENUM_ENT(SourceLanguage, Swift), }; static const EnumEntry CompileSym2FlagNames[] = { @@ -171,6 +181,7 @@ static const EnumEntry CPUTypeNames[] = { CV_ENUM_CLASS_ENT(CPUType, ARM_XMAC), CV_ENUM_CLASS_ENT(CPUType, ARM_WMMX), CV_ENUM_CLASS_ENT(CPUType, ARM7), + CV_ENUM_CLASS_ENT(CPUType, ARM64), CV_ENUM_CLASS_ENT(CPUType, Omni), CV_ENUM_CLASS_ENT(CPUType, Ia64), CV_ENUM_CLASS_ENT(CPUType, Ia64_2), @@ -300,8 +311,11 @@ ArrayRef> getTypeLeafNames() { return makeArrayRef(TypeLeafNames); } -ArrayRef> getRegisterNames() { - return makeArrayRef(RegisterNames); +ArrayRef> getRegisterNames(CPUType Cpu) { + if (Cpu == CPUType::ARM64) { + return makeArrayRef(RegisterNames_ARM64); + } + return makeArrayRef(RegisterNames_X86); } ArrayRef> getPublicSymFlagNames() { diff --git a/lib/DebugInfo/CodeView/Formatters.cpp b/lib/DebugInfo/CodeView/Formatters.cpp index b8d89c76da3b..a7a8c7ff82bf 100644 --- a/lib/DebugInfo/CodeView/Formatters.cpp +++ b/lib/DebugInfo/CodeView/Formatters.cpp @@ -1,9 +1,8 @@ //===- Formatters.cpp -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/CodeView/GlobalTypeTableBuilder.cpp b/lib/DebugInfo/CodeView/GlobalTypeTableBuilder.cpp index e76f9e12f0af..a7ad1d045f04 100644 --- a/lib/DebugInfo/CodeView/GlobalTypeTableBuilder.cpp +++ b/lib/DebugInfo/CodeView/GlobalTypeTableBuilder.cpp @@ -1,9 +1,8 @@ //===- GlobalTypeTableBuilder.cpp -----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -53,14 +52,7 @@ Optional GlobalTypeTableBuilder::getNext(TypeIndex Prev) { } CVType GlobalTypeTableBuilder::getType(TypeIndex Index) { - CVType Type; - Type.RecordData = SeenRecords[Index.toArrayIndex()]; - if (!Type.RecordData.empty()) { - assert(Type.RecordData.size() >= sizeof(RecordPrefix)); - const RecordPrefix *P = - reinterpret_cast(Type.RecordData.data()); - Type.Type = static_cast(uint16_t(P->RecordKind)); - } + CVType Type(SeenRecords[Index.toArrayIndex()]); return Type; } diff --git a/lib/DebugInfo/CodeView/LazyRandomTypeCollection.cpp b/lib/DebugInfo/CodeView/LazyRandomTypeCollection.cpp index ddcad8c631d7..dc1253b7a39f 100644 --- a/lib/DebugInfo/CodeView/LazyRandomTypeCollection.cpp +++ b/lib/DebugInfo/CodeView/LazyRandomTypeCollection.cpp @@ -1,9 +1,8 @@ //===- LazyRandomTypeCollection.cpp ---------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/CodeView/Line.cpp b/lib/DebugInfo/CodeView/Line.cpp index 4cb766b5fd26..53adc8cac511 100644 --- a/lib/DebugInfo/CodeView/Line.cpp +++ b/lib/DebugInfo/CodeView/Line.cpp @@ -1,9 +1,8 @@ //===-- Line.cpp ----------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/CodeView/MergingTypeTableBuilder.cpp b/lib/DebugInfo/CodeView/MergingTypeTableBuilder.cpp index 8aee4aa2e2ae..4d7cd468f3ee 100644 --- a/lib/DebugInfo/CodeView/MergingTypeTableBuilder.cpp +++ b/lib/DebugInfo/CodeView/MergingTypeTableBuilder.cpp @@ -1,9 +1,8 @@ //===- MergingTypeTableBuilder.cpp ----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -53,11 +52,7 @@ Optional MergingTypeTableBuilder::getNext(TypeIndex Prev) { } CVType MergingTypeTableBuilder::getType(TypeIndex Index) { - CVType Type; - Type.RecordData = SeenRecords[Index.toArrayIndex()]; - const RecordPrefix *P = - reinterpret_cast(Type.RecordData.data()); - Type.Type = static_cast(uint16_t(P->RecordKind)); + CVType Type(SeenRecords[Index.toArrayIndex()]); return Type; } diff --git a/lib/DebugInfo/CodeView/RecordName.cpp b/lib/DebugInfo/CodeView/RecordName.cpp index d868ae237a44..cfaad1581159 100644 --- a/lib/DebugInfo/CodeView/RecordName.cpp +++ b/lib/DebugInfo/CodeView/RecordName.cpp @@ -1,9 +1,8 @@ //===- RecordName.cpp ----------------------------------------- *- C++ --*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/CodeView/RecordSerialization.cpp b/lib/DebugInfo/CodeView/RecordSerialization.cpp index bff9a619a846..e7f032f9c670 100644 --- a/lib/DebugInfo/CodeView/RecordSerialization.cpp +++ b/lib/DebugInfo/CodeView/RecordSerialization.cpp @@ -1,9 +1,8 @@ //===-- RecordSerialization.cpp -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/DebugInfo/CodeView/SimpleTypeSerializer.cpp b/lib/DebugInfo/CodeView/SimpleTypeSerializer.cpp index d28b7c3c2d83..654c40a7470d 100644 --- a/lib/DebugInfo/CodeView/SimpleTypeSerializer.cpp +++ b/lib/DebugInfo/CodeView/SimpleTypeSerializer.cpp @@ -3,13 +3,6 @@ using namespace llvm; using namespace llvm::codeview; -static void writeRecordPrefix(BinaryStreamWriter &Writer, TypeLeafKind Kind) { - RecordPrefix Prefix; - Prefix.RecordKind = Kind; - Prefix.RecordLen = 0; - cantFail(Writer.writeObject(Prefix)); -} - static void addPadding(BinaryStreamWriter &Writer) { uint32_t Align = Writer.getOffset() % 4; if (Align == 0) @@ -32,10 +25,12 @@ ArrayRef SimpleTypeSerializer::serialize(T &Record) { BinaryStreamWriter Writer(ScratchBuffer, support::little); TypeRecordMapping Mapping(Writer); - CVType CVT; - CVT.Type = static_cast(Record.getKind()); + // Write the record prefix first with a dummy length but real kind. + RecordPrefix DummyPrefix(uint16_t(Record.getKind())); + cantFail(Writer.writeObject(DummyPrefix)); - writeRecordPrefix(Writer, CVT.Type); + RecordPrefix *Prefix = reinterpret_cast(ScratchBuffer.data()); + CVType CVT(Prefix, sizeof(RecordPrefix)); cantFail(Mapping.visitTypeBegin(CVT)); cantFail(Mapping.visitKnownRecord(CVT, Record)); @@ -43,8 +38,7 @@ ArrayRef SimpleTypeSerializer::serialize(T &Record) { addPadding(Writer); - RecordPrefix *Prefix = reinterpret_cast(ScratchBuffer.data()); - + // Update the size and kind after serialization. Prefix->RecordKind = CVT.kind(); Prefix->RecordLen = Writer.getOffset() - sizeof(uint16_t); diff --git a/lib/DebugInfo/CodeView/StringsAndChecksums.cpp b/lib/DebugInfo/CodeView/StringsAndChecksums.cpp index 85d9dbb8c7df..9e204eec8604 100644 --- a/lib/DebugInfo/CodeView/StringsAndChecksums.cpp +++ b/lib/DebugInfo/CodeView/StringsAndChecksums.cpp @@ -1,9 +1,8 @@ //===- StringsAndChecksums.cpp --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/CodeView/SymbolDumper.cpp b/lib/DebugInfo/CodeView/SymbolDumper.cpp index 04e0bab745d3..27cb7e35234b 100644 --- a/lib/DebugInfo/CodeView/SymbolDumper.cpp +++ b/lib/DebugInfo/CodeView/SymbolDumper.cpp @@ -1,9 +1,8 @@ //===-- SymbolDumper.cpp - CodeView symbol info dumper ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -102,10 +101,10 @@ void CVSymbolDumperImpl::printTypeIndex(StringRef FieldName, TypeIndex TI) { } Error CVSymbolDumperImpl::visitSymbolBegin(CVSymbol &CVR) { - W.startLine() << getSymbolKindName(CVR.Type); + W.startLine() << getSymbolKindName(CVR.kind()); W.getOStream() << " {\n"; W.indent(); - W.printEnum("Kind", unsigned(CVR.Type), getSymbolTypeNames()); + W.printEnum("Kind", unsigned(CVR.kind()), getSymbolTypeNames()); return Error::success(); } @@ -326,7 +325,7 @@ Error CVSymbolDumperImpl::visitKnownRecord( Error CVSymbolDumperImpl::visitKnownRecord( CVSymbol &CVR, DefRangeRegisterRelSym &DefRangeRegisterRel) { W.printEnum("BaseRegister", uint16_t(DefRangeRegisterRel.Hdr.Register), - getRegisterNames()); + getRegisterNames(CompilationCPUType)); W.printBoolean("HasSpilledUDTMember", DefRangeRegisterRel.hasSpilledUDTMember()); W.printNumber("OffsetInParent", DefRangeRegisterRel.offsetInParent()); @@ -340,7 +339,7 @@ Error CVSymbolDumperImpl::visitKnownRecord( Error CVSymbolDumperImpl::visitKnownRecord( CVSymbol &CVR, DefRangeRegisterSym &DefRangeRegister) { W.printEnum("Register", uint16_t(DefRangeRegister.Hdr.Register), - getRegisterNames()); + getRegisterNames(CompilationCPUType)); W.printNumber("MayHaveNoName", DefRangeRegister.Hdr.MayHaveNoName); printLocalVariableAddrRange(DefRangeRegister.Range, DefRangeRegister.getRelocationOffset()); @@ -351,7 +350,7 @@ Error CVSymbolDumperImpl::visitKnownRecord( Error CVSymbolDumperImpl::visitKnownRecord( CVSymbol &CVR, DefRangeSubfieldRegisterSym &DefRangeSubfieldRegister) { W.printEnum("Register", uint16_t(DefRangeSubfieldRegister.Hdr.Register), - getRegisterNames()); + getRegisterNames(CompilationCPUType)); W.printNumber("MayHaveNoName", DefRangeSubfieldRegister.Hdr.MayHaveNoName); W.printNumber("OffsetInParent", DefRangeSubfieldRegister.Hdr.OffsetInParent); printLocalVariableAddrRange(DefRangeSubfieldRegister.Range, @@ -404,7 +403,8 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, FrameCookie.getRelocationOffset(), FrameCookie.CodeOffset, &LinkageName); } - W.printEnum("Register", uint16_t(FrameCookie.Register), getRegisterNames()); + W.printEnum("Register", uint16_t(FrameCookie.Register), + getRegisterNames(CompilationCPUType)); W.printEnum("CookieKind", uint16_t(FrameCookie.CookieKind), getFrameCookieKindNames()); W.printHex("Flags", FrameCookie.Flags); @@ -425,10 +425,10 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, getFrameProcSymFlagNames()); W.printEnum("LocalFramePtrReg", uint16_t(FrameProc.getLocalFramePtrReg(CompilationCPUType)), - getRegisterNames()); + getRegisterNames(CompilationCPUType)); W.printEnum("ParamFramePtrReg", uint16_t(FrameProc.getParamFramePtrReg(CompilationCPUType)), - getRegisterNames()); + getRegisterNames(CompilationCPUType)); return Error::success(); } @@ -506,7 +506,8 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, RegisterSym &Register) { printTypeIndex("Type", Register.Index); - W.printEnum("Seg", uint16_t(Register.Register), getRegisterNames()); + W.printEnum("Seg", uint16_t(Register.Register), + getRegisterNames(CompilationCPUType)); W.printString("Name", Register.Name); return Error::success(); } @@ -600,7 +601,8 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, RegRelativeSym &RegRel) { W.printHex("Offset", RegRel.Offset); printTypeIndex("Type", RegRel.Type); - W.printEnum("Register", uint16_t(RegRel.Register), getRegisterNames()); + W.printEnum("Register", uint16_t(RegRel.Register), + getRegisterNames(CompilationCPUType)); W.printString("VarName", RegRel.Name); return Error::success(); } @@ -631,6 +633,18 @@ Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, return Error::success(); } +Error CVSymbolDumperImpl::visitKnownRecord(CVSymbol &CVR, + AnnotationSym &Annot) { + W.printHex("Offset", Annot.CodeOffset); + W.printHex("Segment", Annot.Segment); + + ListScope S(W, "Strings"); + for (StringRef Str : Annot.Strings) + W.printString(Str); + + return Error::success(); +} + Error CVSymbolDumperImpl::visitUnknownSymbol(CVSymbol &CVR) { W.printNumber("Length", CVR.length()); return Error::success(); diff --git a/lib/DebugInfo/CodeView/SymbolRecordHelpers.cpp b/lib/DebugInfo/CodeView/SymbolRecordHelpers.cpp index 01746138ad1f..51a5a9e9243e 100644 --- a/lib/DebugInfo/CodeView/SymbolRecordHelpers.cpp +++ b/lib/DebugInfo/CodeView/SymbolRecordHelpers.cpp @@ -1,9 +1,8 @@ //===- SymbolRecordHelpers.cpp ----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/CodeView/SymbolRecordMapping.cpp b/lib/DebugInfo/CodeView/SymbolRecordMapping.cpp index 2af8205cebc3..70889839ef48 100644 --- a/lib/DebugInfo/CodeView/SymbolRecordMapping.cpp +++ b/lib/DebugInfo/CodeView/SymbolRecordMapping.cpp @@ -1,9 +1,8 @@ //===- SymbolRecordMapping.cpp -----------------------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -472,6 +471,18 @@ Error SymbolRecordMapping::visitKnownRecord(CVSymbol &CVR, return Error::success(); } +Error SymbolRecordMapping::visitKnownRecord(CVSymbol &CVR, + AnnotationSym &Annot) { + + error(IO.mapInteger(Annot.CodeOffset)); + error(IO.mapInteger(Annot.Segment)); + error(IO.mapVectorN( + Annot.Strings, + [](CodeViewRecordIO &IO, StringRef &S) { return IO.mapStringZ(S); })); + + return Error::success(); +} + RegisterId codeview::decodeFramePtrReg(EncodedFramePtrReg EncodedReg, CPUType CPU) { assert(unsigned(EncodedReg) < 4); diff --git a/lib/DebugInfo/CodeView/SymbolSerializer.cpp b/lib/DebugInfo/CodeView/SymbolSerializer.cpp index 0071ecc85685..de9bb42b1798 100644 --- a/lib/DebugInfo/CodeView/SymbolSerializer.cpp +++ b/lib/DebugInfo/CodeView/SymbolSerializer.cpp @@ -1,9 +1,8 @@ //===- SymbolSerializer.cpp -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/CodeView/TypeDumpVisitor.cpp b/lib/DebugInfo/CodeView/TypeDumpVisitor.cpp index f5d3bea43a14..d5fea5ee5e29 100644 --- a/lib/DebugInfo/CodeView/TypeDumpVisitor.cpp +++ b/lib/DebugInfo/CodeView/TypeDumpVisitor.cpp @@ -1,9 +1,8 @@ //===-- TypeDumpVisitor.cpp - CodeView type info dumper ----------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -172,11 +171,11 @@ Error TypeDumpVisitor::visitTypeBegin(CVType &Record) { } Error TypeDumpVisitor::visitTypeBegin(CVType &Record, TypeIndex Index) { - W->startLine() << getLeafTypeName(Record.Type); + W->startLine() << getLeafTypeName(Record.kind()); W->getOStream() << " (" << HexNumber(Index.getIndex()) << ")"; W->getOStream() << " {\n"; W->indent(); - W->printEnum("TypeLeafKind", unsigned(Record.Type), + W->printEnum("TypeLeafKind", unsigned(Record.kind()), makeArrayRef(LeafTypeNames)); return Error::success(); } diff --git a/lib/DebugInfo/CodeView/TypeHashing.cpp b/lib/DebugInfo/CodeView/TypeHashing.cpp index 826faef35875..2dbc11a84f0b 100644 --- a/lib/DebugInfo/CodeView/TypeHashing.cpp +++ b/lib/DebugInfo/CodeView/TypeHashing.cpp @@ -1,9 +1,8 @@ //===- TypeHashing.cpp -------------------------------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -55,10 +54,16 @@ GloballyHashedType::hashType(ArrayRef RecordData, reinterpret_cast(RefData.data()), Ref.Count); for (TypeIndex TI : Indices) { ArrayRef BytesToHash; - if (TI.isSimple() || TI.isNoneType() || TI.toArrayIndex() >= Prev.size()) { + if (TI.isSimple() || TI.isNoneType()) { const uint8_t *IndexBytes = reinterpret_cast(&TI); BytesToHash = makeArrayRef(IndexBytes, sizeof(TypeIndex)); } else { + if (TI.toArrayIndex() >= Prev.size() || + Prev[TI.toArrayIndex()].empty()) { + // There are references to yet-unhashed records. Suspend hashing for + // this record until all the other records are processed. + return {}; + } BytesToHash = Prev[TI.toArrayIndex()].Hash; } S.update(BytesToHash); diff --git a/lib/DebugInfo/CodeView/TypeIndex.cpp b/lib/DebugInfo/CodeView/TypeIndex.cpp index 332d67470da5..604d342448d3 100644 --- a/lib/DebugInfo/CodeView/TypeIndex.cpp +++ b/lib/DebugInfo/CodeView/TypeIndex.cpp @@ -1,9 +1,8 @@ //===-- TypeIndex.cpp - CodeView type index ---------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp b/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp index 839ab6f0a705..e84e1c9cea78 100644 --- a/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp +++ b/lib/DebugInfo/CodeView/TypeIndexDiscovery.cpp @@ -1,9 +1,8 @@ //===- TypeIndexDiscovery.cpp -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "llvm/DebugInfo/CodeView/TypeIndexDiscovery.h" @@ -364,14 +363,16 @@ static bool discoverTypeIndices(ArrayRef Content, SymbolKind Kind, // values. One idea is to define some structures representing these types // that would allow the use of offsetof(). switch (Kind) { - case SymbolKind::S_GPROC32: - case SymbolKind::S_LPROC32: case SymbolKind::S_GPROC32_ID: case SymbolKind::S_LPROC32_ID: case SymbolKind::S_LPROC32_DPC: case SymbolKind::S_LPROC32_DPC_ID: Refs.push_back({TiRefKind::IndexRef, 24, 1}); // LF_FUNC_ID break; + case SymbolKind::S_GPROC32: + case SymbolKind::S_LPROC32: + Refs.push_back({TiRefKind::TypeRef, 24, 1}); // Type + break; case SymbolKind::S_UDT: Refs.push_back({TiRefKind::TypeRef, 0, 1}); // UDT break; diff --git a/lib/DebugInfo/CodeView/TypeRecordHelpers.cpp b/lib/DebugInfo/CodeView/TypeRecordHelpers.cpp index 2a66474cf5b6..8e632f3be460 100644 --- a/lib/DebugInfo/CodeView/TypeRecordHelpers.cpp +++ b/lib/DebugInfo/CodeView/TypeRecordHelpers.cpp @@ -1,9 +1,8 @@ //===- TypeRecordHelpers.cpp ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/CodeView/TypeRecordMapping.cpp b/lib/DebugInfo/CodeView/TypeRecordMapping.cpp index 3203ff64d3b1..47928c2eef64 100644 --- a/lib/DebugInfo/CodeView/TypeRecordMapping.cpp +++ b/lib/DebugInfo/CodeView/TypeRecordMapping.cpp @@ -1,9 +1,8 @@ //===- TypeRecordMapping.cpp ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -22,19 +21,19 @@ struct MapOneMethodRecord { : IsFromOverloadList(IsFromOverloadList) {} Error operator()(CodeViewRecordIO &IO, OneMethodRecord &Method) const { - error(IO.mapInteger(Method.Attrs.Attrs)); + error(IO.mapInteger(Method.Attrs.Attrs, "AccessSpecifier")); if (IsFromOverloadList) { uint16_t Padding = 0; - error(IO.mapInteger(Padding)); + error(IO.mapInteger(Padding, "Padding")); } - error(IO.mapInteger(Method.Type)); + error(IO.mapInteger(Method.Type, "Type")); if (Method.isIntroducingVirtual()) { - error(IO.mapInteger(Method.VFTableOffset)); - } else if (!IO.isWriting()) + error(IO.mapInteger(Method.VFTableOffset, "VFTableOffset")); + } else if (IO.isReading()) Method.VFTableOffset = -1; if (!IsFromOverloadList) - error(IO.mapStringZ(Method.Name)); + error(IO.mapStringZ(Method.Name, "Name")); return Error::success(); } @@ -73,9 +72,12 @@ static Error mapNameAndUniqueName(CodeViewRecordIO &IO, StringRef &Name, error(IO.mapStringZ(N)); } } else { - error(IO.mapStringZ(Name)); + // Reading & Streaming mode come after writing mode is executed for each + // record. Truncating large names are done during writing, so its not + // necessary to do it while reading or streaming. + error(IO.mapStringZ(Name, "Name")); if (HasUniqueName) - error(IO.mapStringZ(UniqueName)); + error(IO.mapStringZ(UniqueName, "LinkageName")); } return Error::success(); @@ -89,14 +91,18 @@ Error TypeRecordMapping::visitTypeBegin(CVType &CVR) { // split with continuation records. All other record types cannot be // longer than the maximum record length. Optional MaxLen; - if (CVR.Type != TypeLeafKind::LF_FIELDLIST && - CVR.Type != TypeLeafKind::LF_METHODLIST) + if (CVR.kind() != TypeLeafKind::LF_FIELDLIST && + CVR.kind() != TypeLeafKind::LF_METHODLIST) MaxLen = MaxRecordLength - sizeof(RecordPrefix); error(IO.beginRecord(MaxLen)); - TypeKind = CVR.Type; + TypeKind = CVR.kind(); return Error::success(); } +Error TypeRecordMapping::visitTypeBegin(CVType &CVR, TypeIndex Index) { + return visitTypeBegin(CVR); +} + Error TypeRecordMapping::visitTypeEnd(CVType &Record) { assert(TypeKind.hasValue() && "Not in a type mapping!"); assert(!MemberKind.hasValue() && "Still in a member mapping!"); @@ -127,7 +133,7 @@ Error TypeRecordMapping::visitMemberEnd(CVMemberRecord &Record) { assert(TypeKind.hasValue() && "Not in a type mapping!"); assert(MemberKind.hasValue() && "Not in a member mapping!"); - if (!IO.isWriting()) { + if (IO.isReading()) { if (auto EC = IO.skipPadding()) return EC; } @@ -138,33 +144,32 @@ Error TypeRecordMapping::visitMemberEnd(CVMemberRecord &Record) { } Error TypeRecordMapping::visitKnownRecord(CVType &CVR, ModifierRecord &Record) { - error(IO.mapInteger(Record.ModifiedType)); - error(IO.mapEnum(Record.Modifiers)); - + error(IO.mapInteger(Record.ModifiedType, "ModifiedType")); + error(IO.mapEnum(Record.Modifiers, "Modifiers")); return Error::success(); } Error TypeRecordMapping::visitKnownRecord(CVType &CVR, ProcedureRecord &Record) { - error(IO.mapInteger(Record.ReturnType)); - error(IO.mapEnum(Record.CallConv)); - error(IO.mapEnum(Record.Options)); - error(IO.mapInteger(Record.ParameterCount)); - error(IO.mapInteger(Record.ArgumentList)); + error(IO.mapInteger(Record.ReturnType, "ReturnType")); + error(IO.mapEnum(Record.CallConv, "CallingConvention")); + error(IO.mapEnum(Record.Options, "FunctionOptions")); + error(IO.mapInteger(Record.ParameterCount, "NumParameters")); + error(IO.mapInteger(Record.ArgumentList, "ArgListType")); return Error::success(); } Error TypeRecordMapping::visitKnownRecord(CVType &CVR, MemberFunctionRecord &Record) { - error(IO.mapInteger(Record.ReturnType)); - error(IO.mapInteger(Record.ClassType)); - error(IO.mapInteger(Record.ThisType)); - error(IO.mapEnum(Record.CallConv)); - error(IO.mapEnum(Record.Options)); - error(IO.mapInteger(Record.ParameterCount)); - error(IO.mapInteger(Record.ArgumentList)); - error(IO.mapInteger(Record.ThisPointerAdjustment)); + error(IO.mapInteger(Record.ReturnType, "ReturnType")); + error(IO.mapInteger(Record.ClassType, "ClassType")); + error(IO.mapInteger(Record.ThisType, "ThisType")); + error(IO.mapEnum(Record.CallConv, "CallingConvention")); + error(IO.mapEnum(Record.Options, "FunctionOptions")); + error(IO.mapInteger(Record.ParameterCount, "NumParameters")); + error(IO.mapInteger(Record.ArgumentList, "ArgListType")); + error(IO.mapInteger(Record.ThisPointerAdjustment, "ThisAdjustment")); return Error::success(); } @@ -172,8 +177,10 @@ Error TypeRecordMapping::visitKnownRecord(CVType &CVR, Error TypeRecordMapping::visitKnownRecord(CVType &CVR, ArgListRecord &Record) { error(IO.mapVectorN( Record.ArgIndices, - [](CodeViewRecordIO &IO, TypeIndex &N) { return IO.mapInteger(N); })); - + [](CodeViewRecordIO &IO, TypeIndex &N) { + return IO.mapInteger(N, "Argument"); + }, + "NumArgs")); return Error::success(); } @@ -181,47 +188,50 @@ Error TypeRecordMapping::visitKnownRecord(CVType &CVR, StringListRecord &Record) { error(IO.mapVectorN( Record.StringIndices, - [](CodeViewRecordIO &IO, TypeIndex &N) { return IO.mapInteger(N); })); + [](CodeViewRecordIO &IO, TypeIndex &N) { + return IO.mapInteger(N, "Strings"); + }, + "NumStrings")); return Error::success(); } Error TypeRecordMapping::visitKnownRecord(CVType &CVR, PointerRecord &Record) { - error(IO.mapInteger(Record.ReferentType)); - error(IO.mapInteger(Record.Attrs)); + error(IO.mapInteger(Record.ReferentType, "PointeeType")); + error(IO.mapInteger(Record.Attrs, "Attributes")); if (Record.isPointerToMember()) { - if (!IO.isWriting()) + if (IO.isReading()) Record.MemberInfo.emplace(); MemberPointerInfo &M = *Record.MemberInfo; - error(IO.mapInteger(M.ContainingType)); - error(IO.mapEnum(M.Representation)); + error(IO.mapInteger(M.ContainingType, "ClassType")); + error(IO.mapEnum(M.Representation, "Representation")); } return Error::success(); } Error TypeRecordMapping::visitKnownRecord(CVType &CVR, ArrayRecord &Record) { - error(IO.mapInteger(Record.ElementType)); - error(IO.mapInteger(Record.IndexType)); - error(IO.mapEncodedInteger(Record.Size)); - error(IO.mapStringZ(Record.Name)); + error(IO.mapInteger(Record.ElementType, "ElementType")); + error(IO.mapInteger(Record.IndexType, "IndexType")); + error(IO.mapEncodedInteger(Record.Size, "SizeOf")); + error(IO.mapStringZ(Record.Name, "Name")); return Error::success(); } Error TypeRecordMapping::visitKnownRecord(CVType &CVR, ClassRecord &Record) { - assert((CVR.Type == TypeLeafKind::LF_STRUCTURE) || - (CVR.Type == TypeLeafKind::LF_CLASS) || - (CVR.Type == TypeLeafKind::LF_INTERFACE)); - - error(IO.mapInteger(Record.MemberCount)); - error(IO.mapEnum(Record.Options)); - error(IO.mapInteger(Record.FieldList)); - error(IO.mapInteger(Record.DerivationList)); - error(IO.mapInteger(Record.VTableShape)); - error(IO.mapEncodedInteger(Record.Size)); + assert((CVR.kind() == TypeLeafKind::LF_STRUCTURE) || + (CVR.kind() == TypeLeafKind::LF_CLASS) || + (CVR.kind() == TypeLeafKind::LF_INTERFACE)); + + error(IO.mapInteger(Record.MemberCount, "MemberCount")); + error(IO.mapEnum(Record.Options, "Properties")); + error(IO.mapInteger(Record.FieldList, "FieldList")); + error(IO.mapInteger(Record.DerivationList, "DerivedFrom")); + error(IO.mapInteger(Record.VTableShape, "VShape")); + error(IO.mapEncodedInteger(Record.Size, "SizeOf")); error(mapNameAndUniqueName(IO, Record.Name, Record.UniqueName, Record.hasUniqueName())); @@ -229,10 +239,10 @@ Error TypeRecordMapping::visitKnownRecord(CVType &CVR, ClassRecord &Record) { } Error TypeRecordMapping::visitKnownRecord(CVType &CVR, UnionRecord &Record) { - error(IO.mapInteger(Record.MemberCount)); - error(IO.mapEnum(Record.Options)); - error(IO.mapInteger(Record.FieldList)); - error(IO.mapEncodedInteger(Record.Size)); + error(IO.mapInteger(Record.MemberCount, "MemberCount")); + error(IO.mapEnum(Record.Options, "Properties")); + error(IO.mapInteger(Record.FieldList, "FieldList")); + error(IO.mapEncodedInteger(Record.Size, "SizeOf")); error(mapNameAndUniqueName(IO, Record.Name, Record.UniqueName, Record.hasUniqueName())); @@ -240,10 +250,10 @@ Error TypeRecordMapping::visitKnownRecord(CVType &CVR, UnionRecord &Record) { } Error TypeRecordMapping::visitKnownRecord(CVType &CVR, EnumRecord &Record) { - error(IO.mapInteger(Record.MemberCount)); - error(IO.mapEnum(Record.Options)); - error(IO.mapInteger(Record.UnderlyingType)); - error(IO.mapInteger(Record.FieldList)); + error(IO.mapInteger(Record.MemberCount, "NumEnumerators")); + error(IO.mapEnum(Record.Options, "Properties")); + error(IO.mapInteger(Record.UnderlyingType, "UnderlyingType")); + error(IO.mapInteger(Record.FieldList, "FieldListType")); error(mapNameAndUniqueName(IO, Record.Name, Record.UniqueName, Record.hasUniqueName())); @@ -251,9 +261,9 @@ Error TypeRecordMapping::visitKnownRecord(CVType &CVR, EnumRecord &Record) { } Error TypeRecordMapping::visitKnownRecord(CVType &CVR, BitFieldRecord &Record) { - error(IO.mapInteger(Record.Type)); - error(IO.mapInteger(Record.BitSize)); - error(IO.mapInteger(Record.BitOffset)); + error(IO.mapInteger(Record.Type, "Type")); + error(IO.mapInteger(Record.BitSize, "BitSize")); + error(IO.mapInteger(Record.BitOffset, "BitOffset")); return Error::success(); } @@ -261,10 +271,10 @@ Error TypeRecordMapping::visitKnownRecord(CVType &CVR, BitFieldRecord &Record) { Error TypeRecordMapping::visitKnownRecord(CVType &CVR, VFTableShapeRecord &Record) { uint16_t Size; - if (IO.isWriting()) { + if (!IO.isReading()) { ArrayRef Slots = Record.getSlots(); Size = Slots.size(); - error(IO.mapInteger(Size)); + error(IO.mapInteger(Size, "VFEntryCount")); for (size_t SlotIndex = 0; SlotIndex < Slots.size(); SlotIndex += 2) { uint8_t Byte = static_cast(Slots[SlotIndex]) << 4; @@ -288,61 +298,64 @@ Error TypeRecordMapping::visitKnownRecord(CVType &CVR, } Error TypeRecordMapping::visitKnownRecord(CVType &CVR, VFTableRecord &Record) { - error(IO.mapInteger(Record.CompleteClass)); - error(IO.mapInteger(Record.OverriddenVFTable)); - error(IO.mapInteger(Record.VFPtrOffset)); + error(IO.mapInteger(Record.CompleteClass, "CompleteClass")); + error(IO.mapInteger(Record.OverriddenVFTable, "OverriddenVFTable")); + error(IO.mapInteger(Record.VFPtrOffset, "VFPtrOffset")); uint32_t NamesLen = 0; - if (IO.isWriting()) { + if (!IO.isReading()) { for (auto Name : Record.MethodNames) NamesLen += Name.size() + 1; } error(IO.mapInteger(NamesLen)); error(IO.mapVectorTail( Record.MethodNames, - [](CodeViewRecordIO &IO, StringRef &S) { return IO.mapStringZ(S); })); + [](CodeViewRecordIO &IO, StringRef &S) { + return IO.mapStringZ(S, "MethodName"); + }, + "VFTableName")); return Error::success(); } Error TypeRecordMapping::visitKnownRecord(CVType &CVR, StringIdRecord &Record) { - error(IO.mapInteger(Record.Id)); - error(IO.mapStringZ(Record.String)); + error(IO.mapInteger(Record.Id, "Id")); + error(IO.mapStringZ(Record.String, "StringData")); return Error::success(); } Error TypeRecordMapping::visitKnownRecord(CVType &CVR, UdtSourceLineRecord &Record) { - error(IO.mapInteger(Record.UDT)); - error(IO.mapInteger(Record.SourceFile)); - error(IO.mapInteger(Record.LineNumber)); + error(IO.mapInteger(Record.UDT, "UDT")); + error(IO.mapInteger(Record.SourceFile, "SourceFile")); + error(IO.mapInteger(Record.LineNumber, "LineNumber")); return Error::success(); } Error TypeRecordMapping::visitKnownRecord(CVType &CVR, UdtModSourceLineRecord &Record) { - error(IO.mapInteger(Record.UDT)); - error(IO.mapInteger(Record.SourceFile)); - error(IO.mapInteger(Record.LineNumber)); - error(IO.mapInteger(Record.Module)); + error(IO.mapInteger(Record.UDT, "UDT")); + error(IO.mapInteger(Record.SourceFile, "SourceFile")); + error(IO.mapInteger(Record.LineNumber, "LineNumber")); + error(IO.mapInteger(Record.Module, "Module")); return Error::success(); } Error TypeRecordMapping::visitKnownRecord(CVType &CVR, FuncIdRecord &Record) { - error(IO.mapInteger(Record.ParentScope)); - error(IO.mapInteger(Record.FunctionType)); - error(IO.mapStringZ(Record.Name)); + error(IO.mapInteger(Record.ParentScope, "ParentScope")); + error(IO.mapInteger(Record.FunctionType, "FunctionType")); + error(IO.mapStringZ(Record.Name, "Name")); return Error::success(); } Error TypeRecordMapping::visitKnownRecord(CVType &CVR, MemberFuncIdRecord &Record) { - error(IO.mapInteger(Record.ClassType)); - error(IO.mapInteger(Record.FunctionType)); - error(IO.mapStringZ(Record.Name)); + error(IO.mapInteger(Record.ClassType, "ClassType")); + error(IO.mapInteger(Record.FunctionType, "FunctionType")); + error(IO.mapStringZ(Record.Name, "Name")); return Error::success(); } @@ -351,7 +364,10 @@ Error TypeRecordMapping::visitKnownRecord(CVType &CVR, BuildInfoRecord &Record) { error(IO.mapVectorN( Record.ArgIndices, - [](CodeViewRecordIO &IO, TypeIndex &N) { return IO.mapInteger(N); })); + [](CodeViewRecordIO &IO, TypeIndex &N) { + return IO.mapInteger(N, "Argument"); + }, + "NumArgs")); return Error::success(); } @@ -360,7 +376,7 @@ Error TypeRecordMapping::visitKnownRecord(CVType &CVR, MethodOverloadListRecord &Record) { // TODO: Split the list into multiple records if it's longer than 64KB, using // a subrecord of TypeRecordKind::Index to chain the records together. - error(IO.mapVectorTail(Record.Methods, MapOneMethodRecord(true))); + error(IO.mapVectorTail(Record.Methods, MapOneMethodRecord(true), "Method")); return Error::success(); } @@ -374,22 +390,22 @@ Error TypeRecordMapping::visitKnownRecord(CVType &CVR, Error TypeRecordMapping::visitKnownRecord(CVType &CVR, TypeServer2Record &Record) { - error(IO.mapGuid(Record.Guid)); - error(IO.mapInteger(Record.Age)); - error(IO.mapStringZ(Record.Name)); + error(IO.mapGuid(Record.Guid, "Guid")); + error(IO.mapInteger(Record.Age, "Age")); + error(IO.mapStringZ(Record.Name, "Name")); return Error::success(); } Error TypeRecordMapping::visitKnownRecord(CVType &CVR, LabelRecord &Record) { - error(IO.mapEnum(Record.Mode)); + error(IO.mapEnum(Record.Mode, "Mode")); return Error::success(); } Error TypeRecordMapping::visitKnownMember(CVMemberRecord &CVR, BaseClassRecord &Record) { - error(IO.mapInteger(Record.Attrs.Attrs)); - error(IO.mapInteger(Record.Type)); - error(IO.mapEncodedInteger(Record.Offset)); + error(IO.mapInteger(Record.Attrs.Attrs, "AccessSpecifier")); + error(IO.mapInteger(Record.Type, "BaseType")); + error(IO.mapEncodedInteger(Record.Offset, "BaseOffset")); return Error::success(); } @@ -399,27 +415,27 @@ Error TypeRecordMapping::visitKnownMember(CVMemberRecord &CVR, error(IO.mapInteger(Record.Attrs.Attrs)); // FIXME: Handle full APInt such as __int128. - error(IO.mapEncodedInteger(Record.Value)); - error(IO.mapStringZ(Record.Name)); + error(IO.mapEncodedInteger(Record.Value, "EnumValue")); + error(IO.mapStringZ(Record.Name, "Name")); return Error::success(); } Error TypeRecordMapping::visitKnownMember(CVMemberRecord &CVR, DataMemberRecord &Record) { - error(IO.mapInteger(Record.Attrs.Attrs)); - error(IO.mapInteger(Record.Type)); - error(IO.mapEncodedInteger(Record.FieldOffset)); - error(IO.mapStringZ(Record.Name)); + error(IO.mapInteger(Record.Attrs.Attrs, "AccessSpecifier")); + error(IO.mapInteger(Record.Type, "Type")); + error(IO.mapEncodedInteger(Record.FieldOffset, "FieldOffset")); + error(IO.mapStringZ(Record.Name, "Name")); return Error::success(); } Error TypeRecordMapping::visitKnownMember(CVMemberRecord &CVR, OverloadedMethodRecord &Record) { - error(IO.mapInteger(Record.NumOverloads)); - error(IO.mapInteger(Record.MethodList)); - error(IO.mapStringZ(Record.Name)); + error(IO.mapInteger(Record.NumOverloads, "MethodCount")); + error(IO.mapInteger(Record.MethodList, "MethodListIndex")); + error(IO.mapStringZ(Record.Name, "Name")); return Error::success(); } @@ -434,9 +450,9 @@ Error TypeRecordMapping::visitKnownMember(CVMemberRecord &CVR, Error TypeRecordMapping::visitKnownMember(CVMemberRecord &CVR, NestedTypeRecord &Record) { uint16_t Padding = 0; - error(IO.mapInteger(Padding)); - error(IO.mapInteger(Record.Type)); - error(IO.mapStringZ(Record.Name)); + error(IO.mapInteger(Padding, "Padding")); + error(IO.mapInteger(Record.Type, "Type")); + error(IO.mapStringZ(Record.Name, "Name")); return Error::success(); } @@ -444,9 +460,9 @@ Error TypeRecordMapping::visitKnownMember(CVMemberRecord &CVR, Error TypeRecordMapping::visitKnownMember(CVMemberRecord &CVR, StaticDataMemberRecord &Record) { - error(IO.mapInteger(Record.Attrs.Attrs)); - error(IO.mapInteger(Record.Type)); - error(IO.mapStringZ(Record.Name)); + error(IO.mapInteger(Record.Attrs.Attrs, "AccessSpecifier")); + error(IO.mapInteger(Record.Type, "Type")); + error(IO.mapStringZ(Record.Name, "Name")); return Error::success(); } @@ -454,11 +470,11 @@ Error TypeRecordMapping::visitKnownMember(CVMemberRecord &CVR, Error TypeRecordMapping::visitKnownMember(CVMemberRecord &CVR, VirtualBaseClassRecord &Record) { - error(IO.mapInteger(Record.Attrs.Attrs)); - error(IO.mapInteger(Record.BaseType)); - error(IO.mapInteger(Record.VBPtrType)); - error(IO.mapEncodedInteger(Record.VBPtrOffset)); - error(IO.mapEncodedInteger(Record.VTableIndex)); + error(IO.mapInteger(Record.Attrs.Attrs, "AccessSpecifier")); + error(IO.mapInteger(Record.BaseType, "BaseType")); + error(IO.mapInteger(Record.VBPtrType, "VBPtrType")); + error(IO.mapEncodedInteger(Record.VBPtrOffset, "VBPtrOffset")); + error(IO.mapEncodedInteger(Record.VTableIndex, "VBTableIndex")); return Error::success(); } @@ -466,8 +482,8 @@ Error TypeRecordMapping::visitKnownMember(CVMemberRecord &CVR, Error TypeRecordMapping::visitKnownMember(CVMemberRecord &CVR, VFPtrRecord &Record) { uint16_t Padding = 0; - error(IO.mapInteger(Padding)); - error(IO.mapInteger(Record.Type)); + error(IO.mapInteger(Padding, "Padding")); + error(IO.mapInteger(Record.Type, "Type")); return Error::success(); } @@ -475,23 +491,23 @@ Error TypeRecordMapping::visitKnownMember(CVMemberRecord &CVR, Error TypeRecordMapping::visitKnownMember(CVMemberRecord &CVR, ListContinuationRecord &Record) { uint16_t Padding = 0; - error(IO.mapInteger(Padding)); - error(IO.mapInteger(Record.ContinuationIndex)); + error(IO.mapInteger(Padding, "Padding")); + error(IO.mapInteger(Record.ContinuationIndex, "ContinuationIndex")); return Error::success(); } Error TypeRecordMapping::visitKnownRecord(CVType &CVR, PrecompRecord &Precomp) { - error(IO.mapInteger(Precomp.StartTypeIndex)); - error(IO.mapInteger(Precomp.TypesCount)); - error(IO.mapInteger(Precomp.Signature)); - error(IO.mapStringZ(Precomp.PrecompFilePath)); + error(IO.mapInteger(Precomp.StartTypeIndex, "StartIndex")); + error(IO.mapInteger(Precomp.TypesCount, "Count")); + error(IO.mapInteger(Precomp.Signature, "Signature")); + error(IO.mapStringZ(Precomp.PrecompFilePath, "PrecompFile")); return Error::success(); } Error TypeRecordMapping::visitKnownRecord(CVType &CVR, EndPrecompRecord &EndPrecomp) { - error(IO.mapInteger(EndPrecomp.Signature)); + error(IO.mapInteger(EndPrecomp.Signature, "Signature")); return Error::success(); } diff --git a/lib/DebugInfo/CodeView/TypeStreamMerger.cpp b/lib/DebugInfo/CodeView/TypeStreamMerger.cpp index bae11ce6a6a1..aba0e96d606e 100644 --- a/lib/DebugInfo/CodeView/TypeStreamMerger.cpp +++ b/lib/DebugInfo/CodeView/TypeStreamMerger.cpp @@ -1,9 +1,8 @@ //===-- TypeStreamMerger.cpp ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/CodeView/TypeTableCollection.cpp b/lib/DebugInfo/CodeView/TypeTableCollection.cpp index cf951baa5111..e13068b5b1eb 100644 --- a/lib/DebugInfo/CodeView/TypeTableCollection.cpp +++ b/lib/DebugInfo/CodeView/TypeTableCollection.cpp @@ -1,9 +1,8 @@ //===- TypeTableCollection.cpp -------------------------------- *- C++ --*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -37,11 +36,7 @@ Optional TypeTableCollection::getNext(TypeIndex Prev) { CVType TypeTableCollection::getType(TypeIndex Index) { assert(Index.toArrayIndex() < Records.size()); - ArrayRef Bytes = Records[Index.toArrayIndex()]; - const RecordPrefix *Prefix = - reinterpret_cast(Bytes.data()); - TypeLeafKind Kind = static_cast(uint16_t(Prefix->RecordKind)); - return CVType(Kind, Bytes); + return CVType(Records[Index.toArrayIndex()]); } StringRef TypeTableCollection::getTypeName(TypeIndex Index) { diff --git a/lib/DebugInfo/DWARF/DWARFAbbreviationDeclaration.cpp b/lib/DebugInfo/DWARF/DWARFAbbreviationDeclaration.cpp index f49ab40fad9a..f4dd79937608 100644 --- a/lib/DebugInfo/DWARF/DWARFAbbreviationDeclaration.cpp +++ b/lib/DebugInfo/DWARF/DWARFAbbreviationDeclaration.cpp @@ -1,9 +1,8 @@ //===- DWARFAbbreviationDeclaration.cpp -----------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -164,11 +163,11 @@ Optional DWARFAbbreviationDeclaration::getAttributeValue( for (const auto &Spec : AttributeSpecs) { if (*MatchAttrIndex == AttrIndex) { // We have arrived at the attribute to extract, extract if from Offset. + if (Spec.isImplicitConst()) + return DWARFFormValue::createFromSValue(Spec.Form, + Spec.getImplicitConstValue()); + DWARFFormValue FormValue(Spec.Form); - if (Spec.isImplicitConst()) { - FormValue.setSValue(Spec.getImplicitConstValue()); - return FormValue; - } if (FormValue.extractValue(DebugInfoData, &Offset, U.getFormParams(), &U)) return FormValue; } diff --git a/lib/DebugInfo/DWARF/DWARFAcceleratorTable.cpp b/lib/DebugInfo/DWARF/DWARFAcceleratorTable.cpp index 54daf34ff253..0721efb40f6a 100644 --- a/lib/DebugInfo/DWARF/DWARFAcceleratorTable.cpp +++ b/lib/DebugInfo/DWARF/DWARFAcceleratorTable.cpp @@ -1,9 +1,8 @@ //===- DWARFAcceleratorTable.cpp ------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -42,7 +41,7 @@ static Atom formatAtom(unsigned Atom) { return {Atom}; } DWARFAcceleratorTable::~DWARFAcceleratorTable() = default; -llvm::Error AppleAcceleratorTable::extract() { +Error AppleAcceleratorTable::extract() { uint32_t Offset = 0; // Check that we can at least read the header. @@ -377,7 +376,7 @@ void DWARFDebugNames::Header::dump(ScopedPrinter &W) const { W.startLine() << "Augmentation: '" << AugmentationString << "'\n"; } -llvm::Error DWARFDebugNames::Header::extract(const DWARFDataExtractor &AS, +Error DWARFDebugNames::Header::extract(const DWARFDataExtractor &AS, uint32_t *Offset) { // Check that we can read the fixed-size part. if (!AS.isValidOffset(*Offset + sizeof(HeaderPOD) - 1)) @@ -519,6 +518,7 @@ Error DWARFDebugNames::NameIndex::extract() { "Duplicate abbreviation code."); } } + DWARFDebugNames::Entry::Entry(const NameIndex &NameIdx, const Abbrev &Abbr) : NameIdx(&NameIdx), Abbr(&Abbr) { // This merely creates form values. It is up to the caller @@ -585,13 +585,14 @@ uint32_t DWARFDebugNames::NameIndex::getCUOffset(uint32_t CU) const { uint32_t DWARFDebugNames::NameIndex::getLocalTUOffset(uint32_t TU) const { assert(TU < Hdr.LocalTypeUnitCount); - uint32_t Offset = CUsBase + Hdr.CompUnitCount * 4; + uint32_t Offset = CUsBase + 4 * (Hdr.CompUnitCount + TU); return Section.AccelSection.getRelocatedValue(4, &Offset); } uint64_t DWARFDebugNames::NameIndex::getForeignTUSignature(uint32_t TU) const { assert(TU < Hdr.ForeignTypeUnitCount); - uint32_t Offset = CUsBase + (Hdr.CompUnitCount + Hdr.LocalTypeUnitCount) * 4; + uint32_t Offset = + CUsBase + 4 * (Hdr.CompUnitCount + Hdr.LocalTypeUnitCount) + 8 * TU; return Section.AccelSection.getU64(&Offset); } @@ -754,11 +755,11 @@ LLVM_DUMP_METHOD void DWARFDebugNames::NameIndex::dump(ScopedPrinter &W) const { dumpName(W, NTE, None); } -llvm::Error DWARFDebugNames::extract() { +Error DWARFDebugNames::extract() { uint32_t Offset = 0; while (AccelSection.isValidOffset(Offset)) { NameIndex Next(*this, Offset); - if (llvm::Error E = Next.extract()) + if (Error E = Next.extract()) return E; Offset = Next.getNextUnitOffset(); NameIndices.push_back(std::move(Next)); diff --git a/lib/DebugInfo/DWARF/DWARFAddressRange.cpp b/lib/DebugInfo/DWARF/DWARFAddressRange.cpp index 86c8d19c02f4..ef6da08d34aa 100644 --- a/lib/DebugInfo/DWARF/DWARFAddressRange.cpp +++ b/lib/DebugInfo/DWARF/DWARFAddressRange.cpp @@ -1,9 +1,8 @@ //===- DWARFDebugAranges.cpp ------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/DWARF/DWARFCompileUnit.cpp b/lib/DebugInfo/DWARF/DWARFCompileUnit.cpp index 00a23b3898fa..74cce42466dd 100644 --- a/lib/DebugInfo/DWARF/DWARFCompileUnit.cpp +++ b/lib/DebugInfo/DWARF/DWARFCompileUnit.cpp @@ -1,9 +1,8 @@ //===-- DWARFCompileUnit.cpp ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/DWARF/DWARFContext.cpp b/lib/DebugInfo/DWARF/DWARFContext.cpp index e6620ee3dd1d..5ede9bf59619 100644 --- a/lib/DebugInfo/DWARF/DWARFContext.cpp +++ b/lib/DebugInfo/DWARF/DWARFContext.cpp @@ -1,9 +1,8 @@ //===- DWARFContext.cpp ---------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -37,11 +36,12 @@ #include "llvm/Object/Decompressor.h" #include "llvm/Object/MachO.h" #include "llvm/Object/ObjectFile.h" -#include "llvm/Object/RelocVisitor.h" +#include "llvm/Object/RelocationResolver.h" #include "llvm/Support/Casting.h" #include "llvm/Support/DataExtractor.h" #include "llvm/Support/Error.h" #include "llvm/Support/Format.h" +#include "llvm/Support/LEB128.h" #include "llvm/Support/MemoryBuffer.h" #include "llvm/Support/Path.h" #include "llvm/Support/TargetRegistry.h" @@ -102,7 +102,8 @@ static ContributionCollection collectContributionData(DWARFContext::unit_iterator_range Units) { ContributionCollection Contributions; for (const auto &U : Units) - Contributions.push_back(U->getStringOffsetsTableContribution()); + if (const auto &C = U->getStringOffsetsTableContribution()) + Contributions.push_back(C); // Sort the contributions so that any invalid ones are placed at // the start of the contributions vector. This way they are reported // first. @@ -158,9 +159,9 @@ static void dumpDWARFv5StringOffsetsSection( // Detect overlapping contributions. if (Offset > ContributionHeader) { - OS << "error: overlapping contributions to string offsets table in " - "section ." - << SectionName << ".\n"; + WithColor::error() + << "overlapping contributions to string offsets table in section ." + << SectionName << ".\n"; return; } // Report a gap in the table. @@ -269,11 +270,11 @@ static void dumpAddrSection(raw_ostream &OS, DWARFDataExtractor &AddrData, } // Dump the .debug_rnglists or .debug_rnglists.dwo section (DWARF v5). -static void -dumpRnglistsSection(raw_ostream &OS, DWARFDataExtractor &rnglistData, - llvm::function_ref(uint32_t)> - LookupPooledAddress, - DIDumpOptions DumpOpts) { +static void dumpRnglistsSection( + raw_ostream &OS, DWARFDataExtractor &rnglistData, + llvm::function_ref(uint32_t)> + LookupPooledAddress, + DIDumpOptions DumpOpts) { uint32_t Offset = 0; while (rnglistData.isValidOffset(Offset)) { llvm::DWARFDebugRnglistTable Rnglists; @@ -926,6 +927,9 @@ DWARFContext::DIEsForAddress DWARFContext::getDIEsForAddress(uint64_t Address) { DWARFDie DIE = Worklist.back(); Worklist.pop_back(); + if (!DIE.isValid()) + continue; + if (DIE.getTag() == DW_TAG_lexical_block && DIE.addressRangeContainsAddress(Address)) { Result.BlockDIE = DIE; @@ -939,6 +943,8 @@ DWARFContext::DIEsForAddress DWARFContext::getDIEsForAddress(uint64_t Address) { return Result; } +/// TODO: change input parameter from "uint64_t Address" +/// into "SectionedAddress Address" static bool getFunctionNameAndStartLineForAddress(DWARFCompileUnit *CU, uint64_t Address, FunctionNameKind Kind, @@ -967,36 +973,155 @@ static bool getFunctionNameAndStartLineForAddress(DWARFCompileUnit *CU, return FoundResult; } -DILineInfo DWARFContext::getLineInfoForAddress(uint64_t Address, +static Optional getTypeSize(DWARFDie Type, uint64_t PointerSize) { + if (auto SizeAttr = Type.find(DW_AT_byte_size)) + if (Optional Size = SizeAttr->getAsUnsignedConstant()) + return Size; + + switch (Type.getTag()) { + case DW_TAG_pointer_type: + case DW_TAG_reference_type: + case DW_TAG_rvalue_reference_type: + return PointerSize; + case DW_TAG_ptr_to_member_type: { + if (DWARFDie BaseType = Type.getAttributeValueAsReferencedDie(DW_AT_type)) + if (BaseType.getTag() == DW_TAG_subroutine_type) + return 2 * PointerSize; + return PointerSize; + } + case DW_TAG_const_type: + case DW_TAG_volatile_type: + case DW_TAG_restrict_type: + case DW_TAG_typedef: { + if (DWARFDie BaseType = Type.getAttributeValueAsReferencedDie(DW_AT_type)) + return getTypeSize(BaseType, PointerSize); + break; + } + case DW_TAG_array_type: { + DWARFDie BaseType = Type.getAttributeValueAsReferencedDie(DW_AT_type); + if (!BaseType) + return Optional(); + Optional BaseSize = getTypeSize(BaseType, PointerSize); + if (!BaseSize) + return Optional(); + uint64_t Size = *BaseSize; + for (DWARFDie Child : Type) { + if (Child.getTag() != DW_TAG_subrange_type) + continue; + + if (auto ElemCountAttr = Child.find(DW_AT_count)) + if (Optional ElemCount = + ElemCountAttr->getAsUnsignedConstant()) + Size *= *ElemCount; + if (auto UpperBoundAttr = Child.find(DW_AT_upper_bound)) + if (Optional UpperBound = + UpperBoundAttr->getAsSignedConstant()) { + int64_t LowerBound = 0; + if (auto LowerBoundAttr = Child.find(DW_AT_lower_bound)) + LowerBound = LowerBoundAttr->getAsSignedConstant().getValueOr(0); + Size *= *UpperBound - LowerBound + 1; + } + } + return Size; + } + default: + break; + } + return Optional(); +} + +void DWARFContext::addLocalsForDie(DWARFCompileUnit *CU, DWARFDie Subprogram, + DWARFDie Die, std::vector &Result) { + if (Die.getTag() == DW_TAG_variable || + Die.getTag() == DW_TAG_formal_parameter) { + DILocal Local; + if (auto NameAttr = Subprogram.find(DW_AT_name)) + if (Optional Name = NameAttr->getAsCString()) + Local.FunctionName = *Name; + if (auto LocationAttr = Die.find(DW_AT_location)) + if (Optional> Location = LocationAttr->getAsBlock()) + if (!Location->empty() && (*Location)[0] == DW_OP_fbreg) + Local.FrameOffset = + decodeSLEB128(Location->data() + 1, nullptr, Location->end()); + if (auto TagOffsetAttr = Die.find(DW_AT_LLVM_tag_offset)) + Local.TagOffset = TagOffsetAttr->getAsUnsignedConstant(); + + if (auto Origin = + Die.getAttributeValueAsReferencedDie(DW_AT_abstract_origin)) + Die = Origin; + if (auto NameAttr = Die.find(DW_AT_name)) + if (Optional Name = NameAttr->getAsCString()) + Local.Name = *Name; + if (auto Type = Die.getAttributeValueAsReferencedDie(DW_AT_type)) + Local.Size = getTypeSize(Type, getCUAddrSize()); + if (auto DeclFileAttr = Die.find(DW_AT_decl_file)) { + if (const auto *LT = CU->getContext().getLineTableForUnit(CU)) + LT->getFileNameByIndex( + DeclFileAttr->getAsUnsignedConstant().getValue(), + CU->getCompilationDir(), + DILineInfoSpecifier::FileLineInfoKind::AbsoluteFilePath, + Local.DeclFile); + } + if (auto DeclLineAttr = Die.find(DW_AT_decl_line)) + Local.DeclLine = DeclLineAttr->getAsUnsignedConstant().getValue(); + + Result.push_back(Local); + return; + } + + if (Die.getTag() == DW_TAG_inlined_subroutine) + if (auto Origin = + Die.getAttributeValueAsReferencedDie(DW_AT_abstract_origin)) + Subprogram = Origin; + + for (auto Child : Die) + addLocalsForDie(CU, Subprogram, Child, Result); +} + +std::vector +DWARFContext::getLocalsForAddress(object::SectionedAddress Address) { + std::vector Result; + DWARFCompileUnit *CU = getCompileUnitForAddress(Address.Address); + if (!CU) + return Result; + + DWARFDie Subprogram = CU->getSubroutineForAddress(Address.Address); + if (Subprogram.isValid()) + addLocalsForDie(CU, Subprogram, Subprogram, Result); + return Result; +} + +DILineInfo DWARFContext::getLineInfoForAddress(object::SectionedAddress Address, DILineInfoSpecifier Spec) { DILineInfo Result; - DWARFCompileUnit *CU = getCompileUnitForAddress(Address); + DWARFCompileUnit *CU = getCompileUnitForAddress(Address.Address); if (!CU) return Result; - getFunctionNameAndStartLineForAddress(CU, Address, Spec.FNKind, - Result.FunctionName, - Result.StartLine); + + getFunctionNameAndStartLineForAddress(CU, Address.Address, Spec.FNKind, + Result.FunctionName, Result.StartLine); if (Spec.FLIKind != FileLineInfoKind::None) { - if (const DWARFLineTable *LineTable = getLineTableForUnit(CU)) - LineTable->getFileLineInfoForAddress(Address, CU->getCompilationDir(), - Spec.FLIKind, Result); + if (const DWARFLineTable *LineTable = getLineTableForUnit(CU)) { + LineTable->getFileLineInfoForAddress( + {Address.Address, Address.SectionIndex}, CU->getCompilationDir(), + Spec.FLIKind, Result); + } } return Result; } -DILineInfoTable -DWARFContext::getLineInfoForAddressRange(uint64_t Address, uint64_t Size, - DILineInfoSpecifier Spec) { +DILineInfoTable DWARFContext::getLineInfoForAddressRange( + object::SectionedAddress Address, uint64_t Size, DILineInfoSpecifier Spec) { DILineInfoTable Lines; - DWARFCompileUnit *CU = getCompileUnitForAddress(Address); + DWARFCompileUnit *CU = getCompileUnitForAddress(Address.Address); if (!CU) return Lines; std::string FunctionName = ""; uint32_t StartLine = 0; - getFunctionNameAndStartLineForAddress(CU, Address, Spec.FNKind, FunctionName, - StartLine); + getFunctionNameAndStartLineForAddress(CU, Address.Address, Spec.FNKind, + FunctionName, StartLine); // If the Specifier says we don't need FileLineInfo, just // return the top-most function at the starting address. @@ -1004,7 +1129,7 @@ DWARFContext::getLineInfoForAddressRange(uint64_t Address, uint64_t Size, DILineInfo Result; Result.FunctionName = FunctionName; Result.StartLine = StartLine; - Lines.push_back(std::make_pair(Address, Result)); + Lines.push_back(std::make_pair(Address.Address, Result)); return Lines; } @@ -1012,8 +1137,10 @@ DWARFContext::getLineInfoForAddressRange(uint64_t Address, uint64_t Size, // Get the index of row we're looking for in the line table. std::vector RowVector; - if (!LineTable->lookupAddressRange(Address, Size, RowVector)) + if (!LineTable->lookupAddressRange({Address.Address, Address.SectionIndex}, + Size, RowVector)) { return Lines; + } for (uint32_t RowIndex : RowVector) { // Take file number and line/column from the row. @@ -1025,33 +1152,33 @@ DWARFContext::getLineInfoForAddressRange(uint64_t Address, uint64_t Size, Result.Line = Row.Line; Result.Column = Row.Column; Result.StartLine = StartLine; - Lines.push_back(std::make_pair(Row.Address, Result)); + Lines.push_back(std::make_pair(Row.Address.Address, Result)); } return Lines; } DIInliningInfo -DWARFContext::getInliningInfoForAddress(uint64_t Address, +DWARFContext::getInliningInfoForAddress(object::SectionedAddress Address, DILineInfoSpecifier Spec) { DIInliningInfo InliningInfo; - DWARFCompileUnit *CU = getCompileUnitForAddress(Address); + DWARFCompileUnit *CU = getCompileUnitForAddress(Address.Address); if (!CU) return InliningInfo; const DWARFLineTable *LineTable = nullptr; SmallVector InlinedChain; - CU->getInlinedChainForAddress(Address, InlinedChain); + CU->getInlinedChainForAddress(Address.Address, InlinedChain); if (InlinedChain.size() == 0) { // If there is no DIE for address (e.g. it is in unavailable .dwo file), // try to at least get file/line info from symbol table. if (Spec.FLIKind != FileLineInfoKind::None) { DILineInfo Frame; LineTable = getLineTableForUnit(CU); - if (LineTable && - LineTable->getFileLineInfoForAddress(Address, CU->getCompilationDir(), - Spec.FLIKind, Frame)) + if (LineTable && LineTable->getFileLineInfoForAddress( + {Address.Address, Address.SectionIndex}, + CU->getCompilationDir(), Spec.FLIKind, Frame)) InliningInfo.addFrame(Frame); } return InliningInfo; @@ -1073,8 +1200,9 @@ DWARFContext::getInliningInfoForAddress(uint64_t Address, LineTable = getLineTableForUnit(CU); // For the topmost routine, get file/line info from line table. if (LineTable) - LineTable->getFileLineInfoForAddress(Address, CU->getCompilationDir(), - Spec.FLIKind, Frame); + LineTable->getFileLineInfoForAddress( + {Address.Address, Address.SectionIndex}, CU->getCompilationDir(), + Spec.FLIKind, Frame); } else { // Otherwise, use call file, call line and call column from // previous DIE in inlined chain. @@ -1402,8 +1530,14 @@ public: // Try to obtain an already relocated version of this section. // Else use the unrelocated section from the object file. We'll have to // apply relocations ourselves later. - if (!L || !L->getLoadedSectionContents(*RelocatedSection, Data)) - Section.getContents(Data); + if (!L || !L->getLoadedSectionContents(*RelocatedSection, Data)) { + Expected E = Section.getContents(); + if (E) + Data = *E; + else + // maybeDecompress below will error. + consumeError(E.takeError()); + } if (auto Err = maybeDecompress(Section, Name, Data)) { ErrorPolicy EP = HandleError(createError( @@ -1495,6 +1629,9 @@ public: // Symbol to [address, section index] cache mapping. std::map AddrCache; + bool (*Supports)(uint64_t); + RelocationResolver Resolver; + std::tie(Supports, Resolver) = getRelocationResolver(Obj); for (const RelocationRef &Reloc : Section.relocations()) { // FIXME: it's not clear how to correctly handle scattered // relocations. @@ -1509,9 +1646,31 @@ public: continue; } - object::RelocVisitor V(Obj); - uint64_t Val = V.visit(Reloc.getType(), Reloc, SymInfoOrErr->Address); - if (V.error()) { + // Check if Resolver can handle this relocation type early so as not to + // handle invalid cases in DWARFDataExtractor. + // + // TODO Don't store Resolver in every RelocAddrEntry. + if (Supports && Supports(Reloc.getType())) { + auto I = Map->try_emplace( + Reloc.getOffset(), + RelocAddrEntry{SymInfoOrErr->SectionIndex, Reloc, + SymInfoOrErr->Address, + Optional(), 0, Resolver}); + // If we didn't successfully insert that's because we already had a + // relocation for that offset. Store it as a second relocation in the + // same RelocAddrEntry instead. + if (!I.second) { + RelocAddrEntry &entry = I.first->getSecond(); + if (entry.Reloc2) { + ErrorPolicy EP = HandleError(createError( + "At most two relocations per offset are supported")); + if (EP == ErrorPolicy::Halt) + return; + } + entry.Reloc2 = Reloc; + entry.SymbolValue2 = SymInfoOrErr->Address; + } + } else { SmallString<32> Type; Reloc.getTypeName(Type); ErrorPolicy EP = HandleError( @@ -1519,10 +1678,7 @@ public: errorCodeToError(object_error::parse_failed))); if (EP == ErrorPolicy::Halt) return; - continue; } - RelocAddrEntry Rel = {SymInfoOrErr->SectionIndex, Val}; - Map->insert({Reloc.getOffset(), Rel}); } } diff --git a/lib/DebugInfo/DWARF/DWARFDataExtractor.cpp b/lib/DebugInfo/DWARF/DWARFDataExtractor.cpp index 03e317461396..b9adf8cb1d99 100644 --- a/lib/DebugInfo/DWARF/DWARFDataExtractor.cpp +++ b/lib/DebugInfo/DWARF/DWARFDataExtractor.cpp @@ -1,9 +1,8 @@ //===- DWARFDataExtractor.cpp ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -16,15 +15,19 @@ using namespace llvm; uint64_t DWARFDataExtractor::getRelocatedValue(uint32_t Size, uint32_t *Off, uint64_t *SecNdx) const { if (SecNdx) - *SecNdx = -1ULL; + *SecNdx = object::SectionedAddress::UndefSection; if (!Section) return getUnsigned(Off, Size); - Optional Rel = Obj->find(*Section, *Off); - if (!Rel) - return getUnsigned(Off, Size); + Optional E = Obj->find(*Section, *Off); + uint64_t A = getUnsigned(Off, Size); + if (!E) + return A; if (SecNdx) - *SecNdx = Rel->SectionIndex; - return getUnsigned(Off, Size) + Rel->Value; + *SecNdx = E->SectionIndex; + uint64_t R = E->Resolver(E->Reloc, E->SymbolValue, A); + if (E->Reloc2) + R = E->Resolver(*E->Reloc2, E->SymbolValue2, R); + return R; } Optional diff --git a/lib/DebugInfo/DWARF/DWARFDebugAbbrev.cpp b/lib/DebugInfo/DWARF/DWARFDebugAbbrev.cpp index 4830c36a8ee7..31b324e5eb27 100644 --- a/lib/DebugInfo/DWARF/DWARFDebugAbbrev.cpp +++ b/lib/DebugInfo/DWARF/DWARFDebugAbbrev.cpp @@ -1,9 +1,8 @@ //===- DWARFDebugAbbrev.cpp -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -84,12 +83,12 @@ void DWARFDebugAbbrev::parse() const { if (!Data) return; uint32_t Offset = 0; - DWARFAbbreviationDeclarationSet AbbrDecls; auto I = AbbrDeclSets.begin(); while (Data->isValidOffset(Offset)) { while (I != AbbrDeclSets.end() && I->first < Offset) ++I; uint32_t CUAbbrOffset = Offset; + DWARFAbbreviationDeclarationSet AbbrDecls; if (!AbbrDecls.extract(*Data, &Offset)) break; AbbrDeclSets.insert(I, std::make_pair(CUAbbrOffset, std::move(AbbrDecls))); diff --git a/lib/DebugInfo/DWARF/DWARFDebugAddr.cpp b/lib/DebugInfo/DWARF/DWARFDebugAddr.cpp index 22759bfac26c..58626539bba4 100644 --- a/lib/DebugInfo/DWARF/DWARFDebugAddr.cpp +++ b/lib/DebugInfo/DWARF/DWARFDebugAddr.cpp @@ -1,9 +1,8 @@ //===- DWARFDebugAddr.cpp -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -148,28 +147,13 @@ void DWARFDebugAddrTable::dump(raw_ostream &OS, DIDumpOptions DumpOpts) const { HeaderData.Length, HeaderData.Version, HeaderData.AddrSize, HeaderData.SegSize); - static const char *Fmt32 = "0x%8.8" PRIx64; - static const char *Fmt64 = "0x%16.16" PRIx64; - std::string AddrFmt = "\n"; - std::string AddrFmtVerbose = " => "; - if (HeaderData.AddrSize == 4) { - AddrFmt.append(Fmt32); - AddrFmtVerbose.append(Fmt32); - } - else { - AddrFmt.append(Fmt64); - AddrFmtVerbose.append(Fmt64); - } - if (Addrs.size() > 0) { - OS << "Addrs: ["; - for (uint64_t Addr : Addrs) { - OS << format(AddrFmt.c_str(), Addr); - if (DumpOpts.Verbose) - OS << format(AddrFmtVerbose.c_str(), - Addr + HeaderOffset + sizeof(HeaderData)); - } - OS << "\n]\n"; + const char *AddrFmt = (HeaderData.AddrSize == 4) ? "0x%8.8" PRIx64 "\n" + : "0x%16.16" PRIx64 "\n"; + OS << "Addrs: [\n"; + for (uint64_t Addr : Addrs) + OS << format(AddrFmt, Addr); + OS << "]\n"; } } diff --git a/lib/DebugInfo/DWARF/DWARFDebugArangeSet.cpp b/lib/DebugInfo/DWARF/DWARFDebugArangeSet.cpp index b9ef6905912a..6551b61accb8 100644 --- a/lib/DebugInfo/DWARF/DWARFDebugArangeSet.cpp +++ b/lib/DebugInfo/DWARF/DWARFDebugArangeSet.cpp @@ -1,9 +1,8 @@ //===- DWARFDebugArangeSet.cpp --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/DWARF/DWARFDebugAranges.cpp b/lib/DebugInfo/DWARF/DWARFDebugAranges.cpp index e8c5dec821b4..6460c9feeab8 100644 --- a/lib/DebugInfo/DWARF/DWARFDebugAranges.cpp +++ b/lib/DebugInfo/DWARF/DWARFDebugAranges.cpp @@ -1,9 +1,8 @@ //===- DWARFDebugAranges.cpp ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -115,20 +114,9 @@ void DWARFDebugAranges::construct() { } uint32_t DWARFDebugAranges::findAddress(uint64_t Address) const { - if (!Aranges.empty()) { - Range range(Address); - RangeCollIterator begin = Aranges.begin(); - RangeCollIterator end = Aranges.end(); - RangeCollIterator pos = - std::lower_bound(begin, end, range); - - if (pos != end && pos->containsAddress(Address)) { - return pos->CUOffset; - } else if (pos != begin) { - --pos; - if (pos->containsAddress(Address)) - return pos->CUOffset; - } - } + RangeCollIterator It = + partition_point(Aranges, [=](Range R) { return R.HighPC() <= Address; }); + if (It != Aranges.end() && It->LowPC <= Address) + return It->CUOffset; return -1U; } diff --git a/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp b/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp index ba55ffc28174..b3f23366f2a2 100644 --- a/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp +++ b/lib/DebugInfo/DWARF/DWARFDebugFrame.cpp @@ -1,9 +1,8 @@ //===- DWARFDebugFrame.h - Parsing of .debug_frame ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -267,7 +266,7 @@ void CFIProgram::printOperand(raw_ostream &OS, const MCRegisterInfo *MRI, case OT_Expression: assert(Instr.Expression && "missing DWARFExpression object"); OS << " "; - Instr.Expression->print(OS, MRI, IsEH); + Instr.Expression->print(OS, MRI, nullptr, IsEH); break; } } @@ -301,7 +300,7 @@ void CIE::dump(raw_ostream &OS, const MCRegisterInfo *MRI, bool IsEH) const { OS << format(" Data alignment factor: %d\n", (int32_t)DataAlignmentFactor); OS << format(" Return address column: %d\n", (int32_t)ReturnAddressRegister); if (Personality) - OS << format(" Personality Address: %08x\n", *Personality); + OS << format(" Personality Address: %016" PRIx64 "\n", *Personality); if (!AugmentationData.empty()) { OS << " Augmentation data: "; for (uint8_t Byte : AugmentationData) @@ -320,7 +319,7 @@ void FDE::dump(raw_ostream &OS, const MCRegisterInfo *MRI, bool IsEH) const { (uint32_t)InitialLocation, (uint32_t)InitialLocation + (uint32_t)AddressRange); if (LSDAAddress) - OS << format(" LSDA Address: %08x\n", *LSDAAddress); + OS << format(" LSDA Address: %016" PRIx64 "\n", *LSDAAddress); CFIs.dump(OS, MRI, IsEH); OS << "\n"; } @@ -533,10 +532,9 @@ void DWARFDebugFrame::parse(DWARFDataExtractor Data) { } FrameEntry *DWARFDebugFrame::getEntryAtOffset(uint64_t Offset) const { - auto It = - std::lower_bound(Entries.begin(), Entries.end(), Offset, - [](const std::unique_ptr &E, - uint64_t Offset) { return E->getOffset() < Offset; }); + auto It = partition_point(Entries, [=](const std::unique_ptr &E) { + return E->getOffset() < Offset; + }); if (It != Entries.end() && (*It)->getOffset() == Offset) return It->get(); return nullptr; diff --git a/lib/DebugInfo/DWARF/DWARFDebugInfoEntry.cpp b/lib/DebugInfo/DWARF/DWARFDebugInfoEntry.cpp index 976bc4651ae6..d8a755e90df4 100644 --- a/lib/DebugInfo/DWARF/DWARFDebugInfoEntry.cpp +++ b/lib/DebugInfo/DWARF/DWARFDebugInfoEntry.cpp @@ -1,9 +1,8 @@ //===- DWARFDebugInfoEntry.cpp --------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/DWARF/DWARFDebugLine.cpp b/lib/DebugInfo/DWARF/DWARFDebugLine.cpp index 1d621ff244f3..a1cb1e8582ed 100644 --- a/lib/DebugInfo/DWARF/DWARFDebugLine.cpp +++ b/lib/DebugInfo/DWARF/DWARFDebugLine.cpp @@ -1,9 +1,8 @@ //===- DWARFDebugLine.cpp -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -67,6 +66,26 @@ void DWARFDebugLine::ContentTypeTracker::trackContentType( DWARFDebugLine::Prologue::Prologue() { clear(); } +bool DWARFDebugLine::Prologue::hasFileAtIndex(uint64_t FileIndex) const { + uint16_t DwarfVersion = getVersion(); + assert(DwarfVersion != 0 && + "line table prologue has no dwarf version information"); + if (DwarfVersion >= 5) + return FileIndex < FileNames.size(); + return FileIndex != 0 && FileIndex <= FileNames.size(); +} + +const llvm::DWARFDebugLine::FileNameEntry & +DWARFDebugLine::Prologue::getFileNameEntry(uint64_t Index) const { + uint16_t DwarfVersion = getVersion(); + assert(DwarfVersion != 0 && + "line table prologue has no dwarf version information"); + // In DWARF v5 the file names are 0-indexed. + if (DwarfVersion >= 5) + return FileNames[Index]; + return FileNames[Index - 1]; +} + void DWARFDebugLine::Prologue::clear() { TotalLength = PrologueLength = 0; SegSelectorSize = 0; @@ -145,8 +164,8 @@ parseV2DirFileTables(const DWARFDataExtractor &DebugLineData, StringRef S = DebugLineData.getCStrRef(OffsetPtr); if (S.empty()) break; - DWARFFormValue Dir(dwarf::DW_FORM_string); - Dir.setPValue(S.data()); + DWARFFormValue Dir = + DWARFFormValue::createFromPValue(dwarf::DW_FORM_string, S.data()); IncludeDirectories.push_back(Dir); } @@ -155,8 +174,8 @@ parseV2DirFileTables(const DWARFDataExtractor &DebugLineData, if (Name.empty()) break; DWARFDebugLine::FileNameEntry FileEntry; - FileEntry.Name.setForm(dwarf::DW_FORM_string); - FileEntry.Name.setPValue(Name.data()); + FileEntry.Name = + DWARFFormValue::createFromPValue(dwarf::DW_FORM_string, Name.data()); FileEntry.DirIdx = DebugLineData.getULEB128(OffsetPtr); FileEntry.ModTime = DebugLineData.getULEB128(OffsetPtr); FileEntry.Length = DebugLineData.getULEB128(OffsetPtr); @@ -281,11 +300,11 @@ Error DWARFDebugLine::Prologue::parse(const DWARFDataExtractor &DebugLineData, const uint64_t PrologueOffset = *OffsetPtr; clear(); - TotalLength = DebugLineData.getU32(OffsetPtr); + TotalLength = DebugLineData.getRelocatedValue(4, OffsetPtr); if (TotalLength == UINT32_MAX) { FormParams.Format = dwarf::DWARF64; TotalLength = DebugLineData.getU64(OffsetPtr); - } else if (TotalLength >= 0xffffff00) { + } else if (TotalLength >= 0xfffffff0) { return createStringError(errc::invalid_argument, "parsing line table prologue at offset 0x%8.8" PRIx64 " unsupported reserved unit length found of value 0x%8.8" PRIx64, @@ -306,7 +325,8 @@ Error DWARFDebugLine::Prologue::parse(const DWARFDataExtractor &DebugLineData, SegSelectorSize = DebugLineData.getU8(OffsetPtr); } - PrologueLength = DebugLineData.getUnsigned(OffsetPtr, sizeofPrologueLength()); + PrologueLength = + DebugLineData.getRelocatedValue(sizeofPrologueLength(), OffsetPtr); const uint64_t EndPrologueOffset = PrologueLength + *OffsetPtr; MinInstLength = DebugLineData.getU8(OffsetPtr); if (getVersion() >= 4) @@ -348,13 +368,15 @@ Error DWARFDebugLine::Prologue::parse(const DWARFDataExtractor &DebugLineData, DWARFDebugLine::Row::Row(bool DefaultIsStmt) { reset(DefaultIsStmt); } void DWARFDebugLine::Row::postAppend() { + Discriminator = 0; BasicBlock = false; PrologueEnd = false; EpilogueBegin = false; } void DWARFDebugLine::Row::reset(bool DefaultIsStmt) { - Address = 0; + Address.Address = 0; + Address.SectionIndex = object::SectionedAddress::UndefSection; Line = 1; Column = 0; File = 1; @@ -374,7 +396,7 @@ void DWARFDebugLine::Row::dumpTableHeader(raw_ostream &OS) { } void DWARFDebugLine::Row::dump(raw_ostream &OS) const { - OS << format("0x%16.16" PRIx64 " %6u %6u", Address, Line, Column) + OS << format("0x%16.16" PRIx64 " %6u %6u", Address.Address, Line, Column) << format(" %6u %3u %13u ", File, Isa, Discriminator) << (IsStmt ? " is_stmt" : "") << (BasicBlock ? " basic_block" : "") << (PrologueEnd ? " prologue_end" : "") @@ -387,6 +409,7 @@ DWARFDebugLine::Sequence::Sequence() { reset(); } void DWARFDebugLine::Sequence::reset() { LowPC = 0; HighPC = 0; + SectionIndex = object::SectionedAddress::UndefSection; FirstRowIndex = 0; LastRowIndex = 0; Empty = true; @@ -423,19 +446,20 @@ void DWARFDebugLine::ParsingState::resetRowAndSequence() { Sequence.reset(); } -void DWARFDebugLine::ParsingState::appendRowToMatrix(uint32_t Offset) { +void DWARFDebugLine::ParsingState::appendRowToMatrix() { + unsigned RowNumber = LineTable->Rows.size(); if (Sequence.Empty) { // Record the beginning of instruction sequence. Sequence.Empty = false; - Sequence.LowPC = Row.Address; + Sequence.LowPC = Row.Address.Address; Sequence.FirstRowIndex = RowNumber; } - ++RowNumber; LineTable->appendRow(Row); if (Row.EndSequence) { // Record the end of instruction sequence. - Sequence.HighPC = Row.Address; - Sequence.LastRowIndex = RowNumber; + Sequence.HighPC = Row.Address.Address; + Sequence.LastRowIndex = RowNumber + 1; + Sequence.SectionIndex = Row.Address.SectionIndex; if (Sequence.isValid()) LineTable->appendSequence(Sequence); Sequence.reset(); @@ -538,7 +562,7 @@ Error DWARFDebugLine::LineTable::parse( // address is that of the byte after the last target machine instruction // of the sequence. State.Row.EndSequence = true; - State.appendRowToMatrix(*OffsetPtr); + State.appendRowToMatrix(); if (OS) { *OS << "\n"; OS->indent(12); @@ -566,9 +590,10 @@ Error DWARFDebugLine::LineTable::parse( ExtOffset, DebugLineData.getAddressSize(), Len - 1); } - State.Row.Address = DebugLineData.getRelocatedAddress(OffsetPtr); + State.Row.Address.Address = DebugLineData.getRelocatedAddress( + OffsetPtr, &State.Row.Address.SectionIndex); if (OS) - *OS << format(" (0x%16.16" PRIx64 ")", State.Row.Address); + *OS << format(" (0x%16.16" PRIx64 ")", State.Row.Address.Address); break; case DW_LNE_define_file: @@ -595,8 +620,8 @@ Error DWARFDebugLine::LineTable::parse( { FileNameEntry FileEntry; const char *Name = DebugLineData.getCStr(OffsetPtr); - FileEntry.Name.setForm(dwarf::DW_FORM_string); - FileEntry.Name.setPValue(Name); + FileEntry.Name = + DWARFFormValue::createFromPValue(dwarf::DW_FORM_string, Name); FileEntry.DirIdx = DebugLineData.getULEB128(OffsetPtr); FileEntry.ModTime = DebugLineData.getULEB128(OffsetPtr); FileEntry.Length = DebugLineData.getULEB128(OffsetPtr); @@ -637,15 +662,14 @@ Error DWARFDebugLine::LineTable::parse( // Standard Opcodes case DW_LNS_copy: // Takes no arguments. Append a row to the matrix using the - // current values of the state-machine registers. Then set - // the basic_block register to false. - State.appendRowToMatrix(*OffsetPtr); + // current values of the state-machine registers. if (OS) { *OS << "\n"; OS->indent(12); State.Row.dump(*OS); *OS << "\n"; } + State.appendRowToMatrix(); break; case DW_LNS_advance_pc: @@ -655,7 +679,7 @@ Error DWARFDebugLine::LineTable::parse( { uint64_t AddrOffset = DebugLineData.getULEB128(OffsetPtr) * Prologue.MinInstLength; - State.Row.Address += AddrOffset; + State.Row.Address.Address += AddrOffset; if (OS) *OS << " (" << AddrOffset << ")"; } @@ -713,7 +737,7 @@ Error DWARFDebugLine::LineTable::parse( uint8_t AdjustOpcode = 255 - Prologue.OpcodeBase; uint64_t AddrOffset = (AdjustOpcode / Prologue.LineRange) * Prologue.MinInstLength; - State.Row.Address += AddrOffset; + State.Row.Address.Address += AddrOffset; if (OS) *OS << format(" (0x%16.16" PRIx64 ")", AddrOffset); @@ -731,11 +755,11 @@ Error DWARFDebugLine::LineTable::parse( // requires the use of DW_LNS_advance_pc. Such assemblers, however, // can use DW_LNS_fixed_advance_pc instead, sacrificing compression. { - uint16_t PCOffset = DebugLineData.getU16(OffsetPtr); - State.Row.Address += PCOffset; + uint16_t PCOffset = DebugLineData.getRelocatedValue(2, OffsetPtr); + State.Row.Address.Address += PCOffset; if (OS) *OS - << format(" (0x%16.16" PRIx64 ")", PCOffset); + << format(" (0x%4.4" PRIx16 ")", PCOffset); } break; @@ -815,18 +839,16 @@ Error DWARFDebugLine::LineTable::parse( int32_t LineOffset = Prologue.LineBase + (AdjustOpcode % Prologue.LineRange); State.Row.Line += LineOffset; - State.Row.Address += AddrOffset; + State.Row.Address.Address += AddrOffset; if (OS) { - *OS << "address += " << ((uint32_t)AdjustOpcode) - << ", line += " << LineOffset << "\n"; + *OS << "address += " << AddrOffset << ", line += " << LineOffset + << "\n"; OS->indent(12); State.Row.dump(*OS); } - State.appendRowToMatrix(*OffsetPtr); - // Reset discriminator to 0. - State.Row.Discriminator = 0; + State.appendRowToMatrix(); } if(OS) *OS << "\n"; @@ -839,7 +861,7 @@ Error DWARFDebugLine::LineTable::parse( // Sort all sequences so that address lookup will work faster. if (!Sequences.empty()) { - llvm::sort(Sequences, Sequence::orderByLowPC); + llvm::sort(Sequences, Sequence::orderByHighPC); // Note: actually, instruction address ranges of sequences should not // overlap (in shared objects and executables). If they do, the address // lookup would still work, though, but result would be ambiguous. @@ -851,74 +873,88 @@ Error DWARFDebugLine::LineTable::parse( return Error::success(); } -uint32_t -DWARFDebugLine::LineTable::findRowInSeq(const DWARFDebugLine::Sequence &Seq, - uint64_t Address) const { +uint32_t DWARFDebugLine::LineTable::findRowInSeq( + const DWARFDebugLine::Sequence &Seq, + object::SectionedAddress Address) const { if (!Seq.containsPC(Address)) return UnknownRowIndex; - // Search for instruction address in the rows describing the sequence. - // Rows are stored in a vector, so we may use arithmetical operations with - // iterators. + assert(Seq.SectionIndex == Address.SectionIndex); + // In some cases, e.g. first instruction in a function, the compiler generates + // two entries, both with the same address. We want the last one. + // + // In general we want a non-empty range: the last row whose address is less + // than or equal to Address. This can be computed as upper_bound - 1. DWARFDebugLine::Row Row; Row.Address = Address; RowIter FirstRow = Rows.begin() + Seq.FirstRowIndex; RowIter LastRow = Rows.begin() + Seq.LastRowIndex; - LineTable::RowIter RowPos = std::lower_bound( - FirstRow, LastRow, Row, DWARFDebugLine::Row::orderByAddress); - if (RowPos == LastRow) { - return Seq.LastRowIndex - 1; - } - uint32_t Index = Seq.FirstRowIndex + (RowPos - FirstRow); - if (RowPos->Address > Address) { - if (RowPos == FirstRow) - return UnknownRowIndex; - else - Index--; - } - return Index; + assert(FirstRow->Address.Address <= Row.Address.Address && + Row.Address.Address < LastRow[-1].Address.Address); + RowIter RowPos = std::upper_bound(FirstRow + 1, LastRow - 1, Row, + DWARFDebugLine::Row::orderByAddress) - + 1; + assert(Seq.SectionIndex == RowPos->Address.SectionIndex); + return RowPos - Rows.begin(); } -uint32_t DWARFDebugLine::LineTable::lookupAddress(uint64_t Address) const { - if (Sequences.empty()) - return UnknownRowIndex; +uint32_t DWARFDebugLine::LineTable::lookupAddress( + object::SectionedAddress Address) const { + + // Search for relocatable addresses + uint32_t Result = lookupAddressImpl(Address); + + if (Result != UnknownRowIndex || + Address.SectionIndex == object::SectionedAddress::UndefSection) + return Result; + + // Search for absolute addresses + Address.SectionIndex = object::SectionedAddress::UndefSection; + return lookupAddressImpl(Address); +} + +uint32_t DWARFDebugLine::LineTable::lookupAddressImpl( + object::SectionedAddress Address) const { // First, find an instruction sequence containing the given address. DWARFDebugLine::Sequence Sequence; - Sequence.LowPC = Address; - SequenceIter FirstSeq = Sequences.begin(); - SequenceIter LastSeq = Sequences.end(); - SequenceIter SeqPos = std::lower_bound( - FirstSeq, LastSeq, Sequence, DWARFDebugLine::Sequence::orderByLowPC); - DWARFDebugLine::Sequence FoundSeq; - if (SeqPos == LastSeq) { - FoundSeq = Sequences.back(); - } else if (SeqPos->LowPC == Address) { - FoundSeq = *SeqPos; - } else { - if (SeqPos == FirstSeq) - return UnknownRowIndex; - FoundSeq = *(SeqPos - 1); - } - return findRowInSeq(FoundSeq, Address); + Sequence.SectionIndex = Address.SectionIndex; + Sequence.HighPC = Address.Address; + SequenceIter It = llvm::upper_bound(Sequences, Sequence, + DWARFDebugLine::Sequence::orderByHighPC); + if (It == Sequences.end() || It->SectionIndex != Address.SectionIndex) + return UnknownRowIndex; + return findRowInSeq(*It, Address); } bool DWARFDebugLine::LineTable::lookupAddressRange( - uint64_t Address, uint64_t Size, std::vector &Result) const { + object::SectionedAddress Address, uint64_t Size, + std::vector &Result) const { + + // Search for relocatable addresses + if (lookupAddressRangeImpl(Address, Size, Result)) + return true; + + if (Address.SectionIndex == object::SectionedAddress::UndefSection) + return false; + + // Search for absolute addresses + Address.SectionIndex = object::SectionedAddress::UndefSection; + return lookupAddressRangeImpl(Address, Size, Result); +} + +bool DWARFDebugLine::LineTable::lookupAddressRangeImpl( + object::SectionedAddress Address, uint64_t Size, + std::vector &Result) const { if (Sequences.empty()) return false; - uint64_t EndAddr = Address + Size; + uint64_t EndAddr = Address.Address + Size; // First, find an instruction sequence containing the given address. DWARFDebugLine::Sequence Sequence; - Sequence.LowPC = Address; - SequenceIter FirstSeq = Sequences.begin(); + Sequence.SectionIndex = Address.SectionIndex; + Sequence.HighPC = Address.Address; SequenceIter LastSeq = Sequences.end(); - SequenceIter SeqPos = std::lower_bound( - FirstSeq, LastSeq, Sequence, DWARFDebugLine::Sequence::orderByLowPC); - if (SeqPos == LastSeq || SeqPos->LowPC != Address) { - if (SeqPos == FirstSeq) - return false; - SeqPos--; - } - if (!SeqPos->containsPC(Address)) + SequenceIter SeqPos = llvm::upper_bound( + Sequences, Sequence, DWARFDebugLine::Sequence::orderByHighPC); + if (SeqPos == LastSeq || !SeqPos->containsPC(Address)) return false; SequenceIter StartPos = SeqPos; @@ -935,7 +971,8 @@ bool DWARFDebugLine::LineTable::lookupAddressRange( FirstRowIndex = findRowInSeq(CurSeq, Address); // Figure out the last row in the range. - uint32_t LastRowIndex = findRowInSeq(CurSeq, EndAddr - 1); + uint32_t LastRowIndex = + findRowInSeq(CurSeq, {EndAddr - 1, Address.SectionIndex}); if (LastRowIndex == UnknownRowIndex) LastRowIndex = CurSeq.LastRowIndex - 1; @@ -952,15 +989,11 @@ bool DWARFDebugLine::LineTable::lookupAddressRange( return true; } -bool DWARFDebugLine::LineTable::hasFileAtIndex(uint64_t FileIndex) const { - return FileIndex != 0 && FileIndex <= Prologue.FileNames.size(); -} - Optional DWARFDebugLine::LineTable::getSourceByIndex(uint64_t FileIndex, FileLineInfoKind Kind) const { - if (Kind == FileLineInfoKind::None || !hasFileAtIndex(FileIndex)) + if (Kind == FileLineInfoKind::None || !Prologue.hasFileAtIndex(FileIndex)) return None; - const FileNameEntry &Entry = Prologue.FileNames[FileIndex - 1]; + const FileNameEntry &Entry = Prologue.getFileNameEntry(FileIndex); if (Optional source = Entry.Source.getAsCString()) return StringRef(*source); return None; @@ -974,13 +1007,13 @@ static bool isPathAbsoluteOnWindowsOrPosix(const Twine &Path) { sys::path::is_absolute(Path, sys::path::Style::windows); } -bool DWARFDebugLine::LineTable::getFileNameByIndex(uint64_t FileIndex, - const char *CompDir, - FileLineInfoKind Kind, - std::string &Result) const { +bool DWARFDebugLine::Prologue::getFileNameByIndex(uint64_t FileIndex, + StringRef CompDir, + FileLineInfoKind Kind, + std::string &Result) const { if (Kind == FileLineInfoKind::None || !hasFileAtIndex(FileIndex)) return false; - const FileNameEntry &Entry = Prologue.FileNames[FileIndex - 1]; + const FileNameEntry &Entry = getFileNameEntry(FileIndex); StringRef FileName = Entry.Name.getAsCString().getValue(); if (Kind != FileLineInfoKind::AbsoluteFilePath || isPathAbsoluteOnWindowsOrPosix(FileName)) { @@ -989,21 +1022,22 @@ bool DWARFDebugLine::LineTable::getFileNameByIndex(uint64_t FileIndex, } SmallString<16> FilePath; - uint64_t IncludeDirIndex = Entry.DirIdx; StringRef IncludeDir; // Be defensive about the contents of Entry. - if (IncludeDirIndex > 0 && - IncludeDirIndex <= Prologue.IncludeDirectories.size()) - IncludeDir = Prologue.IncludeDirectories[IncludeDirIndex - 1] - .getAsCString() - .getValue(); - - // We may still need to append compilation directory of compile unit. - // We know that FileName is not absolute, the only way to have an - // absolute path at this point would be if IncludeDir is absolute. - if (CompDir && Kind == FileLineInfoKind::AbsoluteFilePath && - !isPathAbsoluteOnWindowsOrPosix(IncludeDir)) - sys::path::append(FilePath, CompDir); + if (getVersion() >= 5) { + if (Entry.DirIdx < IncludeDirectories.size()) + IncludeDir = IncludeDirectories[Entry.DirIdx].getAsCString().getValue(); + } else { + if (0 < Entry.DirIdx && Entry.DirIdx <= IncludeDirectories.size()) + IncludeDir = + IncludeDirectories[Entry.DirIdx - 1].getAsCString().getValue(); + + // We may still need to append compilation directory of compile unit. + // We know that FileName is not absolute, the only way to have an + // absolute path at this point would be if IncludeDir is absolute. + if (!CompDir.empty() && !isPathAbsoluteOnWindowsOrPosix(IncludeDir)) + sys::path::append(FilePath, CompDir); + } // sys::path::append skips empty strings. sys::path::append(FilePath, IncludeDir, FileName); @@ -1012,8 +1046,8 @@ bool DWARFDebugLine::LineTable::getFileNameByIndex(uint64_t FileIndex, } bool DWARFDebugLine::LineTable::getFileLineInfoForAddress( - uint64_t Address, const char *CompDir, FileLineInfoKind Kind, - DILineInfo &Result) const { + object::SectionedAddress Address, const char *CompDir, + FileLineInfoKind Kind, DILineInfo &Result) const { // Get the index of row we're looking for in the line table. uint32_t RowIndex = lookupAddress(Address); if (RowIndex == -1U) @@ -1058,7 +1092,7 @@ DWARFDebugLine::SectionParser::SectionParser(DWARFDataExtractor &Data, } bool DWARFDebugLine::Prologue::totalLengthIsValid() const { - return TotalLength == 0xffffffff || TotalLength < 0xffffff00; + return TotalLength == 0xffffffff || TotalLength < 0xfffffff0; } DWARFDebugLine::LineTable DWARFDebugLine::SectionParser::parseNext( diff --git a/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp b/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp index f8b5ff6ec8fb..6d8f4bee77c4 100644 --- a/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp +++ b/lib/DebugInfo/DWARF/DWARFDebugLoc.cpp @@ -1,9 +1,8 @@ //===- DWARFDebugLoc.cpp --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -31,15 +30,16 @@ using namespace llvm; // non-LLVM tools. static void dumpExpression(raw_ostream &OS, ArrayRef Data, bool IsLittleEndian, unsigned AddressSize, - const MCRegisterInfo *MRI) { + const MCRegisterInfo *MRI, DWARFUnit *U) { DWARFDataExtractor Extractor(StringRef(Data.data(), Data.size()), IsLittleEndian, AddressSize); - DWARFExpression(Extractor, dwarf::DWARF_VERSION, AddressSize).print(OS, MRI); + DWARFExpression(Extractor, dwarf::DWARF_VERSION, AddressSize).print(OS, MRI, U); } void DWARFDebugLoc::LocationList::dump(raw_ostream &OS, bool IsLittleEndian, unsigned AddressSize, const MCRegisterInfo *MRI, + DWARFUnit *U, uint64_t BaseAddress, unsigned Indent) const { for (const Entry &E : Entries) { @@ -51,15 +51,14 @@ void DWARFDebugLoc::LocationList::dump(raw_ostream &OS, bool IsLittleEndian, BaseAddress + E.End); OS << ": "; - dumpExpression(OS, E.Loc, IsLittleEndian, AddressSize, MRI); + dumpExpression(OS, E.Loc, IsLittleEndian, AddressSize, MRI, U); } } DWARFDebugLoc::LocationList const * DWARFDebugLoc::getLocationListAtOffset(uint64_t Offset) const { - auto It = std::lower_bound( - Locations.begin(), Locations.end(), Offset, - [](const LocationList &L, uint64_t Offset) { return L.Offset < Offset; }); + auto It = partition_point( + Locations, [=](const LocationList &L) { return L.Offset < Offset; }); if (It != Locations.end() && It->Offset == Offset) return &(*It); return nullptr; @@ -69,7 +68,7 @@ void DWARFDebugLoc::dump(raw_ostream &OS, const MCRegisterInfo *MRI, Optional Offset) const { auto DumpLocationList = [&](const LocationList &L) { OS << format("0x%8.8x: ", L.Offset); - L.dump(OS, IsLittleEndian, AddressSize, MRI, 0, 12); + L.dump(OS, IsLittleEndian, AddressSize, MRI, nullptr, 0, 12); OS << "\n\n"; }; @@ -184,7 +183,8 @@ DWARFDebugLoclists::parseOneLocationList(DataExtractor Data, unsigned *Offset, } if (Kind != dwarf::DW_LLE_base_address) { - unsigned Bytes = Data.getU16(Offset); + unsigned Bytes = + Version >= 5 ? Data.getULEB128(Offset) : Data.getU16(Offset); // A single location description describing the location of the object... StringRef str = Data.getData().substr(*Offset, Bytes); *Offset += Bytes; @@ -212,9 +212,8 @@ void DWARFDebugLoclists::parse(DataExtractor data, unsigned Version) { DWARFDebugLoclists::LocationList const * DWARFDebugLoclists::getLocationListAtOffset(uint64_t Offset) const { - auto It = std::lower_bound( - Locations.begin(), Locations.end(), Offset, - [](const LocationList &L, uint64_t Offset) { return L.Offset < Offset; }); + auto It = partition_point( + Locations, [=](const LocationList &L) { return L.Offset < Offset; }); if (It != Locations.end() && It->Offset == Offset) return &(*It); return nullptr; @@ -224,6 +223,7 @@ void DWARFDebugLoclists::LocationList::dump(raw_ostream &OS, uint64_t BaseAddr, bool IsLittleEndian, unsigned AddressSize, const MCRegisterInfo *MRI, + DWARFUnit *U, unsigned Indent) const { for (const Entry &E : Entries) { switch (E.Kind) { @@ -253,7 +253,7 @@ void DWARFDebugLoclists::LocationList::dump(raw_ostream &OS, uint64_t BaseAddr, llvm_unreachable("unreachable locations list kind"); } - dumpExpression(OS, E.Loc, IsLittleEndian, AddressSize, MRI); + dumpExpression(OS, E.Loc, IsLittleEndian, AddressSize, MRI, U); } } @@ -262,7 +262,7 @@ void DWARFDebugLoclists::dump(raw_ostream &OS, uint64_t BaseAddr, Optional Offset) const { auto DumpLocationList = [&](const LocationList &L) { OS << format("0x%8.8x: ", L.Offset); - L.dump(OS, BaseAddr, IsLittleEndian, AddressSize, MRI, /*Indent=*/12); + L.dump(OS, BaseAddr, IsLittleEndian, AddressSize, MRI, nullptr, /*Indent=*/12); OS << "\n\n"; }; diff --git a/lib/DebugInfo/DWARF/DWARFDebugMacro.cpp b/lib/DebugInfo/DWARF/DWARFDebugMacro.cpp index 6d789c3027a5..3317a778cc70 100644 --- a/lib/DebugInfo/DWARF/DWARFDebugMacro.cpp +++ b/lib/DebugInfo/DWARF/DWARFDebugMacro.cpp @@ -1,9 +1,8 @@ //===- DWARFDebugMacro.cpp ------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/DWARF/DWARFDebugPubTable.cpp b/lib/DebugInfo/DWARF/DWARFDebugPubTable.cpp index abd1ad59a9c1..963ec64f5e91 100644 --- a/lib/DebugInfo/DWARF/DWARFDebugPubTable.cpp +++ b/lib/DebugInfo/DWARF/DWARFDebugPubTable.cpp @@ -1,9 +1,8 @@ //===- DWARFDebugPubTable.cpp ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/DWARF/DWARFDebugRangeList.cpp b/lib/DebugInfo/DWARF/DWARFDebugRangeList.cpp index dfb913000a46..d8df81a0aa0b 100644 --- a/lib/DebugInfo/DWARF/DWARFDebugRangeList.cpp +++ b/lib/DebugInfo/DWARF/DWARFDebugRangeList.cpp @@ -1,9 +1,8 @@ //===- DWARFDebugRangesList.cpp -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -69,7 +68,7 @@ void DWARFDebugRangeList::dump(raw_ostream &OS) const { } DWARFAddressRangesVector DWARFDebugRangeList::getAbsoluteRanges( - llvm::Optional BaseAddr) const { + llvm::Optional BaseAddr) const { DWARFAddressRangesVector Res; for (const RangeListEntry &RLE : Entries) { if (RLE.isBaseAddressSelectionEntry(AddressSize)) { diff --git a/lib/DebugInfo/DWARF/DWARFDebugRnglists.cpp b/lib/DebugInfo/DWARF/DWARFDebugRnglists.cpp index 60c6eb30857f..5ac3326f6681 100644 --- a/lib/DebugInfo/DWARF/DWARFDebugRnglists.cpp +++ b/lib/DebugInfo/DWARF/DWARFDebugRnglists.cpp @@ -1,9 +1,8 @@ //===- DWARFDebugRnglists.cpp ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -113,9 +112,8 @@ Error RangeListEntry::extract(DWARFDataExtractor Data, uint32_t End, return Error::success(); } -DWARFAddressRangesVector -DWARFDebugRnglist::getAbsoluteRanges(llvm::Optional BaseAddr, - DWARFUnit &U) const { +DWARFAddressRangesVector DWARFDebugRnglist::getAbsoluteRanges( + llvm::Optional BaseAddr, DWARFUnit &U) const { DWARFAddressRangesVector Res; for (const RangeListEntry &RLE : Entries) { if (RLE.EntryKind == dwarf::DW_RLE_end_of_list) @@ -175,7 +173,7 @@ DWARFDebugRnglist::getAbsoluteRanges(llvm::Optional BaseAddr, void RangeListEntry::dump( raw_ostream &OS, uint8_t AddrSize, uint8_t MaxEncodingStringLength, uint64_t &CurrentBase, DIDumpOptions DumpOpts, - llvm::function_ref(uint32_t)> + llvm::function_ref(uint32_t)> LookupPooledAddress) const { auto PrintRawEntry = [](raw_ostream &OS, const RangeListEntry &Entry, uint8_t AddrSize, DIDumpOptions DumpOpts) { @@ -203,7 +201,6 @@ void RangeListEntry::dump( case dwarf::DW_RLE_end_of_list: OS << (DumpOpts.Verbose ? "" : ""); break; - // case dwarf::DW_RLE_base_addressx: case dwarf::DW_RLE_base_addressx: { if (auto SA = LookupPooledAddress(Value0)) CurrentBase = SA->Address; @@ -240,7 +237,7 @@ void RangeListEntry::dump( Start = SA->Address; DWARFAddressRange(Start, Start + Value1).dump(OS, AddrSize, DumpOpts); break; - } break; + } default: llvm_unreachable("Unsupported range list encoding"); } diff --git a/lib/DebugInfo/DWARF/DWARFDie.cpp b/lib/DebugInfo/DWARF/DWARFDie.cpp index 81ef0c8c7aec..d638dc4239f4 100644 --- a/lib/DebugInfo/DWARF/DWARFDie.cpp +++ b/lib/DebugInfo/DWARF/DWARFDie.cpp @@ -1,9 +1,8 @@ //===- DWARFDie.cpp -------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -87,7 +86,7 @@ static void dumpLocation(raw_ostream &OS, DWARFFormValue &FormValue, DataExtractor Data(StringRef((const char *)Expr.data(), Expr.size()), Ctx.isLittleEndian(), 0); DWARFExpression(Data, U->getVersion(), U->getAddressByteSize()) - .print(OS, MRI); + .print(OS, MRI, U); return; } @@ -101,10 +100,10 @@ static void dumpLocation(raw_ostream &OS, DWARFFormValue &FormValue, auto LL = DebugLoc.parseOneLocationList(Data, &Offset); if (LL) { uint64_t BaseAddr = 0; - if (Optional BA = U->getBaseAddress()) + if (Optional BA = U->getBaseAddress()) BaseAddr = BA->Address; - LL->dump(OS, Ctx.isLittleEndian(), Obj.getAddressSize(), MRI, BaseAddr, - Indent); + LL->dump(OS, Ctx.isLittleEndian(), Obj.getAddressSize(), MRI, U, + BaseAddr, Indent); } else OS << "error extracting location list."; return; @@ -126,12 +125,12 @@ static void dumpLocation(raw_ostream &OS, DWARFFormValue &FormValue, Data, &Offset, UseLocLists ? U->getVersion() : 4); uint64_t BaseAddr = 0; - if (Optional BA = U->getBaseAddress()) + if (Optional BA = U->getBaseAddress()) BaseAddr = BA->Address; if (LL) LL->dump(OS, BaseAddr, Ctx.isLittleEndian(), Obj.getAddressSize(), MRI, - Indent); + U, Indent); else OS << "error extracting location list."; } @@ -279,11 +278,7 @@ static void dumpAttribute(raw_ostream &OS, const DWARFDie &Die, OS << formatv(" [{0}]", Form); DWARFUnit *U = Die.getDwarfUnit(); - DWARFFormValue formValue(Form); - - if (!formValue.extractValue(U->getDebugInfoExtractor(), OffsetPtr, - U->getFormParams(), U)) - return; + DWARFFormValue FormValue = DWARFFormValue::createFromUnit(Form, U, OffsetPtr); OS << "\t("; @@ -294,35 +289,33 @@ static void dumpAttribute(raw_ostream &OS, const DWARFDie &Die, Color = HighlightColor::String; if (const auto *LT = U->getContext().getLineTableForUnit(U)) if (LT->getFileNameByIndex( - formValue.getAsUnsignedConstant().getValue(), + FormValue.getAsUnsignedConstant().getValue(), U->getCompilationDir(), DILineInfoSpecifier::FileLineInfoKind::AbsoluteFilePath, File)) { File = '"' + File + '"'; Name = File; } - } else if (Optional Val = formValue.getAsUnsignedConstant()) + } else if (Optional Val = FormValue.getAsUnsignedConstant()) Name = AttributeValueString(Attr, *Val); if (!Name.empty()) WithColor(OS, Color) << Name; else if (Attr == DW_AT_decl_line || Attr == DW_AT_call_line) - OS << *formValue.getAsUnsignedConstant(); + OS << *FormValue.getAsUnsignedConstant(); else if (Attr == DW_AT_high_pc && !DumpOpts.ShowForm && !DumpOpts.Verbose && - formValue.getAsUnsignedConstant()) { + FormValue.getAsUnsignedConstant()) { if (DumpOpts.ShowAddresses) { // Print the actual address rather than the offset. uint64_t LowPC, HighPC, Index; if (Die.getLowAndHighPC(LowPC, HighPC, Index)) OS << format("0x%016" PRIx64, HighPC); else - formValue.dump(OS, DumpOpts); + FormValue.dump(OS, DumpOpts); } - } else if (Attr == DW_AT_location || Attr == DW_AT_frame_base || - Attr == DW_AT_data_member_location || - Attr == DW_AT_GNU_call_site_value) - dumpLocation(OS, formValue, U, sizeof(BaseIndent) + Indent + 4, DumpOpts); + } else if (DWARFAttribute::mayHaveLocationDescription(Attr)) + dumpLocation(OS, FormValue, U, sizeof(BaseIndent) + Indent + 4, DumpOpts); else - formValue.dump(OS, DumpOpts); + FormValue.dump(OS, DumpOpts); std::string Space = DumpOpts.ShowAddresses ? " " : ""; @@ -331,25 +324,25 @@ static void dumpAttribute(raw_ostream &OS, const DWARFDie &Die, // interesting. These attributes are handled below. if (Attr == DW_AT_specification || Attr == DW_AT_abstract_origin) { if (const char *Name = - Die.getAttributeValueAsReferencedDie(formValue).getName( + Die.getAttributeValueAsReferencedDie(FormValue).getName( DINameKind::LinkageName)) OS << Space << "\"" << Name << '\"'; } else if (Attr == DW_AT_type) { OS << Space << "\""; - dumpTypeName(OS, Die.getAttributeValueAsReferencedDie(formValue)); + dumpTypeName(OS, Die.getAttributeValueAsReferencedDie(FormValue)); OS << '"'; } else if (Attr == DW_AT_APPLE_property_attribute) { - if (Optional OptVal = formValue.getAsUnsignedConstant()) + if (Optional OptVal = FormValue.getAsUnsignedConstant()) dumpApplePropertyAttribute(OS, *OptVal); } else if (Attr == DW_AT_ranges) { const DWARFObject &Obj = Die.getDwarfUnit()->getContext().getDWARFObj(); // For DW_FORM_rnglistx we need to dump the offset separately, since // we have only dumped the index so far. - if (formValue.getForm() == DW_FORM_rnglistx) + if (FormValue.getForm() == DW_FORM_rnglistx) if (auto RangeListOffset = - U->getRnglistOffset(*formValue.getAsSectionOffset())) { - DWARFFormValue FV(dwarf::DW_FORM_sec_offset); - FV.setUValue(*RangeListOffset); + U->getRnglistOffset(*FormValue.getAsSectionOffset())) { + DWARFFormValue FV = DWARFFormValue::createFromUValue( + dwarf::DW_FORM_sec_offset, *RangeListOffset); FV.dump(OS, DumpOpts); } if (auto RangesOrError = Die.getAddressRanges()) @@ -403,6 +396,7 @@ DWARFDie::findRecursively(ArrayRef Attrs) const { // DWARF. This corresponds to following the DW_AT_abstract_origin and // DW_AT_specification just once. SmallSet Seen; + Seen.insert(*this); while (!Worklist.empty()) { DWARFDie Die = Worklist.back(); @@ -411,19 +405,16 @@ DWARFDie::findRecursively(ArrayRef Attrs) const { if (!Die.isValid()) continue; - if (Seen.count(Die)) - continue; - - Seen.insert(Die); - if (auto Value = Die.find(Attrs)) return Value; if (auto D = Die.getAttributeValueAsReferencedDie(DW_AT_abstract_origin)) - Worklist.push_back(D); + if (Seen.insert(D).second) + Worklist.push_back(D); if (auto D = Die.getAttributeValueAsReferencedDie(DW_AT_specification)) - Worklist.push_back(D); + if (Seen.insert(D).second) + Worklist.push_back(D); } return None; @@ -438,9 +429,11 @@ DWARFDie::getAttributeValueAsReferencedDie(dwarf::Attribute Attr) const { DWARFDie DWARFDie::getAttributeValueAsReferencedDie(const DWARFFormValue &V) const { - if (auto SpecRef = toReference(V)) { - if (auto SpecUnit = U->getUnitVector().getUnitForOffset(*SpecRef)) - return SpecUnit->getDIEForOffset(*SpecRef); + if (auto SpecRef = V.getAsRelativeReference()) { + if (SpecRef->Unit) + return SpecRef->Unit->getDIEForOffset(SpecRef->Unit->getOffset() + SpecRef->Offset); + if (auto SpecUnit = U->getUnitVector().getUnitForOffset(SpecRef->Offset)) + return SpecUnit->getDIEForOffset(SpecRef->Offset); } return DWARFDie(); } @@ -560,10 +553,12 @@ void DWARFDie::getCallerFrame(uint32_t &CallFile, uint32_t &CallLine, /// Helper to dump a DIE with all of its parents, but no siblings. static unsigned dumpParentChain(DWARFDie Die, raw_ostream &OS, unsigned Indent, - DIDumpOptions DumpOpts) { + DIDumpOptions DumpOpts, unsigned Depth = 0) { if (!Die) return Indent; - Indent = dumpParentChain(Die.getParent(), OS, Indent, DumpOpts); + if (DumpOpts.ParentRecurseDepth > 0 && Depth >= DumpOpts.ParentRecurseDepth) + return Indent; + Indent = dumpParentChain(Die.getParent(), OS, Indent, DumpOpts, Depth + 1); Die.dump(OS, Indent, DumpOpts); return Indent + 2; } @@ -611,8 +606,8 @@ void DWARFDie::dump(raw_ostream &OS, unsigned Indent, } DWARFDie child = getFirstChild(); - if (DumpOpts.ShowChildren && DumpOpts.RecurseDepth > 0 && child) { - DumpOpts.RecurseDepth--; + if (DumpOpts.ShowChildren && DumpOpts.ChildRecurseDepth > 0 && child) { + DumpOpts.ChildRecurseDepth--; DIDumpOptions ChildDumpOpts = DumpOpts; ChildDumpOpts.ShowParents = false; while (child) { @@ -668,7 +663,7 @@ iterator_range DWARFDie::attributes() const { } DWARFDie::attribute_iterator::attribute_iterator(DWARFDie D, bool End) - : Die(D), AttrValue(0), Index(0) { + : Die(D), Index(0) { auto AbbrDecl = Die.getAbbreviationDeclarationPtr(); assert(AbbrDecl && "Must have abbreviation declaration"); if (End) { @@ -690,18 +685,15 @@ void DWARFDie::attribute_iterator::updateForIndex( AttrValue.Attr = AbbrDecl.getAttrByIndex(Index); // Add the previous byte size of any previous attribute value. AttrValue.Offset += AttrValue.ByteSize; - AttrValue.Value.setForm(AbbrDecl.getFormByIndex(Index)); uint32_t ParseOffset = AttrValue.Offset; auto U = Die.getDwarfUnit(); assert(U && "Die must have valid DWARF unit"); - bool b = AttrValue.Value.extractValue(U->getDebugInfoExtractor(), - &ParseOffset, U->getFormParams(), U); - (void)b; - assert(b && "extractValue cannot fail on fully parsed DWARF"); + AttrValue.Value = DWARFFormValue::createFromUnit( + AbbrDecl.getFormByIndex(Index), U, &ParseOffset); AttrValue.ByteSize = ParseOffset - AttrValue.Offset; } else { assert(Index == NumAttrs && "Indexes should be [0, NumAttrs) only"); - AttrValue.clear(); + AttrValue = {}; } } @@ -710,3 +702,39 @@ DWARFDie::attribute_iterator &DWARFDie::attribute_iterator::operator++() { updateForIndex(*AbbrDecl, Index + 1); return *this; } + +bool DWARFAttribute::mayHaveLocationDescription(dwarf::Attribute Attr) { + switch (Attr) { + // From the DWARF v5 specification. + case DW_AT_location: + case DW_AT_byte_size: + case DW_AT_bit_size: + case DW_AT_string_length: + case DW_AT_lower_bound: + case DW_AT_return_addr: + case DW_AT_bit_stride: + case DW_AT_upper_bound: + case DW_AT_count: + case DW_AT_data_member_location: + case DW_AT_frame_base: + case DW_AT_segment: + case DW_AT_static_link: + case DW_AT_use_location: + case DW_AT_vtable_elem_location: + case DW_AT_allocated: + case DW_AT_associated: + case DW_AT_byte_stride: + case DW_AT_rank: + case DW_AT_call_value: + case DW_AT_call_origin: + case DW_AT_call_target: + case DW_AT_call_target_clobbered: + case DW_AT_call_data_location: + case DW_AT_call_data_value: + // Extensions. + case DW_AT_GNU_call_site_value: + return true; + default: + return false; + } +} diff --git a/lib/DebugInfo/DWARF/DWARFExpression.cpp b/lib/DebugInfo/DWARF/DWARFExpression.cpp index 2df4456053fb..470d4b5364b4 100644 --- a/lib/DebugInfo/DWARF/DWARFExpression.cpp +++ b/lib/DebugInfo/DWARF/DWARFExpression.cpp @@ -1,13 +1,13 @@ //===-- DWARFExpression.cpp -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "llvm/DebugInfo/DWARF/DWARFExpression.h" +#include "llvm/DebugInfo/DWARF/DWARFUnit.h" #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/Support/Format.h" @@ -97,6 +97,11 @@ static DescVector getDescriptions() { Descriptions[DW_OP_addrx] = Desc(Op::Dwarf4, Op::SizeLEB); Descriptions[DW_OP_GNU_addr_index] = Desc(Op::Dwarf4, Op::SizeLEB); Descriptions[DW_OP_GNU_const_index] = Desc(Op::Dwarf4, Op::SizeLEB); + Descriptions[DW_OP_GNU_entry_value] = Desc(Op::Dwarf4, Op::SizeLEB); + + Descriptions[DW_OP_convert] = Desc(Op::Dwarf5, Op::BaseTypeRef); + Descriptions[DW_OP_entry_value] = Desc(Op::Dwarf5, Op::SizeLEB); + return Descriptions; } @@ -152,17 +157,21 @@ bool DWARFExpression::Operation::extract(DataExtractor Data, uint16_t Version, case Operation::SizeAddr: if (AddressSize == 8) { Operands[Operand] = Data.getU64(&Offset); - } else { - assert(AddressSize == 4); + } else if (AddressSize == 4) { Operands[Operand] = Data.getU32(&Offset); + } else { + assert(AddressSize == 2); + Operands[Operand] = Data.getU16(&Offset); } break; case Operation::SizeRefAddr: if (getRefAddrSize(AddressSize, Version) == 8) { Operands[Operand] = Data.getU64(&Offset); - } else { - assert(getRefAddrSize(AddressSize, Version) == 4); + } else if (getRefAddrSize(AddressSize, Version) == 4) { Operands[Operand] = Data.getU32(&Offset); + } else { + assert(getRefAddrSize(AddressSize, Version) == 2); + Operands[Operand] = Data.getU16(&Offset); } break; case Operation::SizeLEB: @@ -171,6 +180,9 @@ bool DWARFExpression::Operation::extract(DataExtractor Data, uint16_t Version, else Operands[Operand] = Data.getULEB128(&Offset); break; + case Operation::BaseTypeRef: + Operands[Operand] = Data.getULEB128(&Offset); + break; case Operation::SizeBlock: // We need a size, so this cannot be the first operand if (Operand == 0) @@ -182,6 +194,8 @@ bool DWARFExpression::Operation::extract(DataExtractor Data, uint16_t Version, default: llvm_unreachable("Unknown DWARFExpression Op size"); } + + OperandEndOffsets[Operand] = Offset; } EndOffset = Offset; @@ -222,6 +236,7 @@ static bool prettyPrintRegisterOp(raw_ostream &OS, uint8_t Opcode, bool DWARFExpression::Operation::print(raw_ostream &OS, const DWARFExpression *Expr, const MCRegisterInfo *RegInfo, + DWARFUnit *U, bool isEH) { if (Error) { OS << ""; @@ -245,14 +260,25 @@ bool DWARFExpression::Operation::print(raw_ostream &OS, if (Size == Operation::SizeNA) break; - if (Size == Operation::SizeBlock) { + if (Size == Operation::BaseTypeRef && U) { + auto Die = U->getDIEForOffset(U->getOffset() + Operands[Operand]); + if (Die && Die.getTag() == dwarf::DW_TAG_base_type) { + OS << format(" (0x%08x)", U->getOffset() + Operands[Operand]); + if (auto Name = Die.find(dwarf::DW_AT_name)) + OS << " \"" << Name->getAsCString() << "\""; + } else { + OS << format(" ", + Operands[Operand]); + } + } else if (Size == Operation::SizeBlock) { uint32_t Offset = Operands[Operand]; for (unsigned i = 0; i < Operands[Operand - 1]; ++i) OS << format(" 0x%02x", Expr->Data.getU8(&Offset)); } else { if (Signed) OS << format(" %+" PRId64, (int64_t)Operands[Operand]); - else + else if (Opcode != DW_OP_entry_value && + Opcode != DW_OP_GNU_entry_value) OS << format(" 0x%" PRIx64, Operands[Operand]); } } @@ -260,17 +286,60 @@ bool DWARFExpression::Operation::print(raw_ostream &OS, } void DWARFExpression::print(raw_ostream &OS, const MCRegisterInfo *RegInfo, - bool IsEH) const { + DWARFUnit *U, bool IsEH) const { + uint32_t EntryValExprSize = 0; for (auto &Op : *this) { - if (!Op.print(OS, this, RegInfo, IsEH)) { + if (!Op.print(OS, this, RegInfo, U, IsEH)) { uint32_t FailOffset = Op.getEndOffset(); while (FailOffset < Data.getData().size()) OS << format(" %02x", Data.getU8(&FailOffset)); return; } + + if (Op.getCode() == DW_OP_entry_value || + Op.getCode() == DW_OP_GNU_entry_value) { + OS << "("; + EntryValExprSize = Op.getRawOperand(0); + continue; + } + + if (EntryValExprSize) { + EntryValExprSize--; + if (EntryValExprSize == 0) + OS << ")"; + } + if (Op.getEndOffset() < Data.getData().size()) OS << ", "; } } +bool DWARFExpression::Operation::verify(DWARFUnit *U) { + + for (unsigned Operand = 0; Operand < 2; ++Operand) { + unsigned Size = Desc.Op[Operand]; + + if (Size == Operation::SizeNA) + break; + + if (Size == Operation::BaseTypeRef) { + auto Die = U->getDIEForOffset(U->getOffset() + Operands[Operand]); + if (!Die || Die.getTag() != dwarf::DW_TAG_base_type) { + Error = true; + return false; + } + } + } + + return true; +} + +bool DWARFExpression::verify(DWARFUnit *U) { + for (auto &Op : *this) + if (!Op.verify(U)) + return false; + + return true; +} + } // namespace llvm diff --git a/lib/DebugInfo/DWARF/DWARFFormValue.cpp b/lib/DebugInfo/DWARF/DWARFFormValue.cpp index 7719fea63120..290d35511cdb 100644 --- a/lib/DebugInfo/DWARF/DWARFFormValue.cpp +++ b/lib/DebugInfo/DWARF/DWARFFormValue.cpp @@ -1,9 +1,8 @@ //===- DWARFFormValue.cpp -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -78,6 +77,34 @@ static const DWARFFormValue::FormClass DWARF5FormClasses[] = { }; +DWARFFormValue DWARFFormValue::createFromSValue(dwarf::Form F, int64_t V) { + return DWARFFormValue(F, ValueType(V)); +} + +DWARFFormValue DWARFFormValue::createFromUValue(dwarf::Form F, uint64_t V) { + return DWARFFormValue(F, ValueType(V)); +} + +DWARFFormValue DWARFFormValue::createFromPValue(dwarf::Form F, const char *V) { + return DWARFFormValue(F, ValueType(V)); +} + +DWARFFormValue DWARFFormValue::createFromBlockValue(dwarf::Form F, + ArrayRef D) { + ValueType V; + V.uval = D.size(); + V.data = D.data(); + return DWARFFormValue(F, V); +} + +DWARFFormValue DWARFFormValue::createFromUnit(dwarf::Form F, const DWARFUnit *U, + uint32_t *OffsetPtr) { + DWARFFormValue FormValue(F); + FormValue.extractValue(U->getDebugInfoExtractor(), OffsetPtr, + U->getFormParams(), U); + return FormValue; +} + bool DWARFFormValue::skipValue(dwarf::Form Form, DataExtractor DebugInfoData, uint32_t *OffsetPtr, const dwarf::FormParams Params) { @@ -193,13 +220,17 @@ bool DWARFFormValue::isFormClass(DWARFFormValue::FormClass FC) const { default: break; } - // In DWARF3 DW_FORM_data4 and DW_FORM_data8 served also as a section offset. - // Don't check for DWARF version here, as some producers may still do this - // by mistake. Also accept DW_FORM_[line_]strp since these are - // .debug_[line_]str section offsets. - return (Form == DW_FORM_data4 || Form == DW_FORM_data8 || - Form == DW_FORM_strp || Form == DW_FORM_line_strp) && - FC == FC_SectionOffset; + + if (FC == FC_SectionOffset) { + if (Form == DW_FORM_strp || Form == DW_FORM_line_strp) + return true; + // In DWARF3 DW_FORM_data4 and DW_FORM_data8 served also as a section + // offset. If we don't have a DWARFUnit, default to the old behavior. + if (Form == DW_FORM_data4 || Form == DW_FORM_data8) + return !U || U->getVersion() <= 3; + } + + return false; } bool DWARFFormValue::extractValue(const DWARFDataExtractor &Data, @@ -268,7 +299,7 @@ bool DWARFFormValue::extractValue(const DWARFDataExtractor &Data, case DW_FORM_data8: case DW_FORM_ref8: case DW_FORM_ref_sup8: - Value.uval = Data.getU64(OffsetPtr); + Value.uval = Data.getRelocatedValue(8, OffsetPtr); break; case DW_FORM_data16: // Treat this like a 16-byte block. @@ -323,7 +354,7 @@ bool DWARFFormValue::extractValue(const DWARFDataExtractor &Data, StringRef Str = Data.getData().substr(*OffsetPtr, Value.uval); Value.data = nullptr; if (!Str.empty()) { - Value.data = reinterpret_cast(Str.data()); + Value.data = Str.bytes_begin(); *OffsetPtr += Value.uval; } } @@ -333,7 +364,7 @@ bool DWARFFormValue::extractValue(const DWARFDataExtractor &Data, void DWARFFormValue::dumpSectionedAddress(raw_ostream &OS, DIDumpOptions DumpOpts, - SectionedAddress SA) const { + object::SectionedAddress SA) const { OS << format("0x%016" PRIx64, SA.Address); dumpAddressSection(U->getContext().getDWARFObj(), OS, DumpOpts, SA.SectionIndex); @@ -370,12 +401,14 @@ void DWARFFormValue::dump(raw_ostream &OS, DIDumpOptions DumpOpts) const { case DW_FORM_addrx3: case DW_FORM_addrx4: case DW_FORM_GNU_addr_index: { - Optional A = U->getAddrOffsetSectionItem(UValue); + if (U == nullptr) { + OS << ""; + break; + } + Optional A = U->getAddrOffsetSectionItem(UValue); if (!A || DumpOpts.Verbose) AddrOS << format("indexed (%8.8x) address = ", (uint32_t)UValue); - if (U == nullptr) - OS << ""; - else if (A) + if (A) dumpSectionedAddress(AddrOS, DumpOpts, *A); else OS << ""; @@ -591,14 +624,15 @@ Optional DWARFFormValue::getAsAddress() const { return SA->Address; return None; } -Optional DWARFFormValue::getAsSectionedAddress() const { +Optional +DWARFFormValue::getAsSectionedAddress() const { if (!isFormClass(FC_Address)) return None; if (Form == DW_FORM_GNU_addr_index || Form == DW_FORM_addrx) { uint32_t Index = Value.uval; if (!U) return None; - Optional SA = U->getAddrOffsetSectionItem(Index); + Optional SA = U->getAddrOffsetSectionItem(Index); if (!SA) return None; return SA; @@ -607,6 +641,12 @@ Optional DWARFFormValue::getAsSectionedAddress() const { } Optional DWARFFormValue::getAsReference() const { + if (auto R = getAsRelativeReference()) + return R->Unit ? R->Unit->getOffset() + R->Offset : R->Offset; + return None; +} + +Optional DWARFFormValue::getAsRelativeReference() const { if (!isFormClass(FC_Reference)) return None; switch (Form) { @@ -617,11 +657,11 @@ Optional DWARFFormValue::getAsReference() const { case DW_FORM_ref_udata: if (!U) return None; - return Value.uval + U->getOffset(); + return UnitOffset{const_cast(U), Value.uval}; case DW_FORM_ref_addr: case DW_FORM_ref_sig8: case DW_FORM_GNU_ref_alt: - return Value.uval; + return UnitOffset{nullptr, Value.uval}; default: return None; } diff --git a/lib/DebugInfo/DWARF/DWARFGdbIndex.cpp b/lib/DebugInfo/DWARF/DWARFGdbIndex.cpp index 1abd931e3b8b..f5f975578082 100644 --- a/lib/DebugInfo/DWARF/DWARFGdbIndex.cpp +++ b/lib/DebugInfo/DWARF/DWARFGdbIndex.cpp @@ -1,9 +1,8 @@ //===- DWARFGdbIndex.cpp --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -121,7 +120,7 @@ bool DWARFGdbIndex::parseImpl(DataExtractor Data) { return false; CuListOffset = Data.getU32(&Offset); - uint32_t CuTypesOffset = Data.getU32(&Offset); + TuListOffset = Data.getU32(&Offset); AddressAreaOffset = Data.getU32(&Offset); SymbolTableOffset = Data.getU32(&Offset); ConstantPoolOffset = Data.getU32(&Offset); @@ -129,7 +128,7 @@ bool DWARFGdbIndex::parseImpl(DataExtractor Data) { if (Offset != CuListOffset) return false; - uint32_t CuListSize = (CuTypesOffset - CuListOffset) / 16; + uint32_t CuListSize = (TuListOffset - CuListOffset) / 16; CuList.reserve(CuListSize); for (uint32_t i = 0; i < CuListSize; ++i) { uint64_t CuOffset = Data.getU64(&Offset); @@ -139,7 +138,7 @@ bool DWARFGdbIndex::parseImpl(DataExtractor Data) { // CU Types are no longer needed as DWARF skeleton type units never made it // into the standard. - uint32_t TuListSize = (AddressAreaOffset - CuTypesOffset) / 24; + uint32_t TuListSize = (AddressAreaOffset - TuListOffset) / 24; TuList.resize(TuListSize); for (uint32_t I = 0; I < TuListSize; ++I) { uint64_t CuOffset = Data.getU64(&Offset); diff --git a/lib/DebugInfo/DWARF/DWARFListTable.cpp b/lib/DebugInfo/DWARF/DWARFListTable.cpp index 462c036d73ad..e38e706227da 100644 --- a/lib/DebugInfo/DWARF/DWARFListTable.cpp +++ b/lib/DebugInfo/DWARF/DWARFListTable.cpp @@ -1,9 +1,8 @@ //===- DWARFListTable.cpp ---------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -26,7 +25,7 @@ Error DWARFListTableHeader::extract(DWARFDataExtractor Data, "%s table length at offset 0x%" PRIx32, SectionName.data(), *OffsetPtr); // TODO: Add support for DWARF64. - HeaderData.Length = Data.getU32(OffsetPtr); + HeaderData.Length = Data.getRelocatedValue(4, OffsetPtr); if (HeaderData.Length == 0xffffffffu) return createStringError(errc::not_supported, "DWARF64 is not supported in %s at offset 0x%" PRIx32, @@ -74,7 +73,7 @@ Error DWARFListTableHeader::extract(DWARFDataExtractor Data, SectionName.data(), HeaderOffset, HeaderData.OffsetEntryCount); Data.setAddressSize(HeaderData.AddrSize); for (uint32_t I = 0; I < HeaderData.OffsetEntryCount; ++I) - Offsets.push_back(Data.getU32(OffsetPtr)); + Offsets.push_back(Data.getRelocatedValue(4, OffsetPtr)); return Error::success(); } diff --git a/lib/DebugInfo/DWARF/DWARFTypeUnit.cpp b/lib/DebugInfo/DWARF/DWARFTypeUnit.cpp index 00be75e1a94d..844920ba5b11 100644 --- a/lib/DebugInfo/DWARF/DWARFTypeUnit.cpp +++ b/lib/DebugInfo/DWARF/DWARFTypeUnit.cpp @@ -1,9 +1,8 @@ //===- DWARFTypeUnit.cpp --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/DWARF/DWARFUnit.cpp b/lib/DebugInfo/DWARF/DWARFUnit.cpp index 80234665bdeb..b74acf60c747 100644 --- a/lib/DebugInfo/DWARF/DWARFUnit.cpp +++ b/lib/DebugInfo/DWARF/DWARFUnit.cpp @@ -1,9 +1,8 @@ //===- DWARFUnit.cpp ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -198,7 +197,7 @@ DWARFDataExtractor DWARFUnit::getDebugInfoExtractor() const { getAddressByteSize()); } -Optional +Optional DWARFUnit::getAddrOffsetSectionItem(uint32_t Index) const { if (IsDWO) { auto R = Context.info_section_units(); @@ -242,17 +241,21 @@ bool DWARFUnitHeader::extract(DWARFContext &Context, IndexEntry = Entry; if (!IndexEntry && Index) IndexEntry = Index->getFromOffset(*offset_ptr); - Length = debug_info.getU32(offset_ptr); - // FIXME: Support DWARF64. - unsigned SizeOfLength = 4; + Length = debug_info.getRelocatedValue(4, offset_ptr); FormParams.Format = DWARF32; + unsigned SizeOfLength = 4; + if (Length == 0xffffffff) { + Length = debug_info.getU64(offset_ptr); + FormParams.Format = DWARF64; + SizeOfLength = 8; + } FormParams.Version = debug_info.getU16(offset_ptr); if (FormParams.Version >= 5) { UnitType = debug_info.getU8(offset_ptr); FormParams.AddrSize = debug_info.getU8(offset_ptr); - AbbrOffset = debug_info.getU32(offset_ptr); + AbbrOffset = debug_info.getRelocatedValue(FormParams.getDwarfOffsetByteSize(), offset_ptr); } else { - AbbrOffset = debug_info.getRelocatedValue(4, offset_ptr); + AbbrOffset = debug_info.getRelocatedValue(FormParams.getDwarfOffsetByteSize(), offset_ptr); FormParams.AddrSize = debug_info.getU8(offset_ptr); // Fake a unit type based on the section type. This isn't perfect, // but distinguishing compile and type units is generally enough. @@ -432,12 +435,17 @@ size_t DWARFUnit::extractDIEsIfNeeded(bool CUDieOnly) { // which may differ from the unit's format. DWARFDataExtractor DA(Context.getDWARFObj(), StringOffsetSection, isLittleEndian, 0); - if (IsDWO) - StringOffsetsTableContribution = - determineStringOffsetsTableContributionDWO(DA); - else if (getVersion() >= 5) - StringOffsetsTableContribution = - determineStringOffsetsTableContribution(DA); + if (IsDWO || getVersion() >= 5) { + auto StringOffsetOrError = + IsDWO ? determineStringOffsetsTableContributionDWO(DA) + : determineStringOffsetsTableContribution(DA); + if (!StringOffsetOrError) { + WithColor::error() << "invalid contribution to string offsets table in section .debug_str_offsets[.dwo]: " + << toString(StringOffsetOrError.takeError()) << '\n'; + } else { + StringOffsetsTableContribution = *StringOffsetOrError; + } + } // DWARF v5 uses the .debug_rnglists and .debug_rnglists.dwo sections to // describe address ranges. @@ -634,7 +642,7 @@ DWARFUnit::getInlinedChainForAddress(uint64_t Address, // First, find the subroutine that contains the given address (the leaf // of inlined chain). DWARFDie SubroutineDIE = - (DWO ? DWO.get() : this)->getSubroutineForAddress(Address); + (DWO ? *DWO : *this).getSubroutineForAddress(Address); if (!SubroutineDIE) return; @@ -745,7 +753,7 @@ const DWARFAbbreviationDeclarationSet *DWARFUnit::getAbbreviations() const { return Abbrevs; } -llvm::Optional DWARFUnit::getBaseAddress() { +llvm::Optional DWARFUnit::getBaseAddress() { if (BaseAddr) return BaseAddr; @@ -755,7 +763,7 @@ llvm::Optional DWARFUnit::getBaseAddress() { return BaseAddr; } -Optional +Expected StrOffsetsContributionDescriptor::validateContributionSize( DWARFDataExtractor &DA) { uint8_t EntrySize = getDwarfOffsetByteSize(); @@ -766,58 +774,94 @@ StrOffsetsContributionDescriptor::validateContributionSize( if (ValidationSize >= Size) if (DA.isValidOffsetForDataOfSize((uint32_t)Base, ValidationSize)) return *this; - return None; + return createStringError(errc::invalid_argument, "length exceeds section size"); } // Look for a DWARF64-formatted contribution to the string offsets table // starting at a given offset and record it in a descriptor. -static Optional +static Expected parseDWARF64StringOffsetsTableHeader(DWARFDataExtractor &DA, uint32_t Offset) { if (!DA.isValidOffsetForDataOfSize(Offset, 16)) - return None; + return createStringError(errc::invalid_argument, "section offset exceeds section size"); if (DA.getU32(&Offset) != 0xffffffff) - return None; + return createStringError(errc::invalid_argument, "32 bit contribution referenced from a 64 bit unit"); uint64_t Size = DA.getU64(&Offset); uint8_t Version = DA.getU16(&Offset); (void)DA.getU16(&Offset); // padding // The encoded length includes the 2-byte version field and the 2-byte // padding, so we need to subtract them out when we populate the descriptor. - return {{Offset, Size - 4, Version, DWARF64}}; + return StrOffsetsContributionDescriptor(Offset, Size - 4, Version, DWARF64); } // Look for a DWARF32-formatted contribution to the string offsets table // starting at a given offset and record it in a descriptor. -static Optional +static Expected parseDWARF32StringOffsetsTableHeader(DWARFDataExtractor &DA, uint32_t Offset) { if (!DA.isValidOffsetForDataOfSize(Offset, 8)) - return None; + return createStringError(errc::invalid_argument, "section offset exceeds section size"); + uint32_t ContributionSize = DA.getU32(&Offset); if (ContributionSize >= 0xfffffff0) - return None; + return createStringError(errc::invalid_argument, "invalid length"); + uint8_t Version = DA.getU16(&Offset); (void)DA.getU16(&Offset); // padding // The encoded length includes the 2-byte version field and the 2-byte // padding, so we need to subtract them out when we populate the descriptor. - return {{Offset, ContributionSize - 4, Version, DWARF32}}; + return StrOffsetsContributionDescriptor(Offset, ContributionSize - 4, Version, + DWARF32); +} + +static Expected +parseDWARFStringOffsetsTableHeader(DWARFDataExtractor &DA, + llvm::dwarf::DwarfFormat Format, + uint64_t Offset) { + StrOffsetsContributionDescriptor Desc; + switch (Format) { + case dwarf::DwarfFormat::DWARF64: { + if (Offset < 16) + return createStringError(errc::invalid_argument, "insufficient space for 64 bit header prefix"); + auto DescOrError = parseDWARF64StringOffsetsTableHeader(DA, (uint32_t)Offset - 16); + if (!DescOrError) + return DescOrError.takeError(); + Desc = *DescOrError; + break; + } + case dwarf::DwarfFormat::DWARF32: { + if (Offset < 8) + return createStringError(errc::invalid_argument, "insufficient space for 32 bit header prefix"); + auto DescOrError = parseDWARF32StringOffsetsTableHeader(DA, (uint32_t)Offset - 8); + if (!DescOrError) + return DescOrError.takeError(); + Desc = *DescOrError; + break; + } + } + return Desc.validateContributionSize(DA); } -Optional +Expected> DWARFUnit::determineStringOffsetsTableContribution(DWARFDataExtractor &DA) { - auto Offset = toSectionOffset(getUnitDIE().find(DW_AT_str_offsets_base), 0); - Optional Descriptor; - // Attempt to find a DWARF64 contribution 16 bytes before the base. - if (Offset >= 16) - Descriptor = - parseDWARF64StringOffsetsTableHeader(DA, (uint32_t)Offset - 16); - // Try to find a DWARF32 contribution 8 bytes before the base. - if (!Descriptor && Offset >= 8) - Descriptor = parseDWARF32StringOffsetsTableHeader(DA, (uint32_t)Offset - 8); - return Descriptor ? Descriptor->validateContributionSize(DA) : Descriptor; -} - -Optional + uint64_t Offset; + if (IsDWO) { + Offset = 0; + if (DA.getData().data() == nullptr) + return None; + } else { + auto OptOffset = toSectionOffset(getUnitDIE().find(DW_AT_str_offsets_base)); + if (!OptOffset) + return None; + Offset = *OptOffset; + } + auto DescOrError = parseDWARFStringOffsetsTableHeader(DA, Header.getFormat(), Offset); + if (!DescOrError) + return DescOrError.takeError(); + return *DescOrError; +} + +Expected> DWARFUnit::determineStringOffsetsTableContributionDWO(DWARFDataExtractor & DA) { uint64_t Offset = 0; auto IndexEntry = Header.getIndexEntry(); @@ -826,19 +870,24 @@ DWARFUnit::determineStringOffsetsTableContributionDWO(DWARFDataExtractor & DA) { if (C) Offset = C->Offset; if (getVersion() >= 5) { + if (DA.getData().data() == nullptr) + return None; + Offset += Header.getFormat() == dwarf::DwarfFormat::DWARF32 ? 8 : 16; // Look for a valid contribution at the given offset. - auto Descriptor = - parseDWARF64StringOffsetsTableHeader(DA, (uint32_t)Offset); - if (!Descriptor) - Descriptor = parseDWARF32StringOffsetsTableHeader(DA, (uint32_t)Offset); - return Descriptor ? Descriptor->validateContributionSize(DA) : Descriptor; + auto DescOrError = parseDWARFStringOffsetsTableHeader(DA, Header.getFormat(), Offset); + if (!DescOrError) + return DescOrError.takeError(); + return *DescOrError; } // Prior to DWARF v5, we derive the contribution size from the // index table (in a package file). In a .dwo file it is simply // the length of the string offsets section. if (!IndexEntry) - return {{0, StringOffsetSection.Data.size(), 4, DWARF32}}; + return { + Optional( + {0, StringOffsetSection.Data.size(), 4, DWARF32})}; if (C) - return {{C->Offset, C->Length, 4, DWARF32}}; + return {Optional( + {C->Offset, C->Length, 4, DWARF32})}; return None; } diff --git a/lib/DebugInfo/DWARF/DWARFUnitIndex.cpp b/lib/DebugInfo/DWARF/DWARFUnitIndex.cpp index 84b6c4b81817..047c63461ccf 100644 --- a/lib/DebugInfo/DWARF/DWARFUnitIndex.cpp +++ b/lib/DebugInfo/DWARF/DWARFUnitIndex.cpp @@ -1,9 +1,8 @@ //===- DWARFUnitIndex.cpp -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -173,10 +172,9 @@ DWARFUnitIndex::getFromOffset(uint32_t Offset) const { E2->Contributions[InfoColumn].Offset; }); } - auto I = - llvm::upper_bound(OffsetLookup, Offset, [&](uint32_t Offset, Entry *E2) { - return Offset < E2->Contributions[InfoColumn].Offset; - }); + auto I = partition_point(OffsetLookup, [&](Entry *E2) { + return E2->Contributions[InfoColumn].Offset <= Offset; + }); if (I == OffsetLookup.begin()) return nullptr; --I; diff --git a/lib/DebugInfo/DWARF/DWARFVerifier.cpp b/lib/DebugInfo/DWARF/DWARFVerifier.cpp index f8370178b627..c2b3189514a8 100644 --- a/lib/DebugInfo/DWARF/DWARFVerifier.cpp +++ b/lib/DebugInfo/DWARF/DWARFVerifier.cpp @@ -1,9 +1,8 @@ //===- DWARFVerifier.cpp --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "llvm/DebugInfo/DWARF/DWARFVerifier.h" @@ -61,55 +60,47 @@ DWARFVerifier::DieRangeInfo::insert(const DieRangeInfo &RI) { } bool DWARFVerifier::DieRangeInfo::contains(const DieRangeInfo &RHS) const { - // Both list of ranges are sorted so we can make this fast. - - if (Ranges.empty() || RHS.Ranges.empty()) - return false; - - // Since the ranges are sorted we can advance where we start searching with - // this object's ranges as we traverse RHS.Ranges. - auto End = Ranges.end(); - auto Iter = findRange(RHS.Ranges.front()); + auto I1 = Ranges.begin(), E1 = Ranges.end(); + auto I2 = RHS.Ranges.begin(), E2 = RHS.Ranges.end(); + if (I2 == E2) + return true; - // Now linearly walk the ranges in this object and see if they contain each - // ranges from RHS.Ranges. - for (const auto &R : RHS.Ranges) { - while (Iter != End) { - if (Iter->contains(R)) - break; - ++Iter; + DWARFAddressRange R = *I2; + while (I1 != E1) { + bool Covered = I1->LowPC <= R.LowPC; + if (R.LowPC == R.HighPC || (Covered && R.HighPC <= I1->HighPC)) { + if (++I2 == E2) + return true; + R = *I2; + continue; } - if (Iter == End) + if (!Covered) return false; + if (R.LowPC < I1->HighPC) + R.LowPC = I1->HighPC; + ++I1; } - return true; + return false; } bool DWARFVerifier::DieRangeInfo::intersects(const DieRangeInfo &RHS) const { - if (Ranges.empty() || RHS.Ranges.empty()) - return false; - - auto End = Ranges.end(); - auto Iter = findRange(RHS.Ranges.front()); - for (const auto &R : RHS.Ranges) { - if (Iter == End) - return false; - if (R.HighPC <= Iter->LowPC) - continue; - while (Iter != End) { - if (Iter->intersects(R)) - return true; - ++Iter; - } + auto I1 = Ranges.begin(), E1 = Ranges.end(); + auto I2 = RHS.Ranges.begin(), E2 = RHS.Ranges.end(); + while (I1 != E1 && I2 != E2) { + if (I1->intersects(*I2)) + return true; + if (I1->LowPC < I2->LowPC) + ++I1; + else + ++I2; } - return false; } bool DWARFVerifier::verifyUnitHeader(const DWARFDataExtractor DebugInfoData, uint32_t *Offset, unsigned UnitIndex, uint8_t &UnitType, bool &isUnitDWARF64) { - uint32_t AbbrOffset, Length; + uint64_t AbbrOffset, Length; uint8_t AddrSize = 0; uint16_t Version; bool Success = true; @@ -123,22 +114,19 @@ bool DWARFVerifier::verifyUnitHeader(const DWARFDataExtractor DebugInfoData, uint32_t OffsetStart = *Offset; Length = DebugInfoData.getU32(Offset); if (Length == UINT32_MAX) { + Length = DebugInfoData.getU64(Offset); isUnitDWARF64 = true; - OS << format( - "Unit[%d] is in 64-bit DWARF format; cannot verify from this point.\n", - UnitIndex); - return false; } Version = DebugInfoData.getU16(Offset); if (Version >= 5) { UnitType = DebugInfoData.getU8(Offset); AddrSize = DebugInfoData.getU8(Offset); - AbbrOffset = DebugInfoData.getU32(Offset); + AbbrOffset = isUnitDWARF64 ? DebugInfoData.getU64(Offset) : DebugInfoData.getU32(Offset); ValidType = dwarf::isUnitType(UnitType); } else { UnitType = 0; - AbbrOffset = DebugInfoData.getU32(Offset); + AbbrOffset = isUnitDWARF64 ? DebugInfoData.getU64(Offset) : DebugInfoData.getU32(Offset); AddrSize = DebugInfoData.getU8(Offset); } @@ -166,7 +154,7 @@ bool DWARFVerifier::verifyUnitHeader(const DWARFDataExtractor DebugInfoData, if (!ValidAddrSize) note() << "The address size is unsupported.\n"; } - *Offset = OffsetStart + Length + 4; + *Offset = OffsetStart + Length + (isUnitDWARF64 ? 12 : 4); return Success; } @@ -179,21 +167,11 @@ unsigned DWARFVerifier::verifyUnitContents(DWARFUnit &Unit) { if (Die.getTag() == DW_TAG_null) continue; - bool HasTypeAttr = false; for (auto AttrValue : Die.attributes()) { NumUnitErrors += verifyDebugInfoAttribute(Die, AttrValue); NumUnitErrors += verifyDebugInfoForm(Die, AttrValue); - HasTypeAttr |= (AttrValue.Attr == DW_AT_type); } - if (!HasTypeAttr && (Die.getTag() == DW_TAG_formal_parameter || - Die.getTag() == DW_TAG_variable || - Die.getTag() == DW_TAG_array_type)) { - error() << "DIE with tag " << TagString(Die.getTag()) - << " is missing type attribute:\n"; - dump(Die) << '\n'; - NumUnitErrors++; - } NumUnitErrors += verifyDebugInfoCallSite(Die); } @@ -281,19 +259,12 @@ bool DWARFVerifier::handleDebugAbbrev() { OS << "Verifying .debug_abbrev...\n"; const DWARFObject &DObj = DCtx.getDWARFObj(); - bool noDebugAbbrev = DObj.getAbbrevSection().empty(); - bool noDebugAbbrevDWO = DObj.getAbbrevDWOSection().empty(); - - if (noDebugAbbrev && noDebugAbbrevDWO) { - return true; - } - unsigned NumErrors = 0; - if (!noDebugAbbrev) + if (!DObj.getAbbrevSection().empty()) NumErrors += verifyAbbrevSection(DCtx.getDebugAbbrev()); - - if (!noDebugAbbrevDWO) + if (!DObj.getAbbrevDWOSection().empty()) NumErrors += verifyAbbrevSection(DCtx.getDebugAbbrevDWO()); + return NumErrors == 0; } @@ -503,7 +474,7 @@ unsigned DWARFVerifier::verifyDebugInfoAttribute(const DWARFDie &Die, bool Error = llvm::any_of(Expression, [](DWARFExpression::Operation &Op) { return Op.isError(); }); - if (Error) + if (Error || !Expression.verify(U)) ReportError("DIE contains invalid DWARF expression:"); }; if (Optional> Expr = AttrValue.Value.getAsBlock()) { @@ -629,7 +600,7 @@ unsigned DWARFVerifier::verifyDebugInfoForm(const DWARFDie &Die, dump(Die) << '\n'; break; } - // Check that the index is within the bounds of the section. + // Check that the index is within the bounds of the section. unsigned ItemSize = DieCU->getDwarfStringOffsetsByteSize(); // Use a 64-bit type to calculate the offset to guard against overflow. uint64_t Offset = @@ -664,9 +635,9 @@ unsigned DWARFVerifier::verifyDebugInfoReferences() { // getting the DIE by offset and emitting an error OS << "Verifying .debug_info references...\n"; unsigned NumErrors = 0; - for (auto Pair : ReferenceToDIEOffsets) { - auto Die = DCtx.getDIEForOffset(Pair.first); - if (Die) + for (const std::pair> &Pair : + ReferenceToDIEOffsets) { + if (DCtx.getDIEForOffset(Pair.first)) continue; ++NumErrors; error() << "invalid DIE reference " << format("0x%08" PRIx64, Pair.first) @@ -731,7 +702,6 @@ void DWARFVerifier::verifyDebugLineRows() { continue; // Verify prologue. - uint32_t MaxFileIndex = LineTable->Prologue.FileNames.size(); uint32_t MaxDirIndex = LineTable->Prologue.IncludeDirectories.size(); uint32_t FileIndex = 1; StringMap FullPathMap; @@ -773,7 +743,7 @@ void DWARFVerifier::verifyDebugLineRows() { uint32_t RowIndex = 0; for (const auto &Row : LineTable->Rows) { // Verify row address. - if (Row.Address < PrevAddress) { + if (Row.Address.Address < PrevAddress) { ++NumDebugLineErrors; error() << ".debug_line[" << format("0x%08" PRIx64, @@ -789,13 +759,16 @@ void DWARFVerifier::verifyDebugLineRows() { } // Verify file index. - if (Row.File > MaxFileIndex) { + if (!LineTable->hasFileAtIndex(Row.File)) { ++NumDebugLineErrors; + bool isDWARF5 = LineTable->Prologue.getVersion() >= 5; error() << ".debug_line[" << format("0x%08" PRIx64, *toSectionOffset(Die.find(DW_AT_stmt_list))) << "][" << RowIndex << "] has invalid file index " << Row.File - << " (valid values are [1," << MaxFileIndex << "]):\n"; + << " (valid values are [" << (isDWARF5 ? "0," : "1,") + << LineTable->Prologue.FileNames.size() + << (isDWARF5 ? ")" : "]") << "):\n"; DWARFDebugLine::Row::dumpTableHeader(OS); Row.dump(OS); OS << '\n'; @@ -803,7 +776,7 @@ void DWARFVerifier::verifyDebugLineRows() { if (Row.EndSequence) PrevAddress = 0; else - PrevAddress = Row.Address; + PrevAddress = Row.Address.Address; ++RowIndex; } } diff --git a/lib/DebugInfo/GSYM/FunctionInfo.cpp b/lib/DebugInfo/GSYM/FunctionInfo.cpp new file mode 100644 index 000000000000..55c36a55b4be --- /dev/null +++ b/lib/DebugInfo/GSYM/FunctionInfo.cpp @@ -0,0 +1,22 @@ +//===- FunctionInfo.cpp -----------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/DebugInfo/GSYM/FunctionInfo.h" + +using namespace llvm; +using namespace gsym; + +raw_ostream &llvm::gsym::operator<<(raw_ostream &OS, const FunctionInfo &FI) { + OS << '[' << HEX64(FI.Range.Start) << '-' << HEX64(FI.Range.End) << "): " + << "Name=" << HEX32(FI.Name) << '\n'; + for (const auto &Line : FI.Lines) + OS << Line << '\n'; + OS << FI.Inline; + return OS; +} diff --git a/lib/DebugInfo/GSYM/InlineInfo.cpp b/lib/DebugInfo/GSYM/InlineInfo.cpp new file mode 100644 index 000000000000..781c1755241d --- /dev/null +++ b/lib/DebugInfo/GSYM/InlineInfo.cpp @@ -0,0 +1,59 @@ +//===- InlineInfo.cpp -------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/DebugInfo/GSYM/FileEntry.h" +#include "llvm/DebugInfo/GSYM/InlineInfo.h" +#include +#include + +using namespace llvm; +using namespace gsym; + + +raw_ostream &llvm::gsym::operator<<(raw_ostream &OS, const InlineInfo &II) { + if (!II.isValid()) + return OS; + bool First = true; + for (auto Range : II.Ranges) { + if (First) + First = false; + else + OS << ' '; + OS << Range; + } + OS << " Name = " << HEX32(II.Name) << ", CallFile = " << II.CallFile + << ", CallLine = " << II.CallFile << '\n'; + for (const auto &Child : II.Children) + OS << Child; + return OS; +} + +static bool getInlineStackHelper(const InlineInfo &II, uint64_t Addr, + std::vector &InlineStack) { + if (II.Ranges.contains(Addr)) { + // If this is the top level that represents the concrete function, + // there will be no name and we shoud clear the inline stack. Otherwise + // we have found an inline call stack that we need to insert. + if (II.Name != 0) + InlineStack.insert(InlineStack.begin(), &II); + for (const auto &Child : II.Children) { + if (::getInlineStackHelper(Child, Addr, InlineStack)) + break; + } + return !InlineStack.empty(); + } + return false; +} + +llvm::Optional InlineInfo::getInlineStack(uint64_t Addr) const { + InlineArray Result; + if (getInlineStackHelper(*this, Addr, Result)) + return Result; + return llvm::None; +} diff --git a/lib/DebugInfo/GSYM/Range.cpp b/lib/DebugInfo/GSYM/Range.cpp new file mode 100644 index 000000000000..ca61984dacbd --- /dev/null +++ b/lib/DebugInfo/GSYM/Range.cpp @@ -0,0 +1,55 @@ +//===- Range.cpp ------------------------------------------------*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/DebugInfo/GSYM/Range.h" +#include +#include + +using namespace llvm; +using namespace gsym; + + +void AddressRanges::insert(AddressRange Range) { + if (Range.size() == 0) + return; + + auto It = llvm::upper_bound(Ranges, Range); + auto It2 = It; + while (It2 != Ranges.end() && It2->Start < Range.End) + ++It2; + if (It != It2) { + Range.End = std::max(Range.End, It2[-1].End); + It = Ranges.erase(It, It2); + } + if (It != Ranges.begin() && Range.Start < It[-1].End) + It[-1].End = std::max(It[-1].End, Range.End); + else + Ranges.insert(It, Range); +} + +bool AddressRanges::contains(uint64_t Addr) const { + auto It = std::partition_point( + Ranges.begin(), Ranges.end(), + [=](const AddressRange &R) { return R.Start <= Addr; }); + return It != Ranges.begin() && Addr < It[-1].End; +} + +raw_ostream &llvm::gsym::operator<<(raw_ostream &OS, const AddressRange &R) { + return OS << '[' << HEX64(R.Start) << " - " << HEX64(R.End) << ")"; +} + +raw_ostream &llvm::gsym::operator<<(raw_ostream &OS, const AddressRanges &AR) { + size_t Size = AR.size(); + for (size_t I = 0; I < Size; ++I) { + if (I) + OS << ' '; + OS << AR[I]; + } + return OS; +} diff --git a/lib/DebugInfo/MSF/MSFBuilder.cpp b/lib/DebugInfo/MSF/MSFBuilder.cpp index 71609919558a..c6fe764ab7e0 100644 --- a/lib/DebugInfo/MSF/MSFBuilder.cpp +++ b/lib/DebugInfo/MSF/MSFBuilder.cpp @@ -1,9 +1,8 @@ //===- MSFBuilder.cpp -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/MSF/MSFCommon.cpp b/lib/DebugInfo/MSF/MSFCommon.cpp index d398304375ac..fb4f0700059c 100644 --- a/lib/DebugInfo/MSF/MSFCommon.cpp +++ b/lib/DebugInfo/MSF/MSFCommon.cpp @@ -1,9 +1,8 @@ //===- MSFCommon.cpp - Common types and functions for MSF files -----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/MSF/MSFError.cpp b/lib/DebugInfo/MSF/MSFError.cpp index bfac6bebba3f..b368b802c564 100644 --- a/lib/DebugInfo/MSF/MSFError.cpp +++ b/lib/DebugInfo/MSF/MSFError.cpp @@ -1,9 +1,8 @@ //===- MSFError.cpp - Error extensions for MSF files ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -14,6 +13,7 @@ using namespace llvm; using namespace llvm::msf; +namespace { // FIXME: This class is only here to support the transition to llvm::Error. It // will be removed once this transition is complete. Clients should prefer to // deal with the Error value directly, rather than converting to error_code. @@ -39,6 +39,7 @@ public: llvm_unreachable("Unrecognized msf_error_code"); } }; +} // namespace static llvm::ManagedStatic MSFCategory; const std::error_category &llvm::msf::MSFErrCategory() { return *MSFCategory; } diff --git a/lib/DebugInfo/MSF/MappedBlockStream.cpp b/lib/DebugInfo/MSF/MappedBlockStream.cpp index dec28eb30697..df925771f0d9 100644 --- a/lib/DebugInfo/MSF/MappedBlockStream.cpp +++ b/lib/DebugInfo/MSF/MappedBlockStream.cpp @@ -1,9 +1,8 @@ //===- MappedBlockStream.cpp - Reads stream data from an MSF file ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/DIA/DIADataStream.cpp b/lib/DebugInfo/PDB/DIA/DIADataStream.cpp index 6a10513fad97..8a806f298d0f 100644 --- a/lib/DebugInfo/PDB/DIA/DIADataStream.cpp +++ b/lib/DebugInfo/PDB/DIA/DIADataStream.cpp @@ -1,9 +1,8 @@ //===- DIADataStream.cpp - DIA implementation of IPDBDataStream -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/DIA/DIAEnumDebugStreams.cpp b/lib/DebugInfo/PDB/DIA/DIAEnumDebugStreams.cpp index d2451f13e6cb..e4cb4daf94b1 100644 --- a/lib/DebugInfo/PDB/DIA/DIAEnumDebugStreams.cpp +++ b/lib/DebugInfo/PDB/DIA/DIAEnumDebugStreams.cpp @@ -1,9 +1,8 @@ //==- DIAEnumDebugStreams.cpp - DIA Debug Stream Enumerator impl -*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/DIA/DIAEnumFrameData.cpp b/lib/DebugInfo/PDB/DIA/DIAEnumFrameData.cpp index f873f3525df5..8a181b448a27 100644 --- a/lib/DebugInfo/PDB/DIA/DIAEnumFrameData.cpp +++ b/lib/DebugInfo/PDB/DIA/DIAEnumFrameData.cpp @@ -1,9 +1,8 @@ //==- DIAEnumFrameData.cpp ---------------------------------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/DIA/DIAEnumInjectedSources.cpp b/lib/DebugInfo/PDB/DIA/DIAEnumInjectedSources.cpp index 6c361b81e33d..7226ab2ba0a0 100644 --- a/lib/DebugInfo/PDB/DIA/DIAEnumInjectedSources.cpp +++ b/lib/DebugInfo/PDB/DIA/DIAEnumInjectedSources.cpp @@ -1,9 +1,8 @@ //==- DIAEnumSourceFiles.cpp - DIA Source File Enumerator impl ---*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/DIA/DIAEnumLineNumbers.cpp b/lib/DebugInfo/PDB/DIA/DIAEnumLineNumbers.cpp index 0820d9dc7c9f..6f1d7733fb2d 100644 --- a/lib/DebugInfo/PDB/DIA/DIAEnumLineNumbers.cpp +++ b/lib/DebugInfo/PDB/DIA/DIAEnumLineNumbers.cpp @@ -1,9 +1,8 @@ //==- DIAEnumLineNumbers.cpp - DIA Line Number Enumerator impl ---*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/DIA/DIAEnumSectionContribs.cpp b/lib/DebugInfo/PDB/DIA/DIAEnumSectionContribs.cpp index 90c857aa5713..4f9b232a024a 100644 --- a/lib/DebugInfo/PDB/DIA/DIAEnumSectionContribs.cpp +++ b/lib/DebugInfo/PDB/DIA/DIAEnumSectionContribs.cpp @@ -1,9 +1,8 @@ //==- DIAEnumSectionContribs.cpp ---------------------------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/DIA/DIAEnumSourceFiles.cpp b/lib/DebugInfo/PDB/DIA/DIAEnumSourceFiles.cpp index 06595e7ec1c8..943e9e1b4d58 100644 --- a/lib/DebugInfo/PDB/DIA/DIAEnumSourceFiles.cpp +++ b/lib/DebugInfo/PDB/DIA/DIAEnumSourceFiles.cpp @@ -1,9 +1,8 @@ //==- DIAEnumSourceFiles.cpp - DIA Source File Enumerator impl ---*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/DIA/DIAEnumSymbols.cpp b/lib/DebugInfo/PDB/DIA/DIAEnumSymbols.cpp index 48bc32767e6c..5153596d52ae 100644 --- a/lib/DebugInfo/PDB/DIA/DIAEnumSymbols.cpp +++ b/lib/DebugInfo/PDB/DIA/DIAEnumSymbols.cpp @@ -1,9 +1,8 @@ //==- DIAEnumSymbols.cpp - DIA Symbol Enumerator impl ------------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/DIA/DIAEnumTables.cpp b/lib/DebugInfo/PDB/DIA/DIAEnumTables.cpp index 6fa096156d48..335b575d6542 100644 --- a/lib/DebugInfo/PDB/DIA/DIAEnumTables.cpp +++ b/lib/DebugInfo/PDB/DIA/DIAEnumTables.cpp @@ -1,9 +1,8 @@ //===- DIAEnumTables.cpp - DIA Table Enumerator Impl ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/DIA/DIAFrameData.cpp b/lib/DebugInfo/PDB/DIA/DIAFrameData.cpp index 533cce7923c0..7975156b1abd 100644 --- a/lib/DebugInfo/PDB/DIA/DIAFrameData.cpp +++ b/lib/DebugInfo/PDB/DIA/DIAFrameData.cpp @@ -1,9 +1,8 @@ //===- DIAFrameData.cpp - DIA impl. of IPDBFrameData -------------- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/DIA/DIAInjectedSource.cpp b/lib/DebugInfo/PDB/DIA/DIAInjectedSource.cpp index 1d642f221d79..032b230b5faa 100644 --- a/lib/DebugInfo/PDB/DIA/DIAInjectedSource.cpp +++ b/lib/DebugInfo/PDB/DIA/DIAInjectedSource.cpp @@ -1,9 +1,8 @@ //===- DIAInjectedSource.cpp - DIA impl for IPDBInjectedSource --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -42,11 +41,11 @@ std::string DIAInjectedSource::getVirtualFileName() const { &IDiaInjectedSource::get_virtualFilename); } -PDB_SourceCompression DIAInjectedSource::getCompression() const { +uint32_t DIAInjectedSource::getCompression() const { DWORD Compression = 0; if (S_OK != SourceFile->get_sourceCompression(&Compression)) return PDB_SourceCompression::None; - return static_cast(Compression); + return static_cast(Compression); } std::string DIAInjectedSource::getCode() const { diff --git a/lib/DebugInfo/PDB/DIA/DIALineNumber.cpp b/lib/DebugInfo/PDB/DIA/DIALineNumber.cpp index b19be6b595ab..3af02ea36c7b 100644 --- a/lib/DebugInfo/PDB/DIA/DIALineNumber.cpp +++ b/lib/DebugInfo/PDB/DIA/DIALineNumber.cpp @@ -1,9 +1,8 @@ //===- DIALineNumber.cpp - DIA implementation of IPDBLineNumber -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/DIA/DIARawSymbol.cpp b/lib/DebugInfo/PDB/DIA/DIARawSymbol.cpp index cd4d00a13b18..a8ae076e1d6c 100644 --- a/lib/DebugInfo/PDB/DIA/DIARawSymbol.cpp +++ b/lib/DebugInfo/PDB/DIA/DIARawSymbol.cpp @@ -1,9 +1,8 @@ //===- DIARawSymbol.cpp - DIA implementation of IPDBRawSymbol ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/DIA/DIASectionContrib.cpp b/lib/DebugInfo/PDB/DIA/DIASectionContrib.cpp index 8e233ca15161..e2d928f2c4b2 100644 --- a/lib/DebugInfo/PDB/DIA/DIASectionContrib.cpp +++ b/lib/DebugInfo/PDB/DIA/DIASectionContrib.cpp @@ -1,9 +1,8 @@ //===- DIASectionContrib.cpp - DIA impl. of IPDBSectionContrib ---- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/DIA/DIASession.cpp b/lib/DebugInfo/PDB/DIA/DIASession.cpp index bd375e172ac0..4e0b8587c613 100644 --- a/lib/DebugInfo/PDB/DIA/DIASession.cpp +++ b/lib/DebugInfo/PDB/DIA/DIASession.cpp @@ -1,9 +1,8 @@ //===- DIASession.cpp - DIA implementation of IPDBSession -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "llvm/DebugInfo/PDB/DIA/DIASession.h" diff --git a/lib/DebugInfo/PDB/DIA/DIASourceFile.cpp b/lib/DebugInfo/PDB/DIA/DIASourceFile.cpp index d3e408166a87..21e757c3a060 100644 --- a/lib/DebugInfo/PDB/DIA/DIASourceFile.cpp +++ b/lib/DebugInfo/PDB/DIA/DIASourceFile.cpp @@ -1,9 +1,8 @@ //===- DIASourceFile.cpp - DIA implementation of IPDBSourceFile -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/DIA/DIATable.cpp b/lib/DebugInfo/PDB/DIA/DIATable.cpp index 6017081b2cb6..33d74abd740e 100644 --- a/lib/DebugInfo/PDB/DIA/DIATable.cpp +++ b/lib/DebugInfo/PDB/DIA/DIATable.cpp @@ -1,9 +1,8 @@ //===- DIATable.cpp - DIA implementation of IPDBTable -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/GenericError.cpp b/lib/DebugInfo/PDB/GenericError.cpp index 256952073e88..70dc094c42ec 100644 --- a/lib/DebugInfo/PDB/GenericError.cpp +++ b/lib/DebugInfo/PDB/GenericError.cpp @@ -1,9 +1,8 @@ //===- Error.cpp - system_error extensions for PDB --------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -14,6 +13,7 @@ using namespace llvm; using namespace llvm::pdb; +namespace { // FIXME: This class is only here to support the transition to llvm::Error. It // will be removed once this transition is complete. Clients should prefer to // deal with the Error value directly, rather than converting to error_code. @@ -40,6 +40,7 @@ public: llvm_unreachable("Unrecognized generic_error_code"); } }; +} // namespace static llvm::ManagedStatic PDBCategory; const std::error_category &llvm::pdb::PDBErrCategory() { return *PDBCategory; } diff --git a/lib/DebugInfo/PDB/IPDBSourceFile.cpp b/lib/DebugInfo/PDB/IPDBSourceFile.cpp index 8cb1fbef51f4..113ee04bab95 100644 --- a/lib/DebugInfo/PDB/IPDBSourceFile.cpp +++ b/lib/DebugInfo/PDB/IPDBSourceFile.cpp @@ -1,9 +1,8 @@ //===- IPDBSourceFile.cpp - base interface for a PDB source file ----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/DbiModuleDescriptor.cpp b/lib/DebugInfo/PDB/Native/DbiModuleDescriptor.cpp index 931ac7bb81db..5095efcdee3c 100644 --- a/lib/DebugInfo/PDB/Native/DbiModuleDescriptor.cpp +++ b/lib/DebugInfo/PDB/Native/DbiModuleDescriptor.cpp @@ -1,9 +1,8 @@ //===- DbiModuleDescriptor.cpp - PDB module information -------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.cpp b/lib/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.cpp index ab93efc839a9..20b6c6142547 100644 --- a/lib/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.cpp +++ b/lib/DebugInfo/PDB/Native/DbiModuleDescriptorBuilder.cpp @@ -1,9 +1,8 @@ //===- DbiModuleDescriptorBuilder.cpp - PDB Mod Info Creation ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -104,7 +103,6 @@ uint32_t DbiModuleDescriptorBuilder::calculateSerializedLength() const { } void DbiModuleDescriptorBuilder::finalize() { - Layout.SC.Imod = Layout.Mod; Layout.FileNameOffs = 0; // TODO: Fix this Layout.Flags = 0; // TODO: Fix this Layout.C11Bytes = 0; @@ -117,12 +115,15 @@ void DbiModuleDescriptorBuilder::finalize() { // This value includes both the signature field as well as the record bytes // from the symbol stream. - Layout.SymBytes = SymbolByteSize + sizeof(uint32_t); + Layout.SymBytes = + Layout.ModDiStream == kInvalidStreamIndex ? 0 : getNextSymbolOffset(); } Error DbiModuleDescriptorBuilder::finalizeMsfLayout() { this->Layout.ModDiStream = kInvalidStreamIndex; uint32_t C13Size = calculateC13DebugInfoSize(); + if (!C13Size && !SymbolByteSize) + return Error::success(); auto ExpectedSN = MSF.addStream(calculateDiSymbolStreamSize(SymbolByteSize, C13Size)); if (!ExpectedSN) diff --git a/lib/DebugInfo/PDB/Native/DbiModuleList.cpp b/lib/DebugInfo/PDB/Native/DbiModuleList.cpp index eea70b229c67..5cf014e881cd 100644 --- a/lib/DebugInfo/PDB/Native/DbiModuleList.cpp +++ b/lib/DebugInfo/PDB/Native/DbiModuleList.cpp @@ -1,9 +1,8 @@ //===- DbiModuleList.cpp - PDB module information list --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/DbiStream.cpp b/lib/DebugInfo/PDB/Native/DbiStream.cpp index 60ac17b655a7..4eb16804171d 100644 --- a/lib/DebugInfo/PDB/Native/DbiStream.cpp +++ b/lib/DebugInfo/PDB/Native/DbiStream.cpp @@ -1,9 +1,8 @@ //===- DbiStream.cpp - PDB Dbi Stream (Stream 3) Access -------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -127,8 +126,10 @@ Error DbiStream::reload(PDBFile *Pdb) { return EC; if (auto EC = initializeSectionMapData()) return EC; - if (auto EC = initializeFpoRecords(Pdb)) + if (auto EC = initializeOldFpoRecords(Pdb)) return EC; + if (auto EC = initializeNewFpoRecords(Pdb)) + return EC; if (Reader.bytesRemaining() > 0) return make_error(raw_error_code::corrupt_file, @@ -201,8 +202,16 @@ FixedStreamArray DbiStream::getSectionHeaders() const { return SectionHeaders; } -FixedStreamArray DbiStream::getFpoRecords() { - return FpoRecords; +bool DbiStream::hasOldFpoRecords() const { return OldFpoStream != nullptr; } + +FixedStreamArray DbiStream::getOldFpoRecords() const { + return OldFpoRecords; +} + +bool DbiStream::hasNewFpoRecords() const { return NewFpoStream != nullptr; } + +const DebugFrameDataSubsectionRef &DbiStream::getNewFpoRecords() const { + return NewFpoRecords; } const DbiModuleList &DbiStream::modules() const { return Modules; } @@ -247,22 +256,15 @@ Error DbiStream::initializeSectionContributionData() { // Initializes this->SectionHeaders. Error DbiStream::initializeSectionHeadersData(PDBFile *Pdb) { - if (!Pdb) - return Error::success(); - - if (DbgStreams.size() == 0) - return Error::success(); + Expected> ExpectedStream = + createIndexedStreamForHeaderType(Pdb, DbgHeaderType::SectionHdr); + if (auto EC = ExpectedStream.takeError()) + return EC; - uint32_t StreamNum = getDebugStreamIndex(DbgHeaderType::SectionHdr); - if (StreamNum == kInvalidStreamIndex) + auto &SHS = *ExpectedStream; + if (!SHS) return Error::success(); - if (StreamNum >= Pdb->getNumStreams()) - return make_error(raw_error_code::no_stream); - - auto SHS = MappedBlockStream::createIndexedStream( - Pdb->getMsfLayout(), Pdb->getMsfBuffer(), StreamNum, Pdb->getAllocator()); - size_t StreamLen = SHS->getLength(); if (StreamLen % sizeof(object::coff_section)) return make_error(raw_error_code::corrupt_file, @@ -279,39 +281,65 @@ Error DbiStream::initializeSectionHeadersData(PDBFile *Pdb) { } // Initializes this->Fpos. -Error DbiStream::initializeFpoRecords(PDBFile *Pdb) { - if (!Pdb) - return Error::success(); - - if (DbgStreams.size() == 0) - return Error::success(); - - uint32_t StreamNum = getDebugStreamIndex(DbgHeaderType::NewFPO); +Error DbiStream::initializeOldFpoRecords(PDBFile *Pdb) { + Expected> ExpectedStream = + createIndexedStreamForHeaderType(Pdb, DbgHeaderType::FPO); + if (auto EC = ExpectedStream.takeError()) + return EC; - // This means there is no FPO data. - if (StreamNum == kInvalidStreamIndex) + auto &FS = *ExpectedStream; + if (!FS) return Error::success(); - if (StreamNum >= Pdb->getNumStreams()) - return make_error(raw_error_code::no_stream); - - auto FS = MappedBlockStream::createIndexedStream( - Pdb->getMsfLayout(), Pdb->getMsfBuffer(), StreamNum, Pdb->getAllocator()); - size_t StreamLen = FS->getLength(); if (StreamLen % sizeof(object::FpoData)) return make_error(raw_error_code::corrupt_file, - "Corrupted New FPO stream."); + "Corrupted Old FPO stream."); size_t NumRecords = StreamLen / sizeof(object::FpoData); BinaryStreamReader Reader(*FS); - if (auto EC = Reader.readArray(FpoRecords, NumRecords)) + if (auto EC = Reader.readArray(OldFpoRecords, NumRecords)) return make_error(raw_error_code::corrupt_file, - "Corrupted New FPO stream."); - FpoStream = std::move(FS); + "Corrupted Old FPO stream."); + OldFpoStream = std::move(FS); return Error::success(); } +Error DbiStream::initializeNewFpoRecords(PDBFile *Pdb) { + Expected> ExpectedStream = + createIndexedStreamForHeaderType(Pdb, DbgHeaderType::NewFPO); + if (auto EC = ExpectedStream.takeError()) + return EC; + + auto &FS = *ExpectedStream; + if (!FS) + return Error::success(); + + if (auto EC = NewFpoRecords.initialize(*FS)) + return EC; + + NewFpoStream = std::move(FS); + return Error::success(); +} + +Expected> +DbiStream::createIndexedStreamForHeaderType(PDBFile *Pdb, + DbgHeaderType Type) const { + if (!Pdb) + return nullptr; + + if (DbgStreams.empty()) + return nullptr; + + uint32_t StreamNum = getDebugStreamIndex(Type); + + // This means there is no such stream. + if (StreamNum == kInvalidStreamIndex) + return nullptr; + + return Pdb->safelyCreateIndexedStream(StreamNum); +} + BinarySubstreamRef DbiStream::getSectionContributionData() const { return SecContrSubstream; } diff --git a/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp b/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp index 094216ea800a..b7ade0072ee5 100644 --- a/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp +++ b/lib/DebugInfo/PDB/Native/DbiStreamBuilder.cpp @@ -1,9 +1,8 @@ //===- DbiStreamBuilder.cpp - PDB Dbi Stream Creation -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/EnumTables.cpp b/lib/DebugInfo/PDB/Native/EnumTables.cpp index b3837dc72e5b..f5125393695b 100644 --- a/lib/DebugInfo/PDB/Native/EnumTables.cpp +++ b/lib/DebugInfo/PDB/Native/EnumTables.cpp @@ -1,9 +1,8 @@ //===- EnumTables.cpp - Enum to string conversion tables --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp b/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp index 57da7003da2b..8ed5b8b44c59 100644 --- a/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp +++ b/lib/DebugInfo/PDB/Native/GSIStreamBuilder.cpp @@ -1,9 +1,8 @@ //===- DbiStreamBuilder.cpp - PDB Dbi Stream Creation -----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -31,14 +30,14 @@ using namespace llvm::pdb; using namespace llvm::codeview; struct llvm::pdb::GSIHashStreamBuilder { - struct UdtDenseMapInfo { + struct SymbolDenseMapInfo { static inline CVSymbol getEmptyKey() { static CVSymbol Empty; return Empty; } static inline CVSymbol getTombstoneKey() { - static CVSymbol Tombstone(static_cast(-1), - ArrayRef()); + static CVSymbol Tombstone( + DenseMapInfo>::getTombstoneKey()); return Tombstone; } static unsigned getHashValue(const CVSymbol &Val) { @@ -51,7 +50,7 @@ struct llvm::pdb::GSIHashStreamBuilder { std::vector Records; uint32_t StreamIndex; - llvm::DenseSet UdtHashes; + llvm::DenseSet SymbolHashes; std::vector HashRecords; std::array HashBitmap; std::vector HashBuckets; @@ -67,8 +66,8 @@ struct llvm::pdb::GSIHashStreamBuilder { CodeViewContainer::Pdb)); } void addSymbol(const CVSymbol &Symbol) { - if (Symbol.kind() == S_UDT) { - auto Iter = UdtHashes.insert(Symbol); + if (Symbol.kind() == S_UDT || Symbol.kind() == S_CONSTANT) { + auto Iter = SymbolHashes.insert(Symbol); if (!Iter.second) return; } @@ -263,8 +262,7 @@ static std::vector computeAddrMap(ArrayRef Records) { SymOffsets.push_back(SymOffset); SymOffset += Sym.length(); } - std::stable_sort(PublicsByAddr.begin(), PublicsByAddr.end(), - comparePubSymByAddrAndName); + llvm::stable_sort(PublicsByAddr, comparePubSymByAddrAndName); // Fill in the symbol offsets in the appropriate order. std::vector AddrMap; diff --git a/lib/DebugInfo/PDB/Native/GlobalsStream.cpp b/lib/DebugInfo/PDB/Native/GlobalsStream.cpp index e36319566821..f27d60f46815 100644 --- a/lib/DebugInfo/PDB/Native/GlobalsStream.cpp +++ b/lib/DebugInfo/PDB/Native/GlobalsStream.cpp @@ -1,9 +1,8 @@ //===- GlobalsStream.cpp - PDB Index of Symbols by Name ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/DebugInfo/PDB/Native/Hash.cpp b/lib/DebugInfo/PDB/Native/Hash.cpp index 61188ece2dcb..b5c139ecbec0 100644 --- a/lib/DebugInfo/PDB/Native/Hash.cpp +++ b/lib/DebugInfo/PDB/Native/Hash.cpp @@ -1,9 +1,8 @@ //===- Hash.cpp - PDB Hash Functions --------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/HashTable.cpp b/lib/DebugInfo/PDB/Native/HashTable.cpp index cfabc9cd1ad8..dfdcdf1f4eaf 100644 --- a/lib/DebugInfo/PDB/Native/HashTable.cpp +++ b/lib/DebugInfo/PDB/Native/HashTable.cpp @@ -1,9 +1,8 @@ //===- HashTable.cpp - PDB Hash Table -------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/InfoStream.cpp b/lib/DebugInfo/PDB/Native/InfoStream.cpp index 973a520ffca9..f41bb32d69af 100644 --- a/lib/DebugInfo/PDB/Native/InfoStream.cpp +++ b/lib/DebugInfo/PDB/Native/InfoStream.cpp @@ -1,9 +1,8 @@ //===- InfoStream.cpp - PDB Info Stream (Stream 1) Access -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/InfoStreamBuilder.cpp b/lib/DebugInfo/PDB/Native/InfoStreamBuilder.cpp index 3b5a2accdba6..42daa7cae799 100644 --- a/lib/DebugInfo/PDB/Native/InfoStreamBuilder.cpp +++ b/lib/DebugInfo/PDB/Native/InfoStreamBuilder.cpp @@ -1,9 +1,8 @@ //===- InfoStreamBuilder.cpp - PDB Info Stream Creation ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/InjectedSourceStream.cpp b/lib/DebugInfo/PDB/Native/InjectedSourceStream.cpp new file mode 100644 index 000000000000..3f4101db7b93 --- /dev/null +++ b/lib/DebugInfo/PDB/Native/InjectedSourceStream.cpp @@ -0,0 +1,65 @@ +//===- InjectedSourceStream.cpp - PDB Headerblock Stream Access -----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/DebugInfo/PDB/Native/InjectedSourceStream.h" + +#include "llvm/DebugInfo/MSF/MappedBlockStream.h" +#include "llvm/DebugInfo/PDB/Native/Hash.h" +#include "llvm/DebugInfo/PDB/Native/PDBStringTable.h" +#include "llvm/DebugInfo/PDB/Native/RawConstants.h" +#include "llvm/DebugInfo/PDB/Native/RawTypes.h" +#include "llvm/Support/BinaryStreamReader.h" +#include "llvm/Support/Endian.h" + +using namespace llvm; +using namespace llvm::msf; +using namespace llvm::support; +using namespace llvm::pdb; + +InjectedSourceStream::InjectedSourceStream( + std::unique_ptr Stream) + : Stream(std::move(Stream)) {} + +Error InjectedSourceStream::reload(const PDBStringTable &Strings) { + BinaryStreamReader Reader(*Stream); + + if (auto EC = Reader.readObject(Header)) + return EC; + + if (Header->Version != + static_cast(PdbRaw_SrcHeaderBlockVer::SrcVerOne)) + return make_error(raw_error_code::corrupt_file, + "Invalid headerblock header version"); + + if (auto EC = InjectedSourceTable.load(Reader)) + return EC; + + for (const auto& Entry : *this) { + if (Entry.second.Size != sizeof(SrcHeaderBlockEntry)) + return make_error(raw_error_code::corrupt_file, + "Invalid headerbock entry size"); + if (Entry.second.Version != + static_cast(PdbRaw_SrcHeaderBlockVer::SrcVerOne)) + return make_error(raw_error_code::corrupt_file, + "Invalid headerbock entry version"); + + // Check that all name references are valid. + auto Name = Strings.getStringForID(Entry.second.FileNI); + if (!Name) + return Name.takeError(); + auto ObjName = Strings.getStringForID(Entry.second.ObjNI); + if (!ObjName) + return ObjName.takeError(); + auto VName = Strings.getStringForID(Entry.second.VFileNI); + if (!VName) + return VName.takeError(); + } + + assert(Reader.bytesRemaining() == 0); + return Error::success(); +} diff --git a/lib/DebugInfo/PDB/Native/ModuleDebugStream.cpp b/lib/DebugInfo/PDB/Native/ModuleDebugStream.cpp index 8c97f4a012f0..1445f0bd9e1b 100644 --- a/lib/DebugInfo/PDB/Native/ModuleDebugStream.cpp +++ b/lib/DebugInfo/PDB/Native/ModuleDebugStream.cpp @@ -1,9 +1,8 @@ //===- ModuleDebugStream.cpp - PDB Module Info Stream Access --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -15,6 +14,7 @@ #include "llvm/DebugInfo/CodeView/SymbolRecord.h" #include "llvm/DebugInfo/CodeView/SymbolRecordHelpers.h" #include "llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h" +#include "llvm/DebugInfo/PDB/Native/RawConstants.h" #include "llvm/DebugInfo/PDB/Native/RawError.h" #include "llvm/Support/BinaryStreamReader.h" #include "llvm/Support/BinaryStreamRef.h" @@ -37,6 +37,17 @@ ModuleDebugStreamRef::~ModuleDebugStreamRef() = default; Error ModuleDebugStreamRef::reload() { BinaryStreamReader Reader(*Stream); + if (Mod.getModuleStreamIndex() != llvm::pdb::kInvalidStreamIndex) { + if (Error E = reloadSerialize(Reader)) + return E; + } + if (Reader.bytesRemaining() > 0) + return make_error(raw_error_code::corrupt_file, + "Unexpected bytes in module stream."); + return Error::success(); +} + +Error ModuleDebugStreamRef::reloadSerialize(BinaryStreamReader &Reader) { uint32_t SymbolSize = Mod.getSymbolDebugInfoByteSize(); uint32_t C11Size = Mod.getC11LineInfoByteSize(); uint32_t C13Size = Mod.getC13LineInfoByteSize(); @@ -72,10 +83,6 @@ Error ModuleDebugStreamRef::reload() { return EC; if (auto EC = Reader.readSubstream(GlobalRefsSubstream, GlobalRefsSize)) return EC; - if (Reader.bytesRemaining() > 0) - return make_error(raw_error_code::corrupt_file, - "Unexpected bytes in module stream."); - return Error::success(); } diff --git a/lib/DebugInfo/PDB/Native/NamedStreamMap.cpp b/lib/DebugInfo/PDB/Native/NamedStreamMap.cpp index a4eaed90837d..4a88391494cd 100644 --- a/lib/DebugInfo/PDB/Native/NamedStreamMap.cpp +++ b/lib/DebugInfo/PDB/Native/NamedStreamMap.cpp @@ -1,9 +1,8 @@ //===- NamedStreamMap.cpp - PDB Named Stream Map --------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -35,6 +34,7 @@ uint16_t NamedStreamMapTraits::hashLookupKey(StringRef S) const { // Here, the type HASH is a typedef of unsigned short. // ** It is not a bug that we truncate the result of hashStringV1, in fact // it is a bug if we do not! ** + // See NMTNI::hash() in the reference implementation. return static_cast(hashStringV1(S)); } @@ -46,8 +46,7 @@ uint32_t NamedStreamMapTraits::lookupKeyToStorageKey(StringRef S) { return NS->appendStringData(S); } -NamedStreamMap::NamedStreamMap() - : HashTraits(*this), OffsetIndexMap(1, HashTraits) {} +NamedStreamMap::NamedStreamMap() : HashTraits(*this), OffsetIndexMap(1) {} Error NamedStreamMap::load(BinaryStreamReader &Stream) { uint32_t StringBufferSize; @@ -99,7 +98,7 @@ uint32_t NamedStreamMap::hashString(uint32_t Offset) const { } bool NamedStreamMap::get(StringRef Stream, uint32_t &StreamNo) const { - auto Iter = OffsetIndexMap.find_as(Stream); + auto Iter = OffsetIndexMap.find_as(Stream, HashTraits); if (Iter == OffsetIndexMap.end()) return false; StreamNo = (*Iter).second; @@ -123,5 +122,5 @@ uint32_t NamedStreamMap::appendStringData(StringRef S) { } void NamedStreamMap::set(StringRef Stream, uint32_t StreamNo) { - OffsetIndexMap.set_as(Stream, support::ulittle32_t(StreamNo)); + OffsetIndexMap.set_as(Stream, support::ulittle32_t(StreamNo), HashTraits); } diff --git a/lib/DebugInfo/PDB/Native/NativeCompilandSymbol.cpp b/lib/DebugInfo/PDB/Native/NativeCompilandSymbol.cpp index efa70b0e7bd8..39ae84acba20 100644 --- a/lib/DebugInfo/PDB/Native/NativeCompilandSymbol.cpp +++ b/lib/DebugInfo/PDB/Native/NativeCompilandSymbol.cpp @@ -1,9 +1,8 @@ //===- NativeCompilandSymbol.cpp - Native impl for compilands ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/NativeEnumGlobals.cpp b/lib/DebugInfo/PDB/Native/NativeEnumGlobals.cpp index 6eece3df2db3..54646867bc5f 100644 --- a/lib/DebugInfo/PDB/Native/NativeEnumGlobals.cpp +++ b/lib/DebugInfo/PDB/Native/NativeEnumGlobals.cpp @@ -1,9 +1,8 @@ //==- NativeEnumGlobals.cpp - Native Global Enumerator impl ------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/NativeEnumInjectedSources.cpp b/lib/DebugInfo/PDB/Native/NativeEnumInjectedSources.cpp new file mode 100644 index 000000000000..f17ff5bb01f2 --- /dev/null +++ b/lib/DebugInfo/PDB/Native/NativeEnumInjectedSources.cpp @@ -0,0 +1,120 @@ +//==- NativeEnumInjectedSources.cpp - Native Injected Source Enumerator --*-==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/DebugInfo/PDB/Native/NativeEnumInjectedSources.h" + +#include "llvm/DebugInfo/PDB/Native/InfoStream.h" +#include "llvm/DebugInfo/PDB/Native/PDBFile.h" +#include "llvm/DebugInfo/PDB/Native/PDBStringTable.h" + +namespace llvm { +namespace pdb { + +namespace { + +Expected readStreamData(BinaryStream &Stream, uint32_t Limit) { + uint32_t Offset = 0, DataLength = std::min(Limit, Stream.getLength()); + std::string Result; + Result.reserve(DataLength); + while (Offset < DataLength) { + ArrayRef Data; + if (auto E = Stream.readLongestContiguousChunk(Offset, Data)) + return std::move(E); + Data = Data.take_front(DataLength - Offset); + Offset += Data.size(); + Result += toStringRef(Data); + } + return Result; +} + +class NativeInjectedSource final : public IPDBInjectedSource { + const SrcHeaderBlockEntry &Entry; + const PDBStringTable &Strings; + PDBFile &File; + +public: + NativeInjectedSource(const SrcHeaderBlockEntry &Entry, + PDBFile &File, const PDBStringTable &Strings) + : Entry(Entry), Strings(Strings), File(File) {} + + uint32_t getCrc32() const override { return Entry.CRC; } + uint64_t getCodeByteSize() const override { return Entry.FileSize; } + + std::string getFileName() const override { + auto Name = Strings.getStringForID(Entry.FileNI); + assert(Name && "InjectedSourceStream should have rejected this"); + return *Name; + } + + std::string getObjectFileName() const override { + auto ObjName = Strings.getStringForID(Entry.ObjNI); + assert(ObjName && "InjectedSourceStream should have rejected this"); + return *ObjName; + } + + std::string getVirtualFileName() const override { + auto VName = Strings.getStringForID(Entry.VFileNI); + assert(VName && "InjectedSourceStream should have rejected this"); + return *VName; + } + + uint32_t getCompression() const override { return Entry.Compression; } + + std::string getCode() const override { + // Get name of stream storing the data. + auto VName = Strings.getStringForID(Entry.VFileNI); + assert(VName && "InjectedSourceStream should have rejected this"); + std::string StreamName = ("/src/files/" + *VName).str(); + + // Find stream with that name and read its data. + // FIXME: Consider validating (or even loading) all this in + // InjectedSourceStream so that no error can happen here. + auto ExpectedFileStream = File.safelyCreateNamedStream(StreamName); + if (!ExpectedFileStream) { + consumeError(ExpectedFileStream.takeError()); + return "(failed to open data stream)"; + } + + auto Data = readStreamData(**ExpectedFileStream, Entry.FileSize); + if (!Data) { + consumeError(Data.takeError()); + return "(failed to read data)"; + } + return *Data; + } +}; + +} // namespace + +NativeEnumInjectedSources::NativeEnumInjectedSources( + PDBFile &File, const InjectedSourceStream &IJS, + const PDBStringTable &Strings) + : File(File), Stream(IJS), Strings(Strings), Cur(Stream.begin()) {} + +uint32_t NativeEnumInjectedSources::getChildCount() const { + return static_cast(Stream.size()); +} + +std::unique_ptr +NativeEnumInjectedSources::getChildAtIndex(uint32_t N) const { + if (N >= getChildCount()) + return nullptr; + return make_unique(std::next(Stream.begin(), N)->second, + File, Strings); +} + +std::unique_ptr NativeEnumInjectedSources::getNext() { + if (Cur == Stream.end()) + return nullptr; + return make_unique((Cur++)->second, File, Strings); +} + +void NativeEnumInjectedSources::reset() { Cur = Stream.begin(); } + +} +} diff --git a/lib/DebugInfo/PDB/Native/NativeEnumModules.cpp b/lib/DebugInfo/PDB/Native/NativeEnumModules.cpp index 6e4d56443a07..c6621924b516 100644 --- a/lib/DebugInfo/PDB/Native/NativeEnumModules.cpp +++ b/lib/DebugInfo/PDB/Native/NativeEnumModules.cpp @@ -1,9 +1,8 @@ //==- NativeEnumModules.cpp - Native Symbol Enumerator impl ------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/NativeEnumTypes.cpp b/lib/DebugInfo/PDB/Native/NativeEnumTypes.cpp index 288a9128147a..ac217df1ee48 100644 --- a/lib/DebugInfo/PDB/Native/NativeEnumTypes.cpp +++ b/lib/DebugInfo/PDB/Native/NativeEnumTypes.cpp @@ -1,9 +1,8 @@ //==- NativeEnumTypes.cpp - Native Type Enumerator impl ----------*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/NativeExeSymbol.cpp b/lib/DebugInfo/PDB/Native/NativeExeSymbol.cpp index 6dde5d08a500..3f393409129b 100644 --- a/lib/DebugInfo/PDB/Native/NativeExeSymbol.cpp +++ b/lib/DebugInfo/PDB/Native/NativeExeSymbol.cpp @@ -1,9 +1,8 @@ //===- NativeExeSymbol.cpp - native impl for PDBSymbolExe -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/NativeRawSymbol.cpp b/lib/DebugInfo/PDB/Native/NativeRawSymbol.cpp index 62950cb3e52a..8e43cf24495a 100644 --- a/lib/DebugInfo/PDB/Native/NativeRawSymbol.cpp +++ b/lib/DebugInfo/PDB/Native/NativeRawSymbol.cpp @@ -1,9 +1,8 @@ //===- NativeRawSymbol.cpp - Native implementation of IPDBRawSymbol -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/NativeSession.cpp b/lib/DebugInfo/PDB/Native/NativeSession.cpp index 7807e312365c..8a49cb1c5963 100644 --- a/lib/DebugInfo/PDB/Native/NativeSession.cpp +++ b/lib/DebugInfo/PDB/Native/NativeSession.cpp @@ -1,9 +1,8 @@ //===- NativeSession.cpp - Native implementation of IPDBSession -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -14,6 +13,7 @@ #include "llvm/DebugInfo/PDB/IPDBEnumChildren.h" #include "llvm/DebugInfo/PDB/IPDBSourceFile.h" #include "llvm/DebugInfo/PDB/Native/NativeCompilandSymbol.h" +#include "llvm/DebugInfo/PDB/Native/NativeEnumInjectedSources.h" #include "llvm/DebugInfo/PDB/Native/NativeEnumTypes.h" #include "llvm/DebugInfo/PDB/Native/NativeExeSymbol.h" #include "llvm/DebugInfo/PDB/Native/NativeTypeBuiltin.h" @@ -192,7 +192,17 @@ std::unique_ptr NativeSession::getEnumTables() const { std::unique_ptr NativeSession::getInjectedSources() const { - return nullptr; + auto ISS = Pdb->getInjectedSourceStream(); + if (!ISS) { + consumeError(ISS.takeError()); + return nullptr; + } + auto Strings = Pdb->getStringTable(); + if (!Strings) { + consumeError(Strings.takeError()); + return nullptr; + } + return make_unique(*Pdb, *ISS, *Strings); } std::unique_ptr diff --git a/lib/DebugInfo/PDB/Native/NativeSymbolEnumerator.cpp b/lib/DebugInfo/PDB/Native/NativeSymbolEnumerator.cpp index 6ebb8cae3a65..704c1254afbf 100644 --- a/lib/DebugInfo/PDB/Native/NativeSymbolEnumerator.cpp +++ b/lib/DebugInfo/PDB/Native/NativeSymbolEnumerator.cpp @@ -1,9 +1,8 @@ //===- NativeSymbolEnumerator.cpp - info about enumerators ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/NativeTypeArray.cpp b/lib/DebugInfo/PDB/Native/NativeTypeArray.cpp index a52561728a98..80d455ad66e9 100644 --- a/lib/DebugInfo/PDB/Native/NativeTypeArray.cpp +++ b/lib/DebugInfo/PDB/Native/NativeTypeArray.cpp @@ -1,9 +1,8 @@ //===- NativeTypeArray.cpp - info about arrays ------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/NativeTypeBuiltin.cpp b/lib/DebugInfo/PDB/Native/NativeTypeBuiltin.cpp index 7b0f13f3c075..a08663aa91ba 100644 --- a/lib/DebugInfo/PDB/Native/NativeTypeBuiltin.cpp +++ b/lib/DebugInfo/PDB/Native/NativeTypeBuiltin.cpp @@ -1,9 +1,8 @@ //===- NativeTypeBuiltin.cpp -------------------------------------- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/NativeTypeEnum.cpp b/lib/DebugInfo/PDB/Native/NativeTypeEnum.cpp index 37176fe083b9..9f5e86281a23 100644 --- a/lib/DebugInfo/PDB/Native/NativeTypeEnum.cpp +++ b/lib/DebugInfo/PDB/Native/NativeTypeEnum.cpp @@ -1,9 +1,8 @@ //===- NativeTypeEnum.cpp - info about enum type ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/NativeTypeFunctionSig.cpp b/lib/DebugInfo/PDB/Native/NativeTypeFunctionSig.cpp index a9590fffdb87..405303469c18 100644 --- a/lib/DebugInfo/PDB/Native/NativeTypeFunctionSig.cpp +++ b/lib/DebugInfo/PDB/Native/NativeTypeFunctionSig.cpp @@ -1,9 +1,8 @@ //===- NativeTypeFunctionSig.cpp - info about function signature -*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/NativeTypePointer.cpp b/lib/DebugInfo/PDB/Native/NativeTypePointer.cpp index bd8ecb6c4007..32dcfc235954 100644 --- a/lib/DebugInfo/PDB/Native/NativeTypePointer.cpp +++ b/lib/DebugInfo/PDB/Native/NativeTypePointer.cpp @@ -1,9 +1,8 @@ //===- NativeTypePointer.cpp - info about pointer type ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/NativeTypeUDT.cpp b/lib/DebugInfo/PDB/Native/NativeTypeUDT.cpp index 3abf91dcc6a3..be67846c0b24 100644 --- a/lib/DebugInfo/PDB/Native/NativeTypeUDT.cpp +++ b/lib/DebugInfo/PDB/Native/NativeTypeUDT.cpp @@ -1,9 +1,8 @@ //===- NativeTypeUDT.cpp - info about class/struct type ---------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/PDBFile.cpp b/lib/DebugInfo/PDB/Native/PDBFile.cpp index a1f8786ff12f..983031dfcb78 100644 --- a/lib/DebugInfo/PDB/Native/PDBFile.cpp +++ b/lib/DebugInfo/PDB/Native/PDBFile.cpp @@ -1,9 +1,8 @@ //===- PDBFile.cpp - Low level interface to a PDB file ----------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -15,6 +14,7 @@ #include "llvm/DebugInfo/PDB/Native/DbiStream.h" #include "llvm/DebugInfo/PDB/Native/GlobalsStream.h" #include "llvm/DebugInfo/PDB/Native/InfoStream.h" +#include "llvm/DebugInfo/PDB/Native/InjectedSourceStream.h" #include "llvm/DebugInfo/PDB/Native/PDBStringTable.h" #include "llvm/DebugInfo/PDB/Native/PublicsStream.h" #include "llvm/DebugInfo/PDB/Native/RawError.h" @@ -234,7 +234,8 @@ ArrayRef PDBFile::getDirectoryBlockArray() const { return ContainerLayout.DirectoryBlocks; } -std::unique_ptr PDBFile::createIndexedStream(uint16_t SN) { +std::unique_ptr +PDBFile::createIndexedStream(uint16_t SN) const { if (SN == kInvalidStreamIndex) return nullptr; return MappedBlockStream::createIndexedStream(ContainerLayout, *Buffer, SN, @@ -259,8 +260,8 @@ Expected PDBFile::getPDBGlobalsStream() { if (!DbiS) return DbiS.takeError(); - auto GlobalS = safelyCreateIndexedStream( - ContainerLayout, *Buffer, DbiS->getGlobalSymbolStreamIndex()); + auto GlobalS = + safelyCreateIndexedStream(DbiS->getGlobalSymbolStreamIndex()); if (!GlobalS) return GlobalS.takeError(); auto TempGlobals = llvm::make_unique(std::move(*GlobalS)); @@ -273,7 +274,7 @@ Expected PDBFile::getPDBGlobalsStream() { Expected PDBFile::getPDBInfoStream() { if (!Info) { - auto InfoS = safelyCreateIndexedStream(ContainerLayout, *Buffer, StreamPDB); + auto InfoS = safelyCreateIndexedStream(StreamPDB); if (!InfoS) return InfoS.takeError(); auto TempInfo = llvm::make_unique(std::move(*InfoS)); @@ -286,7 +287,7 @@ Expected PDBFile::getPDBInfoStream() { Expected PDBFile::getPDBDbiStream() { if (!Dbi) { - auto DbiS = safelyCreateIndexedStream(ContainerLayout, *Buffer, StreamDBI); + auto DbiS = safelyCreateIndexedStream(StreamDBI); if (!DbiS) return DbiS.takeError(); auto TempDbi = llvm::make_unique(std::move(*DbiS)); @@ -299,7 +300,7 @@ Expected PDBFile::getPDBDbiStream() { Expected PDBFile::getPDBTpiStream() { if (!Tpi) { - auto TpiS = safelyCreateIndexedStream(ContainerLayout, *Buffer, StreamTPI); + auto TpiS = safelyCreateIndexedStream(StreamTPI); if (!TpiS) return TpiS.takeError(); auto TempTpi = llvm::make_unique(*this, std::move(*TpiS)); @@ -315,7 +316,7 @@ Expected PDBFile::getPDBIpiStream() { if (!hasPDBIpiStream()) return make_error(raw_error_code::no_stream); - auto IpiS = safelyCreateIndexedStream(ContainerLayout, *Buffer, StreamIPI); + auto IpiS = safelyCreateIndexedStream(StreamIPI); if (!IpiS) return IpiS.takeError(); auto TempIpi = llvm::make_unique(*this, std::move(*IpiS)); @@ -332,8 +333,8 @@ Expected PDBFile::getPDBPublicsStream() { if (!DbiS) return DbiS.takeError(); - auto PublicS = safelyCreateIndexedStream( - ContainerLayout, *Buffer, DbiS->getPublicSymbolStreamIndex()); + auto PublicS = + safelyCreateIndexedStream(DbiS->getPublicSymbolStreamIndex()); if (!PublicS) return PublicS.takeError(); auto TempPublics = llvm::make_unique(std::move(*PublicS)); @@ -351,8 +352,7 @@ Expected PDBFile::getPDBSymbolStream() { return DbiS.takeError(); uint32_t SymbolStreamNum = DbiS->getSymRecordStreamIndex(); - auto SymbolS = - safelyCreateIndexedStream(ContainerLayout, *Buffer, SymbolStreamNum); + auto SymbolS = safelyCreateIndexedStream(SymbolStreamNum); if (!SymbolS) return SymbolS.takeError(); @@ -366,17 +366,7 @@ Expected PDBFile::getPDBSymbolStream() { Expected PDBFile::getStringTable() { if (!Strings) { - auto IS = getPDBInfoStream(); - if (!IS) - return IS.takeError(); - - Expected ExpectedNSI = IS->getNamedStreamIndex("/names"); - if (!ExpectedNSI) - return ExpectedNSI.takeError(); - uint32_t NameStreamIndex = *ExpectedNSI; - - auto NS = - safelyCreateIndexedStream(ContainerLayout, *Buffer, NameStreamIndex); + auto NS = safelyCreateNamedStream("/names"); if (!NS) return NS.takeError(); @@ -391,6 +381,24 @@ Expected PDBFile::getStringTable() { return *Strings; } +Expected PDBFile::getInjectedSourceStream() { + if (!InjectedSources) { + auto IJS = safelyCreateNamedStream("/src/headerblock"); + if (!IJS) + return IJS.takeError(); + + auto Strings = getStringTable(); + if (!Strings) + return Strings.takeError(); + + auto IJ = llvm::make_unique(std::move(*IJS)); + if (auto EC = IJ->reload(*Strings)) + return std::move(EC); + InjectedSources = std::move(IJ); + } + return *InjectedSources; +} + uint32_t PDBFile::getPointerSize() { auto DbiS = getPDBDbiStream(); if (!DbiS) @@ -459,16 +467,41 @@ bool PDBFile::hasPDBStringTable() { return true; } +bool PDBFile::hasPDBInjectedSourceStream() { + auto IS = getPDBInfoStream(); + if (!IS) + return false; + Expected ExpectedNSI = IS->getNamedStreamIndex("/src/headerblock"); + if (!ExpectedNSI) { + consumeError(ExpectedNSI.takeError()); + return false; + } + assert(*ExpectedNSI < getNumStreams()); + return true; +} + /// Wrapper around MappedBlockStream::createIndexedStream() that checks if a /// stream with that index actually exists. If it does not, the return value /// will have an MSFError with code msf_error_code::no_stream. Else, the return /// value will contain the stream returned by createIndexedStream(). Expected> -PDBFile::safelyCreateIndexedStream(const MSFLayout &Layout, - BinaryStreamRef MsfData, - uint32_t StreamIndex) const { +PDBFile::safelyCreateIndexedStream(uint32_t StreamIndex) const { if (StreamIndex >= getNumStreams()) + // This rejects kInvalidStreamIndex with an error as well. return make_error(raw_error_code::no_stream); - return MappedBlockStream::createIndexedStream(Layout, MsfData, StreamIndex, - Allocator); + return createIndexedStream(StreamIndex); +} + +Expected> +PDBFile::safelyCreateNamedStream(StringRef Name) { + auto IS = getPDBInfoStream(); + if (!IS) + return IS.takeError(); + + Expected ExpectedNSI = IS->getNamedStreamIndex(Name); + if (!ExpectedNSI) + return ExpectedNSI.takeError(); + uint32_t NameStreamIndex = *ExpectedNSI; + + return safelyCreateIndexedStream(NameStreamIndex); } diff --git a/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp b/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp index e0ceb7499ee5..8f5a048ea4b5 100644 --- a/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp +++ b/lib/DebugInfo/PDB/Native/PDBFileBuilder.cpp @@ -1,9 +1,8 @@ //===- PDBFileBuilder.cpp - PDB File Creation -------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -35,7 +34,7 @@ using namespace llvm::support; PDBFileBuilder::PDBFileBuilder(BumpPtrAllocator &Allocator) : Allocator(Allocator), InjectedSourceHashTraits(Strings), - InjectedSourceTable(2, InjectedSourceHashTraits) {} + InjectedSourceTable(2) {} PDBFileBuilder::~PDBFileBuilder() {} @@ -190,7 +189,8 @@ Error PDBFileBuilder::finalizeMsfLayout() { static_cast(PdbRaw_SrcHeaderBlockVer::SrcVerOne); Entry.CRC = CRC.getCRC(); StringRef VName = getStringTableBuilder().getStringForId(IS.VNameIndex); - InjectedSourceTable.set_as(VName, std::move(Entry)); + InjectedSourceTable.set_as(VName, std::move(Entry), + InjectedSourceHashTraits); } uint32_t SrcHeaderBlockSize = diff --git a/lib/DebugInfo/PDB/Native/PDBStringTable.cpp b/lib/DebugInfo/PDB/Native/PDBStringTable.cpp index afeea32043dd..2be1656e06bb 100644 --- a/lib/DebugInfo/PDB/Native/PDBStringTable.cpp +++ b/lib/DebugInfo/PDB/Native/PDBStringTable.cpp @@ -1,9 +1,8 @@ //===- PDBStringTable.cpp - PDB String Table ---------------------*- C++-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/PDBStringTableBuilder.cpp b/lib/DebugInfo/PDB/Native/PDBStringTableBuilder.cpp index d9dcabf3d958..f7f36901e4d4 100644 --- a/lib/DebugInfo/PDB/Native/PDBStringTableBuilder.cpp +++ b/lib/DebugInfo/PDB/Native/PDBStringTableBuilder.cpp @@ -1,9 +1,8 @@ //===- PDBStringTableBuilder.cpp - PDB String Table -------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -27,7 +26,13 @@ StringTableHashTraits::StringTableHashTraits(PDBStringTableBuilder &Table) : Table(&Table) {} uint32_t StringTableHashTraits::hashLookupKey(StringRef S) const { - return Table->getIdForString(S); + // The reference implementation doesn't include code for /src/headerblock + // handling, but it can only read natvis entries lld's PDB files if + // this hash function truncates the hash to 16 bit. + // PDB/include/misc.h in the reference implementation has a hashSz() function + // that returns an unsigned short, that seems what's being used for + // /src/headerblock. + return static_cast(Table->getIdForString(S)); } StringRef StringTableHashTraits::storageKeyToLookupKey(uint32_t Offset) const { @@ -50,63 +55,75 @@ StringRef PDBStringTableBuilder::getStringForId(uint32_t Id) const { return Strings.getStringForId(Id); } -// This is a precomputed list of Buckets given the specified number of -// strings. Matching the reference algorithm exactly is not strictly -// necessary for correctness, but it helps when comparing LLD's PDBs with -// Microsoft's PDBs so as to eliminate superfluous differences. -static std::map StringsToBuckets = { - {1, 2}, - {2, 4}, - {4, 7}, - {6, 11}, - {9, 17}, - {13, 26}, - {20, 40}, - {31, 61}, - {46, 92}, - {70, 139}, - {105, 209}, - {157, 314}, - {236, 472}, - {355, 709}, - {532, 1064}, - {799, 1597}, - {1198, 2396}, - {1798, 3595}, - {2697, 5393}, - {4045, 8090}, - {6068, 12136}, - {9103, 18205}, - {13654, 27308}, - {20482, 40963}, - {30723, 61445}, - {46084, 92168}, - {69127, 138253}, - {103690, 207380}, - {155536, 311071}, - {233304, 466607}, - {349956, 699911}, - {524934, 1049867}, - {787401, 1574801}, - {1181101, 2362202}, - {1771652, 3543304}, - {2657479, 5314957}, - {3986218, 7972436}, - {5979328, 11958655}, - {8968992, 17937983}, - {13453488, 26906975}, - {20180232, 40360463}, - {30270348, 60540695}, - {45405522, 90811043}, - {68108283, 136216565}, - {102162424, 204324848}, - {153243637, 306487273}, - {229865455, 459730910}, - {344798183, 689596366}, - {517197275, 1034394550}, - {775795913, 1551591826}}; - static uint32_t computeBucketCount(uint32_t NumStrings) { + // This is a precomputed list of Buckets given the specified number of + // strings. Matching the reference algorithm exactly is not strictly + // necessary for correctness, but it helps when comparing LLD's PDBs with + // Microsoft's PDBs so as to eliminate superfluous differences. + // The reference implementation does (in nmt.h, NMT::grow()): + // unsigned StringCount = 0; + // unsigned BucketCount = 1; + // fn insert() { + // ++StringCount; + // if (BucketCount * 3 / 4 < StringCount) + // BucketCount = BucketCount * 3 / 2 + 1; + // } + // This list contains all StringCount, BucketCount pairs where BucketCount was + // just incremented. It ends before the first BucketCount entry where + // BucketCount * 3 would overflow a 32-bit unsigned int. + static std::map StringsToBuckets = { + {0, 1}, + {1, 2}, + {2, 4}, + {4, 7}, + {6, 11}, + {9, 17}, + {13, 26}, + {20, 40}, + {31, 61}, + {46, 92}, + {70, 139}, + {105, 209}, + {157, 314}, + {236, 472}, + {355, 709}, + {532, 1064}, + {799, 1597}, + {1198, 2396}, + {1798, 3595}, + {2697, 5393}, + {4045, 8090}, + {6068, 12136}, + {9103, 18205}, + {13654, 27308}, + {20482, 40963}, + {30723, 61445}, + {46084, 92168}, + {69127, 138253}, + {103690, 207380}, + {155536, 311071}, + {233304, 466607}, + {349956, 699911}, + {524934, 1049867}, + {787401, 1574801}, + {1181101, 2362202}, + {1771652, 3543304}, + {2657479, 5314957}, + {3986218, 7972436}, + {5979328, 11958655}, + {8968992, 17937983}, + {13453488, 26906975}, + {20180232, 40360463}, + {30270348, 60540695}, + {45405522, 90811043}, + {68108283, 136216565}, + {102162424, 204324848}, + {153243637, 306487273}, + {229865455, 459730910}, + {344798183, 689596366}, + {517197275, 1034394550}, + {775795913, 1551591826}, + {1163693870, 2327387740}}; auto Entry = StringsToBuckets.lower_bound(NumStrings); assert(Entry != StringsToBuckets.end()); return Entry->second; diff --git a/lib/DebugInfo/PDB/Native/PublicsStream.cpp b/lib/DebugInfo/PDB/Native/PublicsStream.cpp index f6466eb80464..a33bf03bf8fb 100644 --- a/lib/DebugInfo/PDB/Native/PublicsStream.cpp +++ b/lib/DebugInfo/PDB/Native/PublicsStream.cpp @@ -1,9 +1,8 @@ //===- PublicsStream.cpp - PDB Public Symbol Stream -----------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/DebugInfo/PDB/Native/RawError.cpp b/lib/DebugInfo/PDB/Native/RawError.cpp index dec9797088f2..ed6cf0839675 100644 --- a/lib/DebugInfo/PDB/Native/RawError.cpp +++ b/lib/DebugInfo/PDB/Native/RawError.cpp @@ -5,6 +5,7 @@ using namespace llvm; using namespace llvm::pdb; +namespace { // FIXME: This class is only here to support the transition to llvm::Error. It // will be removed once this transition is complete. Clients should prefer to // deal with the Error value directly, rather than converting to error_code. @@ -44,6 +45,7 @@ public: llvm_unreachable("Unrecognized raw_error_code"); } }; +} // namespace static llvm::ManagedStatic RawCategory; const std::error_category &llvm::pdb::RawErrCategory() { return *RawCategory; } diff --git a/lib/DebugInfo/PDB/Native/SymbolStream.cpp b/lib/DebugInfo/PDB/Native/SymbolStream.cpp index 2d8d04ceca4d..003840b6e67e 100644 --- a/lib/DebugInfo/PDB/Native/SymbolStream.cpp +++ b/lib/DebugInfo/PDB/Native/SymbolStream.cpp @@ -1,9 +1,8 @@ //===- SymbolStream.cpp - PDB Symbol Stream Access ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/TpiHashing.cpp b/lib/DebugInfo/PDB/Native/TpiHashing.cpp index 18708826ffc7..b21b82bf76fd 100644 --- a/lib/DebugInfo/PDB/Native/TpiHashing.cpp +++ b/lib/DebugInfo/PDB/Native/TpiHashing.cpp @@ -1,9 +1,8 @@ //===- TpiHashing.cpp -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/Native/TpiStream.cpp b/lib/DebugInfo/PDB/Native/TpiStream.cpp index f234d446e6a0..8ee7f897b8bb 100644 --- a/lib/DebugInfo/PDB/Native/TpiStream.cpp +++ b/lib/DebugInfo/PDB/Native/TpiStream.cpp @@ -1,9 +1,8 @@ //===- TpiStream.cpp - PDB Type Info (TPI) Stream 2 Access ----------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -79,14 +78,13 @@ Error TpiStream::reload() { // Hash indices, hash values, etc come from the hash stream. if (Header->HashStreamIndex != kInvalidStreamIndex) { - if (Header->HashStreamIndex >= Pdb.getNumStreams()) + auto HS = Pdb.safelyCreateIndexedStream(Header->HashStreamIndex); + if (!HS) { + consumeError(HS.takeError()); return make_error(raw_error_code::corrupt_file, "Invalid TPI hash stream index."); - - auto HS = MappedBlockStream::createIndexedStream( - Pdb.getMsfLayout(), Pdb.getMsfBuffer(), Header->HashStreamIndex, - Pdb.getAllocator()); - BinaryStreamReader HSR(*HS); + } + BinaryStreamReader HSR(**HS); // There should be a hash value for every type record, or no hashes at all. uint32_t NumHashValues = @@ -111,7 +109,7 @@ Error TpiStream::reload() { return EC; } - HashStream = std::move(HS); + HashStream = std::move(*HS); } Types = llvm::make_unique( diff --git a/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp b/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp index 8dd30018028e..6b308453c2de 100644 --- a/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp +++ b/lib/DebugInfo/PDB/Native/TpiStreamBuilder.cpp @@ -1,9 +1,8 @@ //===- TpiStreamBuilder.cpp - -------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -77,7 +76,7 @@ Error TpiStreamBuilder::finalize() { H->HashStreamIndex = HashStreamIndex; H->HashAuxStreamIndex = kInvalidStreamIndex; H->HashKeySize = sizeof(ulittle32_t); - H->NumHashBuckets = MinTpiHashBuckets; + H->NumHashBuckets = MaxTpiHashBuckets - 1; // Recall that hash values go into a completely different stream identified by // the `HashStreamIndex` field of the `TpiStreamHeader`. Therefore, the data @@ -130,7 +129,7 @@ Error TpiStreamBuilder::finalizeMsfLayout() { ulittle32_t *H = Allocator.Allocate(TypeHashes.size()); MutableArrayRef HashBuffer(H, TypeHashes.size()); for (uint32_t I = 0; I < TypeHashes.size(); ++I) { - HashBuffer[I] = TypeHashes[I] % MinTpiHashBuckets; + HashBuffer[I] = TypeHashes[I] % (MaxTpiHashBuckets - 1); } ArrayRef Bytes( reinterpret_cast(HashBuffer.data()), @@ -153,9 +152,12 @@ Error TpiStreamBuilder::commit(const msf::MSFLayout &Layout, if (auto EC = Writer.writeObject(*Header)) return EC; - for (auto Rec : TypeRecords) + for (auto Rec : TypeRecords) { + assert(!Rec.empty()); // An empty record will not write anything, but it + // would shift all offsets from here on. if (auto EC = Writer.writeBytes(Rec)) return EC; + } if (HashStreamIndex != kInvalidStreamIndex) { auto HVS = WritableMappedBlockStream::createIndexedStream( diff --git a/lib/DebugInfo/PDB/PDB.cpp b/lib/DebugInfo/PDB/PDB.cpp index fc1ad8bcd7cd..e7b968cb7bea 100644 --- a/lib/DebugInfo/PDB/PDB.cpp +++ b/lib/DebugInfo/PDB/PDB.cpp @@ -1,9 +1,8 @@ //===- PDB.cpp - base header file for creating a PDB reader ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBContext.cpp b/lib/DebugInfo/PDB/PDBContext.cpp index df0feac2bc40..e452f1d4ced7 100644 --- a/lib/DebugInfo/PDB/PDBContext.cpp +++ b/lib/DebugInfo/PDB/PDBContext.cpp @@ -1,9 +1,8 @@ //===-- PDBContext.cpp ------------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===/ @@ -31,14 +30,14 @@ PDBContext::PDBContext(const COFFObjectFile &Object, void PDBContext::dump(raw_ostream &OS, DIDumpOptions DumpOpts){} -DILineInfo PDBContext::getLineInfoForAddress(uint64_t Address, +DILineInfo PDBContext::getLineInfoForAddress(object::SectionedAddress Address, DILineInfoSpecifier Specifier) { DILineInfo Result; - Result.FunctionName = getFunctionName(Address, Specifier.FNKind); + Result.FunctionName = getFunctionName(Address.Address, Specifier.FNKind); uint32_t Length = 1; std::unique_ptr Symbol = - Session->findSymbolByAddress(Address, PDB_SymType::None); + Session->findSymbolByAddress(Address.Address, PDB_SymType::None); if (auto Func = dyn_cast_or_null(Symbol.get())) { Length = Func->getLength(); } else if (auto Data = dyn_cast_or_null(Symbol.get())) { @@ -47,7 +46,7 @@ DILineInfo PDBContext::getLineInfoForAddress(uint64_t Address, // If we couldn't find a symbol, then just assume 1 byte, so that we get // only the line number of the first instruction. - auto LineNumbers = Session->findLineNumbersByAddress(Address, Length); + auto LineNumbers = Session->findLineNumbersByAddress(Address.Address, Length); if (!LineNumbers || LineNumbers->getChildCount() == 0) return Result; @@ -64,26 +63,27 @@ DILineInfo PDBContext::getLineInfoForAddress(uint64_t Address, } DILineInfoTable -PDBContext::getLineInfoForAddressRange(uint64_t Address, uint64_t Size, +PDBContext::getLineInfoForAddressRange(object::SectionedAddress Address, + uint64_t Size, DILineInfoSpecifier Specifier) { if (Size == 0) return DILineInfoTable(); DILineInfoTable Table; - auto LineNumbers = Session->findLineNumbersByAddress(Address, Size); + auto LineNumbers = Session->findLineNumbersByAddress(Address.Address, Size); if (!LineNumbers || LineNumbers->getChildCount() == 0) return Table; while (auto LineInfo = LineNumbers->getNext()) { - DILineInfo LineEntry = - getLineInfoForAddress(LineInfo->getVirtualAddress(), Specifier); + DILineInfo LineEntry = getLineInfoForAddress( + {LineInfo->getVirtualAddress(), Address.SectionIndex}, Specifier); Table.push_back(std::make_pair(LineInfo->getVirtualAddress(), LineEntry)); } return Table; } DIInliningInfo -PDBContext::getInliningInfoForAddress(uint64_t Address, +PDBContext::getInliningInfoForAddress(object::SectionedAddress Address, DILineInfoSpecifier Specifier) { DIInliningInfo InlineInfo; DILineInfo Frame = getLineInfoForAddress(Address, Specifier); @@ -91,6 +91,11 @@ PDBContext::getInliningInfoForAddress(uint64_t Address, return InlineInfo; } +std::vector +PDBContext::getLocalsForAddress(object::SectionedAddress Address) { + return std::vector(); +} + std::string PDBContext::getFunctionName(uint64_t Address, DINameKind NameKind) const { if (NameKind == DINameKind::None) diff --git a/lib/DebugInfo/PDB/PDBExtras.cpp b/lib/DebugInfo/PDB/PDBExtras.cpp index 0d8af232cd92..354a99476c4b 100644 --- a/lib/DebugInfo/PDB/PDBExtras.cpp +++ b/lib/DebugInfo/PDB/PDBExtras.cpp @@ -1,9 +1,8 @@ //===- PDBExtras.cpp - helper functions and classes for PDBs --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -118,13 +117,37 @@ raw_ostream &llvm::pdb::operator<<(raw_ostream &OS, const PDB_DataKind &Data) { } raw_ostream &llvm::pdb::operator<<(raw_ostream &OS, - const codeview::RegisterId &Reg) { - switch (Reg) { -#define CV_REGISTER(name, val) case codeview::RegisterId::name: OS << #name; return OS; + const llvm::codeview::CPURegister &CpuReg) { + if (CpuReg.Cpu == llvm::codeview::CPUType::ARM64) { + switch (CpuReg.Reg) { +#define CV_REGISTERS_ARM64 +#define CV_REGISTER(name, val) \ + case codeview::RegisterId::name: \ + OS << #name; \ + return OS; +#include "llvm/DebugInfo/CodeView/CodeViewRegisters.def" +#undef CV_REGISTER +#undef CV_REGISTERS_ARM64 + + default: + break; + } + } else { + switch (CpuReg.Reg) { +#define CV_REGISTERS_X86 +#define CV_REGISTER(name, val) \ + case codeview::RegisterId::name: \ + OS << #name; \ + return OS; #include "llvm/DebugInfo/CodeView/CodeViewRegisters.def" #undef CV_REGISTER +#undef CV_REGISTERS_X86 + + default: + break; + } } - OS << static_cast(Reg); + OS << static_cast(CpuReg.Reg); return OS; } @@ -193,6 +216,7 @@ raw_ostream &llvm::pdb::operator<<(raw_ostream &OS, const PDB_Lang &Lang) { CASE_OUTPUT_ENUM_CLASS_NAME(PDB_Lang, MSIL, OS) CASE_OUTPUT_ENUM_CLASS_NAME(PDB_Lang, HLSL, OS) CASE_OUTPUT_ENUM_CLASS_NAME(PDB_Lang, D, OS) + CASE_OUTPUT_ENUM_CLASS_NAME(PDB_Lang, Swift, OS) } return OS; } @@ -296,14 +320,17 @@ raw_ostream &llvm::pdb::operator<<(raw_ostream &OS, return OS; } -raw_ostream &llvm::pdb::operator<<(raw_ostream &OS, - const PDB_SourceCompression &Compression) { +raw_ostream &llvm::pdb::dumpPDBSourceCompression(raw_ostream &OS, + uint32_t Compression) { switch (Compression) { CASE_OUTPUT_ENUM_CLASS_NAME(PDB_SourceCompression, None, OS) CASE_OUTPUT_ENUM_CLASS_NAME(PDB_SourceCompression, Huffman, OS) CASE_OUTPUT_ENUM_CLASS_NAME(PDB_SourceCompression, LZ, OS) CASE_OUTPUT_ENUM_CLASS_STR(PDB_SourceCompression, RunLengthEncoded, "RLE", OS) + CASE_OUTPUT_ENUM_CLASS_NAME(PDB_SourceCompression, DotNet, OS) + default: + OS << "Unknown (" << Compression << ")"; } return OS; } diff --git a/lib/DebugInfo/PDB/PDBInterfaceAnchors.cpp b/lib/DebugInfo/PDB/PDBInterfaceAnchors.cpp index 951909295d13..8eb3311b09e3 100644 --- a/lib/DebugInfo/PDB/PDBInterfaceAnchors.cpp +++ b/lib/DebugInfo/PDB/PDBInterfaceAnchors.cpp @@ -1,9 +1,8 @@ //===- PDBInterfaceAnchors.h - defines class anchor funcions ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // Class anchors are necessary per the LLVM Coding style guide, to ensure that diff --git a/lib/DebugInfo/PDB/PDBSymDumper.cpp b/lib/DebugInfo/PDB/PDBSymDumper.cpp index 2f819312e54e..0956a32f4a49 100644 --- a/lib/DebugInfo/PDB/PDBSymDumper.cpp +++ b/lib/DebugInfo/PDB/PDBSymDumper.cpp @@ -1,9 +1,8 @@ //===- PDBSymDumper.cpp - ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbol.cpp b/lib/DebugInfo/PDB/PDBSymbol.cpp index d492edafdafe..34c8ac41d45b 100644 --- a/lib/DebugInfo/PDB/PDBSymbol.cpp +++ b/lib/DebugInfo/PDB/PDBSymbol.cpp @@ -1,9 +1,8 @@ //===- PDBSymbol.cpp - base class for user-facing symbol types --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolAnnotation.cpp b/lib/DebugInfo/PDB/PDBSymbolAnnotation.cpp index cb1a9bee8024..0fa83efb7ae0 100644 --- a/lib/DebugInfo/PDB/PDBSymbolAnnotation.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolAnnotation.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolAnnotation.cpp - --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolBlock.cpp b/lib/DebugInfo/PDB/PDBSymbolBlock.cpp index 13eec9734d02..9452282a8817 100644 --- a/lib/DebugInfo/PDB/PDBSymbolBlock.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolBlock.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolBlock.cpp - -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolCompiland.cpp b/lib/DebugInfo/PDB/PDBSymbolCompiland.cpp index bbc5e6dd2a17..9b2883546305 100644 --- a/lib/DebugInfo/PDB/PDBSymbolCompiland.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolCompiland.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolCompiland.cpp - compiland details ---------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -91,16 +90,16 @@ std::string PDBSymbolCompiland::getSourceFileFullPath() const { PDB_Lang Lang = Details ? Details->getLanguage() : PDB_Lang::Cpp; auto SrcFiles = Session.getSourceFilesForCompiland(*this); if (SrcFiles) { - bool LangC = (Lang == PDB_Lang::Cpp || Lang == PDB_Lang::C); while (auto File = SrcFiles->getNext()) { std::string FileName = File->getFileName(); auto file_extension = sys::path::extension(FileName); if (StringSwitch(file_extension.lower()) - .Case(".cpp", LangC) - .Case(".c", LangC) - .Case(".cc", LangC) - .Case(".cxx", LangC) + .Case(".cpp", Lang == PDB_Lang::Cpp) + .Case(".cc", Lang == PDB_Lang::Cpp) + .Case(".cxx", Lang == PDB_Lang::Cpp) + .Case(".c", Lang == PDB_Lang::C) .Case(".asm", Lang == PDB_Lang::Masm) + .Case(".swift", Lang == PDB_Lang::Swift) .Default(false)) return File->getFileName(); } diff --git a/lib/DebugInfo/PDB/PDBSymbolCompilandDetails.cpp b/lib/DebugInfo/PDB/PDBSymbolCompilandDetails.cpp index bdd8535a3ef3..0d86dfe1e632 100644 --- a/lib/DebugInfo/PDB/PDBSymbolCompilandDetails.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolCompilandDetails.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolCompilandDetails.cpp - compiland details --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolCompilandEnv.cpp b/lib/DebugInfo/PDB/PDBSymbolCompilandEnv.cpp index f88df2df6be4..61f119405fd9 100644 --- a/lib/DebugInfo/PDB/PDBSymbolCompilandEnv.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolCompilandEnv.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolCompilandEnv.cpp - compiland env variables ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolCustom.cpp b/lib/DebugInfo/PDB/PDBSymbolCustom.cpp index 10a21806adb6..6c9a4aa76c3d 100644 --- a/lib/DebugInfo/PDB/PDBSymbolCustom.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolCustom.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolCustom.cpp - compiler-specific types ------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolData.cpp b/lib/DebugInfo/PDB/PDBSymbolData.cpp index 7de94670bcb3..d2b82111ccd5 100644 --- a/lib/DebugInfo/PDB/PDBSymbolData.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolData.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolData.cpp - PDB data (e.g. variable) accessors ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolExe.cpp b/lib/DebugInfo/PDB/PDBSymbolExe.cpp index eb409412af59..c85756c43e47 100644 --- a/lib/DebugInfo/PDB/PDBSymbolExe.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolExe.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolExe.cpp - ---------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolFunc.cpp b/lib/DebugInfo/PDB/PDBSymbolFunc.cpp index 75063cb3e7f8..7c3ba981fd6b 100644 --- a/lib/DebugInfo/PDB/PDBSymbolFunc.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolFunc.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolFunc.cpp - --------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolFuncDebugEnd.cpp b/lib/DebugInfo/PDB/PDBSymbolFuncDebugEnd.cpp index af8aafa7be96..66433dc17b49 100644 --- a/lib/DebugInfo/PDB/PDBSymbolFuncDebugEnd.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolFuncDebugEnd.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolFuncDebugEnd.cpp - ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolFuncDebugStart.cpp b/lib/DebugInfo/PDB/PDBSymbolFuncDebugStart.cpp index 77b510873bea..fe32c93c0121 100644 --- a/lib/DebugInfo/PDB/PDBSymbolFuncDebugStart.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolFuncDebugStart.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolFuncDebugStart.cpp - ----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolLabel.cpp b/lib/DebugInfo/PDB/PDBSymbolLabel.cpp index c802b97925e6..1fffe69a0c83 100644 --- a/lib/DebugInfo/PDB/PDBSymbolLabel.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolLabel.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolLabel.cpp - -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolPublicSymbol.cpp b/lib/DebugInfo/PDB/PDBSymbolPublicSymbol.cpp index a2dd2ab92dd9..08697683f641 100644 --- a/lib/DebugInfo/PDB/PDBSymbolPublicSymbol.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolPublicSymbol.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolPublicSymbol.cpp - ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolThunk.cpp b/lib/DebugInfo/PDB/PDBSymbolThunk.cpp index d227e3a7a60c..6483858183e5 100644 --- a/lib/DebugInfo/PDB/PDBSymbolThunk.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolThunk.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolThunk.cpp - -------------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeArray.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeArray.cpp index a2064d1ac1eb..a0d521abe43f 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeArray.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeArray.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolTypeArray.cpp - ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeBaseClass.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeBaseClass.cpp index f0376c05557f..08467059b5e1 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeBaseClass.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeBaseClass.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolTypeBaseClass.cpp - -----------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeBuiltin.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeBuiltin.cpp index a9f59e5f9d4d..a0dd9ef601c0 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeBuiltin.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeBuiltin.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolTypeBuiltin.cpp - ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeCustom.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeCustom.cpp index cfb347fbac55..6723894c90ea 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeCustom.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeCustom.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolTypeCustom.cpp - --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeDimension.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeDimension.cpp index 4eb48997635a..4a25a391f278 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeDimension.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeDimension.cpp @@ -1,10 +1,9 @@ //===- PDBSymbolTypeDimension.cpp - --------------------------------*- C++ //-*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeEnum.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeEnum.cpp index 2e88d9eb284a..b9fdf6aec811 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeEnum.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeEnum.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolTypeEnum.cpp - --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeFriend.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeFriend.cpp index 00d2d51aa8a7..4ffea42cbb0a 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeFriend.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeFriend.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolTypeFriend.cpp - --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeFunctionArg.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeFunctionArg.cpp index 0399e110d592..683e93548fb1 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeFunctionArg.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeFunctionArg.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolTypeFunctionArg.cpp - --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeFunctionSig.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeFunctionSig.cpp index c0564d3941dd..292320a6fe6d 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeFunctionSig.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeFunctionSig.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolTypeFunctionSig.cpp - --------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeManaged.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeManaged.cpp index 1faaf9c67a2c..e80e6c716572 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeManaged.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeManaged.cpp @@ -1,9 +1,8 @@ //===- PDBSymboTypelManaged.cpp - ------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolTypePointer.cpp b/lib/DebugInfo/PDB/PDBSymbolTypePointer.cpp index cf5a369116a9..462fc315359b 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypePointer.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypePointer.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolTypePointer.cpp -----------------------------------*- C++ -===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeTypedef.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeTypedef.cpp index 1838f1612b49..70749d9bf5f5 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeTypedef.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeTypedef.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolTypeTypedef.cpp ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeUDT.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeUDT.cpp index 2f5222f34fe4..d302c29a3bec 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeUDT.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeUDT.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolTypeUDT.cpp - --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeVTable.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeVTable.cpp index 0262f91e8336..4e2a45116d51 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeVTable.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeVTable.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolTypeVTable.cpp - --------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolTypeVTableShape.cpp b/lib/DebugInfo/PDB/PDBSymbolTypeVTableShape.cpp index 16c3a3606981..78957620e083 100644 --- a/lib/DebugInfo/PDB/PDBSymbolTypeVTableShape.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolTypeVTableShape.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolTypeVTableShape.cpp - ---------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolUnknown.cpp b/lib/DebugInfo/PDB/PDBSymbolUnknown.cpp index 7bcf9457a2b6..650d01183171 100644 --- a/lib/DebugInfo/PDB/PDBSymbolUnknown.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolUnknown.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolUnknown.cpp - -----------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/PDBSymbolUsingNamespace.cpp b/lib/DebugInfo/PDB/PDBSymbolUsingNamespace.cpp index ecf2126f8802..74afbdb18086 100644 --- a/lib/DebugInfo/PDB/PDBSymbolUsingNamespace.cpp +++ b/lib/DebugInfo/PDB/PDBSymbolUsingNamespace.cpp @@ -1,9 +1,8 @@ //===- PDBSymbolUsingNamespace.cpp - ------------------- --------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/PDB/UDTLayout.cpp b/lib/DebugInfo/PDB/UDTLayout.cpp index 5f4390bbaf12..acb1599480b0 100644 --- a/lib/DebugInfo/PDB/UDTLayout.cpp +++ b/lib/DebugInfo/PDB/UDTLayout.cpp @@ -1,9 +1,8 @@ //===- UDTLayout.cpp ------------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/DebugInfo/Symbolize/DIPrinter.cpp b/lib/DebugInfo/Symbolize/DIPrinter.cpp index c1e2536d6e20..b2bfef251485 100644 --- a/lib/DebugInfo/Symbolize/DIPrinter.cpp +++ b/lib/DebugInfo/Symbolize/DIPrinter.cpp @@ -1,9 +1,8 @@ //===- lib/DebugInfo/Symbolize/DIPrinter.cpp ------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -19,6 +18,7 @@ #include "llvm/Support/Format.h" #include "llvm/Support/LineIterator.h" #include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Path.h" #include "llvm/Support/raw_ostream.h" #include #include @@ -78,8 +78,13 @@ void DIPrinter::print(const DILineInfo &Info, bool Inlined) { std::string Filename = Info.FileName; if (Filename == kDILineInfoBadString) Filename = kBadString; + else if (Basenames) + Filename = llvm::sys::path::filename(Filename); if (!Verbose) { - OS << Filename << ":" << Info.Line << ":" << Info.Column << "\n"; + OS << Filename << ":" << Info.Line; + if (Style == OutputStyle::LLVM) + OS << ":" << Info.Column; + OS << "\n"; printContext(Filename, Info.Line); return; } @@ -117,5 +122,28 @@ DIPrinter &DIPrinter::operator<<(const DIGlobal &Global) { return *this; } +DIPrinter &DIPrinter::operator<<(const DILocal &Local) { + OS << Local.FunctionName << '\n'; + OS << Local.Name << '\n'; + if (Local.DeclFile.empty()) + OS << "??"; + else + OS << Local.DeclFile; + OS << ':' << Local.DeclLine << '\n'; + if (Local.FrameOffset) + OS << *Local.FrameOffset << ' '; + else + OS << "?? "; + if (Local.Size) + OS << *Local.Size << ' '; + else + OS << "?? "; + if (Local.TagOffset) + OS << *Local.TagOffset << '\n'; + else + OS << "??\n"; + return *this; +} + } // end namespace symbolize } // end namespace llvm diff --git a/lib/DebugInfo/Symbolize/SymbolizableObjectFile.cpp b/lib/DebugInfo/Symbolize/SymbolizableObjectFile.cpp index 08be524ab464..2765bf44d504 100644 --- a/lib/DebugInfo/Symbolize/SymbolizableObjectFile.cpp +++ b/lib/DebugInfo/Symbolize/SymbolizableObjectFile.cpp @@ -1,9 +1,8 @@ //===- SymbolizableObjectFile.cpp -----------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -43,8 +42,9 @@ getDILineInfoSpecifier(FunctionNameKind FNKind) { } ErrorOr> -SymbolizableObjectFile::create(object::ObjectFile *Obj, +SymbolizableObjectFile::create(const object::ObjectFile *Obj, std::unique_ptr DICtx) { + assert(DICtx); std::unique_ptr res( new SymbolizableObjectFile(Obj, std::move(DICtx))); std::unique_ptr OpdExtractor; @@ -54,13 +54,13 @@ SymbolizableObjectFile::create(object::ObjectFile *Obj, if (Obj->getArch() == Triple::ppc64) { for (section_iterator Section : Obj->sections()) { StringRef Name; - StringRef Data; if (auto EC = Section->getName(Name)) return EC; if (Name == ".opd") { - if (auto EC = Section->getContents(Data)) - return EC; - OpdExtractor.reset(new DataExtractor(Data, Obj->isLittleEndian(), + Expected E = Section->getContents(); + if (!E) + return errorToErrorCode(E.takeError()); + OpdExtractor.reset(new DataExtractor(*E, Obj->isLittleEndian(), Obj->getBytesInAddress())); OpdAddress = Section->getAddress(); break; @@ -79,10 +79,30 @@ SymbolizableObjectFile::create(object::ObjectFile *Obj, if (auto EC = res->addCoffExportSymbols(CoffObj)) return EC; } + + std::vector> &Fs = res->Functions, + &Os = res->Objects; + auto Uniquify = [](std::vector> &S) { + // Sort by (Addr,Size,Name). If several SymbolDescs share the same Addr, + // pick the one with the largest Size. This helps us avoid symbols with no + // size information (Size=0). + llvm::sort(S); + auto I = S.begin(), E = S.end(), J = S.begin(); + while (I != E) { + auto OI = I; + while (++I != E && OI->first.Addr == I->first.Addr) { + } + *J++ = I[-1]; + } + S.erase(J, S.end()); + }; + Uniquify(Fs); + Uniquify(Os); + return std::move(res); } -SymbolizableObjectFile::SymbolizableObjectFile(ObjectFile *Obj, +SymbolizableObjectFile::SymbolizableObjectFile(const ObjectFile *Obj, std::unique_ptr DICtx) : Module(Obj), DebugInfoContext(std::move(DICtx)) {} @@ -128,7 +148,7 @@ std::error_code SymbolizableObjectFile::addCoffExportSymbols( uint64_t SymbolStart = ImageBase + Export.Offset; uint64_t SymbolSize = NextOffset - Export.Offset; SymbolDesc SD = {SymbolStart, SymbolSize}; - Functions.insert(std::make_pair(SD, Export.Name)); + Functions.emplace_back(SD, Export.Name); } return std::error_code(); } @@ -137,6 +157,11 @@ std::error_code SymbolizableObjectFile::addSymbol(const SymbolRef &Symbol, uint64_t SymbolSize, DataExtractor *OpdExtractor, uint64_t OpdAddress) { + // Avoid adding symbols from an unknown/undefined section. + const ObjectFile *Obj = Symbol.getObject(); + Expected Sec = Symbol.getSection(); + if (!Sec || (Obj && Obj->section_end() == *Sec)) + return std::error_code(); Expected SymbolTypeOrErr = Symbol.getType(); if (!SymbolTypeOrErr) return errorToErrorCode(SymbolTypeOrErr.takeError()); @@ -170,7 +195,7 @@ std::error_code SymbolizableObjectFile::addSymbol(const SymbolRef &Symbol, // with same address size. Make sure we choose the correct one. auto &M = SymbolType == SymbolRef::ST_Function ? Functions : Objects; SymbolDesc SD = { SymbolAddress, SymbolSize }; - M.insert(std::make_pair(SD, SymbolName)); + M.emplace_back(SD, SymbolName); return std::error_code(); } @@ -191,12 +216,10 @@ bool SymbolizableObjectFile::getNameFromSymbolTable(SymbolRef::Type Type, std::string &Name, uint64_t &Addr, uint64_t &Size) const { - const auto &SymbolMap = Type == SymbolRef::ST_Function ? Functions : Objects; - if (SymbolMap.empty()) - return false; - SymbolDesc SD = { Address, Address }; - auto SymbolIterator = SymbolMap.upper_bound(SD); - if (SymbolIterator == SymbolMap.begin()) + const auto &Symbols = Type == SymbolRef::ST_Function ? Functions : Objects; + std::pair SD{{Address, UINT64_C(-1)}, StringRef()}; + auto SymbolIterator = llvm::upper_bound(Symbols, SD); + if (SymbolIterator == Symbols.begin()) return false; --SymbolIterator; if (SymbolIterator->first.Size != 0 && @@ -218,19 +241,21 @@ bool SymbolizableObjectFile::shouldOverrideWithSymbolTable( isa(DebugInfoContext.get()); } -DILineInfo SymbolizableObjectFile::symbolizeCode(uint64_t ModuleOffset, - FunctionNameKind FNKind, - bool UseSymbolTable) const { - DILineInfo LineInfo; - if (DebugInfoContext) { - LineInfo = DebugInfoContext->getLineInfoForAddress( - ModuleOffset, getDILineInfoSpecifier(FNKind)); - } +DILineInfo +SymbolizableObjectFile::symbolizeCode(object::SectionedAddress ModuleOffset, + FunctionNameKind FNKind, + bool UseSymbolTable) const { + if (ModuleOffset.SectionIndex == object::SectionedAddress::UndefSection) + ModuleOffset.SectionIndex = + getModuleSectionIndexForAddress(ModuleOffset.Address); + DILineInfo LineInfo = DebugInfoContext->getLineInfoForAddress( + ModuleOffset, getDILineInfoSpecifier(FNKind)); + // Override function name from symbol table if necessary. if (shouldOverrideWithSymbolTable(FNKind, UseSymbolTable)) { std::string FunctionName; uint64_t Start, Size; - if (getNameFromSymbolTable(SymbolRef::ST_Function, ModuleOffset, + if (getNameFromSymbolTable(SymbolRef::ST_Function, ModuleOffset.Address, FunctionName, Start, Size)) { LineInfo.FunctionName = FunctionName; } @@ -239,12 +264,14 @@ DILineInfo SymbolizableObjectFile::symbolizeCode(uint64_t ModuleOffset, } DIInliningInfo SymbolizableObjectFile::symbolizeInlinedCode( - uint64_t ModuleOffset, FunctionNameKind FNKind, bool UseSymbolTable) const { - DIInliningInfo InlinedContext; + object::SectionedAddress ModuleOffset, FunctionNameKind FNKind, + bool UseSymbolTable) const { + if (ModuleOffset.SectionIndex == object::SectionedAddress::UndefSection) + ModuleOffset.SectionIndex = + getModuleSectionIndexForAddress(ModuleOffset.Address); + DIInliningInfo InlinedContext = DebugInfoContext->getInliningInfoForAddress( + ModuleOffset, getDILineInfoSpecifier(FNKind)); - if (DebugInfoContext) - InlinedContext = DebugInfoContext->getInliningInfoForAddress( - ModuleOffset, getDILineInfoSpecifier(FNKind)); // Make sure there is at least one frame in context. if (InlinedContext.getNumberOfFrames() == 0) InlinedContext.addFrame(DILineInfo()); @@ -253,7 +280,7 @@ DIInliningInfo SymbolizableObjectFile::symbolizeInlinedCode( if (shouldOverrideWithSymbolTable(FNKind, UseSymbolTable)) { std::string FunctionName; uint64_t Start, Size; - if (getNameFromSymbolTable(SymbolRef::ST_Function, ModuleOffset, + if (getNameFromSymbolTable(SymbolRef::ST_Function, ModuleOffset.Address, FunctionName, Start, Size)) { InlinedContext.getMutableFrame(InlinedContext.getNumberOfFrames() - 1) ->FunctionName = FunctionName; @@ -263,9 +290,34 @@ DIInliningInfo SymbolizableObjectFile::symbolizeInlinedCode( return InlinedContext; } -DIGlobal SymbolizableObjectFile::symbolizeData(uint64_t ModuleOffset) const { +DIGlobal SymbolizableObjectFile::symbolizeData( + object::SectionedAddress ModuleOffset) const { DIGlobal Res; - getNameFromSymbolTable(SymbolRef::ST_Data, ModuleOffset, Res.Name, Res.Start, - Res.Size); + getNameFromSymbolTable(SymbolRef::ST_Data, ModuleOffset.Address, Res.Name, + Res.Start, Res.Size); return Res; } + +std::vector SymbolizableObjectFile::symbolizeFrame( + object::SectionedAddress ModuleOffset) const { + if (ModuleOffset.SectionIndex == object::SectionedAddress::UndefSection) + ModuleOffset.SectionIndex = + getModuleSectionIndexForAddress(ModuleOffset.Address); + return DebugInfoContext->getLocalsForAddress(ModuleOffset); +} + +/// Search for the first occurence of specified Address in ObjectFile. +uint64_t SymbolizableObjectFile::getModuleSectionIndexForAddress( + uint64_t Address) const { + + for (SectionRef Sec : Module->sections()) { + if (!Sec.isText() || Sec.isVirtual()) + continue; + + if (Address >= Sec.getAddress() && + Address < Sec.getAddress() + Sec.getSize()) + return Sec.getIndex(); + } + + return object::SectionedAddress::UndefSection; +} diff --git a/lib/DebugInfo/Symbolize/SymbolizableObjectFile.h b/lib/DebugInfo/Symbolize/SymbolizableObjectFile.h index 216cca8de4f5..9cab94178c1b 100644 --- a/lib/DebugInfo/Symbolize/SymbolizableObjectFile.h +++ b/lib/DebugInfo/Symbolize/SymbolizableObjectFile.h @@ -1,9 +1,8 @@ //===- SymbolizableObjectFile.h ---------------------------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -32,14 +31,17 @@ namespace symbolize { class SymbolizableObjectFile : public SymbolizableModule { public: static ErrorOr> - create(object::ObjectFile *Obj, std::unique_ptr DICtx); + create(const object::ObjectFile *Obj, std::unique_ptr DICtx); - DILineInfo symbolizeCode(uint64_t ModuleOffset, FunctionNameKind FNKind, + DILineInfo symbolizeCode(object::SectionedAddress ModuleOffset, + FunctionNameKind FNKind, bool UseSymbolTable) const override; - DIInliningInfo symbolizeInlinedCode(uint64_t ModuleOffset, + DIInliningInfo symbolizeInlinedCode(object::SectionedAddress ModuleOffset, FunctionNameKind FNKind, bool UseSymbolTable) const override; - DIGlobal symbolizeData(uint64_t ModuleOffset) const override; + DIGlobal symbolizeData(object::SectionedAddress ModuleOffset) const override; + std::vector + symbolizeFrame(object::SectionedAddress ModuleOffset) const override; // Return true if this is a 32-bit x86 PE COFF module. bool isWin32Module() const override; @@ -63,7 +65,10 @@ private: uint64_t OpdAddress = 0); std::error_code addCoffExportSymbols(const object::COFFObjectFile *CoffObj); - object::ObjectFile *Module; + /// Search for the first occurence of specified Address in ObjectFile. + uint64_t getModuleSectionIndexForAddress(uint64_t Address) const; + + const object::ObjectFile *Module; std::unique_ptr DebugInfoContext; struct SymbolDesc { @@ -72,14 +77,14 @@ private: // the following symbol. uint64_t Size; - friend bool operator<(const SymbolDesc &s1, const SymbolDesc &s2) { - return s1.Addr < s2.Addr; + bool operator<(const SymbolDesc &RHS) const { + return Addr != RHS.Addr ? Addr < RHS.Addr : Size < RHS.Size; } }; - std::map Functions; - std::map Objects; + std::vector> Functions; + std::vector> Objects; - SymbolizableObjectFile(object::ObjectFile *Obj, + SymbolizableObjectFile(const object::ObjectFile *Obj, std::unique_ptr DICtx); }; diff --git a/lib/DebugInfo/Symbolize/Symbolize.cpp b/lib/DebugInfo/Symbolize/Symbolize.cpp index 59a85d6c3fcf..6a619f8f2f37 100644 --- a/lib/DebugInfo/Symbolize/Symbolize.cpp +++ b/lib/DebugInfo/Symbolize/Symbolize.cpp @@ -1,9 +1,8 @@ //===-- LLVMSymbolize.cpp -------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -17,7 +16,6 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/BinaryFormat/COFF.h" -#include "llvm/Config/config.h" #include "llvm/DebugInfo/DWARF/DWARFContext.h" #include "llvm/DebugInfo/PDB/PDB.h" #include "llvm/DebugInfo/PDB/PDBContext.h" @@ -25,6 +23,7 @@ #include "llvm/Object/COFF.h" #include "llvm/Object/MachO.h" #include "llvm/Object/MachOUniversal.h" +#include "llvm/Support/CRC.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Compression.h" #include "llvm/Support/DataExtractor.h" @@ -34,7 +33,6 @@ #include "llvm/Support/Path.h" #include #include -#include #include #if defined(_MSC_VER) @@ -54,14 +52,8 @@ namespace llvm { namespace symbolize { Expected -LLVMSymbolizer::symbolizeCode(const std::string &ModuleName, - uint64_t ModuleOffset, StringRef DWPName) { - SymbolizableModule *Info; - if (auto InfoOrErr = getOrCreateModuleInfo(ModuleName, DWPName)) - Info = InfoOrErr.get(); - else - return InfoOrErr.takeError(); - +LLVMSymbolizer::symbolizeCodeCommon(SymbolizableModule *Info, + object::SectionedAddress ModuleOffset) { // A null module means an error has already been reported. Return an empty // result. if (!Info) @@ -70,7 +62,7 @@ LLVMSymbolizer::symbolizeCode(const std::string &ModuleName, // If the user is giving us relative addresses, add the preferred base of the // object to the offset before we do the query. It's what DIContext expects. if (Opts.RelativeAddresses) - ModuleOffset += Info->getModulePreferredBase(); + ModuleOffset.Address += Info->getModulePreferredBase(); DILineInfo LineInfo = Info->symbolizeCode(ModuleOffset, Opts.PrintFunctions, Opts.UseSymbolTable); @@ -79,11 +71,37 @@ LLVMSymbolizer::symbolizeCode(const std::string &ModuleName, return LineInfo; } +Expected +LLVMSymbolizer::symbolizeCode(const ObjectFile &Obj, + object::SectionedAddress ModuleOffset) { + StringRef ModuleName = Obj.getFileName(); + auto I = Modules.find(ModuleName); + if (I != Modules.end()) + return symbolizeCodeCommon(I->second.get(), ModuleOffset); + + std::unique_ptr Context = + DWARFContext::create(Obj, nullptr, DWARFContext::defaultErrorHandler); + Expected InfoOrErr = + createModuleInfo(&Obj, std::move(Context), ModuleName); + if (!InfoOrErr) + return InfoOrErr.takeError(); + return symbolizeCodeCommon(*InfoOrErr, ModuleOffset); +} + +Expected +LLVMSymbolizer::symbolizeCode(const std::string &ModuleName, + object::SectionedAddress ModuleOffset) { + Expected InfoOrErr = getOrCreateModuleInfo(ModuleName); + if (!InfoOrErr) + return InfoOrErr.takeError(); + return symbolizeCodeCommon(*InfoOrErr, ModuleOffset); +} + Expected LLVMSymbolizer::symbolizeInlinedCode(const std::string &ModuleName, - uint64_t ModuleOffset, StringRef DWPName) { + object::SectionedAddress ModuleOffset) { SymbolizableModule *Info; - if (auto InfoOrErr = getOrCreateModuleInfo(ModuleName, DWPName)) + if (auto InfoOrErr = getOrCreateModuleInfo(ModuleName)) Info = InfoOrErr.get(); else return InfoOrErr.takeError(); @@ -96,7 +114,7 @@ LLVMSymbolizer::symbolizeInlinedCode(const std::string &ModuleName, // If the user is giving us relative addresses, add the preferred base of the // object to the offset before we do the query. It's what DIContext expects. if (Opts.RelativeAddresses) - ModuleOffset += Info->getModulePreferredBase(); + ModuleOffset.Address += Info->getModulePreferredBase(); DIInliningInfo InlinedContext = Info->symbolizeInlinedCode( ModuleOffset, Opts.PrintFunctions, Opts.UseSymbolTable); @@ -109,8 +127,9 @@ LLVMSymbolizer::symbolizeInlinedCode(const std::string &ModuleName, return InlinedContext; } -Expected LLVMSymbolizer::symbolizeData(const std::string &ModuleName, - uint64_t ModuleOffset) { +Expected +LLVMSymbolizer::symbolizeData(const std::string &ModuleName, + object::SectionedAddress ModuleOffset) { SymbolizableModule *Info; if (auto InfoOrErr = getOrCreateModuleInfo(ModuleName)) Info = InfoOrErr.get(); @@ -126,7 +145,7 @@ Expected LLVMSymbolizer::symbolizeData(const std::string &ModuleName, // the object to the offset before we do the query. It's what DIContext // expects. if (Opts.RelativeAddresses) - ModuleOffset += Info->getModulePreferredBase(); + ModuleOffset.Address += Info->getModulePreferredBase(); DIGlobal Global = Info->symbolizeData(ModuleOffset); if (Opts.Demangle) @@ -134,6 +153,29 @@ Expected LLVMSymbolizer::symbolizeData(const std::string &ModuleName, return Global; } +Expected> +LLVMSymbolizer::symbolizeFrame(const std::string &ModuleName, + object::SectionedAddress ModuleOffset) { + SymbolizableModule *Info; + if (auto InfoOrErr = getOrCreateModuleInfo(ModuleName)) + Info = InfoOrErr.get(); + else + return InfoOrErr.takeError(); + + // A null module means an error has already been reported. Return an empty + // result. + if (!Info) + return std::vector(); + + // If the user is giving us relative addresses, add the preferred base of + // the object to the offset before we do the query. It's what DIContext + // expects. + if (Opts.RelativeAddresses) + ModuleOffset.Address += Info->getModulePreferredBase(); + + return Info->symbolizeFrame(ModuleOffset); +} + void LLVMSymbolizer::flush() { ObjectForUBPathAndArch.clear(); BinaryForPath.clear(); @@ -163,42 +205,45 @@ bool checkFileCRC(StringRef Path, uint32_t CRCHash) { MemoryBuffer::getFileOrSTDIN(Path); if (!MB) return false; - return !zlib::isAvailable() || CRCHash == zlib::crc32(MB.get()->getBuffer()); + return CRCHash == llvm::crc32(0, MB.get()->getBuffer()); } bool findDebugBinary(const std::string &OrigPath, const std::string &DebuglinkName, uint32_t CRCHash, + const std::string &FallbackDebugPath, std::string &Result) { - std::string OrigRealPath = OrigPath; -#if defined(HAVE_REALPATH) - if (char *RP = realpath(OrigPath.c_str(), nullptr)) { - OrigRealPath = RP; - free(RP); - } -#endif - SmallString<16> OrigDir(OrigRealPath); + SmallString<16> OrigDir(OrigPath); llvm::sys::path::remove_filename(OrigDir); SmallString<16> DebugPath = OrigDir; - // Try /path/to/original_binary/debuglink_name + // Try relative/path/to/original_binary/debuglink_name llvm::sys::path::append(DebugPath, DebuglinkName); if (checkFileCRC(DebugPath, CRCHash)) { Result = DebugPath.str(); return true; } - // Try /path/to/original_binary/.debug/debuglink_name + // Try relative/path/to/original_binary/.debug/debuglink_name DebugPath = OrigDir; llvm::sys::path::append(DebugPath, ".debug", DebuglinkName); if (checkFileCRC(DebugPath, CRCHash)) { Result = DebugPath.str(); return true; } + // Make the path absolute so that lookups will go to + // "/usr/lib/debug/full/path/to/debug", not + // "/usr/lib/debug/to/debug" + llvm::sys::fs::make_absolute(OrigDir); + if (!FallbackDebugPath.empty()) { + // Try /absolute/path/to/original_binary/debuglink_name + DebugPath = FallbackDebugPath; + } else { #if defined(__NetBSD__) - // Try /usr/libdata/debug/path/to/original_binary/debuglink_name - DebugPath = "/usr/libdata/debug"; + // Try /usr/libdata/debug/absolute/path/to/original_binary/debuglink_name + DebugPath = "/usr/libdata/debug"; #else - // Try /usr/lib/debug/path/to/original_binary/debuglink_name - DebugPath = "/usr/lib/debug"; + // Try /usr/lib/debug/absolute/path/to/original_binary/debuglink_name + DebugPath = "/usr/lib/debug"; #endif + } llvm::sys::path::append(DebugPath, llvm::sys::path::relative_path(OrigDir), DebuglinkName); if (checkFileCRC(DebugPath, CRCHash)) { @@ -217,9 +262,12 @@ bool getGNUDebuglinkContents(const ObjectFile *Obj, std::string &DebugName, Section.getName(Name); Name = Name.substr(Name.find_first_not_of("._")); if (Name == "gnu_debuglink") { - StringRef Data; - Section.getContents(Data); - DataExtractor DE(Data, Obj->isLittleEndian(), 0); + Expected ContentsOrErr = Section.getContents(); + if (!ContentsOrErr) { + consumeError(ContentsOrErr.takeError()); + return false; + } + DataExtractor DE(*ContentsOrErr, Obj->isLittleEndian(), 0); uint32_t Offset = 0; if (const char *DebugNameStr = DE.getCStr(&Offset)) { // 4-byte align the offset. @@ -284,7 +332,8 @@ ObjectFile *LLVMSymbolizer::lookUpDebuglinkObject(const std::string &Path, std::string DebugBinaryPath; if (!getGNUDebuglinkContents(Obj, DebuglinkName, CRCHash)) return nullptr; - if (!findDebugBinary(Path, DebuglinkName, CRCHash, DebugBinaryPath)) + if (!findDebugBinary(Path, DebuglinkName, CRCHash, Opts.FallbackDebugPath, + DebugBinaryPath)) return nullptr; auto DbgObjOrErr = getOrCreateObject(DebugBinaryPath, ArchName); if (!DbgObjOrErr) { @@ -298,15 +347,14 @@ ObjectFile *LLVMSymbolizer::lookUpDebuglinkObject(const std::string &Path, Expected LLVMSymbolizer::getOrCreateObjectPair(const std::string &Path, const std::string &ArchName) { - const auto &I = ObjectPairForPathArch.find(std::make_pair(Path, ArchName)); - if (I != ObjectPairForPathArch.end()) { + auto I = ObjectPairForPathArch.find(std::make_pair(Path, ArchName)); + if (I != ObjectPairForPathArch.end()) return I->second; - } auto ObjOrErr = getOrCreateObject(Path, ArchName); if (!ObjOrErr) { - ObjectPairForPathArch.insert(std::make_pair(std::make_pair(Path, ArchName), - ObjectPair(nullptr, nullptr))); + ObjectPairForPathArch.emplace(std::make_pair(Path, ArchName), + ObjectPair(nullptr, nullptr)); return ObjOrErr.takeError(); } @@ -321,46 +369,43 @@ LLVMSymbolizer::getOrCreateObjectPair(const std::string &Path, if (!DbgObj) DbgObj = Obj; ObjectPair Res = std::make_pair(Obj, DbgObj); - ObjectPairForPathArch.insert( - std::make_pair(std::make_pair(Path, ArchName), Res)); + ObjectPairForPathArch.emplace(std::make_pair(Path, ArchName), Res); return Res; } Expected LLVMSymbolizer::getOrCreateObject(const std::string &Path, const std::string &ArchName) { - const auto &I = BinaryForPath.find(Path); - Binary *Bin = nullptr; - if (I == BinaryForPath.end()) { + Binary *Bin; + auto Pair = BinaryForPath.emplace(Path, OwningBinary()); + if (!Pair.second) { + Bin = Pair.first->second.getBinary(); + } else { Expected> BinOrErr = createBinary(Path); - if (!BinOrErr) { - BinaryForPath.insert(std::make_pair(Path, OwningBinary())); + if (!BinOrErr) return BinOrErr.takeError(); - } - Bin = BinOrErr->getBinary(); - BinaryForPath.insert(std::make_pair(Path, std::move(BinOrErr.get()))); - } else { - Bin = I->second.getBinary(); + Pair.first->second = std::move(BinOrErr.get()); + Bin = Pair.first->second.getBinary(); } if (!Bin) return static_cast(nullptr); if (MachOUniversalBinary *UB = dyn_cast_or_null(Bin)) { - const auto &I = ObjectForUBPathAndArch.find(std::make_pair(Path, ArchName)); - if (I != ObjectForUBPathAndArch.end()) { + auto I = ObjectForUBPathAndArch.find(std::make_pair(Path, ArchName)); + if (I != ObjectForUBPathAndArch.end()) return I->second.get(); - } + Expected> ObjOrErr = UB->getObjectForArch(ArchName); if (!ObjOrErr) { - ObjectForUBPathAndArch.insert(std::make_pair( - std::make_pair(Path, ArchName), std::unique_ptr())); + ObjectForUBPathAndArch.emplace(std::make_pair(Path, ArchName), + std::unique_ptr()); return ObjOrErr.takeError(); } ObjectFile *Res = ObjOrErr->get(); - ObjectForUBPathAndArch.insert(std::make_pair(std::make_pair(Path, ArchName), - std::move(ObjOrErr.get()))); + ObjectForUBPathAndArch.emplace(std::make_pair(Path, ArchName), + std::move(ObjOrErr.get())); return Res; } if (Bin->isObject()) { @@ -370,12 +415,28 @@ LLVMSymbolizer::getOrCreateObject(const std::string &Path, } Expected -LLVMSymbolizer::getOrCreateModuleInfo(const std::string &ModuleName, - StringRef DWPName) { - const auto &I = Modules.find(ModuleName); - if (I != Modules.end()) { +LLVMSymbolizer::createModuleInfo(const ObjectFile *Obj, + std::unique_ptr Context, + StringRef ModuleName) { + auto InfoOrErr = + SymbolizableObjectFile::create(Obj, std::move(Context)); + std::unique_ptr SymMod; + if (InfoOrErr) + SymMod = std::move(*InfoOrErr); + auto InsertResult = + Modules.insert(std::make_pair(ModuleName, std::move(SymMod))); + assert(InsertResult.second); + if (std::error_code EC = InfoOrErr.getError()) + return errorCodeToError(EC); + return InsertResult.first->second.get(); +} + +Expected +LLVMSymbolizer::getOrCreateModuleInfo(const std::string &ModuleName) { + auto I = Modules.find(ModuleName); + if (I != Modules.end()) return I->second.get(); - } + std::string BinaryName = ModuleName; std::string ArchName = Opts.DefaultArch; size_t ColonPos = ModuleName.find_last_of(':'); @@ -390,8 +451,7 @@ LLVMSymbolizer::getOrCreateModuleInfo(const std::string &ModuleName, auto ObjectsOrErr = getOrCreateObjectPair(BinaryName, ArchName); if (!ObjectsOrErr) { // Failed to find valid object file. - Modules.insert( - std::make_pair(ModuleName, std::unique_ptr())); + Modules.emplace(ModuleName, std::unique_ptr()); return ObjectsOrErr.takeError(); } ObjectPair Objects = ObjectsOrErr.get(); @@ -408,8 +468,7 @@ LLVMSymbolizer::getOrCreateModuleInfo(const std::string &ModuleName, std::unique_ptr Session; if (auto Err = loadDataForEXE(PDB_ReaderType::DIA, Objects.first->getFileName(), Session)) { - Modules.insert( - std::make_pair(ModuleName, std::unique_ptr())); + Modules.emplace(ModuleName, std::unique_ptr()); // Return along the PDB filename to provide more context return createFileError(PDBFileName, std::move(Err)); } @@ -417,20 +476,10 @@ LLVMSymbolizer::getOrCreateModuleInfo(const std::string &ModuleName, } } if (!Context) - Context = DWARFContext::create(*Objects.second, nullptr, - DWARFContext::defaultErrorHandler, DWPName); - assert(Context); - auto InfoOrErr = - SymbolizableObjectFile::create(Objects.first, std::move(Context)); - std::unique_ptr SymMod; - if (InfoOrErr) - SymMod = std::move(InfoOrErr.get()); - auto InsertResult = - Modules.insert(std::make_pair(ModuleName, std::move(SymMod))); - assert(InsertResult.second); - if (auto EC = InfoOrErr.getError()) - return errorCodeToError(EC); - return InsertResult.first->second.get(); + Context = + DWARFContext::create(*Objects.second, nullptr, + DWARFContext::defaultErrorHandler, Opts.DWPName); + return createModuleInfo(Objects.first, std::move(Context), ModuleName); } namespace { diff --git a/lib/Demangle/Demangle.cpp b/lib/Demangle/Demangle.cpp new file mode 100644 index 000000000000..5f921537b9bd --- /dev/null +++ b/lib/Demangle/Demangle.cpp @@ -0,0 +1,36 @@ +//===-- Demangle.cpp - Common demangling functions ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file This file contains definitions of common demangling functions. +/// +//===----------------------------------------------------------------------===// + +#include "llvm/Demangle/Demangle.h" +#include + +static bool isItaniumEncoding(const std::string &MangledName) { + size_t Pos = MangledName.find_first_not_of('_'); + // A valid Itanium encoding requires 1-4 leading underscores, followed by 'Z'. + return Pos > 0 && Pos <= 4 && MangledName[Pos] == 'Z'; +} + +std::string llvm::demangle(const std::string &MangledName) { + char *Demangled; + if (isItaniumEncoding(MangledName)) + Demangled = itaniumDemangle(MangledName.c_str(), nullptr, nullptr, nullptr); + else + Demangled = + microsoftDemangle(MangledName.c_str(), nullptr, nullptr, nullptr); + + if (!Demangled) + return MangledName; + + std::string Ret = Demangled; + free(Demangled); + return Ret; +} diff --git a/lib/Demangle/ItaniumDemangle.cpp b/lib/Demangle/ItaniumDemangle.cpp index b2de0be2b70c..5c99c70e3cc6 100644 --- a/lib/Demangle/ItaniumDemangle.cpp +++ b/lib/Demangle/ItaniumDemangle.cpp @@ -1,9 +1,8 @@ //===------------------------- ItaniumDemangle.cpp ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/Demangle/MicrosoftDemangle.cpp b/lib/Demangle/MicrosoftDemangle.cpp index 51ffa0bff7f3..bf7d77638f34 100644 --- a/lib/Demangle/MicrosoftDemangle.cpp +++ b/lib/Demangle/MicrosoftDemangle.cpp @@ -1,9 +1,8 @@ //===- MicrosoftDemangle.cpp ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -18,7 +17,7 @@ #include "llvm/Demangle/Demangle.h" #include "llvm/Demangle/MicrosoftDemangleNodes.h" -#include "llvm/Demangle/Compiler.h" +#include "llvm/Demangle/DemangleConfig.h" #include "llvm/Demangle/StringView.h" #include "llvm/Demangle/Utility.h" @@ -59,14 +58,18 @@ static bool isMemberPointer(StringView MangledName, bool &Error) { // what. break; default: - Error = true; - return false; + // isMemberPointer() is called only if isPointerType() returns true, + // and it rejects other prefixes. + DEMANGLE_UNREACHABLE; } // If it starts with a number, then 6 indicates a non-member function // pointer, and 8 indicates a member function pointer. if (startsWithDigit(MangledName)) { - assert(MangledName[0] == '6' || MangledName[0] == '8'); + if (MangledName[0] != '6' && MangledName[0] != '8') { + Error = true; + return false; + } return (MangledName[0] == '8'); } @@ -76,7 +79,10 @@ static bool isMemberPointer(StringView MangledName, bool &Error) { MangledName.consumeFront('I'); // restrict MangledName.consumeFront('F'); // unaligned - assert(!MangledName.empty()); + if (MangledName.empty()) { + Error = true; + return false; + } // The next value should be either ABCD (non-member) or QRST (member). switch (MangledName.front()) { @@ -136,8 +142,6 @@ consumeSpecialIntrinsicKind(StringView &MangledName) { static bool startsWithLocalScopePattern(StringView S) { if (!S.consumeFront('?')) return false; - if (S.size() < 2) - return false; size_t End = S.find('?'); if (End == StringView::npos) @@ -234,10 +238,10 @@ demanglePointerCVQualifiers(StringView &MangledName) { case 'S': return std::make_pair(Qualifiers(Q_Const | Q_Volatile), PointerAffinity::Pointer); - default: - assert(false && "Ty is not a pointer type!"); } - return std::make_pair(Q_None, PointerAffinity::Pointer); + // This function is only called if isPointerType() returns true, + // and it only returns true for the six cases listed above. + DEMANGLE_UNREACHABLE; } StringView Demangler::copyString(StringView Borrowed) { @@ -265,12 +269,16 @@ Demangler::demangleSpecialTableSymbolNode(StringView &MangledName, NI->Name = "`RTTI Complete Object Locator'"; break; default: - LLVM_BUILTIN_UNREACHABLE; + DEMANGLE_UNREACHABLE; } QualifiedNameNode *QN = demangleNameScopeChain(MangledName, NI); SpecialTableSymbolNode *STSN = Arena.alloc(); STSN->Name = QN; bool IsMember = false; + if (MangledName.empty()) { + Error = true; + return nullptr; + } char Front = MangledName.popFront(); if (Front != '6' && Front != '7') { Error = true; @@ -284,9 +292,10 @@ Demangler::demangleSpecialTableSymbolNode(StringView &MangledName, } LocalStaticGuardVariableNode * -Demangler::demangleLocalStaticGuard(StringView &MangledName) { +Demangler::demangleLocalStaticGuard(StringView &MangledName, bool IsThread) { LocalStaticGuardIdentifierNode *LSGI = Arena.alloc(); + LSGI->IsThread = IsThread; QualifiedNameNode *QN = demangleNameScopeChain(MangledName, LSGI); LocalStaticGuardVariableNode *LSGVN = Arena.alloc(); @@ -379,11 +388,11 @@ FunctionSymbolNode *Demangler::demangleInitFiniStub(StringView &MangledName, if (MangledName.consumeFront('?')) IsKnownStaticDataMember = true; - QualifiedNameNode *QN = demangleFullyQualifiedSymbolName(MangledName); + SymbolNode *Symbol = demangleDeclarator(MangledName); + if (Error) + return nullptr; - SymbolNode *Symbol = demangleEncodedSymbol(MangledName, QN); FunctionSymbolNode *FSN = nullptr; - Symbol->Name = QN; if (Symbol->kind() == NodeKind::VariableSymbol) { DSIN->Variable = static_cast(Symbol); @@ -401,7 +410,8 @@ FunctionSymbolNode *Demangler::demangleInitFiniStub(StringView &MangledName, } FSN = demangleFunctionEncoding(MangledName); - FSN->Name = synthesizeQualifiedName(Arena, DSIN); + if (FSN) + FSN->Name = synthesizeQualifiedName(Arena, DSIN); } else { if (IsKnownStaticDataMember) { // This was supposed to be a static data member, but we got a function. @@ -419,10 +429,10 @@ FunctionSymbolNode *Demangler::demangleInitFiniStub(StringView &MangledName, SymbolNode *Demangler::demangleSpecialIntrinsic(StringView &MangledName) { SpecialIntrinsicKind SIK = consumeSpecialIntrinsicKind(MangledName); - if (SIK == SpecialIntrinsicKind::None) - return nullptr; switch (SIK) { + case SpecialIntrinsicKind::None: + return nullptr; case SpecialIntrinsicKind::StringLiteralSymbol: return demangleStringLiteral(MangledName); case SpecialIntrinsicKind::Vftable: @@ -433,7 +443,9 @@ SymbolNode *Demangler::demangleSpecialIntrinsic(StringView &MangledName) { case SpecialIntrinsicKind::VcallThunk: return demangleVcallThunkNode(MangledName); case SpecialIntrinsicKind::LocalStaticGuard: - return demangleLocalStaticGuard(MangledName); + return demangleLocalStaticGuard(MangledName, /*IsThread=*/false); + case SpecialIntrinsicKind::LocalStaticThreadGuard: + return demangleLocalStaticGuard(MangledName, /*IsThread=*/true); case SpecialIntrinsicKind::RttiTypeDescriptor: { TypeNode *T = demangleType(MangledName, QualifierMangleMode::Result); if (Error) @@ -453,11 +465,16 @@ SymbolNode *Demangler::demangleSpecialIntrinsic(StringView &MangledName) { case SpecialIntrinsicKind::RttiBaseClassDescriptor: return demangleRttiBaseClassDescriptorNode(Arena, MangledName); case SpecialIntrinsicKind::DynamicInitializer: - return demangleInitFiniStub(MangledName, false); + return demangleInitFiniStub(MangledName, /*IsDestructor=*/false); case SpecialIntrinsicKind::DynamicAtexitDestructor: - return demangleInitFiniStub(MangledName, true); - default: + return demangleInitFiniStub(MangledName, /*IsDestructor=*/true); + case SpecialIntrinsicKind::Typeof: + case SpecialIntrinsicKind::UdtReturning: + // It's unclear which tools produces these manglings, so demangling + // support is not (yet?) implemented. break; + case SpecialIntrinsicKind::Unknown: + DEMANGLE_UNREACHABLE; // Never returned by consumeSpecialIntrinsicKind. } Error = true; return nullptr; @@ -467,11 +484,15 @@ IdentifierNode * Demangler::demangleFunctionIdentifierCode(StringView &MangledName) { assert(MangledName.startsWith('?')); MangledName = MangledName.dropFront(); + if (MangledName.empty()) { + Error = true; + return nullptr; + } if (MangledName.consumeFront("__")) return demangleFunctionIdentifierCode( MangledName, FunctionIdentifierCodeGroup::DoubleUnder); - else if (MangledName.consumeFront("_")) + if (MangledName.consumeFront("_")) return demangleFunctionIdentifierCode(MangledName, FunctionIdentifierCodeGroup::Under); return demangleFunctionIdentifierCode(MangledName, @@ -497,16 +518,22 @@ LiteralOperatorIdentifierNode * Demangler::demangleLiteralOperatorIdentifier(StringView &MangledName) { LiteralOperatorIdentifierNode *N = Arena.alloc(); - N->Name = demangleSimpleString(MangledName, false); + N->Name = demangleSimpleString(MangledName, /*Memorize=*/false); return N; } -static IntrinsicFunctionKind -translateIntrinsicFunctionCode(char CH, FunctionIdentifierCodeGroup Group) { +IntrinsicFunctionKind +Demangler::translateIntrinsicFunctionCode(char CH, + FunctionIdentifierCodeGroup Group) { + using IFK = IntrinsicFunctionKind; + if (!(CH >= '0' && CH <= '9') && !(CH >= 'A' && CH <= 'Z')) { + Error = true; + return IFK::None; + } + // Not all ? identifiers are intrinsics *functions*. This function only maps // operator codes for the special functions, all others are handled elsewhere, // hence the IFK::None entries in the table. - using IFK = IntrinsicFunctionKind; static IFK Basic[36] = { IFK::None, // ?0 # Foo::Foo() IFK::None, // ?1 # Foo::~Foo() @@ -606,8 +633,8 @@ translateIntrinsicFunctionCode(char CH, FunctionIdentifierCodeGroup Group) { // iter IFK::None, // ?__J local static thread guard IFK::None, // ?__K operator ""_name - IFK::CoAwait, // ?__L co_await - IFK::None, // ?__M + IFK::CoAwait, // ?__L operator co_await + IFK::Spaceship, // ?__M operator<=> IFK::None, // ?__N IFK::None, // ?__O IFK::None, // ?__P @@ -632,12 +659,16 @@ translateIntrinsicFunctionCode(char CH, FunctionIdentifierCodeGroup Group) { case FunctionIdentifierCodeGroup::DoubleUnder: return DoubleUnder[Index]; } - LLVM_BUILTIN_UNREACHABLE; + DEMANGLE_UNREACHABLE; } IdentifierNode * Demangler::demangleFunctionIdentifierCode(StringView &MangledName, FunctionIdentifierCodeGroup Group) { + if (MangledName.empty()) { + Error = true; + return nullptr; + } switch (Group) { case FunctionIdentifierCodeGroup::Basic: switch (char CH = MangledName.popFront()) { @@ -650,7 +681,6 @@ Demangler::demangleFunctionIdentifierCode(StringView &MangledName, return Arena.alloc( translateIntrinsicFunctionCode(CH, Group)); } - break; case FunctionIdentifierCodeGroup::Under: return Arena.alloc( translateIntrinsicFunctionCode(MangledName.popFront(), Group)); @@ -663,13 +693,17 @@ Demangler::demangleFunctionIdentifierCode(StringView &MangledName, translateIntrinsicFunctionCode(CH, Group)); } } - // No Mangling Yet: Spaceship, // operator<=> - return nullptr; + DEMANGLE_UNREACHABLE; } SymbolNode *Demangler::demangleEncodedSymbol(StringView &MangledName, QualifiedNameNode *Name) { + if (MangledName.empty()) { + Error = true; + return nullptr; + } + // Read a variable. switch (MangledName.front()) { case '0': @@ -680,8 +714,6 @@ SymbolNode *Demangler::demangleEncodedSymbol(StringView &MangledName, StorageClass SC = demangleVariableStorageClass(MangledName); return demangleVariableEncoding(MangledName, SC); } - case '8': - return nullptr; } FunctionSymbolNode *FSN = demangleFunctionEncoding(MangledName); @@ -689,23 +721,74 @@ SymbolNode *Demangler::demangleEncodedSymbol(StringView &MangledName, if (UQN->kind() == NodeKind::ConversionOperatorIdentifier) { ConversionOperatorIdentifierNode *COIN = static_cast(UQN); - COIN->TargetType = FSN->Signature->ReturnType; + if (FSN) + COIN->TargetType = FSN->Signature->ReturnType; } return FSN; } +SymbolNode *Demangler::demangleDeclarator(StringView &MangledName) { + // What follows is a main symbol name. This may include namespaces or class + // back references. + QualifiedNameNode *QN = demangleFullyQualifiedSymbolName(MangledName); + if (Error) + return nullptr; + + SymbolNode *Symbol = demangleEncodedSymbol(MangledName, QN); + if (Error) + return nullptr; + Symbol->Name = QN; + + IdentifierNode *UQN = QN->getUnqualifiedIdentifier(); + if (UQN->kind() == NodeKind::ConversionOperatorIdentifier) { + ConversionOperatorIdentifierNode *COIN = + static_cast(UQN); + if (!COIN->TargetType) { + Error = true; + return nullptr; + } + } + return Symbol; +} + +SymbolNode *Demangler::demangleMD5Name(StringView &MangledName) { + assert(MangledName.startsWith("??@")); + // This is an MD5 mangled name. We can't demangle it, just return the + // mangled name. + // An MD5 mangled name is ??@ followed by 32 characters and a terminating @. + size_t MD5Last = MangledName.find('@', strlen("??@")); + if (MD5Last == StringView::npos) { + Error = true; + return nullptr; + } + const char *Start = MangledName.begin(); + MangledName = MangledName.dropFront(MD5Last + 1); + + // There are two additional special cases for MD5 names: + // 1. For complete object locators where the object name is long enough + // for the object to have an MD5 name, the complete object locator is + // called ??@...@??_R4@ (with a trailing "??_R4@" instead of the usual + // leading "??_R4". This is handled here. + // 2. For catchable types, in versions of MSVC before 2015 (<1900) or after + // 2017.2 (>= 1914), the catchable type mangling is _CT??@...@??@...@8 + // instead of_CT??@...@8 with just one MD5 name. Since we don't yet + // demangle catchable types anywhere, this isn't handled for MD5 names + // either. + MangledName.consumeFront("??_R4@"); + + StringView MD5(Start, MangledName.begin()); + SymbolNode *S = Arena.alloc(NodeKind::Md5Symbol); + S->Name = synthesizeQualifiedName(Arena, MD5); + + return S; +} + // Parser entry point. SymbolNode *Demangler::parse(StringView &MangledName) { - // We can't demangle MD5 names, just output them as-is. - // Also, MSVC-style mangled symbols must start with '?'. - if (MangledName.startsWith("??@")) { - // This is an MD5 mangled name. We can't demangle it, just return the - // mangled name. - SymbolNode *S = Arena.alloc(NodeKind::Md5Symbol); - S->Name = synthesizeQualifiedName(Arena, MangledName); - return S; - } + if (MangledName.startsWith("??@")) + return demangleMD5Name(MangledName); + // MSVC-style mangled symbols must start with '?'. if (!MangledName.startsWith('?')) { Error = true; return nullptr; @@ -718,21 +801,7 @@ SymbolNode *Demangler::parse(StringView &MangledName) { if (SymbolNode *SI = demangleSpecialIntrinsic(MangledName)) return SI; - // What follows is a main symbol name. This may include namespaces or class - // back references. - QualifiedNameNode *QN = demangleFullyQualifiedSymbolName(MangledName); - if (Error) - return nullptr; - - SymbolNode *Symbol = demangleEncodedSymbol(MangledName, QN); - if (Symbol) { - Symbol->Name = QN; - } - - if (Error) - return nullptr; - - return Symbol; + return demangleDeclarator(MangledName); } TagTypeNode *Demangler::parseTagUniqueName(StringView &MangledName) { @@ -759,6 +828,9 @@ VariableSymbolNode *Demangler::demangleVariableEncoding(StringView &MangledName, VSN->Type = demangleType(MangledName, QualifierMangleMode::Drop); VSN->SC = SC; + if (Error) + return nullptr; + // ::= // ::= # pointers, references switch (VSN->Type->kind()) { @@ -797,7 +869,7 @@ VariableSymbolNode *Demangler::demangleVariableEncoding(StringView &MangledName, // ::= [?] // // ::= # when 1 <= Number <= 10 -// ::= + @ # when Numbrer == 0 or >= 10 +// ::= + @ # when Number == 0 or >= 10 // // ::= [A-P] # A = 0, B = 1, ... std::pair Demangler::demangleNumber(StringView &MangledName) { @@ -906,8 +978,18 @@ Demangler::demangleTemplateInstantiationName(StringView &MangledName, if (Error) return nullptr; - if (NBB & NBB_Template) + if (NBB & NBB_Template) { + // NBB_Template is only set for types and non-leaf names ("a::" in "a::b"). + // Structors and conversion operators only makes sense in a leaf name, so + // reject them in NBB_Template contexts. + if (Identifier->kind() == NodeKind::ConversionOperatorIdentifier || + Identifier->kind() == NodeKind::StructorIdentifier) { + Error = true; + return nullptr; + } + memorizeIdentifier(Identifier); + } return Identifier; } @@ -931,6 +1013,7 @@ static uint8_t rebasedHexDigitToNumber(char C) { } uint8_t Demangler::demangleCharLiteral(StringView &MangledName) { + assert(!MangledName.empty()); if (!MangledName.startsWith('?')) return MangledName.popFront(); @@ -988,7 +1071,7 @@ wchar_t Demangler::demangleWcharLiteral(StringView &MangledName) { uint8_t C1, C2; C1 = demangleCharLiteral(MangledName); - if (Error) + if (Error || MangledName.empty()) goto WCharLiteralError; C2 = demangleCharLiteral(MangledName); if (Error) @@ -1007,10 +1090,8 @@ static void writeHexDigit(char *Buffer, uint8_t Digit) { } static void outputHex(OutputStream &OS, unsigned C) { - if (C == 0) { - OS << "\\x00"; - return; - } + assert (C != 0); + // It's easier to do the math if we can work from right to left, but we need // to print the numbers from left to right. So render this into a temporary // buffer first, then output the temporary buffer. Each byte is of the form @@ -1019,23 +1100,26 @@ static void outputHex(OutputStream &OS, unsigned C) { char TempBuffer[17]; ::memset(TempBuffer, 0, sizeof(TempBuffer)); - constexpr int MaxPos = 15; + constexpr int MaxPos = sizeof(TempBuffer) - 1; - int Pos = MaxPos - 1; + int Pos = MaxPos - 1; // TempBuffer[MaxPos] is the terminating \0. while (C != 0) { for (int I = 0; I < 2; ++I) { writeHexDigit(&TempBuffer[Pos--], C % 16); C /= 16; } - TempBuffer[Pos--] = 'x'; - TempBuffer[Pos--] = '\\'; - assert(Pos >= 0); } + TempBuffer[Pos--] = 'x'; + assert(Pos >= 0); + TempBuffer[Pos--] = '\\'; OS << StringView(&TempBuffer[Pos + 1]); } static void outputEscapedChar(OutputStream &OS, unsigned C) { switch (C) { + case '\0': // nul + OS << "\\0"; + return; case '\'': // single quote OS << "\\\'"; return; @@ -1100,8 +1184,11 @@ static unsigned countEmbeddedNulls(const uint8_t *StringBytes, return Result; } +// A mangled (non-wide) string literal stores the total length of the string it +// refers to (passed in NumBytes), and it contains up to 32 bytes of actual text +// (passed in StringBytes, NumChars). static unsigned guessCharByteSize(const uint8_t *StringBytes, unsigned NumChars, - unsigned NumBytes) { + uint64_t NumBytes) { assert(NumBytes > 0); // If the number of bytes is odd, this is guaranteed to be a char string. @@ -1113,7 +1200,7 @@ static unsigned guessCharByteSize(const uint8_t *StringBytes, unsigned NumChars, // 2-byte, or 4-byte null terminator. if (NumBytes < 32) { unsigned TrailingNulls = countTrailingNullBytes(StringBytes, NumChars); - if (TrailingNulls >= 4) + if (TrailingNulls >= 4 && NumBytes % 4 == 0) return 4; if (TrailingNulls >= 2) return 2; @@ -1127,7 +1214,7 @@ static unsigned guessCharByteSize(const uint8_t *StringBytes, unsigned NumChars, // perfect and is biased towards languages that have ascii alphabets, but this // was always going to be best effort since the encoding is lossy. unsigned Nulls = countEmbeddedNulls(StringBytes, NumChars); - if (Nulls >= 2 * NumChars / 3) + if (Nulls >= 2 * NumChars / 3 && NumBytes % 4 == 0) return 4; if (Nulls >= NumChars / 3) return 2; @@ -1178,6 +1265,11 @@ Demangler::demangleStringLiteral(StringView &MangledName) { EncodedStringLiteralNode *Result = Arena.alloc(); + // Must happen before the first `goto StringLiteralError`. + if (!initializeOutputStream(nullptr, nullptr, OS, 1024)) + // FIXME: Propagate out-of-memory as an error? + std::terminate(); + // Prefix indicating the beginning of a string literal if (!MangledName.consumeFront("@_")) goto StringLiteralError; @@ -1188,7 +1280,7 @@ Demangler::demangleStringLiteral(StringView &MangledName) { switch (MangledName.popFront()) { case '1': IsWcharT = true; - LLVM_FALLTHROUGH; + DEMANGLE_FALLTHROUGH; case '0': break; default: @@ -1197,7 +1289,7 @@ Demangler::demangleStringLiteral(StringView &MangledName) { // Encoded Length std::tie(StringByteSize, IsNegative) = demangleNumber(MangledName); - if (Error || IsNegative) + if (Error || IsNegative || StringByteSize < (IsWcharT ? 2 : 1)) goto StringLiteralError; // CRC 32 (always 8 characters plus a terminator) @@ -1209,16 +1301,14 @@ Demangler::demangleStringLiteral(StringView &MangledName) { if (MangledName.empty()) goto StringLiteralError; - if (!initializeOutputStream(nullptr, nullptr, OS, 1024)) - // FIXME: Propagate out-of-memory as an error? - std::terminate(); if (IsWcharT) { Result->Char = CharKind::Wchar; if (StringByteSize > 64) Result->IsTruncated = true; while (!MangledName.consumeFront('@')) { - assert(StringByteSize >= 2); + if (MangledName.size() < 2) + goto StringLiteralError; wchar_t W = demangleWcharLiteral(MangledName); if (StringByteSize != 2 || Result->IsTruncated) outputEscapedChar(OS, W); @@ -1234,7 +1324,8 @@ Demangler::demangleStringLiteral(StringView &MangledName) { unsigned BytesDecoded = 0; while (!MangledName.consumeFront('@')) { - assert(StringByteSize >= 1); + if (MangledName.size() < 1 || BytesDecoded >= MaxStringByteLength) + goto StringLiteralError; StringBytes[BytesDecoded++] = demangleCharLiteral(MangledName); } @@ -1255,7 +1346,7 @@ Demangler::demangleStringLiteral(StringView &MangledName) { Result->Char = CharKind::Char32; break; default: - LLVM_BUILTIN_UNREACHABLE; + DEMANGLE_UNREACHABLE; } const unsigned NumChars = BytesDecoded / CharBytes; for (unsigned CharIndex = 0; CharIndex < NumChars; ++CharIndex) { @@ -1274,15 +1365,20 @@ Demangler::demangleStringLiteral(StringView &MangledName) { StringLiteralError: Error = true; + std::free(OS.getBuffer()); return nullptr; } +// Returns MangledName's prefix before the first '@', or an error if +// MangledName contains no '@' or the prefix has length 0. StringView Demangler::demangleSimpleString(StringView &MangledName, bool Memorize) { StringView S; for (size_t i = 0; i < MangledName.size(); ++i) { if (MangledName[i] != '@') continue; + if (i == 0) + break; S = MangledName.substr(0, i); MangledName = MangledName.dropFront(i + 1); @@ -1319,8 +1415,10 @@ Demangler::demangleLocallyScopedNamePiece(StringView &MangledName) { NamedIdentifierNode *Identifier = Arena.alloc(); MangledName.consumeFront('?'); - auto Number = demangleNumber(MangledName); - assert(!Number.second); + uint64_t Number = 0; + bool IsNegative = false; + std::tie(Number, IsNegative) = demangleNumber(MangledName); + assert(!IsNegative); // One ? to terminate the number MangledName.consumeFront('?'); @@ -1338,7 +1436,7 @@ Demangler::demangleLocallyScopedNamePiece(StringView &MangledName) { OS << '`'; Scope->output(OS, OF_Default); OS << '\''; - OS << "::`" << Number.first << "'"; + OS << "::`" << Number << "'"; OS << '\0'; char *Result = OS.getBuffer(); Identifier->Name = copyString(Result); @@ -1349,7 +1447,8 @@ Demangler::demangleLocallyScopedNamePiece(StringView &MangledName) { // Parses a type name in the form of A@B@C@@ which represents C::B::A. QualifiedNameNode * Demangler::demangleFullyQualifiedTypeName(StringView &MangledName) { - IdentifierNode *Identifier = demangleUnqualifiedTypeName(MangledName, true); + IdentifierNode *Identifier = + demangleUnqualifiedTypeName(MangledName, /*Memorize=*/true); if (Error) return nullptr; assert(Identifier); @@ -1381,9 +1480,12 @@ Demangler::demangleFullyQualifiedSymbolName(StringView &MangledName) { return nullptr; if (Identifier->kind() == NodeKind::StructorIdentifier) { + if (QN->Components->Count < 2) { + Error = true; + return nullptr; + } StructorIdentifierNode *SIN = static_cast(Identifier); - assert(QN->Components->Count >= 2); Node *ClassNode = QN->Components->Nodes[QN->Components->Count - 2]; SIN->Class = static_cast(ClassNode); } @@ -1415,7 +1517,7 @@ Demangler::demangleUnqualifiedSymbolName(StringView &MangledName, return demangleTemplateInstantiationName(MangledName, NBB); if (MangledName.startsWith('?')) return demangleFunctionIdentifierCode(MangledName); - return demangleSimpleName(MangledName, (NBB & NBB_Simple) != 0); + return demangleSimpleName(MangledName, /*Memorize=*/(NBB & NBB_Simple) != 0); } IdentifierNode *Demangler::demangleNameScopePiece(StringView &MangledName) { @@ -1431,7 +1533,7 @@ IdentifierNode *Demangler::demangleNameScopePiece(StringView &MangledName) { if (startsWithLocalScopePattern(MangledName)) return demangleLocallyScopedNamePiece(MangledName); - return demangleSimpleName(MangledName, true); + return demangleSimpleName(MangledName, /*Memorize=*/true); } static NodeArrayNode *nodeListToNodeArray(ArenaAllocator &Arena, NodeList *Head, @@ -1489,11 +1591,11 @@ FuncClass Demangler::demangleFunctionClass(StringView &MangledName) { case 'C': return FuncClass(FC_Private | FC_Static); case 'D': - return FuncClass(FC_Private | FC_Static); + return FuncClass(FC_Private | FC_Static | FC_Far); case 'E': return FuncClass(FC_Private | FC_Virtual); case 'F': - return FuncClass(FC_Private | FC_Virtual); + return FuncClass(FC_Private | FC_Virtual | FC_Far); case 'G': return FuncClass(FC_Private | FC_StaticThisAdjust); case 'H': @@ -1538,7 +1640,8 @@ FuncClass Demangler::demangleFunctionClass(StringView &MangledName) { FuncClass VFlag = FC_VirtualThisAdjust; if (MangledName.consumeFront('R')) VFlag = FuncClass(VFlag | FC_VirtualThisAdjustEx); - + if (MangledName.empty()) + break; switch (MangledName.popFront()) { case '0': return FuncClass(FC_Private | FC_Virtual | VFlag); @@ -1561,6 +1664,11 @@ FuncClass Demangler::demangleFunctionClass(StringView &MangledName) { } CallingConv Demangler::demangleCallingConvention(StringView &MangledName) { + if (MangledName.empty()) { + Error = true; + return CallingConv::None; + } + switch (MangledName.popFront()) { case 'A': case 'B': @@ -1591,7 +1699,7 @@ CallingConv Demangler::demangleCallingConvention(StringView &MangledName) { } StorageClass Demangler::demangleVariableStorageClass(StringView &MangledName) { - assert(std::isdigit(MangledName.front())); + assert(MangledName.front() >= '0' && MangledName.front() <= '4'); switch (MangledName.popFront()) { case '0': @@ -1605,12 +1713,15 @@ StorageClass Demangler::demangleVariableStorageClass(StringView &MangledName) { case '4': return StorageClass::FunctionLocalStatic; } - Error = true; - return StorageClass::None; + DEMANGLE_UNREACHABLE; } std::pair Demangler::demangleQualifiers(StringView &MangledName) { + if (MangledName.empty()) { + Error = true; + return std::make_pair(Q_None, false); + } switch (MangledName.popFront()) { // Member qualifiers @@ -1649,6 +1760,11 @@ TypeNode *Demangler::demangleType(StringView &MangledName, std::tie(Quals, IsMember) = demangleQualifiers(MangledName); } + if (MangledName.empty()) { + Error = true; + return nullptr; + } + TypeNode *Ty = nullptr; if (isTagType(MangledName)) Ty = demangleClassType(MangledName); @@ -1710,7 +1826,7 @@ FunctionSignatureNode *Demangler::demangleFunctionType(StringView &MangledName, if (!IsStructor) FTy->ReturnType = demangleType(MangledName, QualifierMangleMode::Result); - FTy->Params = demangleFunctionParameterList(MangledName); + FTy->Params = demangleFunctionParameterList(MangledName, FTy->IsVariadic); FTy->IsNoexcept = demangleThrowSpecification(MangledName); @@ -1723,6 +1839,11 @@ Demangler::demangleFunctionEncoding(StringView &MangledName) { if (MangledName.consumeFront("$$J0")) ExtraFlags = FC_ExternC; + if (MangledName.empty()) { + Error = true; + return nullptr; + } + FuncClass FC = demangleFunctionClass(MangledName); FC = FuncClass(ExtraFlags | FC); @@ -1750,6 +1871,10 @@ Demangler::demangleFunctionEncoding(StringView &MangledName) { bool HasThisQuals = !(FC & (FC_Global | FC_Static)); FSN = demangleFunctionType(MangledName, HasThisQuals); } + + if (Error) + return nullptr; + if (TTN) { *static_cast(TTN) = *FSN; FSN = TTN; @@ -1766,7 +1891,7 @@ CustomTypeNode *Demangler::demangleCustomType(StringView &MangledName) { MangledName.popFront(); CustomTypeNode *CTN = Arena.alloc(); - CTN->Identifier = demangleUnqualifiedTypeName(MangledName, true); + CTN->Identifier = demangleUnqualifiedTypeName(MangledName, /*Memorize=*/true); if (!MangledName.consumeFront('@')) Error = true; if (Error) @@ -1820,6 +1945,8 @@ PrimitiveTypeNode *Demangler::demanglePrimitiveType(StringView &MangledName) { return Arena.alloc(PrimitiveKind::Uint64); case 'W': return Arena.alloc(PrimitiveKind::Wchar); + case 'Q': + return Arena.alloc(PrimitiveKind::Char8); case 'S': return Arena.alloc(PrimitiveKind::Char16); case 'U': @@ -1846,7 +1973,7 @@ TagTypeNode *Demangler::demangleClassType(StringView &MangledName) { TT = Arena.alloc(TagKind::Class); break; case 'W': - if (MangledName.popFront() != '4') { + if (!MangledName.consumeFront('4')) { Error = true; return nullptr; } @@ -1890,6 +2017,8 @@ PointerTypeNode *Demangler::demangleMemberPointerType(StringView &MangledName) { Qualifiers ExtQuals = demanglePointerExtQualifiers(MangledName); Pointer->Quals = Qualifiers(Pointer->Quals | ExtQuals); + // isMemberPointer() only returns true if there is at least one character + // after the qualifiers. if (MangledName.consumeFront("8")) { Pointer->ClassParent = demangleFullyQualifiedTypeName(MangledName); Pointer->Pointee = demangleFunctionType(MangledName, true); @@ -1897,11 +2026,12 @@ PointerTypeNode *Demangler::demangleMemberPointerType(StringView &MangledName) { Qualifiers PointeeQuals = Q_None; bool IsMember = false; std::tie(PointeeQuals, IsMember) = demangleQualifiers(MangledName); - assert(IsMember); + assert(IsMember || Error); Pointer->ClassParent = demangleFullyQualifiedTypeName(MangledName); Pointer->Pointee = demangleType(MangledName, QualifierMangleMode::Drop); - Pointer->Pointee->Quals = PointeeQuals; + if (Pointer->Pointee) + Pointer->Pointee->Quals = PointeeQuals; } return Pointer; @@ -1938,7 +2068,7 @@ ArrayTypeNode *Demangler::demangleArrayType(StringView &MangledName) { for (uint64_t I = 0; I < Rank; ++I) { uint64_t D = 0; std::tie(D, IsNegative) = demangleNumber(MangledName); - if (IsNegative) { + if (Error || IsNegative) { Error = true; return nullptr; } @@ -1963,12 +2093,12 @@ ArrayTypeNode *Demangler::demangleArrayType(StringView &MangledName) { return ATy; } -// Reads a function or a template parameters. -NodeArrayNode * -Demangler::demangleFunctionParameterList(StringView &MangledName) { +// Reads a function's parameters. +NodeArrayNode *Demangler::demangleFunctionParameterList(StringView &MangledName, + bool &IsVariadic) { // Empty parameter list. if (MangledName.consumeFront('X')) - return {}; + return nullptr; NodeList *Head = Arena.alloc(); NodeList **Current = &Head; @@ -1981,7 +2111,7 @@ Demangler::demangleFunctionParameterList(StringView &MangledName) { size_t N = MangledName[0] - '0'; if (N >= Backrefs.FunctionParamCount) { Error = true; - return {}; + return nullptr; } MangledName = MangledName.dropFront(); @@ -2012,7 +2142,7 @@ Demangler::demangleFunctionParameterList(StringView &MangledName) { } if (Error) - return {}; + return nullptr; NodeArrayNode *NA = nodeListToNodeArray(Arena, Head, Count); // A non-empty parameter list is terminated by either 'Z' (variadic) parameter @@ -2022,13 +2152,11 @@ Demangler::demangleFunctionParameterList(StringView &MangledName) { return NA; if (MangledName.consumeFront('Z')) { - // This is a variadic parameter list. We probably need a variadic node to - // append to the end. + IsVariadic = true; return NA; } - Error = true; - return {}; + DEMANGLE_UNREACHABLE; } NodeArrayNode * @@ -2037,7 +2165,7 @@ Demangler::demangleTemplateParameterList(StringView &MangledName) { NodeList **Current = &Head; size_t Count = 0; - while (!Error && !MangledName.startsWith('@')) { + while (!MangledName.startsWith('@')) { if (MangledName.consumeFront("$S") || MangledName.consumeFront("$$V") || MangledName.consumeFront("$$$V") || MangledName.consumeFront("$$Z")) { // parameter pack separator @@ -2070,12 +2198,16 @@ Demangler::demangleTemplateParameterList(StringView &MangledName) { MangledName = MangledName.dropFront(); // 1 - single inheritance // H - multiple inheritance - // I - virtual inheritance + // I - virtual inheritance // J - unspecified inheritance char InheritanceSpecifier = MangledName.popFront(); SymbolNode *S = nullptr; if (MangledName.startsWith('?')) { S = parse(MangledName); + if (Error || !S->Name) { + Error = true; + return nullptr; + } memorizeIdentifier(S->Name->getUnqualifiedIdentifier()); } @@ -2083,20 +2215,19 @@ Demangler::demangleTemplateParameterList(StringView &MangledName) { case 'J': TPRN->ThunkOffsets[TPRN->ThunkOffsetCount++] = demangleSigned(MangledName); - LLVM_FALLTHROUGH; + DEMANGLE_FALLTHROUGH; case 'I': TPRN->ThunkOffsets[TPRN->ThunkOffsetCount++] = demangleSigned(MangledName); - LLVM_FALLTHROUGH; + DEMANGLE_FALLTHROUGH; case 'H': TPRN->ThunkOffsets[TPRN->ThunkOffsetCount++] = demangleSigned(MangledName); - LLVM_FALLTHROUGH; + DEMANGLE_FALLTHROUGH; case '1': break; default: - Error = true; - break; + DEMANGLE_UNREACHABLE; } TPRN->Affinity = PointerAffinity::Pointer; TPRN->Symbol = S; @@ -2117,18 +2248,15 @@ Demangler::demangleTemplateParameterList(StringView &MangledName) { case 'G': TPRN->ThunkOffsets[TPRN->ThunkOffsetCount++] = demangleSigned(MangledName); - LLVM_FALLTHROUGH; + DEMANGLE_FALLTHROUGH; case 'F': TPRN->ThunkOffsets[TPRN->ThunkOffsetCount++] = demangleSigned(MangledName); TPRN->ThunkOffsets[TPRN->ThunkOffsetCount++] = demangleSigned(MangledName); - LLVM_FALLTHROUGH; - case '0': break; default: - Error = true; - break; + DEMANGLE_UNREACHABLE; } TPRN->IsMemberPointer = true; @@ -2148,15 +2276,14 @@ Demangler::demangleTemplateParameterList(StringView &MangledName) { Current = &TP.Next; } - if (Error) - return nullptr; + // The loop above returns nullptr on Error. + assert(!Error); // Template parameter lists cannot be variadic, so it can only be terminated - // by @. - if (MangledName.consumeFront('@')) - return nodeListToNodeArray(Arena, Head, Count); - Error = true; - return nullptr; + // by @ (as opposed to 'Z' in the function parameter case). + assert(MangledName.startsWith('@')); // The above loop exits only on '@'. + MangledName.consumeFront('@'); + return nodeListToNodeArray(Arena, Head, Count); } void Demangler::dumpBackReferences() { diff --git a/lib/Demangle/MicrosoftDemangleNodes.cpp b/lib/Demangle/MicrosoftDemangleNodes.cpp index 622f8e75e351..63ca475ec1fe 100644 --- a/lib/Demangle/MicrosoftDemangleNodes.cpp +++ b/lib/Demangle/MicrosoftDemangleNodes.cpp @@ -1,9 +1,8 @@ //===- MicrosoftDemangle.cpp ----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is dual licensed under the MIT and the University of Illinois Open -// Source Licenses. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -12,7 +11,7 @@ //===----------------------------------------------------------------------===// #include "llvm/Demangle/MicrosoftDemangleNodes.h" -#include "llvm/Demangle/Compiler.h" +#include "llvm/Demangle/DemangleConfig.h" #include "llvm/Demangle/Utility.h" #include #include @@ -35,21 +34,20 @@ static void outputSpaceIfNecessary(OutputStream &OS) { OS << " "; } -static bool outputSingleQualifier(OutputStream &OS, Qualifiers Q) { +static void outputSingleQualifier(OutputStream &OS, Qualifiers Q) { switch (Q) { case Q_Const: OS << "const"; - return true; + break; case Q_Volatile: OS << "volatile"; - return true; + break; case Q_Restrict: OS << "__restrict"; - return true; + break; default: break; } - return false; } static bool outputQualifierIfPresent(OutputStream &OS, Qualifiers Q, @@ -131,6 +129,7 @@ void PrimitiveTypeNode::outputPre(OutputStream &OS, OutputFlags Flags) const { OUTPUT_ENUM_CLASS_VALUE(PrimitiveKind, Char, "char"); OUTPUT_ENUM_CLASS_VALUE(PrimitiveKind, Schar, "signed char"); OUTPUT_ENUM_CLASS_VALUE(PrimitiveKind, Uchar, "unsigned char"); + OUTPUT_ENUM_CLASS_VALUE(PrimitiveKind, Char8, "char8_t"); OUTPUT_ENUM_CLASS_VALUE(PrimitiveKind, Char16, "char16_t"); OUTPUT_ENUM_CLASS_VALUE(PrimitiveKind, Char32, "char32_t"); OUTPUT_ENUM_CLASS_VALUE(PrimitiveKind, Short, "short"); @@ -338,8 +337,9 @@ void IntrinsicFunctionIdentifierNode::output(OutputStream &OS, "`vector vbase copy constructor iterator'"); OUTPUT_ENUM_CLASS_VALUE(IntrinsicFunctionKind, ManVectorVbaseCopyCtorIter, "`managed vector vbase copy constructor iterator'"); - OUTPUT_ENUM_CLASS_VALUE(IntrinsicFunctionKind, CoAwait, "co_await"); - OUTPUT_ENUM_CLASS_VALUE(IntrinsicFunctionKind, Spaceship, "operator <=>"); + OUTPUT_ENUM_CLASS_VALUE(IntrinsicFunctionKind, CoAwait, + "operator co_await"); + OUTPUT_ENUM_CLASS_VALUE(IntrinsicFunctionKind, Spaceship, "operator<=>"); case IntrinsicFunctionKind::MaxIntrinsic: case IntrinsicFunctionKind::None: break; @@ -349,7 +349,10 @@ void IntrinsicFunctionIdentifierNode::output(OutputStream &OS, void LocalStaticGuardIdentifierNode::output(OutputStream &OS, OutputFlags Flags) const { - OS << "`local static guard'"; + if (IsThread) + OS << "`local static thread guard'"; + else + OS << "`local static guard'"; if (ScopeIndex > 0) OS << "{" << ScopeIndex << "}"; } @@ -411,6 +414,12 @@ void FunctionSignatureNode::outputPost(OutputStream &OS, Params->output(OS, Flags); else OS << "void"; + + if (IsVariadic) { + if (OS.back() != '(') + OS << ", "; + OS << "..."; + } OS << ")"; } diff --git a/lib/ExecutionEngine/ExecutionEngine.cpp b/lib/ExecutionEngine/ExecutionEngine.cpp index ae96c7f5955f..1c6c0406d048 100644 --- a/lib/ExecutionEngine/ExecutionEngine.cpp +++ b/lib/ExecutionEngine/ExecutionEngine.cpp @@ -1,9 +1,8 @@ //===-- ExecutionEngine.cpp - Common Implementation shared by EEs ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -1020,32 +1019,6 @@ GenericValue ExecutionEngine::getConstantValue(const Constant *C) { return Result; } -/// StoreIntToMemory - Fills the StoreBytes bytes of memory starting from Dst -/// with the integer held in IntVal. -static void StoreIntToMemory(const APInt &IntVal, uint8_t *Dst, - unsigned StoreBytes) { - assert((IntVal.getBitWidth()+7)/8 >= StoreBytes && "Integer too small!"); - const uint8_t *Src = (const uint8_t *)IntVal.getRawData(); - - if (sys::IsLittleEndianHost) { - // Little-endian host - the source is ordered from LSB to MSB. Order the - // destination from LSB to MSB: Do a straight copy. - memcpy(Dst, Src, StoreBytes); - } else { - // Big-endian host - the source is an array of 64 bit words ordered from - // LSW to MSW. Each word is ordered from MSB to LSB. Order the destination - // from MSB to LSB: Reverse the word order, but not the bytes in a word. - while (StoreBytes > sizeof(uint64_t)) { - StoreBytes -= sizeof(uint64_t); - // May not be aligned so use memcpy. - memcpy(Dst + StoreBytes, Src, sizeof(uint64_t)); - Src += sizeof(uint64_t); - } - - memcpy(Dst, Src + sizeof(uint64_t) - StoreBytes, StoreBytes); - } -} - void ExecutionEngine::StoreValueToMemory(const GenericValue &Val, GenericValue *Ptr, Type *Ty) { const unsigned StoreBytes = getDataLayout().getTypeStoreSize(Ty); @@ -1093,33 +1066,6 @@ void ExecutionEngine::StoreValueToMemory(const GenericValue &Val, std::reverse((uint8_t*)Ptr, StoreBytes + (uint8_t*)Ptr); } -/// LoadIntFromMemory - Loads the integer stored in the LoadBytes bytes starting -/// from Src into IntVal, which is assumed to be wide enough and to hold zero. -static void LoadIntFromMemory(APInt &IntVal, uint8_t *Src, unsigned LoadBytes) { - assert((IntVal.getBitWidth()+7)/8 >= LoadBytes && "Integer too small!"); - uint8_t *Dst = reinterpret_cast( - const_cast(IntVal.getRawData())); - - if (sys::IsLittleEndianHost) - // Little-endian host - the destination must be ordered from LSB to MSB. - // The source is ordered from LSB to MSB: Do a straight copy. - memcpy(Dst, Src, LoadBytes); - else { - // Big-endian - the destination is an array of 64 bit words ordered from - // LSW to MSW. Each word must be ordered from MSB to LSB. The source is - // ordered from MSB to LSB: Reverse the word order, but not the bytes in - // a word. - while (LoadBytes > sizeof(uint64_t)) { - LoadBytes -= sizeof(uint64_t); - // May not be aligned so use memcpy. - memcpy(Dst, Src + LoadBytes, sizeof(uint64_t)); - Dst += sizeof(uint64_t); - } - - memcpy(Dst + sizeof(uint64_t) - LoadBytes, Src, LoadBytes); - } -} - /// FIXME: document /// void ExecutionEngine::LoadValueFromMemory(GenericValue &Result, diff --git a/lib/ExecutionEngine/ExecutionEngineBindings.cpp b/lib/ExecutionEngine/ExecutionEngineBindings.cpp index 3be4bec566a0..c741fe2b3778 100644 --- a/lib/ExecutionEngine/ExecutionEngineBindings.cpp +++ b/lib/ExecutionEngine/ExecutionEngineBindings.cpp @@ -1,9 +1,8 @@ //===-- ExecutionEngineBindings.cpp - C bindings for EEs ------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ExecutionEngine/GDBRegistrationListener.cpp b/lib/ExecutionEngine/GDBRegistrationListener.cpp index 8204f5a90268..08d20156a590 100644 --- a/lib/ExecutionEngine/GDBRegistrationListener.cpp +++ b/lib/ExecutionEngine/GDBRegistrationListener.cpp @@ -1,9 +1,8 @@ //===----- GDBRegistrationListener.cpp - Registers objects with GDB -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp b/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp index e9051c198506..1ebc820a8b49 100644 --- a/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp +++ b/lib/ExecutionEngine/IntelJITEvents/IntelJITEventListener.cpp @@ -1,9 +1,8 @@ //===-- IntelJITEventListener.cpp - Tell Intel profiler about JITed code --===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -142,13 +141,25 @@ void IntelJITEventListener::notifyObjectLoaded( uint64_t Addr = *AddrOrErr; uint64_t Size = P.second; + auto SecOrErr = Sym.getSection(); + if (!SecOrErr) { + // TODO: Actually report errors helpfully. + consumeError(SecOrErr.takeError()); + continue; + } + object::section_iterator Sec = *SecOrErr; + if (Sec == Obj.section_end()) + continue; + uint64_t Index = Sec->getIndex(); + // Record this address in a local vector Functions.push_back((void*)Addr); // Build the function loaded notification message iJIT_Method_Load FunctionMessage = FunctionDescToIntelJITFormat(*Wrapper, Name->data(), Addr, Size); - DILineInfoTable Lines = Context->getLineInfoForAddressRange(Addr, Size); + DILineInfoTable Lines = + Context->getLineInfoForAddressRange({Addr, Index}, Size); DILineInfoTable::iterator Begin = Lines.begin(); DILineInfoTable::iterator End = Lines.end(); for (DILineInfoTable::iterator It = Begin; It != End; ++It) { diff --git a/lib/ExecutionEngine/IntelJITEvents/IntelJITEventsWrapper.h b/lib/ExecutionEngine/IntelJITEvents/IntelJITEventsWrapper.h index 777d0f179cb5..68699c6a2200 100644 --- a/lib/ExecutionEngine/IntelJITEvents/IntelJITEventsWrapper.h +++ b/lib/ExecutionEngine/IntelJITEvents/IntelJITEventsWrapper.h @@ -1,9 +1,8 @@ //===-- IntelJITEventsWrapper.h - Intel JIT Events API Wrapper --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ExecutionEngine/IntelJITEvents/ittnotify_config.h b/lib/ExecutionEngine/IntelJITEvents/ittnotify_config.h index 61d8cc75d9f2..16ce672150cc 100644 --- a/lib/ExecutionEngine/IntelJITEvents/ittnotify_config.h +++ b/lib/ExecutionEngine/IntelJITEvents/ittnotify_config.h @@ -1,9 +1,8 @@ /*===-- ittnotify_config.h - JIT Profiling API internal config-----*- C -*-===* * - * The LLVM Compiler Infrastructure - * - * This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * *===----------------------------------------------------------------------===* * diff --git a/lib/ExecutionEngine/IntelJITEvents/ittnotify_types.h b/lib/ExecutionEngine/IntelJITEvents/ittnotify_types.h index 5df752f66f10..15008fe93e60 100644 --- a/lib/ExecutionEngine/IntelJITEvents/ittnotify_types.h +++ b/lib/ExecutionEngine/IntelJITEvents/ittnotify_types.h @@ -1,9 +1,8 @@ /*===-- ittnotify_types.h - JIT Profiling API internal types--------*- C -*-===* * - * The LLVM Compiler Infrastructure - * - * This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * *===----------------------------------------------------------------------===* * diff --git a/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c b/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c index bc8fea148749..074e0735628a 100644 --- a/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c +++ b/lib/ExecutionEngine/IntelJITEvents/jitprofiling.c @@ -1,9 +1,8 @@ /*===-- jitprofiling.c - JIT (Just-In-Time) Profiling API----------*- C -*-===* * - * The LLVM Compiler Infrastructure - * - * This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * *===----------------------------------------------------------------------===* * diff --git a/lib/ExecutionEngine/IntelJITEvents/jitprofiling.h b/lib/ExecutionEngine/IntelJITEvents/jitprofiling.h index efd2b1a33f75..ba627b430ff1 100644 --- a/lib/ExecutionEngine/IntelJITEvents/jitprofiling.h +++ b/lib/ExecutionEngine/IntelJITEvents/jitprofiling.h @@ -1,9 +1,8 @@ /*===-- jitprofiling.h - JIT Profiling API-------------------------*- C -*-===* * - * The LLVM Compiler Infrastructure - * - * This file is distributed under the University of Illinois Open Source - * License. See LICENSE.TXT for details. + * Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. + * See https://llvm.org/LICENSE.txt for license information. + * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception * *===----------------------------------------------------------------------===* * diff --git a/lib/ExecutionEngine/Interpreter/Execution.cpp b/lib/ExecutionEngine/Interpreter/Execution.cpp index 98dca1102759..51f31d3d5d8f 100644 --- a/lib/ExecutionEngine/Interpreter/Execution.cpp +++ b/lib/ExecutionEngine/Interpreter/Execution.cpp @@ -1,9 +1,8 @@ //===-- Execution.cpp - Implement code to simulate the program ------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -43,6 +42,60 @@ static void SetValue(Value *V, GenericValue Val, ExecutionContext &SF) { SF.Values[V] = Val; } +//===----------------------------------------------------------------------===// +// Unary Instruction Implementations +//===----------------------------------------------------------------------===// + +static void executeFNegInst(GenericValue &Dest, GenericValue Src, Type *Ty) { + switch (Ty->getTypeID()) { + case Type::FloatTyID: + Dest.FloatVal = -Src.FloatVal; + break; + case Type::DoubleTyID: + Dest.DoubleVal = -Src.DoubleVal; + break; + default: + llvm_unreachable("Unhandled type for FNeg instruction"); + } +} + +void Interpreter::visitUnaryOperator(UnaryOperator &I) { + ExecutionContext &SF = ECStack.back(); + Type *Ty = I.getOperand(0)->getType(); + GenericValue Src = getOperandValue(I.getOperand(0), SF); + GenericValue R; // Result + + // First process vector operation + if (Ty->isVectorTy()) { + R.AggregateVal.resize(Src.AggregateVal.size()); + + switch(I.getOpcode()) { + default: + llvm_unreachable("Don't know how to handle this unary operator"); + break; + case Instruction::FNeg: + if (cast(Ty)->getElementType()->isFloatTy()) { + for (unsigned i = 0; i < R.AggregateVal.size(); ++i) + R.AggregateVal[i].FloatVal = -Src.AggregateVal[i].FloatVal; + } else if (cast(Ty)->getElementType()->isDoubleTy()) { + for (unsigned i = 0; i < R.AggregateVal.size(); ++i) + R.AggregateVal[i].DoubleVal = -Src.AggregateVal[i].DoubleVal; + } else { + llvm_unreachable("Unhandled type for FNeg instruction"); + } + break; + } + } else { + switch (I.getOpcode()) { + default: + llvm_unreachable("Don't know how to handle this unary operator"); + break; + case Instruction::FNeg: executeFNegInst(R, Src, Ty); break; + } + } + SetValue(&I, R, SF); +} + //===----------------------------------------------------------------------===// // Binary Instruction Implementations //===----------------------------------------------------------------------===// @@ -2113,7 +2166,7 @@ void Interpreter::run() { // Track the number of dynamic instructions executed. ++NumDynamicInsts; - LLVM_DEBUG(dbgs() << "About to interpret: " << I); + LLVM_DEBUG(dbgs() << "About to interpret: " << I << "\n"); visit(I); // Dispatch to one of the visit* methods... } } diff --git a/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp b/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp index 334fcacf8078..c3a2ccc582c9 100644 --- a/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp +++ b/lib/ExecutionEngine/Interpreter/ExternalFunctions.cpp @@ -1,9 +1,8 @@ //===-- ExternalFunctions.cpp - Implement External Functions --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ExecutionEngine/Interpreter/Interpreter.cpp b/lib/ExecutionEngine/Interpreter/Interpreter.cpp index 9818adfff82e..5727f7adb49c 100644 --- a/lib/ExecutionEngine/Interpreter/Interpreter.cpp +++ b/lib/ExecutionEngine/Interpreter/Interpreter.cpp @@ -1,9 +1,8 @@ //===- Interpreter.cpp - Top-Level LLVM Interpreter Implementation --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ExecutionEngine/Interpreter/Interpreter.h b/lib/ExecutionEngine/Interpreter/Interpreter.h index 33542e7e43ad..e72d778317d6 100644 --- a/lib/ExecutionEngine/Interpreter/Interpreter.h +++ b/lib/ExecutionEngine/Interpreter/Interpreter.h @@ -1,9 +1,8 @@ //===-- Interpreter.h ------------------------------------------*- C++ -*--===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -125,6 +124,7 @@ public: void visitSwitchInst(SwitchInst &I); void visitIndirectBrInst(IndirectBrInst &I); + void visitUnaryOperator(UnaryOperator &I); void visitBinaryOperator(BinaryOperator &I); void visitICmpInst(ICmpInst &I); void visitFCmpInst(FCmpInst &I); diff --git a/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h b/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h new file mode 100644 index 000000000000..1271ad962b38 --- /dev/null +++ b/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h @@ -0,0 +1,82 @@ +//===--- BasicGOTAndStubsBuilder.h - Generic GOT/Stub creation --*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// A base for simple GOT and stub creation. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_EXECUTIONENGINE_JITLINK_BASICGOTANDSTUBSBUILDER_H +#define LLVM_LIB_EXECUTIONENGINE_JITLINK_BASICGOTANDSTUBSBUILDER_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +namespace llvm { +namespace jitlink { + +template class BasicGOTAndStubsBuilder { +public: + BasicGOTAndStubsBuilder(AtomGraph &G) : G(G) {} + + void run() { + // We're going to be adding new atoms, but we don't want to iterate over + // the newly added ones, so just copy the existing atoms out. + std::vector DAs(G.defined_atoms().begin(), + G.defined_atoms().end()); + + for (auto *DA : DAs) + for (auto &E : DA->edges()) + if (impl().isGOTEdge(E)) + impl().fixGOTEdge(E, getGOTEntryAtom(E.getTarget())); + else if (impl().isExternalBranchEdge(E)) + impl().fixExternalBranchEdge(E, getStubAtom(E.getTarget())); + } + +protected: + Atom &getGOTEntryAtom(Atom &Target) { + assert(Target.hasName() && "GOT edge cannot point to anonymous target"); + + auto GOTEntryI = GOTEntries.find(Target.getName()); + + // Build the entry if it doesn't exist. + if (GOTEntryI == GOTEntries.end()) { + auto &GOTEntry = impl().createGOTEntry(Target); + GOTEntryI = + GOTEntries.insert(std::make_pair(Target.getName(), &GOTEntry)).first; + } + + assert(GOTEntryI != GOTEntries.end() && "Could not get GOT entry atom"); + return *GOTEntryI->second; + } + + Atom &getStubAtom(Atom &Target) { + assert(Target.hasName() && + "External branch edge can not point to an anonymous target"); + auto StubI = Stubs.find(Target.getName()); + + if (StubI == Stubs.end()) { + auto &StubAtom = impl().createStub(Target); + StubI = Stubs.insert(std::make_pair(Target.getName(), &StubAtom)).first; + } + + assert(StubI != Stubs.end() && "Count not get stub atom"); + return *StubI->second; + } + + AtomGraph &G; + +private: + BuilderImpl &impl() { return static_cast(*this); } + + DenseMap GOTEntries; + DenseMap Stubs; +}; + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_LIB_EXECUTIONENGINE_JITLINK_BASICGOTANDSTUBSBUILDER_H diff --git a/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp new file mode 100644 index 000000000000..25f0e9040ffe --- /dev/null +++ b/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp @@ -0,0 +1,544 @@ +//===-------- JITLink_EHFrameSupport.cpp - JITLink eh-frame utils ---------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "EHFrameSupportImpl.h" + +#include "llvm/BinaryFormat/Dwarf.h" +#include "llvm/Support/DynamicLibrary.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +EHFrameParser::EHFrameParser(AtomGraph &G, Section &EHFrameSection, + StringRef EHFrameContent, + JITTargetAddress EHFrameAddress, + Edge::Kind FDEToCIERelocKind, + Edge::Kind FDEToTargetRelocKind) + : G(G), EHFrameSection(EHFrameSection), EHFrameContent(EHFrameContent), + EHFrameAddress(EHFrameAddress), + EHFrameReader(EHFrameContent, G.getEndianness()), + FDEToCIERelocKind(FDEToCIERelocKind), + FDEToTargetRelocKind(FDEToTargetRelocKind) {} + +Error EHFrameParser::atomize() { + while (!EHFrameReader.empty()) { + size_t RecordOffset = EHFrameReader.getOffset(); + + LLVM_DEBUG({ + dbgs() << "Processing eh-frame record at " + << format("0x%016" PRIx64, EHFrameAddress + RecordOffset) + << " (offset " << RecordOffset << ")\n"; + }); + + size_t CIELength = 0; + uint32_t CIELengthField; + if (auto Err = EHFrameReader.readInteger(CIELengthField)) + return Err; + + // Process CIE length/extended-length fields to build the atom. + // + // The value of these fields describe the length of the *rest* of the CIE + // (not including data up to the end of the field itself) so we have to + // bump CIELength to include the data up to the end of the field: 4 bytes + // for Length, or 12 bytes (4 bytes + 8 bytes) for ExtendedLength. + if (CIELengthField == 0) // Length 0 means end of __eh_frame section. + break; + + // If the regular length field's value is 0xffffffff, use extended length. + if (CIELengthField == 0xffffffff) { + uint64_t CIEExtendedLengthField; + if (auto Err = EHFrameReader.readInteger(CIEExtendedLengthField)) + return Err; + if (CIEExtendedLengthField > EHFrameReader.bytesRemaining()) + return make_error("CIE record extends past the end of " + "the __eh_frame section"); + if (CIEExtendedLengthField + 12 > std::numeric_limits::max()) + return make_error("CIE record too large to process"); + CIELength = CIEExtendedLengthField + 12; + } else { + if (CIELengthField > EHFrameReader.bytesRemaining()) + return make_error("CIE record extends past the end of " + "the __eh_frame section"); + CIELength = CIELengthField + 4; + } + + LLVM_DEBUG(dbgs() << " length: " << CIELength << "\n"); + + // Add an atom for this record. + CurRecordAtom = &G.addAnonymousAtom( + EHFrameSection, EHFrameAddress + RecordOffset, G.getPointerSize()); + CurRecordAtom->setContent(EHFrameContent.substr(RecordOffset, CIELength)); + + // Read the CIE Pointer. + size_t CIEPointerAddress = EHFrameAddress + EHFrameReader.getOffset(); + uint32_t CIEPointer; + if (auto Err = EHFrameReader.readInteger(CIEPointer)) + return Err; + + // Based on the CIE pointer value, parse this as a CIE or FDE record. + if (CIEPointer == 0) { + if (auto Err = processCIE()) + return Err; + } else { + if (auto Err = processFDE(CIEPointerAddress, CIEPointer)) + return Err; + } + + EHFrameReader.setOffset(RecordOffset + CIELength); + } + + return Error::success(); +} + +Expected +EHFrameParser::parseAugmentationString() { + AugmentationInfo AugInfo; + uint8_t NextChar; + uint8_t *NextField = &AugInfo.Fields[0]; + + if (auto Err = EHFrameReader.readInteger(NextChar)) + return std::move(Err); + + while (NextChar != 0) { + switch (NextChar) { + case 'z': + AugInfo.AugmentationDataPresent = true; + break; + case 'e': + if (auto Err = EHFrameReader.readInteger(NextChar)) + return std::move(Err); + if (NextChar != 'h') + return make_error("Unrecognized substring e" + + Twine(NextChar) + + " in augmentation string"); + AugInfo.EHDataFieldPresent = true; + break; + case 'L': + case 'P': + case 'R': + *NextField++ = NextChar; + break; + default: + return make_error("Unrecognized character " + + Twine(NextChar) + + " in augmentation string"); + } + + if (auto Err = EHFrameReader.readInteger(NextChar)) + return std::move(Err); + } + + return std::move(AugInfo); +} + +Expected EHFrameParser::readAbsolutePointer() { + static_assert(sizeof(JITTargetAddress) == sizeof(uint64_t), + "Result must be able to hold a uint64_t"); + JITTargetAddress Addr; + if (G.getPointerSize() == 8) { + if (auto Err = EHFrameReader.readInteger(Addr)) + return std::move(Err); + } else if (G.getPointerSize() == 4) { + uint32_t Addr32; + if (auto Err = EHFrameReader.readInteger(Addr32)) + return std::move(Err); + Addr = Addr32; + } else + llvm_unreachable("Pointer size is not 32-bit or 64-bit"); + return Addr; +} + +Error EHFrameParser::processCIE() { + // Use the dwarf namespace for convenient access to pointer encoding + // constants. + using namespace dwarf; + + LLVM_DEBUG(dbgs() << " Record is CIE\n"); + + CIEInformation CIEInfo(*CurRecordAtom); + + uint8_t Version = 0; + if (auto Err = EHFrameReader.readInteger(Version)) + return Err; + + if (Version != 0x01) + return make_error("Bad CIE version " + Twine(Version) + + " (should be 0x01) in eh-frame"); + + auto AugInfo = parseAugmentationString(); + if (!AugInfo) + return AugInfo.takeError(); + + // Skip the EH Data field if present. + if (AugInfo->EHDataFieldPresent) + if (auto Err = EHFrameReader.skip(G.getPointerSize())) + return Err; + + // Read and sanity check the code alignment factor. + { + uint64_t CodeAlignmentFactor = 0; + if (auto Err = EHFrameReader.readULEB128(CodeAlignmentFactor)) + return Err; + if (CodeAlignmentFactor != 1) + return make_error("Unsupported CIE code alignment factor " + + Twine(CodeAlignmentFactor) + + " (expected 1)"); + } + + // Read and sanity check the data alignment factor. + { + int64_t DataAlignmentFactor = 0; + if (auto Err = EHFrameReader.readSLEB128(DataAlignmentFactor)) + return Err; + if (DataAlignmentFactor != -8) + return make_error("Unsupported CIE data alignment factor " + + Twine(DataAlignmentFactor) + + " (expected -8)"); + } + + // Skip the return address register field. + if (auto Err = EHFrameReader.skip(1)) + return Err; + + uint64_t AugmentationDataLength = 0; + if (auto Err = EHFrameReader.readULEB128(AugmentationDataLength)) + return Err; + + uint32_t AugmentationDataStartOffset = EHFrameReader.getOffset(); + + uint8_t *NextField = &AugInfo->Fields[0]; + while (uint8_t Field = *NextField++) { + switch (Field) { + case 'L': { + CIEInfo.FDEsHaveLSDAField = true; + uint8_t LSDAPointerEncoding; + if (auto Err = EHFrameReader.readInteger(LSDAPointerEncoding)) + return Err; + if (LSDAPointerEncoding != (DW_EH_PE_pcrel | DW_EH_PE_absptr)) + return make_error( + "Unsupported LSDA pointer encoding " + + formatv("{0:x2}", LSDAPointerEncoding) + " in CIE at " + + formatv("{0:x16}", CurRecordAtom->getAddress())); + break; + } + case 'P': { + uint8_t PersonalityPointerEncoding = 0; + if (auto Err = EHFrameReader.readInteger(PersonalityPointerEncoding)) + return Err; + if (PersonalityPointerEncoding != + (DW_EH_PE_indirect | DW_EH_PE_pcrel | DW_EH_PE_sdata4)) + return make_error( + "Unspported personality pointer " + "encoding " + + formatv("{0:x2}", PersonalityPointerEncoding) + " in CIE at " + + formatv("{0:x16}", CurRecordAtom->getAddress())); + uint32_t PersonalityPointerAddress; + if (auto Err = EHFrameReader.readInteger(PersonalityPointerAddress)) + return Err; + break; + } + case 'R': { + uint8_t FDEPointerEncoding; + if (auto Err = EHFrameReader.readInteger(FDEPointerEncoding)) + return Err; + if (FDEPointerEncoding != (DW_EH_PE_pcrel | DW_EH_PE_absptr)) + return make_error( + "Unsupported FDE address pointer " + "encoding " + + formatv("{0:x2}", FDEPointerEncoding) + " in CIE at " + + formatv("{0:x16}", CurRecordAtom->getAddress())); + break; + } + default: + llvm_unreachable("Invalid augmentation string field"); + } + } + + if (EHFrameReader.getOffset() - AugmentationDataStartOffset > + AugmentationDataLength) + return make_error("Read past the end of the augmentation " + "data while parsing fields"); + + assert(!CIEInfos.count(CurRecordAtom->getAddress()) && + "Multiple CIEs recorded at the same address?"); + CIEInfos[CurRecordAtom->getAddress()] = std::move(CIEInfo); + + return Error::success(); +} + +Error EHFrameParser::processFDE(JITTargetAddress CIEPointerAddress, + uint32_t CIEPointer) { + LLVM_DEBUG(dbgs() << " Record is FDE\n"); + + LLVM_DEBUG({ + dbgs() << " CIE pointer: " + << format("0x%016" PRIx64, CIEPointerAddress - CIEPointer) << "\n"; + }); + + auto CIEInfoItr = CIEInfos.find(CIEPointerAddress - CIEPointer); + if (CIEInfoItr == CIEInfos.end()) + return make_error( + "FDE at " + formatv("{0:x16}", CurRecordAtom->getAddress()) + + " points to non-existant CIE at " + + formatv("{0:x16}", CIEPointerAddress - CIEPointer)); + auto &CIEInfo = CIEInfoItr->second; + + // The CIEPointer looks good. Add a relocation. + CurRecordAtom->addEdge(FDEToCIERelocKind, + CIEPointerAddress - CurRecordAtom->getAddress(), + *CIEInfo.CIEAtom, 0); + + // Read and sanity check the PC-start pointer and size. + JITTargetAddress PCBeginAddress = EHFrameAddress + EHFrameReader.getOffset(); + + auto PCBeginDelta = readAbsolutePointer(); + if (!PCBeginDelta) + return PCBeginDelta.takeError(); + + JITTargetAddress PCBegin = PCBeginAddress + *PCBeginDelta; + LLVM_DEBUG({ + dbgs() << " PC begin: " << format("0x%016" PRIx64, PCBegin) << "\n"; + }); + + auto *TargetAtom = G.getAtomByAddress(PCBegin); + + if (!TargetAtom) + return make_error("FDE PC-begin " + + formatv("{0:x16}", PCBegin) + + " does not point at atom"); + + if (TargetAtom->getAddress() != PCBegin) + return make_error( + "FDE PC-begin " + formatv("{0:x16}", PCBegin) + + " does not point to start of atom at " + + formatv("{0:x16}", TargetAtom->getAddress())); + + LLVM_DEBUG(dbgs() << " FDE target: " << *TargetAtom << "\n"); + + // The PC-start pointer and size look good. Add relocations. + CurRecordAtom->addEdge(FDEToTargetRelocKind, + PCBeginAddress - CurRecordAtom->getAddress(), + *TargetAtom, 0); + + // Add a keep-alive relocation from the function to the FDE to ensure it is + // not dead stripped. + TargetAtom->addEdge(Edge::KeepAlive, 0, *CurRecordAtom, 0); + + // Skip over the PC range size field. + if (auto Err = EHFrameReader.skip(G.getPointerSize())) + return Err; + + if (CIEInfo.FDEsHaveLSDAField) { + uint64_t AugmentationDataSize; + if (auto Err = EHFrameReader.readULEB128(AugmentationDataSize)) + return Err; + if (AugmentationDataSize != G.getPointerSize()) + return make_error( + "Unexpected FDE augmentation data size (expected " + + Twine(G.getPointerSize()) + ", got " + Twine(AugmentationDataSize) + + ") for FDE at " + formatv("{0:x16}", CurRecordAtom->getAddress())); + JITTargetAddress LSDAAddress = EHFrameAddress + EHFrameReader.getOffset(); + auto LSDADelta = readAbsolutePointer(); + if (!LSDADelta) + return LSDADelta.takeError(); + + JITTargetAddress LSDA = LSDAAddress + *LSDADelta; + + auto *LSDAAtom = G.getAtomByAddress(LSDA); + + if (!LSDAAtom) + return make_error("FDE LSDA " + formatv("{0:x16}", LSDA) + + " does not point at atom"); + + if (LSDAAtom->getAddress() != LSDA) + return make_error( + "FDE LSDA " + formatv("{0:x16}", LSDA) + + " does not point to start of atom at " + + formatv("{0:x16}", LSDAAtom->getAddress())); + + LLVM_DEBUG(dbgs() << " FDE LSDA: " << *LSDAAtom << "\n"); + + // LSDA looks good. Add relocations. + CurRecordAtom->addEdge(FDEToTargetRelocKind, + LSDAAddress - CurRecordAtom->getAddress(), *LSDAAtom, + 0); + } + + return Error::success(); +} + +Error addEHFrame(AtomGraph &G, Section &EHFrameSection, + StringRef EHFrameContent, JITTargetAddress EHFrameAddress, + Edge::Kind FDEToCIERelocKind, + Edge::Kind FDEToTargetRelocKind) { + return EHFrameParser(G, EHFrameSection, EHFrameContent, EHFrameAddress, + FDEToCIERelocKind, FDEToTargetRelocKind) + .atomize(); +} + +// Determine whether we can register EH tables. +#if (defined(__GNUC__) && !defined(__ARM_EABI__) && !defined(__ia64__) && \ + !(defined(_AIX) && defined(__ibmxl__)) && !defined(__SEH__) && \ + !defined(__USING_SJLJ_EXCEPTIONS__)) +#define HAVE_EHTABLE_SUPPORT 1 +#else +#define HAVE_EHTABLE_SUPPORT 0 +#endif + +#if HAVE_EHTABLE_SUPPORT +extern "C" void __register_frame(const void *); +extern "C" void __deregister_frame(const void *); + +Error registerFrameWrapper(const void *P) { + __register_frame(P); + return Error::success(); +} + +Error deregisterFrameWrapper(const void *P) { + __deregister_frame(P); + return Error::success(); +} + +#else + +// The building compiler does not have __(de)register_frame but +// it may be found at runtime in a dynamically-loaded library. +// For example, this happens when building LLVM with Visual C++ +// but using the MingW runtime. +static Error registerFrameWrapper(const void *P) { + static void((*RegisterFrame)(const void *)) = 0; + + if (!RegisterFrame) + *(void **)&RegisterFrame = + llvm::sys::DynamicLibrary::SearchForAddressOfSymbol("__register_frame"); + + if (RegisterFrame) { + RegisterFrame(P); + return Error::success(); + } + + return make_error("could not register eh-frame: " + "__register_frame function not found"); +} + +static Error deregisterFrameWrapper(const void *P) { + static void((*DeregisterFrame)(const void *)) = 0; + + if (!DeregisterFrame) + *(void **)&DeregisterFrame = + llvm::sys::DynamicLibrary::SearchForAddressOfSymbol( + "__deregister_frame"); + + if (DeregisterFrame) { + DeregisterFrame(P); + return Error::success(); + } + + return make_error("could not deregister eh-frame: " + "__deregister_frame function not found"); +} +#endif + +#ifdef __APPLE__ + +template +Error walkAppleEHFrameSection(const char *const SectionStart, + HandleFDEFn HandleFDE) { + const char *CurCFIRecord = SectionStart; + uint64_t Size = *reinterpret_cast(CurCFIRecord); + + while (Size != 0) { + const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4); + if (Size == 0xffffffff) + Size = *reinterpret_cast(CurCFIRecord + 4) + 12; + else + Size += 4; + uint32_t Offset = *reinterpret_cast(OffsetField); + if (Offset != 0) + if (auto Err = HandleFDE(CurCFIRecord)) + return Err; + + LLVM_DEBUG({ + dbgs() << "Registering eh-frame section:\n"; + dbgs() << "Processing " << (Offset ? "FDE" : "CIE") << " @" + << (void *)CurCFIRecord << ": ["; + for (unsigned I = 0; I < Size; ++I) + dbgs() << format(" 0x%02" PRIx8, *(CurCFIRecord + I)); + dbgs() << " ]\n"; + }); + CurCFIRecord += Size; + + Size = *reinterpret_cast(CurCFIRecord); + } + + return Error::success(); +} + +#endif // __APPLE__ + +Error registerEHFrameSection(const void *EHFrameSectionAddr) { +#ifdef __APPLE__ + // On Darwin __register_frame has to be called for each FDE entry. + return walkAppleEHFrameSection(static_cast(EHFrameSectionAddr), + registerFrameWrapper); +#else + // On Linux __register_frame takes a single argument: + // a pointer to the start of the .eh_frame section. + + // How can it find the end? Because crtendS.o is linked + // in and it has an .eh_frame section with four zero chars. + return registerFrameWrapper(EHFrameSectionAddr); +#endif +} + +Error deregisterEHFrameSection(const void *EHFrameSectionAddr) { +#ifdef __APPLE__ + return walkAppleEHFrameSection(static_cast(EHFrameSectionAddr), + deregisterFrameWrapper); +#else + return deregisterFrameWrapper(EHFrameSectionAddr); +#endif +} + +EHFrameRegistrar::~EHFrameRegistrar() {} + +InProcessEHFrameRegistrar &InProcessEHFrameRegistrar::getInstance() { + static InProcessEHFrameRegistrar Instance; + return Instance; +} + +InProcessEHFrameRegistrar::InProcessEHFrameRegistrar() {} + +AtomGraphPassFunction +createEHFrameRecorderPass(const Triple &TT, + StoreFrameAddressFunction StoreFrameAddress) { + const char *EHFrameSectionName = nullptr; + if (TT.getObjectFormat() == Triple::MachO) + EHFrameSectionName = "__eh_frame"; + else + EHFrameSectionName = ".eh_frame"; + + auto RecordEHFrame = [EHFrameSectionName, + StoreFrameAddress](AtomGraph &G) -> Error { + // Search for a non-empty eh-frame and record the address of the first atom + // in it. + JITTargetAddress Addr = 0; + if (auto *S = G.findSectionByName(EHFrameSectionName)) + Addr = S->getRange().getStart(); + StoreFrameAddress(Addr); + return Error::success(); + }; + + return RecordEHFrame; +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h b/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h new file mode 100644 index 000000000000..d679edef7ea6 --- /dev/null +++ b/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h @@ -0,0 +1,72 @@ +//===------- EHFrameSupportImpl.h - JITLink eh-frame utils ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// EHFrame registration support for JITLink. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_EXECUTIONENGINE_JITLINK_EHFRAMESUPPORTIMPL_H +#define LLVM_LIB_EXECUTIONENGINE_JITLINK_EHFRAMESUPPORTIMPL_H + +#include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/Support/BinaryStreamReader.h" + +namespace llvm { +namespace jitlink { + +/// A generic parser for eh-frame sections. +/// +/// Adds atoms representing CIE and FDE entries, using the given FDE-to-CIE and +/// FDEToTarget relocation kinds. +class EHFrameParser { +public: + EHFrameParser(AtomGraph &G, Section &EHFrameSection, StringRef EHFrameContent, + JITTargetAddress EHFrameAddress, Edge::Kind FDEToCIERelocKind, + Edge::Kind FDEToTargetRelocKind); + Error atomize(); + +private: + struct AugmentationInfo { + bool AugmentationDataPresent = false; + bool EHDataFieldPresent = false; + uint8_t Fields[4] = {0x0, 0x0, 0x0, 0x0}; + }; + + Expected parseAugmentationString(); + Expected readAbsolutePointer(); + Error processCIE(); + Error processFDE(JITTargetAddress CIEPointerAddress, uint32_t CIEPointer); + + struct CIEInformation { + CIEInformation() = default; + CIEInformation(DefinedAtom &CIEAtom) : CIEAtom(&CIEAtom) {} + DefinedAtom *CIEAtom = nullptr; + bool FDEsHaveLSDAField = false; + }; + + AtomGraph &G; + Section &EHFrameSection; + StringRef EHFrameContent; + JITTargetAddress EHFrameAddress; + BinaryStreamReader EHFrameReader; + DefinedAtom *CurRecordAtom = nullptr; + DenseMap CIEInfos; + Edge::Kind FDEToCIERelocKind; + Edge::Kind FDEToTargetRelocKind; +}; + +Error addEHFrame(AtomGraph &G, Section &EHFrameSection, + StringRef EHFrameContent, JITTargetAddress EHFrameAddress, + Edge::Kind FDEToCIERelocKind, Edge::Kind FDEToTargetRelocKind); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_LIB_EXECUTIONENGINE_JITLINK_EHFRAMESUPPORTIMPL_H diff --git a/lib/ExecutionEngine/JITLink/JITLink.cpp b/lib/ExecutionEngine/JITLink/JITLink.cpp new file mode 100644 index 000000000000..9d0a7459dc09 --- /dev/null +++ b/lib/ExecutionEngine/JITLink/JITLink.cpp @@ -0,0 +1,172 @@ +//===------------- JITLink.cpp - Core Run-time JIT linker APIs ------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +#include "llvm/BinaryFormat/Magic.h" +#include "llvm/ExecutionEngine/JITLink/MachO.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; +using namespace llvm::object; + +#define DEBUG_TYPE "jitlink" + +namespace { + +enum JITLinkErrorCode { GenericJITLinkError = 1 }; + +// FIXME: This class is only here to support the transition to llvm::Error. It +// will be removed once this transition is complete. Clients should prefer to +// deal with the Error value directly, rather than converting to error_code. +class JITLinkerErrorCategory : public std::error_category { +public: + const char *name() const noexcept override { return "runtimedyld"; } + + std::string message(int Condition) const override { + switch (static_cast(Condition)) { + case GenericJITLinkError: + return "Generic JITLink error"; + } + llvm_unreachable("Unrecognized JITLinkErrorCode"); + } +}; + +static ManagedStatic JITLinkerErrorCategory; + +} // namespace + +namespace llvm { +namespace jitlink { + +char JITLinkError::ID = 0; + +void JITLinkError::log(raw_ostream &OS) const { OS << ErrMsg << "\n"; } + +std::error_code JITLinkError::convertToErrorCode() const { + return std::error_code(GenericJITLinkError, *JITLinkerErrorCategory); +} + +const StringRef getGenericEdgeKindName(Edge::Kind K) { + switch (K) { + case Edge::Invalid: + return "INVALID RELOCATION"; + case Edge::KeepAlive: + return "Keep-Alive"; + case Edge::LayoutNext: + return "Layout-Next"; + default: + llvm_unreachable("Unrecognized relocation kind"); + } +} + +raw_ostream &operator<<(raw_ostream &OS, const Atom &A) { + OS << "<"; + if (A.getName().empty()) + OS << "anon@" << format("0x%016" PRIx64, A.getAddress()); + else + OS << A.getName(); + OS << " ["; + if (A.isDefined()) { + auto &DA = static_cast(A); + OS << " section=" << DA.getSection().getName(); + if (DA.isLive()) + OS << " live"; + if (DA.shouldDiscard()) + OS << " should-discard"; + } else + OS << " external"; + OS << " ]>"; + return OS; +} + +void printEdge(raw_ostream &OS, const Atom &FixupAtom, const Edge &E, + StringRef EdgeKindName) { + OS << "edge@" << formatv("{0:x16}", FixupAtom.getAddress() + E.getOffset()) + << ": " << FixupAtom << " + " << E.getOffset() << " -- " << EdgeKindName + << " -> " << E.getTarget() << " + " << E.getAddend(); +} + +Section::~Section() { + for (auto *DA : DefinedAtoms) + DA->~DefinedAtom(); +} + +void AtomGraph::dump(raw_ostream &OS, + std::function EdgeKindToName) { + if (!EdgeKindToName) + EdgeKindToName = [](Edge::Kind K) { return StringRef(); }; + + OS << "Defined atoms:\n"; + for (auto *DA : defined_atoms()) { + OS << " " << format("0x%016" PRIx64, DA->getAddress()) << ": " << *DA + << "\n"; + for (auto &E : DA->edges()) { + OS << " "; + StringRef EdgeName = (E.getKind() < Edge::FirstRelocation + ? getGenericEdgeKindName(E.getKind()) + : EdgeKindToName(E.getKind())); + + if (!EdgeName.empty()) + printEdge(OS, *DA, E, EdgeName); + else { + auto EdgeNumberString = std::to_string(E.getKind()); + printEdge(OS, *DA, E, EdgeNumberString); + } + OS << "\n"; + } + } + + OS << "Absolute atoms:\n"; + for (auto *A : absolute_atoms()) + OS << " " << format("0x%016" PRIx64, A->getAddress()) << ": " << *A + << "\n"; + + OS << "External atoms:\n"; + for (auto *A : external_atoms()) + OS << " " << format("0x%016" PRIx64, A->getAddress()) << ": " << *A + << "\n"; +} + +JITLinkContext::~JITLinkContext() {} + +bool JITLinkContext::shouldAddDefaultTargetPasses(const Triple &TT) const { + return true; +} + +AtomGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const { + return AtomGraphPassFunction(); +} + +Error JITLinkContext::modifyPassConfig(const Triple &TT, + PassConfiguration &Config) { + return Error::success(); +} + +Error markAllAtomsLive(AtomGraph &G) { + for (auto *DA : G.defined_atoms()) + DA->setLive(true); + return Error::success(); +} + +void jitLink(std::unique_ptr Ctx) { + auto Magic = identify_magic(Ctx->getObjectBuffer().getBuffer()); + switch (Magic) { + case file_magic::macho_object: + return jitLink_MachO(std::move(Ctx)); + default: + Ctx->notifyFailed(make_error("Unsupported file format")); + }; +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp new file mode 100644 index 000000000000..96e074da122b --- /dev/null +++ b/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp @@ -0,0 +1,481 @@ +//===--------- JITLinkGeneric.cpp - Generic JIT linker utilities ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic JITLinker utility class. +// +//===----------------------------------------------------------------------===// + +#include "JITLinkGeneric.h" +#include "EHFrameSupportImpl.h" + +#include "llvm/Support/BinaryStreamReader.h" +#include "llvm/Support/MemoryBuffer.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +JITLinkerBase::~JITLinkerBase() {} + +void JITLinkerBase::linkPhase1(std::unique_ptr Self) { + + // Build the atom graph. + if (auto GraphOrErr = buildGraph(Ctx->getObjectBuffer())) + G = std::move(*GraphOrErr); + else + return Ctx->notifyFailed(GraphOrErr.takeError()); + assert(G && "Graph should have been created by buildGraph above"); + + // Prune and optimize the graph. + if (auto Err = runPasses(Passes.PrePrunePasses, *G)) + return Ctx->notifyFailed(std::move(Err)); + + LLVM_DEBUG({ + dbgs() << "Atom graph \"" << G->getName() << "\" pre-pruning:\n"; + dumpGraph(dbgs()); + }); + + prune(*G); + + LLVM_DEBUG({ + dbgs() << "Atom graph \"" << G->getName() << "\" post-pruning:\n"; + dumpGraph(dbgs()); + }); + + // Run post-pruning passes. + if (auto Err = runPasses(Passes.PostPrunePasses, *G)) + return Ctx->notifyFailed(std::move(Err)); + + // Sort atoms into segments. + layOutAtoms(); + + // Allocate memory for segments. + if (auto Err = allocateSegments(Layout)) + return Ctx->notifyFailed(std::move(Err)); + + // Notify client that the defined atoms have been assigned addresses. + Ctx->notifyResolved(*G); + + auto ExternalSymbols = getExternalSymbolNames(); + + // We're about to hand off ownership of ourself to the continuation. Grab a + // pointer to the context so that we can call it to initiate the lookup. + // + // FIXME: Once callee expressions are defined to be sequenced before argument + // expressions (c++17) we can simplify all this to: + // + // Ctx->lookup(std::move(UnresolvedExternals), + // [Self=std::move(Self)](Expected Result) { + // Self->linkPhase2(std::move(Self), std::move(Result)); + // }); + // + // FIXME: Use move capture once we have c++14. + auto *TmpCtx = Ctx.get(); + auto *UnownedSelf = Self.release(); + auto Phase2Continuation = + [UnownedSelf](Expected LookupResult) { + std::unique_ptr Self(UnownedSelf); + UnownedSelf->linkPhase2(std::move(Self), std::move(LookupResult)); + }; + TmpCtx->lookup(std::move(ExternalSymbols), std::move(Phase2Continuation)); +} + +void JITLinkerBase::linkPhase2(std::unique_ptr Self, + Expected LR) { + // If the lookup failed, bail out. + if (!LR) + return deallocateAndBailOut(LR.takeError()); + + // Assign addresses to external atoms. + applyLookupResult(*LR); + + LLVM_DEBUG({ + dbgs() << "Atom graph \"" << G->getName() << "\" before copy-and-fixup:\n"; + dumpGraph(dbgs()); + }); + + // Copy atom content to working memory and fix up. + if (auto Err = copyAndFixUpAllAtoms(Layout, *Alloc)) + return deallocateAndBailOut(std::move(Err)); + + LLVM_DEBUG({ + dbgs() << "Atom graph \"" << G->getName() << "\" after copy-and-fixup:\n"; + dumpGraph(dbgs()); + }); + + if (auto Err = runPasses(Passes.PostFixupPasses, *G)) + return deallocateAndBailOut(std::move(Err)); + + // FIXME: Use move capture once we have c++14. + auto *UnownedSelf = Self.release(); + auto Phase3Continuation = [UnownedSelf](Error Err) { + std::unique_ptr Self(UnownedSelf); + UnownedSelf->linkPhase3(std::move(Self), std::move(Err)); + }; + + Alloc->finalizeAsync(std::move(Phase3Continuation)); +} + +void JITLinkerBase::linkPhase3(std::unique_ptr Self, Error Err) { + if (Err) + return deallocateAndBailOut(std::move(Err)); + Ctx->notifyFinalized(std::move(Alloc)); +} + +Error JITLinkerBase::runPasses(AtomGraphPassList &Passes, AtomGraph &G) { + for (auto &P : Passes) + if (auto Err = P(G)) + return Err; + return Error::success(); +} + +void JITLinkerBase::layOutAtoms() { + // Group sections by protections, and whether or not they're zero-fill. + for (auto &S : G->sections()) { + + // Skip empty sections. + if (S.atoms_empty()) + continue; + + auto &SL = Layout[S.getProtectionFlags()]; + if (S.isZeroFill()) + SL.ZeroFillSections.push_back(SegmentLayout::SectionLayout(S)); + else + SL.ContentSections.push_back(SegmentLayout::SectionLayout(S)); + } + + // Sort sections within the layout by ordinal. + { + auto CompareByOrdinal = [](const SegmentLayout::SectionLayout &LHS, + const SegmentLayout::SectionLayout &RHS) { + return LHS.S->getSectionOrdinal() < RHS.S->getSectionOrdinal(); + }; + for (auto &KV : Layout) { + auto &SL = KV.second; + std::sort(SL.ContentSections.begin(), SL.ContentSections.end(), + CompareByOrdinal); + std::sort(SL.ZeroFillSections.begin(), SL.ZeroFillSections.end(), + CompareByOrdinal); + } + } + + // Add atoms to the sections. + for (auto &KV : Layout) { + auto &SL = KV.second; + for (auto *SIList : {&SL.ContentSections, &SL.ZeroFillSections}) { + for (auto &SI : *SIList) { + // First build the set of layout-heads (i.e. "heads" of layout-next + // chains) by copying the section atoms, then eliminating any that + // appear as layout-next targets. + DenseSet LayoutHeads; + for (auto *DA : SI.S->atoms()) + LayoutHeads.insert(DA); + + for (auto *DA : SI.S->atoms()) + if (DA->hasLayoutNext()) + LayoutHeads.erase(&DA->getLayoutNext()); + + // Next, sort the layout heads by address order. + std::vector OrderedLayoutHeads; + OrderedLayoutHeads.reserve(LayoutHeads.size()); + for (auto *DA : LayoutHeads) + OrderedLayoutHeads.push_back(DA); + + // Now sort the list of layout heads by address. + std::sort(OrderedLayoutHeads.begin(), OrderedLayoutHeads.end(), + [](const DefinedAtom *LHS, const DefinedAtom *RHS) { + return LHS->getAddress() < RHS->getAddress(); + }); + + // Now populate the SI.Atoms field by appending each of the chains. + for (auto *DA : OrderedLayoutHeads) { + SI.Atoms.push_back(DA); + while (DA->hasLayoutNext()) { + auto &Next = DA->getLayoutNext(); + SI.Atoms.push_back(&Next); + DA = &Next; + } + } + } + } + } + + LLVM_DEBUG({ + dbgs() << "Segment ordering:\n"; + for (auto &KV : Layout) { + dbgs() << " Segment " + << static_cast(KV.first) << ":\n"; + auto &SL = KV.second; + for (auto &SIEntry : + {std::make_pair(&SL.ContentSections, "content sections"), + std::make_pair(&SL.ZeroFillSections, "zero-fill sections")}) { + auto &SIList = *SIEntry.first; + dbgs() << " " << SIEntry.second << ":\n"; + for (auto &SI : SIList) { + dbgs() << " " << SI.S->getName() << ":\n"; + for (auto *DA : SI.Atoms) + dbgs() << " " << *DA << "\n"; + } + } + } + }); +} + +Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) { + + // Compute segment sizes and allocate memory. + LLVM_DEBUG(dbgs() << "JIT linker requesting: { "); + JITLinkMemoryManager::SegmentsRequestMap Segments; + for (auto &KV : Layout) { + auto &Prot = KV.first; + auto &SegLayout = KV.second; + + // Calculate segment content size. + size_t SegContentSize = 0; + for (auto &SI : SegLayout.ContentSections) { + assert(!SI.S->atoms_empty() && "Sections in layout must not be empty"); + assert(!SI.Atoms.empty() && "Section layouts must not be empty"); + + // Bump to section alignment before processing atoms. + SegContentSize = alignTo(SegContentSize, SI.S->getAlignment()); + + for (auto *DA : SI.Atoms) { + SegContentSize = alignTo(SegContentSize, DA->getAlignment()); + SegContentSize += DA->getSize(); + } + } + + // Get segment content alignment. + unsigned SegContentAlign = 1; + if (!SegLayout.ContentSections.empty()) { + auto &FirstContentSection = SegLayout.ContentSections.front(); + SegContentAlign = + std::max(FirstContentSection.S->getAlignment(), + FirstContentSection.Atoms.front()->getAlignment()); + } + + // Calculate segment zero-fill size. + uint64_t SegZeroFillSize = 0; + for (auto &SI : SegLayout.ZeroFillSections) { + assert(!SI.S->atoms_empty() && "Sections in layout must not be empty"); + assert(!SI.Atoms.empty() && "Section layouts must not be empty"); + + // Bump to section alignment before processing atoms. + SegZeroFillSize = alignTo(SegZeroFillSize, SI.S->getAlignment()); + + for (auto *DA : SI.Atoms) { + SegZeroFillSize = alignTo(SegZeroFillSize, DA->getAlignment()); + SegZeroFillSize += DA->getSize(); + } + } + + // Calculate segment zero-fill alignment. + uint32_t SegZeroFillAlign = 1; + + if (!SegLayout.ZeroFillSections.empty()) { + auto &FirstZeroFillSection = SegLayout.ZeroFillSections.front(); + SegZeroFillAlign = + std::max(FirstZeroFillSection.S->getAlignment(), + FirstZeroFillSection.Atoms.front()->getAlignment()); + } + + if (SegContentSize == 0) + SegContentAlign = SegZeroFillAlign; + + if (SegContentAlign % SegZeroFillAlign != 0) + return make_error("First content atom alignment does not " + "accommodate first zero-fill atom " + "alignment"); + + Segments[Prot] = {SegContentSize, SegContentAlign, SegZeroFillSize, + SegZeroFillAlign}; + + LLVM_DEBUG({ + dbgs() << (&KV == &*Layout.begin() ? "" : "; ") + << static_cast(Prot) << ": " + << SegContentSize << " content bytes (alignment " + << SegContentAlign << ") + " << SegZeroFillSize + << " zero-fill bytes (alignment " << SegZeroFillAlign << ")"; + }); + } + LLVM_DEBUG(dbgs() << " }\n"); + + if (auto AllocOrErr = Ctx->getMemoryManager().allocate(Segments)) + Alloc = std::move(*AllocOrErr); + else + return AllocOrErr.takeError(); + + LLVM_DEBUG({ + dbgs() << "JIT linker got working memory:\n"; + for (auto &KV : Layout) { + auto Prot = static_cast(KV.first); + dbgs() << " " << Prot << ": " + << (const void *)Alloc->getWorkingMemory(Prot).data() << "\n"; + } + }); + + // Update atom target addresses. + for (auto &KV : Layout) { + auto &Prot = KV.first; + auto &SL = KV.second; + + JITTargetAddress AtomTargetAddr = + Alloc->getTargetMemory(static_cast(Prot)); + + for (auto *SIList : {&SL.ContentSections, &SL.ZeroFillSections}) + for (auto &SI : *SIList) { + AtomTargetAddr = alignTo(AtomTargetAddr, SI.S->getAlignment()); + for (auto *DA : SI.Atoms) { + AtomTargetAddr = alignTo(AtomTargetAddr, DA->getAlignment()); + DA->setAddress(AtomTargetAddr); + AtomTargetAddr += DA->getSize(); + } + } + } + + return Error::success(); +} + +DenseSet JITLinkerBase::getExternalSymbolNames() const { + // Identify unresolved external atoms. + DenseSet UnresolvedExternals; + for (auto *DA : G->external_atoms()) { + assert(DA->getAddress() == 0 && + "External has already been assigned an address"); + assert(DA->getName() != StringRef() && DA->getName() != "" && + "Externals must be named"); + UnresolvedExternals.insert(DA->getName()); + } + return UnresolvedExternals; +} + +void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) { + for (auto &KV : Result) { + Atom &A = G->getAtomByName(KV.first); + assert(A.getAddress() == 0 && "Atom already resolved"); + A.setAddress(KV.second.getAddress()); + } + + LLVM_DEBUG({ + dbgs() << "Externals after applying lookup result:\n"; + for (auto *A : G->external_atoms()) + dbgs() << " " << A->getName() << ": " + << formatv("{0:x16}", A->getAddress()) << "\n"; + }); + assert(llvm::all_of(G->external_atoms(), + [](Atom *A) { return A->getAddress() != 0; }) && + "All atoms should have been resolved by this point"); +} + +void JITLinkerBase::deallocateAndBailOut(Error Err) { + assert(Err && "Should not be bailing out on success value"); + assert(Alloc && "can not call deallocateAndBailOut before allocation"); + Ctx->notifyFailed(joinErrors(std::move(Err), Alloc->deallocate())); +} + +void JITLinkerBase::dumpGraph(raw_ostream &OS) { + assert(G && "Graph is not set yet"); + G->dump(dbgs(), [this](Edge::Kind K) { return getEdgeKindName(K); }); +} + +void prune(AtomGraph &G) { + std::vector Worklist; + DenseMap> EdgesToUpdate; + + // Build the initial worklist from all atoms initially live. + for (auto *DA : G.defined_atoms()) { + if (!DA->isLive() || DA->shouldDiscard()) + continue; + + for (auto &E : DA->edges()) { + if (!E.getTarget().isDefined()) + continue; + + auto &EDT = static_cast(E.getTarget()); + + if (EDT.shouldDiscard()) + EdgesToUpdate[&EDT].push_back(&E); + else if (E.isKeepAlive() && !EDT.isLive()) + Worklist.push_back(&EDT); + } + } + + // Propagate live flags to all atoms reachable from the initial live set. + while (!Worklist.empty()) { + DefinedAtom &NextLive = *Worklist.back(); + Worklist.pop_back(); + + assert(!NextLive.shouldDiscard() && + "should-discard nodes should never make it into the worklist"); + + // If this atom has already been marked as live, or is marked to be + // discarded, then skip it. + if (NextLive.isLive()) + continue; + + // Otherwise set it as live and add any non-live atoms that it points to + // to the worklist. + NextLive.setLive(true); + + for (auto &E : NextLive.edges()) { + if (!E.getTarget().isDefined()) + continue; + + auto &EDT = static_cast(E.getTarget()); + + if (EDT.shouldDiscard()) + EdgesToUpdate[&EDT].push_back(&E); + else if (E.isKeepAlive() && !EDT.isLive()) + Worklist.push_back(&EDT); + } + } + + // Collect atoms to remove, then remove them from the graph. + std::vector AtomsToRemove; + for (auto *DA : G.defined_atoms()) + if (DA->shouldDiscard() || !DA->isLive()) + AtomsToRemove.push_back(DA); + + LLVM_DEBUG(dbgs() << "Pruning atoms:\n"); + for (auto *DA : AtomsToRemove) { + LLVM_DEBUG(dbgs() << " " << *DA << "... "); + + // Check whether we need to replace this atom with an external atom. + // + // We replace if all of the following hold: + // (1) The atom is marked should-discard, + // (2) it has live edges (i.e. edges from live atoms) pointing to it. + // + // Otherwise we simply delete the atom. + + G.removeDefinedAtom(*DA); + + auto EdgesToUpdateItr = EdgesToUpdate.find(DA); + if (EdgesToUpdateItr != EdgesToUpdate.end()) { + auto &ExternalReplacement = G.addExternalAtom(DA->getName()); + for (auto *EdgeToUpdate : EdgesToUpdateItr->second) + EdgeToUpdate->setTarget(ExternalReplacement); + LLVM_DEBUG(dbgs() << "replaced with " << ExternalReplacement << "\n"); + } else + LLVM_DEBUG(dbgs() << "deleted\n"); + } + + // Finally, discard any absolute symbols that were marked should-discard. + { + std::vector AbsoluteAtomsToRemove; + for (auto *A : G.absolute_atoms()) + if (A->shouldDiscard() || A->isLive()) + AbsoluteAtomsToRemove.push_back(A); + for (auto *A : AbsoluteAtomsToRemove) + G.removeAbsoluteAtom(*A); + } +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/lib/ExecutionEngine/JITLink/JITLinkGeneric.h b/lib/ExecutionEngine/JITLink/JITLinkGeneric.h new file mode 100644 index 000000000000..e6fd6e38f7a6 --- /dev/null +++ b/lib/ExecutionEngine/JITLink/JITLinkGeneric.h @@ -0,0 +1,256 @@ +//===------ JITLinkGeneric.h - Generic JIT linker utilities -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic JITLinker utilities. E.g. graph pruning, eh-frame parsing. +// +//===----------------------------------------------------------------------===// + +#ifndef LIB_EXECUTIONENGINE_JITLINK_JITLINKGENERIC_H +#define LIB_EXECUTIONENGINE_JITLINK_JITLINKGENERIC_H + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { + +class MemoryBufferRef; + +namespace jitlink { + +/// Base class for a JIT linker. +/// +/// A JITLinkerBase instance links one object file into an ongoing JIT +/// session. Symbol resolution and finalization operations are pluggable, +/// and called using continuation passing (passing a continuation for the +/// remaining linker work) to allow them to be performed asynchronously. +class JITLinkerBase { +public: + JITLinkerBase(std::unique_ptr Ctx, PassConfiguration Passes) + : Ctx(std::move(Ctx)), Passes(std::move(Passes)) { + assert(this->Ctx && "Ctx can not be null"); + } + + virtual ~JITLinkerBase(); + +protected: + struct SegmentLayout { + using SectionAtomsList = std::vector; + struct SectionLayout { + SectionLayout(Section &S) : S(&S) {} + + Section *S; + SectionAtomsList Atoms; + }; + + using SectionLayoutList = std::vector; + + SectionLayoutList ContentSections; + SectionLayoutList ZeroFillSections; + }; + + using SegmentLayoutMap = DenseMap; + + // Phase 1: + // 1.1: Build atom graph + // 1.2: Run pre-prune passes + // 1.2: Prune graph + // 1.3: Run post-prune passes + // 1.4: Sort atoms into segments + // 1.5: Allocate segment memory + // 1.6: Identify externals and make an async call to resolve function + void linkPhase1(std::unique_ptr Self); + + // Phase 2: + // 2.1: Apply resolution results + // 2.2: Fix up atom contents + // 2.3: Call OnResolved callback + // 2.3: Make an async call to transfer and finalize memory. + void linkPhase2(std::unique_ptr Self, + Expected LookupResult); + + // Phase 3: + // 3.1: Call OnFinalized callback, handing off allocation. + void linkPhase3(std::unique_ptr Self, Error Err); + + // Build a graph from the given object buffer. + // To be implemented by the client. + virtual Expected> + buildGraph(MemoryBufferRef ObjBuffer) = 0; + + // For debug dumping of the atom graph. + virtual StringRef getEdgeKindName(Edge::Kind K) const = 0; + +private: + // Run all passes in the given pass list, bailing out immediately if any pass + // returns an error. + Error runPasses(AtomGraphPassList &Passes, AtomGraph &G); + + // Copy atom contents and apply relocations. + // Implemented in JITLinker. + virtual Error + copyAndFixUpAllAtoms(const SegmentLayoutMap &Layout, + JITLinkMemoryManager::Allocation &Alloc) const = 0; + + void layOutAtoms(); + Error allocateSegments(const SegmentLayoutMap &Layout); + DenseSet getExternalSymbolNames() const; + void applyLookupResult(AsyncLookupResult LR); + void deallocateAndBailOut(Error Err); + + void dumpGraph(raw_ostream &OS); + + std::unique_ptr Ctx; + PassConfiguration Passes; + std::unique_ptr G; + SegmentLayoutMap Layout; + std::unique_ptr Alloc; +}; + +template class JITLinker : public JITLinkerBase { +public: + using JITLinkerBase::JITLinkerBase; + + /// Link constructs a LinkerImpl instance and calls linkPhase1. + /// Link should be called with the constructor arguments for LinkerImpl, which + /// will be forwarded to the constructor. + template static void link(ArgTs &&... Args) { + auto L = llvm::make_unique(std::forward(Args)...); + + // Ownership of the linker is passed into the linker's doLink function to + // allow it to be passed on to async continuations. + // + // FIXME: Remove LTmp once we have c++17. + // C++17 sequencing rules guarantee that function name expressions are + // sequenced before arguments, so L->linkPhase1(std::move(L), ...) will be + // well formed. + auto <mp = *L; + LTmp.linkPhase1(std::move(L)); + } + +private: + const LinkerImpl &impl() const { + return static_cast(*this); + } + + Error + copyAndFixUpAllAtoms(const SegmentLayoutMap &Layout, + JITLinkMemoryManager::Allocation &Alloc) const override { + LLVM_DEBUG(dbgs() << "Copying and fixing up atoms:\n"); + for (auto &KV : Layout) { + auto &Prot = KV.first; + auto &SegLayout = KV.second; + + auto SegMem = Alloc.getWorkingMemory( + static_cast(Prot)); + char *LastAtomEnd = SegMem.data(); + char *AtomDataPtr = LastAtomEnd; + + LLVM_DEBUG({ + dbgs() << " Processing segment " + << static_cast(Prot) << " [ " + << (const void *)SegMem.data() << " .. " + << (const void *)((char *)SegMem.data() + SegMem.size()) + << " ]\n Processing content sections:\n"; + }); + + for (auto &SI : SegLayout.ContentSections) { + LLVM_DEBUG(dbgs() << " " << SI.S->getName() << ":\n"); + + AtomDataPtr += alignmentAdjustment(AtomDataPtr, SI.S->getAlignment()); + + LLVM_DEBUG({ + dbgs() << " Bumped atom pointer to " << (const void *)AtomDataPtr + << " to meet section alignment " + << " of " << SI.S->getAlignment() << "\n"; + }); + + for (auto *DA : SI.Atoms) { + + // Align. + AtomDataPtr += alignmentAdjustment(AtomDataPtr, DA->getAlignment()); + LLVM_DEBUG({ + dbgs() << " Bumped atom pointer to " + << (const void *)AtomDataPtr << " to meet alignment of " + << DA->getAlignment() << "\n"; + }); + + // Zero pad up to alignment. + LLVM_DEBUG({ + if (LastAtomEnd != AtomDataPtr) + dbgs() << " Zero padding from " << (const void *)LastAtomEnd + << " to " << (const void *)AtomDataPtr << "\n"; + }); + while (LastAtomEnd != AtomDataPtr) + *LastAtomEnd++ = 0; + + // Copy initial atom content. + LLVM_DEBUG({ + dbgs() << " Copying atom " << *DA << " content, " + << DA->getContent().size() << " bytes, from " + << (const void *)DA->getContent().data() << " to " + << (const void *)AtomDataPtr << "\n"; + }); + memcpy(AtomDataPtr, DA->getContent().data(), DA->getContent().size()); + + // Copy atom data and apply fixups. + LLVM_DEBUG(dbgs() << " Applying fixups.\n"); + for (auto &E : DA->edges()) { + + // Skip non-relocation edges. + if (!E.isRelocation()) + continue; + + // Dispatch to LinkerImpl for fixup. + if (auto Err = impl().applyFixup(*DA, E, AtomDataPtr)) + return Err; + } + + // Point the atom's content to the fixed up buffer. + DA->setContent(StringRef(AtomDataPtr, DA->getContent().size())); + + // Update atom end pointer. + LastAtomEnd = AtomDataPtr + DA->getContent().size(); + AtomDataPtr = LastAtomEnd; + } + } + + // Zero pad the rest of the segment. + LLVM_DEBUG({ + dbgs() << " Zero padding end of segment from " + << (const void *)LastAtomEnd << " to " + << (const void *)((char *)SegMem.data() + SegMem.size()) << "\n"; + }); + while (LastAtomEnd != SegMem.data() + SegMem.size()) + *LastAtomEnd++ = 0; + } + + return Error::success(); + } +}; + +/// Dead strips and replaces discarded definitions with external atoms. +/// +/// Finds the set of nodes reachable from any node initially marked live +/// (nodes marked should-discard are treated as not live, even if they are +/// reachable). All nodes not marked as live at the end of this process, +/// are deleted. Nodes that are live, but marked should-discard are replaced +/// with external atoms and all edges to them are re-written. +void prune(AtomGraph &G); + +Error addEHFrame(AtomGraph &G, Section &EHFrameSection, + StringRef EHFrameContent, JITTargetAddress EHFrameAddress, + Edge::Kind FDEToCIERelocKind, Edge::Kind FDEToTargetRelocKind); + +} // end namespace jitlink +} // end namespace llvm + +#undef DEBUG_TYPE // "jitlink" + +#endif // LLVM_EXECUTIONENGINE_JITLINK_JITLINKGENERIC_H diff --git a/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp b/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp new file mode 100644 index 000000000000..267307cfde05 --- /dev/null +++ b/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp @@ -0,0 +1,105 @@ +//===--- JITLinkMemoryManager.cpp - JITLinkMemoryManager implementation ---===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/JITLinkMemoryManager.h" +#include "llvm/Support/Process.h" + +namespace llvm { +namespace jitlink { + +JITLinkMemoryManager::~JITLinkMemoryManager() = default; +JITLinkMemoryManager::Allocation::~Allocation() = default; + +Expected> +InProcessMemoryManager::allocate(const SegmentsRequestMap &Request) { + + using AllocationMap = DenseMap; + + // Local class for allocation. + class IPMMAlloc : public Allocation { + public: + IPMMAlloc(AllocationMap SegBlocks) : SegBlocks(std::move(SegBlocks)) {} + MutableArrayRef getWorkingMemory(ProtectionFlags Seg) override { + assert(SegBlocks.count(Seg) && "No allocation for segment"); + return {static_cast(SegBlocks[Seg].base()), + SegBlocks[Seg].allocatedSize()}; + } + JITTargetAddress getTargetMemory(ProtectionFlags Seg) override { + assert(SegBlocks.count(Seg) && "No allocation for segment"); + return reinterpret_cast(SegBlocks[Seg].base()); + } + void finalizeAsync(FinalizeContinuation OnFinalize) override { + OnFinalize(applyProtections()); + } + Error deallocate() override { + for (auto &KV : SegBlocks) + if (auto EC = sys::Memory::releaseMappedMemory(KV.second)) + return errorCodeToError(EC); + return Error::success(); + } + + private: + Error applyProtections() { + for (auto &KV : SegBlocks) { + auto &Prot = KV.first; + auto &Block = KV.second; + if (auto EC = sys::Memory::protectMappedMemory(Block, Prot)) + return errorCodeToError(EC); + if (Prot & sys::Memory::MF_EXEC) + sys::Memory::InvalidateInstructionCache(Block.base(), + Block.allocatedSize()); + } + return Error::success(); + } + + AllocationMap SegBlocks; + }; + + AllocationMap Blocks; + const sys::Memory::ProtectionFlags ReadWrite = + static_cast(sys::Memory::MF_READ | + sys::Memory::MF_WRITE); + + for (auto &KV : Request) { + auto &Seg = KV.second; + + if (Seg.getContentAlignment() > sys::Process::getPageSizeEstimate()) + return make_error("Cannot request higher than page " + "alignment", + inconvertibleErrorCode()); + + if (sys::Process::getPageSizeEstimate() % Seg.getContentAlignment() != 0) + return make_error("Page size is not a multiple of " + "alignment", + inconvertibleErrorCode()); + + uint64_t ZeroFillStart = + alignTo(Seg.getContentSize(), Seg.getZeroFillAlignment()); + uint64_t SegmentSize = ZeroFillStart + Seg.getZeroFillSize(); + + std::error_code EC; + auto SegMem = + sys::Memory::allocateMappedMemory(SegmentSize, nullptr, ReadWrite, EC); + + if (EC) + return errorCodeToError(EC); + + // Zero out the zero-fill memory. + memset(static_cast(SegMem.base()) + ZeroFillStart, 0, + Seg.getZeroFillSize()); + + // Record the block for this segment. + Blocks[KV.first] = std::move(SegMem); + } + return std::unique_ptr( + new IPMMAlloc(std::move(Blocks))); +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/lib/ExecutionEngine/JITLink/MachO.cpp b/lib/ExecutionEngine/JITLink/MachO.cpp new file mode 100644 index 000000000000..15995b8ce98f --- /dev/null +++ b/lib/ExecutionEngine/JITLink/MachO.cpp @@ -0,0 +1,78 @@ +//===-------------- MachO.cpp - JIT linker function for MachO -------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// MachO jit-link function. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/MachO.h" + +#include "llvm/BinaryFormat/MachO.h" +#include "llvm/ExecutionEngine/JITLink/MachO_x86_64.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/MemoryBuffer.h" + +using namespace llvm; + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +void jitLink_MachO(std::unique_ptr Ctx) { + + // We don't want to do full MachO validation here. Just parse enough of the + // header to find out what MachO linker to use. + + StringRef Data = Ctx->getObjectBuffer().getBuffer(); + if (Data.size() < 4) { + Ctx->notifyFailed(make_error("Truncated MachO buffer")); + return; + } + + uint32_t Magic; + memcpy(&Magic, Data.data(), sizeof(uint32_t)); + LLVM_DEBUG({ + dbgs() << "jitLink_MachO: magic = " << format("0x%08" PRIx32, Magic) + << ", identifier = \"" + << Ctx->getObjectBuffer().getBufferIdentifier() << "\"\n"; + }); + + if (Magic == MachO::MH_MAGIC || Magic == MachO::MH_CIGAM) { + Ctx->notifyFailed( + make_error("MachO 32-bit platforms not supported")); + return; + } else if (Magic == MachO::MH_MAGIC_64 || Magic == MachO::MH_CIGAM_64) { + MachO::mach_header_64 Header; + + memcpy(&Header, Data.data(), sizeof(MachO::mach_header_64)); + if (Magic == MachO::MH_CIGAM_64) + swapStruct(Header); + + LLVM_DEBUG({ + dbgs() << "jitLink_MachO: cputype = " + << format("0x%08" PRIx32, Header.cputype) + << ", cpusubtype = " << format("0x%08" PRIx32, Header.cpusubtype) + << "\n"; + }); + + switch (Header.cputype) { + case MachO::CPU_TYPE_X86_64: + return jitLink_MachO_x86_64(std::move(Ctx)); + } + Ctx->notifyFailed(make_error("MachO-64 CPU type not valid")); + return; + } + + Ctx->notifyFailed(make_error("MachO magic not valid")); +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp b/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp new file mode 100644 index 000000000000..1501c7ad0bc5 --- /dev/null +++ b/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp @@ -0,0 +1,411 @@ +//=--------- MachOAtomGraphBuilder.cpp - MachO AtomGraph builder ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic MachO AtomGraph buliding code. +// +//===----------------------------------------------------------------------===// + +#include "MachOAtomGraphBuilder.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +MachOAtomGraphBuilder::~MachOAtomGraphBuilder() {} + +Expected> MachOAtomGraphBuilder::buildGraph() { + if (auto Err = parseSections()) + return std::move(Err); + + if (auto Err = addAtoms()) + return std::move(Err); + + if (auto Err = addRelocations()) + return std::move(Err); + + return std::move(G); +} + +MachOAtomGraphBuilder::MachOAtomGraphBuilder(const object::MachOObjectFile &Obj) + : Obj(Obj), + G(llvm::make_unique(Obj.getFileName(), getPointerSize(Obj), + getEndianness(Obj))) {} + +void MachOAtomGraphBuilder::addCustomAtomizer(StringRef SectionName, + CustomAtomizeFunction Atomizer) { + assert(!CustomAtomizeFunctions.count(SectionName) && + "Custom atomizer for this section already exists"); + CustomAtomizeFunctions[SectionName] = std::move(Atomizer); +} + +bool MachOAtomGraphBuilder::areLayoutLocked(const Atom &A, const Atom &B) { + // If these atoms are the same then they're trivially "locked". + if (&A == &B) + return true; + + // If A and B are different, check whether either is undefined. (in which + // case they are not locked). + if (!A.isDefined() || !B.isDefined()) + return false; + + // A and B are different, but they're both defined atoms. We need to check + // whether they're part of the same alt_entry chain. + auto &DA = static_cast(A); + auto &DB = static_cast(B); + + auto AStartItr = AltEntryStarts.find(&DA); + if (AStartItr == AltEntryStarts.end()) // If A is not in a chain bail out. + return false; + + auto BStartItr = AltEntryStarts.find(&DB); + if (BStartItr == AltEntryStarts.end()) // If B is not in a chain bail out. + return false; + + // A and B are layout locked if they're in the same chain. + return AStartItr->second == BStartItr->second; +} + +unsigned +MachOAtomGraphBuilder::getPointerSize(const object::MachOObjectFile &Obj) { + return Obj.is64Bit() ? 8 : 4; +} + +support::endianness +MachOAtomGraphBuilder::getEndianness(const object::MachOObjectFile &Obj) { + return Obj.isLittleEndian() ? support::little : support::big; +} + +MachOAtomGraphBuilder::MachOSection &MachOAtomGraphBuilder::getCommonSection() { + if (!CommonSymbolsSection) { + auto Prot = static_cast( + sys::Memory::MF_READ | sys::Memory::MF_WRITE); + auto &GenericSection = G->createSection("", 1, Prot, true); + CommonSymbolsSection = MachOSection(GenericSection); + } + return *CommonSymbolsSection; +} + +Error MachOAtomGraphBuilder::parseSections() { + for (auto &SecRef : Obj.sections()) { + assert((SecRef.getAlignment() <= std::numeric_limits::max()) && + "Section alignment does not fit in 32 bits"); + + StringRef Name; + if (auto EC = SecRef.getName(Name)) + return errorCodeToError(EC); + + unsigned SectionIndex = SecRef.getIndex() + 1; + + uint32_t Align = SecRef.getAlignment(); + if (!isPowerOf2_32(Align)) + return make_error("Section " + Name + + " has non-power-of-2 " + "alignment"); + + // FIXME: Get real section permissions + // How, exactly, on MachO? + sys::Memory::ProtectionFlags Prot; + if (SecRef.isText()) + Prot = static_cast(sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + else + Prot = static_cast(sys::Memory::MF_READ | + sys::Memory::MF_WRITE); + + auto &GenericSection = G->createSection(Name, Align, Prot, SecRef.isBSS()); + + LLVM_DEBUG({ + dbgs() << "Adding section " << Name << ": " + << format("0x%016" PRIx64, SecRef.getAddress()) + << ", align: " << SecRef.getAlignment() << "\n"; + }); + + assert(!Sections.count(SectionIndex) && "Section index already in use"); + + auto &MachOSec = + Sections + .try_emplace(SectionIndex, GenericSection, SecRef.getAddress(), + SecRef.getAlignment()) + .first->second; + + if (!SecRef.isVirtual()) { + // If this section has content then record it. + Expected Content = SecRef.getContents(); + if (!Content) + return Content.takeError(); + if (Content->size() != SecRef.getSize()) + return make_error("Section content size does not match " + "declared size for " + + Name); + MachOSec.setContent(*Content); + } else { + // If this is a zero-fill section then just record the size. + MachOSec.setZeroFill(SecRef.getSize()); + } + + uint32_t SectionFlags = + Obj.is64Bit() ? Obj.getSection64(SecRef.getRawDataRefImpl()).flags + : Obj.getSection(SecRef.getRawDataRefImpl()).flags; + + MachOSec.setNoDeadStrip(SectionFlags & MachO::S_ATTR_NO_DEAD_STRIP); + } + + return Error::success(); +} + +// Adds atoms with identified start addresses (but not lengths) for all named +// atoms. +// Also, for every section that contains named atoms, but does not have an +// atom at offset zero of that section, constructs an anonymous atom covering +// that range. +Error MachOAtomGraphBuilder::addNonCustomAtoms() { + using AddrToAtomMap = std::map; + DenseMap SecToAtoms; + + DenseMap FirstOrdinal; + std::vector AltEntryAtoms; + + DenseSet ProcessedSymbols; // Used to check for duplicate defs. + + for (auto SymI = Obj.symbol_begin(), SymE = Obj.symbol_end(); SymI != SymE; + ++SymI) { + object::SymbolRef Sym(SymI->getRawDataRefImpl(), &Obj); + + auto Name = Sym.getName(); + if (!Name) + return Name.takeError(); + + // Bail out on duplicate definitions: There should never be more than one + // definition for a symbol in a given object file. + if (ProcessedSymbols.count(*Name)) + return make_error("Duplicate definition within object: " + + *Name); + else + ProcessedSymbols.insert(*Name); + + auto Addr = Sym.getAddress(); + if (!Addr) + return Addr.takeError(); + + auto SymType = Sym.getType(); + if (!SymType) + return SymType.takeError(); + + auto Flags = Sym.getFlags(); + + if (Flags & object::SymbolRef::SF_Undefined) { + LLVM_DEBUG(dbgs() << "Adding undef atom \"" << *Name << "\"\n"); + G->addExternalAtom(*Name); + continue; + } else if (Flags & object::SymbolRef::SF_Absolute) { + LLVM_DEBUG(dbgs() << "Adding absolute \"" << *Name << "\" addr: " + << format("0x%016" PRIx64, *Addr) << "\n"); + auto &A = G->addAbsoluteAtom(*Name, *Addr); + A.setGlobal(Flags & object::SymbolRef::SF_Global); + A.setExported(Flags & object::SymbolRef::SF_Exported); + A.setWeak(Flags & object::SymbolRef::SF_Weak); + continue; + } else if (Flags & object::SymbolRef::SF_Common) { + LLVM_DEBUG({ + dbgs() << "Adding common \"" << *Name + << "\" addr: " << format("0x%016" PRIx64, *Addr) << "\n"; + }); + auto &A = + G->addCommonAtom(getCommonSection().getGenericSection(), *Name, *Addr, + std::max(Sym.getAlignment(), 1U), + Obj.getCommonSymbolSize(Sym.getRawDataRefImpl())); + A.setGlobal(Flags & object::SymbolRef::SF_Global); + A.setExported(Flags & object::SymbolRef::SF_Exported); + continue; + } + + LLVM_DEBUG(dbgs() << "Adding defined atom \"" << *Name << "\"\n"); + + // This atom is neither undefined nor absolute, so it must be defined in + // this object. Get its section index. + auto SecItr = Sym.getSection(); + if (!SecItr) + return SecItr.takeError(); + + uint64_t SectionIndex = (*SecItr)->getIndex() + 1; + + LLVM_DEBUG(dbgs() << " to section index " << SectionIndex << "\n"); + + auto SecByIndexItr = Sections.find(SectionIndex); + if (SecByIndexItr == Sections.end()) + return make_error("Unrecognized section index in macho"); + + auto &Sec = SecByIndexItr->second; + + auto &DA = G->addDefinedAtom(Sec.getGenericSection(), *Name, *Addr, + std::max(Sym.getAlignment(), 1U)); + + DA.setGlobal(Flags & object::SymbolRef::SF_Global); + DA.setExported(Flags & object::SymbolRef::SF_Exported); + DA.setWeak(Flags & object::SymbolRef::SF_Weak); + + DA.setCallable(*SymType & object::SymbolRef::ST_Function); + + // Check NDesc flags. + { + uint16_t NDesc = 0; + if (Obj.is64Bit()) + NDesc = Obj.getSymbol64TableEntry(SymI->getRawDataRefImpl()).n_desc; + else + NDesc = Obj.getSymbolTableEntry(SymI->getRawDataRefImpl()).n_desc; + + // Record atom for alt-entry post-processing (where the layout-next + // constraints will be added). + if (NDesc & MachO::N_ALT_ENTRY) + AltEntryAtoms.push_back(&DA); + + // If this atom has a no-dead-strip attr attached then mark it live. + if (NDesc & MachO::N_NO_DEAD_STRIP) + DA.setLive(true); + } + + LLVM_DEBUG({ + dbgs() << " Added " << *Name + << " addr: " << format("0x%016" PRIx64, *Addr) + << ", align: " << DA.getAlignment() + << ", section: " << Sec.getGenericSection().getName() << "\n"; + }); + + auto &SecAtoms = SecToAtoms[&Sec]; + SecAtoms[DA.getAddress() - Sec.getAddress()] = &DA; + } + + // Add anonymous atoms. + for (auto &KV : Sections) { + auto &S = KV.second; + + // Skip empty sections. + if (S.empty()) + continue; + + // Skip sections with custom handling. + if (CustomAtomizeFunctions.count(S.getName())) + continue; + + auto SAI = SecToAtoms.find(&S); + + // If S is not in the SecToAtoms map then it contained no named atom. Add + // one anonymous atom to cover the whole section. + if (SAI == SecToAtoms.end()) { + SecToAtoms[&S][0] = &G->addAnonymousAtom( + S.getGenericSection(), S.getAddress(), S.getAlignment()); + continue; + } + + // Otherwise, check whether this section had an atom covering offset zero. + // If not, add one. + auto &SecAtoms = SAI->second; + if (!SecAtoms.count(0)) + SecAtoms[0] = &G->addAnonymousAtom(S.getGenericSection(), S.getAddress(), + S.getAlignment()); + } + + LLVM_DEBUG(dbgs() << "MachOGraphBuilder setting atom content\n"); + + // Set atom contents and any section-based flags. + for (auto &KV : SecToAtoms) { + auto &S = *KV.first; + auto &SecAtoms = KV.second; + + // Iterate the atoms in reverse order and set up their contents. + JITTargetAddress LastAtomAddr = S.getSize(); + for (auto I = SecAtoms.rbegin(), E = SecAtoms.rend(); I != E; ++I) { + auto Offset = I->first; + auto &A = *I->second; + LLVM_DEBUG({ + dbgs() << " " << A << " to [ " << S.getAddress() + Offset << " .. " + << S.getAddress() + LastAtomAddr << " ]\n"; + }); + + if (S.isZeroFill()) + A.setZeroFill(LastAtomAddr - Offset); + else + A.setContent(S.getContent().substr(Offset, LastAtomAddr - Offset)); + + // If the section has no-dead-strip set then mark the atom as live. + if (S.isNoDeadStrip()) + A.setLive(true); + + LastAtomAddr = Offset; + } + } + + LLVM_DEBUG(dbgs() << "Adding alt-entry starts\n"); + + // Sort alt-entry atoms by address in ascending order. + llvm::sort(AltEntryAtoms.begin(), AltEntryAtoms.end(), + [](const DefinedAtom *LHS, const DefinedAtom *RHS) { + return LHS->getAddress() < RHS->getAddress(); + }); + + // Process alt-entry atoms in address order to build the table of alt-entry + // atoms to alt-entry chain starts. + for (auto *DA : AltEntryAtoms) { + assert(!AltEntryStarts.count(DA) && "Duplicate entry in AltEntryStarts"); + + // DA is an alt-entry atom. Look for the predecessor atom that it is locked + // to, bailing out if we do not find one. + auto AltEntryPred = G->findAtomByAddress(DA->getAddress() - 1); + if (!AltEntryPred) + return AltEntryPred.takeError(); + + // Add a LayoutNext edge from the predecessor to this atom. + AltEntryPred->setLayoutNext(*DA); + + // Check to see whether the predecessor itself is an alt-entry atom. + auto AltEntryStartItr = AltEntryStarts.find(&*AltEntryPred); + if (AltEntryStartItr != AltEntryStarts.end()) { + // If the predecessor was an alt-entry atom then re-use its value. + LLVM_DEBUG({ + dbgs() << " " << *DA << " -> " << *AltEntryStartItr->second + << " (based on existing entry for " << *AltEntryPred << ")\n"; + }); + AltEntryStarts[DA] = AltEntryStartItr->second; + } else { + // If the predecessor does not have an entry then add an entry for this + // atom (i.e. the alt_entry atom) and a self-reference entry for the + /// predecessory atom that is the start of this chain. + LLVM_DEBUG({ + dbgs() << " " << *AltEntryPred << " -> " << *AltEntryPred << "\n" + << " " << *DA << " -> " << *AltEntryPred << "\n"; + }); + AltEntryStarts[&*AltEntryPred] = &*AltEntryPred; + AltEntryStarts[DA] = &*AltEntryPred; + } + } + + return Error::success(); +} + +Error MachOAtomGraphBuilder::addAtoms() { + // Add all named atoms. + if (auto Err = addNonCustomAtoms()) + return Err; + + // Process special sections. + for (auto &KV : Sections) { + auto &S = KV.second; + auto HI = CustomAtomizeFunctions.find(S.getGenericSection().getName()); + if (HI != CustomAtomizeFunctions.end()) { + auto &Atomize = HI->second; + if (auto Err = Atomize(S)) + return Err; + } + } + + return Error::success(); +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h b/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h new file mode 100644 index 000000000000..72d441b24d06 --- /dev/null +++ b/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h @@ -0,0 +1,138 @@ +//===----- MachOAtomGraphBuilder.h - MachO AtomGraph builder ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic MachO AtomGraph building code. +// +//===----------------------------------------------------------------------===// + +#ifndef LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H +#define LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +#include "JITLinkGeneric.h" + +#include "llvm/Object/MachO.h" + +namespace llvm { +namespace jitlink { + +class MachOAtomGraphBuilder { +public: + virtual ~MachOAtomGraphBuilder(); + Expected> buildGraph(); + +protected: + using OffsetToAtomMap = std::map; + + class MachOSection { + public: + MachOSection() = default; + + /// Create a MachO section with the given address and alignment. + MachOSection(Section &GenericSection, JITTargetAddress Address, + unsigned Alignment) + : Address(Address), GenericSection(&GenericSection), + Alignment(Alignment) {} + + /// Create a section without address, content or size (used for common + /// symbol sections). + MachOSection(Section &GenericSection) : GenericSection(&GenericSection) {} + + Section &getGenericSection() const { + assert(GenericSection && "Section is null"); + return *GenericSection; + } + + StringRef getName() const { + assert(GenericSection && "No generic section attached"); + return GenericSection->getName(); + } + + MachOSection &setContent(StringRef Content) { + assert(!ContentPtr && !Size && "Content/zeroFill already set"); + ContentPtr = Content.data(); + Size = Content.size(); + return *this; + } + + MachOSection &setZeroFill(uint64_t Size) { + assert(!ContentPtr && !this->Size && "Content/zeroFill already set"); + this->Size = Size; + return *this; + } + + bool isZeroFill() const { return !ContentPtr; } + + bool empty() const { return getSize() == 0; } + + size_t getSize() const { return Size; } + + StringRef getContent() const { + assert(ContentPtr && "getContent() called on zero-fill section"); + return {ContentPtr, static_cast(Size)}; + } + + JITTargetAddress getAddress() const { return Address; } + + unsigned getAlignment() const { return Alignment; } + + MachOSection &setNoDeadStrip(bool NoDeadStrip) { + this->NoDeadStrip = NoDeadStrip; + return *this; + } + + bool isNoDeadStrip() const { return NoDeadStrip; } + + private: + JITTargetAddress Address = 0; + Section *GenericSection = nullptr; + const char *ContentPtr = nullptr; + uint64_t Size = 0; + unsigned Alignment = 0; + bool NoDeadStrip = false; + }; + + using CustomAtomizeFunction = std::function; + + MachOAtomGraphBuilder(const object::MachOObjectFile &Obj); + + AtomGraph &getGraph() const { return *G; } + + const object::MachOObjectFile &getObject() const { return Obj; } + + void addCustomAtomizer(StringRef SectionName, CustomAtomizeFunction Atomizer); + + virtual Error addRelocations() = 0; + + /// Returns true if Atom A and Atom B are at a fixed offset from one another + /// (i.e. if they're part of the same alt-entry chain). + bool areLayoutLocked(const Atom &A, const Atom &B); + +private: + static unsigned getPointerSize(const object::MachOObjectFile &Obj); + static support::endianness getEndianness(const object::MachOObjectFile &Obj); + + MachOSection &getCommonSection(); + + Error parseSections(); + Error addNonCustomAtoms(); + Error addAtoms(); + + const object::MachOObjectFile &Obj; + std::unique_ptr G; + DenseMap AltEntryStarts; + DenseMap Sections; + StringMap CustomAtomizeFunctions; + Optional CommonSymbolsSection; +}; + +} // end namespace jitlink +} // end namespace llvm + +#endif // LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H diff --git a/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp new file mode 100644 index 000000000000..4010678c6d33 --- /dev/null +++ b/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -0,0 +1,608 @@ +//===---- MachO_x86_64.cpp -JIT linker implementation for MachO/x86-64 ----===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// MachO/x86-64 jit-link implementation. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/MachO_x86_64.h" + +#include "BasicGOTAndStubsBuilder.h" +#include "MachOAtomGraphBuilder.h" + +#define DEBUG_TYPE "jitlink" + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::jitlink::MachO_x86_64_Edges; + +namespace { + +class MachOAtomGraphBuilder_x86_64 : public MachOAtomGraphBuilder { +public: + MachOAtomGraphBuilder_x86_64(const object::MachOObjectFile &Obj) + : MachOAtomGraphBuilder(Obj), + NumSymbols(Obj.getSymtabLoadCommand().nsyms) { + addCustomAtomizer("__eh_frame", [this](MachOSection &EHFrameSection) { + return addEHFrame(getGraph(), EHFrameSection.getGenericSection(), + EHFrameSection.getContent(), + EHFrameSection.getAddress(), NegDelta32, Delta64); + }); + } + +private: + static Expected + getRelocationKind(const MachO::relocation_info &RI) { + switch (RI.r_type) { + case MachO::X86_64_RELOC_UNSIGNED: + if (!RI.r_pcrel && RI.r_length == 3) + return RI.r_extern ? Pointer64 : Pointer64Anon; + break; + case MachO::X86_64_RELOC_SIGNED: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32 : PCRel32Anon; + break; + case MachO::X86_64_RELOC_BRANCH: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return Branch32; + break; + case MachO::X86_64_RELOC_GOT_LOAD: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return PCRel32GOTLoad; + break; + case MachO::X86_64_RELOC_GOT: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return PCRel32GOT; + break; + case MachO::X86_64_RELOC_SUBTRACTOR: + // SUBTRACTOR must be non-pc-rel, extern, with length 2 or 3. + // Initially represent SUBTRACTOR relocations with 'Delta'. They may + // be turned into NegDelta by parsePairRelocation. + if (!RI.r_pcrel && RI.r_extern) { + if (RI.r_length == 2) + return Delta32; + else if (RI.r_length == 3) + return Delta64; + } + break; + case MachO::X86_64_RELOC_SIGNED_1: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32Minus1 : PCRel32Minus1Anon; + break; + case MachO::X86_64_RELOC_SIGNED_2: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32Minus2 : PCRel32Minus2Anon; + break; + case MachO::X86_64_RELOC_SIGNED_4: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32Minus4 : PCRel32Minus4Anon; + break; + case MachO::X86_64_RELOC_TLV: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return PCRel32TLV; + break; + } + + return make_error( + "Unsupported x86-64 relocation: address=" + + formatv("{0:x8}", RI.r_address) + + ", symbolnum=" + formatv("{0:x6}", RI.r_symbolnum) + + ", kind=" + formatv("{0:x1}", RI.r_type) + + ", pc_rel=" + (RI.r_pcrel ? "true" : "false") + + ", extern= " + (RI.r_extern ? "true" : "false") + + ", length=" + formatv("{0:d}", RI.r_length)); + } + + Expected findAtomBySymbolIndex(const MachO::relocation_info &RI) { + auto &Obj = getObject(); + if (RI.r_symbolnum >= NumSymbols) + return make_error("Symbol index out of range"); + auto SymI = Obj.getSymbolByIndex(RI.r_symbolnum); + auto Name = SymI->getName(); + if (!Name) + return Name.takeError(); + return getGraph().getAtomByName(*Name); + } + + MachO::relocation_info + getRelocationInfo(const object::relocation_iterator RelItr) { + MachO::any_relocation_info ARI = + getObject().getRelocation(RelItr->getRawDataRefImpl()); + MachO::relocation_info RI; + memcpy(&RI, &ARI, sizeof(MachO::relocation_info)); + return RI; + } + + using PairRelocInfo = std::tuple; + + // Parses paired SUBTRACTOR/UNSIGNED relocations and, on success, + // returns the edge kind and addend to be used. + Expected + parsePairRelocation(DefinedAtom &AtomToFix, Edge::Kind SubtractorKind, + const MachO::relocation_info &SubRI, + JITTargetAddress FixupAddress, const char *FixupContent, + object::relocation_iterator &UnsignedRelItr, + object::relocation_iterator &RelEnd) { + using namespace support; + + assert(((SubtractorKind == Delta32 && SubRI.r_length == 2) || + (SubtractorKind == Delta64 && SubRI.r_length == 3)) && + "Subtractor kind should match length"); + assert(SubRI.r_extern && "SUBTRACTOR reloc symbol should be extern"); + assert(!SubRI.r_pcrel && "SUBTRACTOR reloc should not be PCRel"); + + if (UnsignedRelItr == RelEnd) + return make_error("x86_64 SUBTRACTOR without paired " + "UNSIGNED relocation"); + + auto UnsignedRI = getRelocationInfo(UnsignedRelItr); + + if (SubRI.r_address != UnsignedRI.r_address) + return make_error("x86_64 SUBTRACTOR and paired UNSIGNED " + "point to different addresses"); + + if (SubRI.r_length != UnsignedRI.r_length) + return make_error("length of x86_64 SUBTRACTOR and paired " + "UNSIGNED reloc must match"); + + auto FromAtom = findAtomBySymbolIndex(SubRI); + if (!FromAtom) + return FromAtom.takeError(); + + // Read the current fixup value. + uint64_t FixupValue = 0; + if (SubRI.r_length == 3) + FixupValue = *(const little64_t *)FixupContent; + else + FixupValue = *(const little32_t *)FixupContent; + + // Find 'ToAtom' using symbol number or address, depending on whether the + // paired UNSIGNED relocation is extern. + Atom *ToAtom = nullptr; + if (UnsignedRI.r_extern) { + // Find target atom by symbol index. + if (auto ToAtomOrErr = findAtomBySymbolIndex(UnsignedRI)) + ToAtom = &*ToAtomOrErr; + else + return ToAtomOrErr.takeError(); + } else { + if (auto ToAtomOrErr = getGraph().findAtomByAddress(FixupValue)) + ToAtom = &*ToAtomOrErr; + else + return ToAtomOrErr.takeError(); + FixupValue -= ToAtom->getAddress(); + } + + MachOX86RelocationKind DeltaKind; + Atom *TargetAtom; + uint64_t Addend; + if (areLayoutLocked(AtomToFix, *FromAtom)) { + TargetAtom = ToAtom; + DeltaKind = (SubRI.r_length == 3) ? Delta64 : Delta32; + Addend = FixupValue + (FixupAddress - FromAtom->getAddress()); + // FIXME: handle extern 'from'. + } else if (areLayoutLocked(AtomToFix, *ToAtom)) { + TargetAtom = &*FromAtom; + DeltaKind = (SubRI.r_length == 3) ? NegDelta64 : NegDelta32; + Addend = FixupValue - (FixupAddress - ToAtom->getAddress()); + } else { + // AtomToFix was neither FromAtom nor ToAtom. + return make_error("SUBTRACTOR relocation must fix up " + "either 'A' or 'B' (or an atom in one " + "of their alt-entry groups)"); + } + + return PairRelocInfo(DeltaKind, TargetAtom, Addend); + } + + Error addRelocations() override { + using namespace support; + auto &G = getGraph(); + auto &Obj = getObject(); + + for (auto &S : Obj.sections()) { + + JITTargetAddress SectionAddress = S.getAddress(); + + for (auto RelItr = S.relocation_begin(), RelEnd = S.relocation_end(); + RelItr != RelEnd; ++RelItr) { + + MachO::relocation_info RI = getRelocationInfo(RelItr); + + // Sanity check the relocation kind. + auto Kind = getRelocationKind(RI); + if (!Kind) + return Kind.takeError(); + + // Find the address of the value to fix up. + JITTargetAddress FixupAddress = SectionAddress + (uint32_t)RI.r_address; + + LLVM_DEBUG({ + dbgs() << "Processing relocation at " + << format("0x%016" PRIx64, FixupAddress) << "\n"; + }); + + // Find the atom that the fixup points to. + DefinedAtom *AtomToFix = nullptr; + { + auto AtomToFixOrErr = G.findAtomByAddress(FixupAddress); + if (!AtomToFixOrErr) + return AtomToFixOrErr.takeError(); + AtomToFix = &*AtomToFixOrErr; + } + + if (FixupAddress + static_cast(1ULL << RI.r_length) > + AtomToFix->getAddress() + AtomToFix->getContent().size()) + return make_error( + "Relocation content extends past end of fixup atom"); + + // Get a pointer to the fixup content. + const char *FixupContent = AtomToFix->getContent().data() + + (FixupAddress - AtomToFix->getAddress()); + + // The target atom and addend will be populated by the switch below. + Atom *TargetAtom = nullptr; + uint64_t Addend = 0; + + switch (*Kind) { + case Branch32: + case PCRel32: + case PCRel32GOTLoad: + case PCRel32GOT: + if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = *(const ulittle32_t *)FixupContent; + break; + case Pointer64: + if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = *(const ulittle64_t *)FixupContent; + break; + case Pointer64Anon: { + JITTargetAddress TargetAddress = *(const ulittle64_t *)FixupContent; + if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = TargetAddress - TargetAtom->getAddress(); + break; + } + case PCRel32Minus1: + case PCRel32Minus2: + case PCRel32Minus4: + if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = *(const ulittle32_t *)FixupContent + + (1 << (*Kind - PCRel32Minus1)); + break; + case PCRel32Anon: { + JITTargetAddress TargetAddress = + FixupAddress + 4 + *(const ulittle32_t *)FixupContent; + if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = TargetAddress - TargetAtom->getAddress(); + break; + } + case PCRel32Minus1Anon: + case PCRel32Minus2Anon: + case PCRel32Minus4Anon: { + JITTargetAddress Delta = + static_cast(1ULL << (*Kind - PCRel32Minus1Anon)); + JITTargetAddress TargetAddress = + FixupAddress + 4 + Delta + *(const ulittle32_t *)FixupContent; + if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = TargetAddress - TargetAtom->getAddress(); + break; + } + case Delta32: + case Delta64: { + // We use Delta32/Delta64 to represent SUBTRACTOR relocations. + // parsePairRelocation handles the paired reloc, and returns the + // edge kind to be used (either Delta32/Delta64, or + // NegDelta32/NegDelta64, depending on the direction of the + // subtraction) along with the addend. + auto PairInfo = + parsePairRelocation(*AtomToFix, *Kind, RI, FixupAddress, + FixupContent, ++RelItr, RelEnd); + if (!PairInfo) + return PairInfo.takeError(); + std::tie(*Kind, TargetAtom, Addend) = *PairInfo; + assert(TargetAtom && "No target atom from parsePairRelocation?"); + break; + } + default: + llvm_unreachable("Special relocation kind should not appear in " + "mach-o file"); + } + + LLVM_DEBUG({ + Edge GE(*Kind, FixupAddress - AtomToFix->getAddress(), *TargetAtom, + Addend); + printEdge(dbgs(), *AtomToFix, GE, + getMachOX86RelocationKindName(*Kind)); + dbgs() << "\n"; + }); + AtomToFix->addEdge(*Kind, FixupAddress - AtomToFix->getAddress(), + *TargetAtom, Addend); + } + } + return Error::success(); + } + + unsigned NumSymbols = 0; +}; + +class MachO_x86_64_GOTAndStubsBuilder + : public BasicGOTAndStubsBuilder { +public: + MachO_x86_64_GOTAndStubsBuilder(AtomGraph &G) + : BasicGOTAndStubsBuilder(G) {} + + bool isGOTEdge(Edge &E) const { + return E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad; + } + + DefinedAtom &createGOTEntry(Atom &Target) { + auto &GOTEntryAtom = G.addAnonymousAtom(getGOTSection(), 0x0, 8); + GOTEntryAtom.setContent( + StringRef(reinterpret_cast(NullGOTEntryContent), 8)); + GOTEntryAtom.addEdge(Pointer64, 0, Target, 0); + return GOTEntryAtom; + } + + void fixGOTEdge(Edge &E, Atom &GOTEntry) { + assert((E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad) && + "Not a GOT edge?"); + E.setKind(PCRel32); + E.setTarget(GOTEntry); + // Leave the edge addend as-is. + } + + bool isExternalBranchEdge(Edge &E) { + return E.getKind() == Branch32 && !E.getTarget().isDefined(); + } + + DefinedAtom &createStub(Atom &Target) { + auto &StubAtom = G.addAnonymousAtom(getStubsSection(), 0x0, 2); + StubAtom.setContent( + StringRef(reinterpret_cast(StubContent), 6)); + + // Re-use GOT entries for stub targets. + auto &GOTEntryAtom = getGOTEntryAtom(Target); + StubAtom.addEdge(PCRel32, 2, GOTEntryAtom, 0); + + return StubAtom; + } + + void fixExternalBranchEdge(Edge &E, Atom &Stub) { + assert(E.getKind() == Branch32 && "Not a Branch32 edge?"); + assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?"); + E.setTarget(Stub); + } + +private: + Section &getGOTSection() { + if (!GOTSection) + GOTSection = &G.createSection("$__GOT", 8, sys::Memory::MF_READ, false); + return *GOTSection; + } + + Section &getStubsSection() { + if (!StubsSection) { + auto StubsProt = static_cast( + sys::Memory::MF_READ | sys::Memory::MF_EXEC); + StubsSection = &G.createSection("$__STUBS", 8, StubsProt, false); + } + return *StubsSection; + } + + static const uint8_t NullGOTEntryContent[8]; + static const uint8_t StubContent[6]; + Section *GOTSection = nullptr; + Section *StubsSection = nullptr; +}; + +const uint8_t MachO_x86_64_GOTAndStubsBuilder::NullGOTEntryContent[8] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +const uint8_t MachO_x86_64_GOTAndStubsBuilder::StubContent[6] = { + 0xFF, 0x25, 0x00, 0x00, 0x00, 0x00}; +} // namespace + +namespace llvm { +namespace jitlink { + +class MachOJITLinker_x86_64 : public JITLinker { + friend class JITLinker; + +public: + MachOJITLinker_x86_64(std::unique_ptr Ctx, + PassConfiguration PassConfig) + : JITLinker(std::move(Ctx), std::move(PassConfig)) {} + +private: + StringRef getEdgeKindName(Edge::Kind R) const override { + return getMachOX86RelocationKindName(R); + } + + Expected> + buildGraph(MemoryBufferRef ObjBuffer) override { + auto MachOObj = object::ObjectFile::createMachOObjectFile(ObjBuffer); + if (!MachOObj) + return MachOObj.takeError(); + return MachOAtomGraphBuilder_x86_64(**MachOObj).buildGraph(); + } + + static Error targetOutOfRangeError(const Atom &A, const Edge &E) { + std::string ErrMsg; + { + raw_string_ostream ErrStream(ErrMsg); + ErrStream << "Relocation target out of range: "; + printEdge(ErrStream, A, E, getMachOX86RelocationKindName(E.getKind())); + ErrStream << "\n"; + } + return make_error(std::move(ErrMsg)); + } + + Error applyFixup(DefinedAtom &A, const Edge &E, char *AtomWorkingMem) const { + using namespace support; + + char *FixupPtr = AtomWorkingMem + E.getOffset(); + JITTargetAddress FixupAddress = A.getAddress() + E.getOffset(); + + switch (E.getKind()) { + case Branch32: + case PCRel32: + case PCRel32Anon: { + int64_t Value = + E.getTarget().getAddress() - (FixupAddress + 4) + E.getAddend(); + if (Value < std::numeric_limits::min() || + Value > std::numeric_limits::max()) + return targetOutOfRangeError(A, E); + *(little32_t *)FixupPtr = Value; + break; + } + case Pointer64: + case Pointer64Anon: { + uint64_t Value = E.getTarget().getAddress() + E.getAddend(); + *(ulittle64_t *)FixupPtr = Value; + break; + } + case PCRel32Minus1: + case PCRel32Minus2: + case PCRel32Minus4: { + int Delta = 4 + (1 << (E.getKind() - PCRel32Minus1)); + int64_t Value = + E.getTarget().getAddress() - (FixupAddress + Delta) + E.getAddend(); + if (Value < std::numeric_limits::min() || + Value > std::numeric_limits::max()) + return targetOutOfRangeError(A, E); + *(little32_t *)FixupPtr = Value; + break; + } + case PCRel32Minus1Anon: + case PCRel32Minus2Anon: + case PCRel32Minus4Anon: { + int Delta = 4 + (1 << (E.getKind() - PCRel32Minus1Anon)); + int64_t Value = + E.getTarget().getAddress() - (FixupAddress + Delta) + E.getAddend(); + if (Value < std::numeric_limits::min() || + Value > std::numeric_limits::max()) + return targetOutOfRangeError(A, E); + *(little32_t *)FixupPtr = Value; + break; + } + case Delta32: + case Delta64: + case NegDelta32: + case NegDelta64: { + int64_t Value; + if (E.getKind() == Delta32 || E.getKind() == Delta64) + Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + else + Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); + + if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { + if (Value < std::numeric_limits::min() || + Value > std::numeric_limits::max()) + return targetOutOfRangeError(A, E); + *(little32_t *)FixupPtr = Value; + } else + *(little64_t *)FixupPtr = Value; + break; + } + default: + llvm_unreachable("Unrecognized edge kind"); + } + + return Error::success(); + } + + uint64_t NullValue = 0; +}; + +void jitLink_MachO_x86_64(std::unique_ptr Ctx) { + PassConfiguration Config; + Triple TT("x86_64-apple-macosx"); + + if (Ctx->shouldAddDefaultTargetPasses(TT)) { + // Add a mark-live pass. + if (auto MarkLive = Ctx->getMarkLivePass(TT)) + Config.PrePrunePasses.push_back(std::move(MarkLive)); + else + Config.PrePrunePasses.push_back(markAllAtomsLive); + + // Add an in-place GOT/Stubs pass. + Config.PostPrunePasses.push_back([](AtomGraph &G) -> Error { + MachO_x86_64_GOTAndStubsBuilder(G).run(); + return Error::success(); + }); + } + + if (auto Err = Ctx->modifyPassConfig(TT, Config)) + return Ctx->notifyFailed(std::move(Err)); + + // Construct a JITLinker and run the link function. + MachOJITLinker_x86_64::link(std::move(Ctx), std::move(Config)); +} + +StringRef getMachOX86RelocationKindName(Edge::Kind R) { + switch (R) { + case Branch32: + return "Branch32"; + case Pointer64: + return "Pointer64"; + case Pointer64Anon: + return "Pointer64Anon"; + case PCRel32: + return "PCRel32"; + case PCRel32Minus1: + return "PCRel32Minus1"; + case PCRel32Minus2: + return "PCRel32Minus2"; + case PCRel32Minus4: + return "PCRel32Minus4"; + case PCRel32Anon: + return "PCRel32Anon"; + case PCRel32Minus1Anon: + return "PCRel32Minus1Anon"; + case PCRel32Minus2Anon: + return "PCRel32Minus2Anon"; + case PCRel32Minus4Anon: + return "PCRel32Minus4Anon"; + case PCRel32GOTLoad: + return "PCRel32GOTLoad"; + case PCRel32GOT: + return "PCRel32GOT"; + case PCRel32TLV: + return "PCRel32TLV"; + case Delta32: + return "Delta32"; + case Delta64: + return "Delta64"; + case NegDelta32: + return "NegDelta32"; + case NegDelta64: + return "NegDelta64"; + default: + return getGenericEdgeKindName(static_cast(R)); + } +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/lib/ExecutionEngine/MCJIT/MCJIT.cpp b/lib/ExecutionEngine/MCJIT/MCJIT.cpp index ffc6707e1488..08815b7a80ae 100644 --- a/lib/ExecutionEngine/MCJIT/MCJIT.cpp +++ b/lib/ExecutionEngine/MCJIT/MCJIT.cpp @@ -1,9 +1,8 @@ //===-- MCJIT.cpp - MC-based Just-in-Time Compiler ------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/ExecutionEngine/MCJIT/MCJIT.h b/lib/ExecutionEngine/MCJIT/MCJIT.h index 1119e138720f..77097fc0d17e 100644 --- a/lib/ExecutionEngine/MCJIT/MCJIT.h +++ b/lib/ExecutionEngine/MCJIT/MCJIT.h @@ -1,9 +1,8 @@ //===-- MCJIT.h - Class definition for the MCJIT ----------------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp b/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp index 21af6b585c41..2ad9d24555f3 100644 --- a/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp +++ b/lib/ExecutionEngine/OProfileJIT/OProfileJITEventListener.cpp @@ -1,9 +1,8 @@ //===-- OProfileJITEventListener.cpp - Tell OProfile about JITted code ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ExecutionEngine/OProfileJIT/OProfileWrapper.cpp b/lib/ExecutionEngine/OProfileJIT/OProfileWrapper.cpp index b473ac3faf4c..1a2667736926 100644 --- a/lib/ExecutionEngine/OProfileJIT/OProfileWrapper.cpp +++ b/lib/ExecutionEngine/OProfileJIT/OProfileWrapper.cpp @@ -1,9 +1,8 @@ //===-- OProfileWrapper.cpp - OProfile JIT API Wrapper implementation -----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp b/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp index 241eb3600da7..99bf53bc3afa 100644 --- a/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp +++ b/lib/ExecutionEngine/Orc/CompileOnDemandLayer.cpp @@ -1,9 +1,8 @@ //===----- CompileOnDemandLayer.cpp - Lazily emit IR on first call --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/ExecutionEngine/Orc/CompileUtils.cpp b/lib/ExecutionEngine/Orc/CompileUtils.cpp new file mode 100644 index 000000000000..d46b6fcf9a5f --- /dev/null +++ b/lib/ExecutionEngine/Orc/CompileUtils.cpp @@ -0,0 +1,86 @@ +//===------ CompileUtils.cpp - Utilities for compiling IR in the JIT ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/CompileUtils.h" + +#include "llvm/ADT/SmallVector.h" +#include "llvm/ExecutionEngine/ObjectCache.h" +#include "llvm/IR/LegacyPassManager.h" +#include "llvm/IR/Module.h" +#include "llvm/Object/ObjectFile.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/SmallVectorMemoryBuffer.h" +#include "llvm/Target/TargetMachine.h" + +#include + +namespace llvm { +namespace orc { + +/// Compile a Module to an ObjectFile. +SimpleCompiler::CompileResult SimpleCompiler::operator()(Module &M) { + CompileResult CachedObject = tryToLoadFromObjectCache(M); + if (CachedObject) + return CachedObject; + + SmallVector ObjBufferSV; + + { + raw_svector_ostream ObjStream(ObjBufferSV); + + legacy::PassManager PM; + MCContext *Ctx; + if (TM.addPassesToEmitMC(PM, Ctx, ObjStream)) + llvm_unreachable("Target does not support MC emission."); + PM.run(M); + } + + auto ObjBuffer = llvm::make_unique( + std::move(ObjBufferSV), + ""); + + auto Obj = object::ObjectFile::createObjectFile(ObjBuffer->getMemBufferRef()); + + if (Obj) { + notifyObjectCompiled(M, *ObjBuffer); + return std::move(ObjBuffer); + } + + // TODO: Actually report errors helpfully. + consumeError(Obj.takeError()); + return nullptr; +} + +SimpleCompiler::CompileResult +SimpleCompiler::tryToLoadFromObjectCache(const Module &M) { + if (!ObjCache) + return CompileResult(); + + return ObjCache->getObject(&M); +} + +void SimpleCompiler::notifyObjectCompiled(const Module &M, + const MemoryBuffer &ObjBuffer) { + if (ObjCache) + ObjCache->notifyObjectCompiled(&M, ObjBuffer.getMemBufferRef()); +} + +ConcurrentIRCompiler::ConcurrentIRCompiler(JITTargetMachineBuilder JTMB, + ObjectCache *ObjCache) + : JTMB(std::move(JTMB)), ObjCache(ObjCache) {} + +std::unique_ptr ConcurrentIRCompiler::operator()(Module &M) { + auto TM = cantFail(JTMB.createTargetMachine()); + SimpleCompiler C(*TM, ObjCache); + return C(M); +} + +} // end namespace orc +} // end namespace llvm diff --git a/lib/ExecutionEngine/Orc/Core.cpp b/lib/ExecutionEngine/Orc/Core.cpp index 73c0bcdf7d28..dac37e030e0c 100644 --- a/lib/ExecutionEngine/Orc/Core.cpp +++ b/lib/ExecutionEngine/Orc/Core.cpp @@ -1,9 +1,8 @@ //===--- Core.cpp - Core ORC APIs (MaterializationUnit, JITDylib, etc.) ---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -27,17 +26,17 @@ namespace { #ifndef NDEBUG -cl::opt PrintHidden("debug-orc-print-hidden", cl::init(false), +cl::opt PrintHidden("debug-orc-print-hidden", cl::init(true), cl::desc("debug print hidden symbols defined by " "materialization units"), cl::Hidden); -cl::opt PrintCallable("debug-orc-print-callable", cl::init(false), +cl::opt PrintCallable("debug-orc-print-callable", cl::init(true), cl::desc("debug print callable symbols defined by " "materialization units"), cl::Hidden); -cl::opt PrintData("debug-orc-print-data", cl::init(false), +cl::opt PrintData("debug-orc-print-data", cl::init(true), cl::desc("debug print data symbols defined by " "materialization units"), cl::Hidden); @@ -134,8 +133,6 @@ struct PrintSymbolMapElemsMatchingCLOpts { namespace llvm { namespace orc { - SymbolStringPool::PoolMapEntry SymbolStringPtr::Tombstone(0); - char FailedToMaterialize::ID = 0; char SymbolsNotFound::ID = 0; char SymbolsCouldNotBeRemoved::ID = 0; @@ -222,6 +219,31 @@ raw_ostream &operator<<(raw_ostream &OS, const JITDylibSearchList &JDs) { return OS; } +raw_ostream &operator<<(raw_ostream &OS, const SymbolAliasMap &Aliases) { + OS << "{"; + for (auto &KV : Aliases) + OS << " " << *KV.first << ": " << KV.second.Aliasee << " " + << KV.second.AliasFlags; + OS << " }\n"; + return OS; +} + +raw_ostream &operator<<(raw_ostream &OS, const SymbolState &S) { + switch (S) { + case SymbolState::Invalid: + return OS << "Invalid"; + case SymbolState::NeverSearched: + return OS << "Never-Searched"; + case SymbolState::Materializing: + return OS << "Materializing"; + case SymbolState::Resolved: + return OS << "Resolved"; + case SymbolState::Ready: + return OS << "Ready"; + } + llvm_unreachable("Invalid state"); +} + FailedToMaterialize::FailedToMaterialize(SymbolNameSet Symbols) : Symbols(std::move(Symbols)) { assert(!this->Symbols.empty() && "Can not fail to resolve an empty set"); @@ -262,85 +284,46 @@ void SymbolsCouldNotBeRemoved::log(raw_ostream &OS) const { } AsynchronousSymbolQuery::AsynchronousSymbolQuery( - const SymbolNameSet &Symbols, SymbolsResolvedCallback NotifySymbolsResolved, - SymbolsReadyCallback NotifySymbolsReady) - : NotifySymbolsResolved(std::move(NotifySymbolsResolved)), - NotifySymbolsReady(std::move(NotifySymbolsReady)) { - NotYetResolvedCount = NotYetReadyCount = Symbols.size(); + const SymbolNameSet &Symbols, SymbolState RequiredState, + SymbolsResolvedCallback NotifyComplete) + : NotifyComplete(std::move(NotifyComplete)), RequiredState(RequiredState) { + assert(RequiredState >= SymbolState::Resolved && + "Cannot query for a symbols that have not reached the resolve state " + "yet"); + + OutstandingSymbolsCount = Symbols.size(); for (auto &S : Symbols) ResolvedSymbols[S] = nullptr; } -void AsynchronousSymbolQuery::resolve(const SymbolStringPtr &Name, - JITEvaluatedSymbol Sym) { +void AsynchronousSymbolQuery::notifySymbolMetRequiredState( + const SymbolStringPtr &Name, JITEvaluatedSymbol Sym) { auto I = ResolvedSymbols.find(Name); assert(I != ResolvedSymbols.end() && "Resolving symbol outside the requested set"); assert(I->second.getAddress() == 0 && "Redundantly resolving symbol Name"); I->second = std::move(Sym); - --NotYetResolvedCount; -} - -void AsynchronousSymbolQuery::handleFullyResolved() { - assert(NotYetResolvedCount == 0 && "Not fully resolved?"); - - if (!NotifySymbolsResolved) { - // handleFullyResolved may be called by handleFullyReady (see comments in - // that method), in which case this is a no-op, so bail out. - assert(!NotifySymbolsReady && - "NotifySymbolsResolved already called or an error occurred"); - return; - } - - auto TmpNotifySymbolsResolved = std::move(NotifySymbolsResolved); - NotifySymbolsResolved = SymbolsResolvedCallback(); - TmpNotifySymbolsResolved(std::move(ResolvedSymbols)); -} - -void AsynchronousSymbolQuery::notifySymbolReady() { - assert(NotYetReadyCount != 0 && "All symbols already emitted"); - --NotYetReadyCount; + --OutstandingSymbolsCount; } -void AsynchronousSymbolQuery::handleFullyReady() { - assert(NotifySymbolsReady && - "NotifySymbolsReady already called or an error occurred"); +void AsynchronousSymbolQuery::handleComplete() { + assert(OutstandingSymbolsCount == 0 && + "Symbols remain, handleComplete called prematurely"); - auto TmpNotifySymbolsReady = std::move(NotifySymbolsReady); - NotifySymbolsReady = SymbolsReadyCallback(); - - if (NotYetResolvedCount == 0 && NotifySymbolsResolved) { - // The NotifyResolved callback of one query must have caused this query to - // become ready (i.e. there is still a handleFullyResolved callback waiting - // to be made back up the stack). Fold the handleFullyResolved call into - // this one before proceeding. This will cause the call further up the - // stack to become a no-op. - handleFullyResolved(); - } - - assert(QueryRegistrations.empty() && - "Query is still registered with some symbols"); - assert(!NotifySymbolsResolved && "Resolution not applied yet"); - TmpNotifySymbolsReady(Error::success()); + auto TmpNotifyComplete = std::move(NotifyComplete); + NotifyComplete = SymbolsResolvedCallback(); + TmpNotifyComplete(std::move(ResolvedSymbols)); } -bool AsynchronousSymbolQuery::canStillFail() { - return (NotifySymbolsResolved || NotifySymbolsReady); -} +bool AsynchronousSymbolQuery::canStillFail() { return !!NotifyComplete; } void AsynchronousSymbolQuery::handleFailed(Error Err) { assert(QueryRegistrations.empty() && ResolvedSymbols.empty() && - NotYetResolvedCount == 0 && NotYetReadyCount == 0 && + OutstandingSymbolsCount == 0 && "Query should already have been abandoned"); - if (NotifySymbolsResolved) { - NotifySymbolsResolved(std::move(Err)); - NotifySymbolsResolved = SymbolsResolvedCallback(); - } else { - assert(NotifySymbolsReady && "Failed after both callbacks issued?"); - NotifySymbolsReady(std::move(Err)); - } - NotifySymbolsReady = SymbolsReadyCallback(); + NotifyComplete(std::move(Err)); + NotifyComplete = SymbolsResolvedCallback(); } void AsynchronousSymbolQuery::addQueryDependence(JITDylib &JD, @@ -363,8 +346,7 @@ void AsynchronousSymbolQuery::removeQueryDependence( void AsynchronousSymbolQuery::detach() { ResolvedSymbols.clear(); - NotYetResolvedCount = 0; - NotYetReadyCount = 0; + OutstandingSymbolsCount = 0; for (auto &KV : QueryRegistrations) KV.first->detachQueryHelper(*this, KV.second); QueryRegistrations.clear(); @@ -374,11 +356,6 @@ MaterializationResponsibility::MaterializationResponsibility( JITDylib &JD, SymbolFlagsMap SymbolFlags, VModuleKey K) : JD(JD), SymbolFlags(std::move(SymbolFlags)), K(std::move(K)) { assert(!this->SymbolFlags.empty() && "Materializing nothing?"); - -#ifndef NDEBUG - for (auto &KV : this->SymbolFlags) - KV.second |= JITSymbolFlags::Materializing; -#endif } MaterializationResponsibility::~MaterializationResponsibility() { @@ -390,16 +367,15 @@ SymbolNameSet MaterializationResponsibility::getRequestedSymbols() const { return JD.getRequestedSymbols(SymbolFlags); } -void MaterializationResponsibility::resolve(const SymbolMap &Symbols) { - LLVM_DEBUG(dbgs() << "In " << JD.getName() << " resolving " << Symbols - << "\n"); +void MaterializationResponsibility::notifyResolved(const SymbolMap &Symbols) { + LLVM_DEBUG({ + dbgs() << "In " << JD.getName() << " resolving " << Symbols << "\n"; + }); #ifndef NDEBUG for (auto &KV : Symbols) { auto I = SymbolFlags.find(KV.first); assert(I != SymbolFlags.end() && "Resolving symbol outside this responsibility set"); - assert(I->second.isMaterializing() && "Duplicate resolution"); - I->second &= ~JITSymbolFlags::Materializing; if (I->second.isWeak()) assert(I->second == (KV.second.getFlags() | JITSymbolFlags::Weak) && "Resolving symbol with incorrect flags"); @@ -412,12 +388,11 @@ void MaterializationResponsibility::resolve(const SymbolMap &Symbols) { JD.resolve(Symbols); } -void MaterializationResponsibility::emit() { -#ifndef NDEBUG - for (auto &KV : SymbolFlags) - assert(!KV.second.isMaterializing() && - "Failed to resolve symbol before emission"); -#endif // NDEBUG +void MaterializationResponsibility::notifyEmitted() { + + LLVM_DEBUG({ + dbgs() << "In " << JD.getName() << " emitting " << SymbolFlags << "\n"; + }); JD.emit(SymbolFlags); SymbolFlags.clear(); @@ -429,19 +404,19 @@ Error MaterializationResponsibility::defineMaterializing( // It's ok if we hit a duplicate here: In that case the new version will be // discarded, and the JITDylib::defineMaterializing method will return a // duplicate symbol error. - for (auto &KV : NewSymbolFlags) { - auto I = SymbolFlags.insert(KV).first; - (void)I; -#ifndef NDEBUG - I->second |= JITSymbolFlags::Materializing; -#endif - } + for (auto &KV : NewSymbolFlags) + SymbolFlags.insert(KV); return JD.defineMaterializing(NewSymbolFlags); } void MaterializationResponsibility::failMaterialization() { + LLVM_DEBUG({ + dbgs() << "In " << JD.getName() << " failing materialization for " + << SymbolFlags << "\n"; + }); + SymbolNameSet FailedSymbols; for (auto &KV : SymbolFlags) FailedSymbols.insert(KV.first); @@ -510,8 +485,8 @@ StringRef AbsoluteSymbolsMaterializationUnit::getName() const { void AbsoluteSymbolsMaterializationUnit::materialize( MaterializationResponsibility R) { - R.resolve(Symbols); - R.emit(); + R.notifyResolved(Symbols); + R.notifyEmitted(); } void AbsoluteSymbolsMaterializationUnit::discard(const JITDylib &JD, @@ -559,6 +534,14 @@ void ReExportsMaterializationUnit::materialize( Aliases.erase(I); } + LLVM_DEBUG({ + ES.runSessionLocked([&]() { + dbgs() << "materializing reexports: target = " << TgtJD.getName() + << ", source = " << SrcJD.getName() << " " << RequestedAliases + << "\n"; + }); + }); + if (!Aliases.empty()) { if (SourceJD) R.replace(reexports(*SourceJD, std::move(Aliases), MatchNonExported)); @@ -641,7 +624,7 @@ void ReExportsMaterializationUnit::materialize( } }; - auto OnResolve = [QueryInfo](Expected Result) { + auto OnComplete = [QueryInfo](Expected Result) { if (Result) { SymbolMap ResolutionMap; for (auto &KV : QueryInfo->Aliases) { @@ -650,8 +633,8 @@ void ReExportsMaterializationUnit::materialize( ResolutionMap[KV.first] = JITEvaluatedSymbol( (*Result)[KV.second.Aliasee].getAddress(), KV.second.AliasFlags); } - QueryInfo->R.resolve(ResolutionMap); - QueryInfo->R.emit(); + QueryInfo->R.notifyResolved(ResolutionMap); + QueryInfo->R.notifyEmitted(); } else { auto &ES = QueryInfo->R.getTargetJITDylib().getExecutionSession(); ES.reportError(Result.takeError()); @@ -659,10 +642,8 @@ void ReExportsMaterializationUnit::materialize( } }; - auto OnReady = [&ES](Error Err) { ES.reportError(std::move(Err)); }; - ES.lookup(JITDylibSearchList({{&SrcJD, MatchNonExported}}), QuerySymbols, - std::move(OnResolve), std::move(OnReady), + SymbolState::Resolved, std::move(OnComplete), std::move(RegisterDependencies)); } } @@ -687,17 +668,20 @@ Expected buildSimpleReexportsAliasMap(JITDylib &SourceJD, const SymbolNameSet &Symbols) { auto Flags = SourceJD.lookupFlags(Symbols); - if (Flags.size() != Symbols.size()) { + if (!Flags) + return Flags.takeError(); + + if (Flags->size() != Symbols.size()) { SymbolNameSet Unresolved = Symbols; - for (auto &KV : Flags) + for (auto &KV : *Flags) Unresolved.erase(KV.first); return make_error(std::move(Unresolved)); } SymbolAliasMap Result; for (auto &Name : Symbols) { - assert(Flags.count(Name) && "Missing entry in flags map"); - Result[Name] = SymbolAliasMapEntry(Name, Flags[Name]); + assert(Flags->count(Name) && "Missing entry in flags map"); + Result[Name] = SymbolAliasMapEntry(Name, (*Flags)[Name]); } return Result; @@ -709,14 +693,17 @@ ReexportsGenerator::ReexportsGenerator(JITDylib &SourceJD, : SourceJD(SourceJD), MatchNonExported(MatchNonExported), Allow(std::move(Allow)) {} -SymbolNameSet ReexportsGenerator::operator()(JITDylib &JD, - const SymbolNameSet &Names) { +Expected +ReexportsGenerator::operator()(JITDylib &JD, const SymbolNameSet &Names) { orc::SymbolNameSet Added; orc::SymbolAliasMap AliasMap; auto Flags = SourceJD.lookupFlags(Names); - for (auto &KV : Flags) { + if (!Flags) + return Flags.takeError(); + + for (auto &KV : *Flags) { if (Allow && !Allow(KV.first)) continue; AliasMap[KV.first] = SymbolAliasMapEntry(KV.first, KV.second); @@ -731,21 +718,19 @@ SymbolNameSet ReexportsGenerator::operator()(JITDylib &JD, Error JITDylib::defineMaterializing(const SymbolFlagsMap &SymbolFlags) { return ES.runSessionLocked([&]() -> Error { - std::vector AddedSyms; + std::vector AddedSyms; for (auto &KV : SymbolFlags) { - SymbolMap::iterator EntryItr; + SymbolTable::iterator EntryItr; bool Added; - auto NewFlags = KV.second; - NewFlags |= JITSymbolFlags::Materializing; - - std::tie(EntryItr, Added) = Symbols.insert( - std::make_pair(KV.first, JITEvaluatedSymbol(0, NewFlags))); + std::tie(EntryItr, Added) = + Symbols.insert(std::make_pair(KV.first, SymbolTableEntry(KV.second))); - if (Added) + if (Added) { AddedSyms.push_back(EntryItr); - else { + EntryItr->second.setState(SymbolState::Materializing); + } else { // Remove any symbols already added. for (auto &SI : AddedSyms) Symbols.erase(SI); @@ -769,9 +754,10 @@ void JITDylib::replace(std::unique_ptr MU) { for (auto &KV : MU->getSymbols()) { auto SymI = Symbols.find(KV.first); assert(SymI != Symbols.end() && "Replacing unknown symbol"); - assert(!SymI->second.getFlags().isLazy() && - SymI->second.getFlags().isMaterializing() && - "Can not replace symbol that is not materializing"); + assert(SymI->second.isInMaterializationPhase() && + "Can not call replace on a symbol that is not materializing"); + assert(!SymI->second.hasMaterializerAttached() && + "Symbol should not have materializer attached already"); assert(UnmaterializedInfos.count(KV.first) == 0 && "Symbol being replaced should have no UnmaterializedInfo"); } @@ -782,7 +768,7 @@ void JITDylib::replace(std::unique_ptr MU) { for (auto &KV : MU->getSymbols()) { auto MII = MaterializingInfos.find(KV.first); if (MII != MaterializingInfos.end()) { - if (!MII->second.PendingQueries.empty()) + if (MII->second.hasQueriesPending()) return std::move(MU); } } @@ -790,16 +776,15 @@ void JITDylib::replace(std::unique_ptr MU) { // Otherwise, make MU responsible for all the symbols. auto UMI = std::make_shared(std::move(MU)); for (auto &KV : UMI->MU->getSymbols()) { - assert(!KV.second.isLazy() && - "Lazy flag should be managed internally."); - assert(!KV.second.isMaterializing() && - "Materializing flags should be managed internally."); - auto SymI = Symbols.find(KV.first); - JITSymbolFlags ReplaceFlags = KV.second; - ReplaceFlags |= JITSymbolFlags::Lazy; - SymI->second = JITEvaluatedSymbol(SymI->second.getAddress(), - std::move(ReplaceFlags)); + assert(SymI->second.getState() == SymbolState::Materializing && + "Can not replace a symbol that is not materializing"); + assert(!SymI->second.hasMaterializerAttached() && + "Can not replace a symbol that has a materializer attached"); + assert(UnmaterializedInfos.count(KV.first) == 0 && + "Unexpected materializer entry in map"); + SymI->second.setAddress(SymI->second.getAddress()); + SymI->second.setMaterializerAttached(true); UnmaterializedInfos[KV.first] = UMI; } @@ -817,14 +802,14 @@ JITDylib::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) const { for (auto &KV : SymbolFlags) { assert(Symbols.count(KV.first) && "JITDylib does not cover this symbol?"); - assert(Symbols.find(KV.first)->second.getFlags().isMaterializing() && - "getRequestedSymbols can only be called for materializing " - "symbols"); + assert(Symbols.find(KV.first)->second.isInMaterializationPhase() && + "getRequestedSymbols can only be called for symbols that have " + "started materializing"); auto I = MaterializingInfos.find(KV.first); if (I == MaterializingInfos.end()) continue; - if (!I->second.PendingQueries.empty()) + if (I->second.hasQueriesPending()) RequestedSymbols.insert(KV.first); } @@ -835,9 +820,8 @@ JITDylib::getRequestedSymbols(const SymbolFlagsMap &SymbolFlags) const { void JITDylib::addDependencies(const SymbolStringPtr &Name, const SymbolDependenceMap &Dependencies) { assert(Symbols.count(Name) && "Name not in symbol table"); - assert((Symbols[Name].getFlags().isLazy() || - Symbols[Name].getFlags().isMaterializing()) && - "Symbol is not lazy or materializing"); + assert(Symbols[Name].isInMaterializationPhase() && + "Can not add dependencies for a symbol that is not materializing"); auto &MI = MaterializingInfos[Name]; assert(!MI.IsEmitted && "Can not add dependencies to an emitted symbol"); @@ -852,9 +836,8 @@ void JITDylib::addDependencies(const SymbolStringPtr &Name, // Assert that this symbol exists and has not been emitted already. auto SymI = OtherJITDylib.Symbols.find(OtherSymbol); assert(SymI != OtherJITDylib.Symbols.end() && - (SymI->second.getFlags().isLazy() || - SymI->second.getFlags().isMaterializing()) && - "Dependency on emitted symbol"); + (SymI->second.getState() != SymbolState::Ready && + "Dependency on emitted symbol")); #endif auto &OtherMI = OtherJITDylib.MaterializingInfos[OtherSymbol]; @@ -873,54 +856,52 @@ void JITDylib::addDependencies(const SymbolStringPtr &Name, } void JITDylib::resolve(const SymbolMap &Resolved) { - auto FullyResolvedQueries = ES.runSessionLocked([&, this]() { - AsynchronousSymbolQuerySet FullyResolvedQueries; + auto CompletedQueries = ES.runSessionLocked([&, this]() { + AsynchronousSymbolQuerySet CompletedQueries; for (const auto &KV : Resolved) { auto &Name = KV.first; auto Sym = KV.second; - assert(!Sym.getFlags().isLazy() && !Sym.getFlags().isMaterializing() && - "Materializing flags should be managed internally"); - auto I = Symbols.find(Name); assert(I != Symbols.end() && "Symbol not found"); - assert(!I->second.getFlags().isLazy() && - I->second.getFlags().isMaterializing() && + assert(!I->second.hasMaterializerAttached() && + "Resolving symbol with materializer attached?"); + assert(I->second.getState() == SymbolState::Materializing && "Symbol should be materializing"); assert(I->second.getAddress() == 0 && "Symbol has already been resolved"); assert((Sym.getFlags() & ~JITSymbolFlags::Weak) == - (JITSymbolFlags::stripTransientFlags(I->second.getFlags()) & - ~JITSymbolFlags::Weak) && + (I->second.getFlags() & ~JITSymbolFlags::Weak) && "Resolved flags should match the declared flags"); // Once resolved, symbols can never be weak. JITSymbolFlags ResolvedFlags = Sym.getFlags(); ResolvedFlags &= ~JITSymbolFlags::Weak; - ResolvedFlags |= JITSymbolFlags::Materializing; - I->second = JITEvaluatedSymbol(Sym.getAddress(), ResolvedFlags); + I->second.setAddress(Sym.getAddress()); + I->second.setFlags(ResolvedFlags); + I->second.setState(SymbolState::Resolved); auto &MI = MaterializingInfos[Name]; - for (auto &Q : MI.PendingQueries) { - Q->resolve(Name, Sym); - if (Q->isFullyResolved()) - FullyResolvedQueries.insert(Q); + for (auto &Q : MI.takeQueriesMeeting(SymbolState::Resolved)) { + Q->notifySymbolMetRequiredState(Name, Sym); + if (Q->isComplete()) + CompletedQueries.insert(std::move(Q)); } } - return FullyResolvedQueries; + return CompletedQueries; }); - for (auto &Q : FullyResolvedQueries) { - assert(Q->isFullyResolved() && "Q not fully resolved"); - Q->handleFullyResolved(); + for (auto &Q : CompletedQueries) { + assert(Q->isComplete() && "Q not completed"); + Q->handleComplete(); } } void JITDylib::emit(const SymbolFlagsMap &Emitted) { - auto FullyReadyQueries = ES.runSessionLocked([&, this]() { - AsynchronousSymbolQuerySet ReadyQueries; + auto CompletedQueries = ES.runSessionLocked([&, this]() { + AsynchronousSymbolQuerySet CompletedQueries; for (const auto &KV : Emitted) { const auto &Name = KV.first; @@ -962,20 +943,22 @@ void JITDylib::emit(const SymbolFlagsMap &Emitted) { DependantMI.UnemittedDependencies.empty()) { assert(DependantMI.Dependants.empty() && "Dependants should be empty by now"); - for (auto &Q : DependantMI.PendingQueries) { - Q->notifySymbolReady(); - if (Q->isFullyReady()) - ReadyQueries.insert(Q); - Q->removeQueryDependence(DependantJD, DependantName); - } // Since this dependant is now ready, we erase its MaterializingInfo // and update its materializing state. - assert(DependantJD.Symbols.count(DependantName) && + auto DependantSymI = DependantJD.Symbols.find(DependantName); + assert(DependantSymI != DependantJD.Symbols.end() && "Dependant has no entry in the Symbols table"); - auto &DependantSym = DependantJD.Symbols[DependantName]; - DependantSym.setFlags(DependantSym.getFlags() & - ~JITSymbolFlags::Materializing); + DependantSymI->second.setState(SymbolState::Ready); + + for (auto &Q : DependantMI.takeQueriesMeeting(SymbolState::Ready)) { + Q->notifySymbolMetRequiredState( + DependantName, DependantSymI->second.getSymbol()); + if (Q->isComplete()) + CompletedQueries.insert(Q); + Q->removeQueryDependence(DependantJD, DependantName); + } + DependantJD.MaterializingInfos.erase(DependantMII); } } @@ -984,26 +967,25 @@ void JITDylib::emit(const SymbolFlagsMap &Emitted) { MI.IsEmitted = true; if (MI.UnemittedDependencies.empty()) { - for (auto &Q : MI.PendingQueries) { - Q->notifySymbolReady(); - if (Q->isFullyReady()) - ReadyQueries.insert(Q); + auto SymI = Symbols.find(Name); + assert(SymI != Symbols.end() && "Symbol has no entry in Symbols table"); + SymI->second.setState(SymbolState::Ready); + for (auto &Q : MI.takeQueriesMeeting(SymbolState::Ready)) { + Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); + if (Q->isComplete()) + CompletedQueries.insert(Q); Q->removeQueryDependence(*this, Name); } - assert(Symbols.count(Name) && - "Symbol has no entry in the Symbols table"); - auto &Sym = Symbols[Name]; - Sym.setFlags(Sym.getFlags() & ~JITSymbolFlags::Materializing); MaterializingInfos.erase(MII); } } - return ReadyQueries; + return CompletedQueries; }); - for (auto &Q : FullyReadyQueries) { - assert(Q->isFullyReady() && "Q is not fully ready"); - Q->handleFullyReady(); + for (auto &Q : CompletedQueries) { + assert(Q->isComplete() && "Q is not complete"); + Q->handleComplete(); } } @@ -1013,6 +995,7 @@ void JITDylib::notifyFailed(const SymbolNameSet &FailedSymbols) { auto FailedQueriesToNotify = ES.runSessionLocked([&, this]() { AsynchronousSymbolQuerySet FailedQueries; + std::vector MIIsToRemove; for (auto &Name : FailedSymbols) { auto I = Symbols.find(Name); @@ -1026,17 +1009,40 @@ void JITDylib::notifyFailed(const SymbolNameSet &FailedSymbols) { if (MII == MaterializingInfos.end()) continue; + // Remove this symbol from the dependants list of any dependencies. + for (auto &KV : MII->second.UnemittedDependencies) { + auto *DependencyJD = KV.first; + auto &Dependencies = KV.second; + for (auto &DependencyName : Dependencies) { + auto DependencyMII = + DependencyJD->MaterializingInfos.find(DependencyName); + assert(DependencyMII != DependencyJD->MaterializingInfos.end() && + "Unemitted dependency must have a MaterializingInfo entry"); + assert(DependencyMII->second.Dependants.count(this) && + "Dependency's dependants list does not contain this JITDylib"); + assert(DependencyMII->second.Dependants[this].count(Name) && + "Dependency's dependants list does not contain dependant"); + DependencyMII->second.Dependants[this].erase(Name); + } + } + // Copy all the queries to the FailedQueries list, then abandon them. // This has to be a copy, and the copy has to come before the abandon // operation: Each Q.detach() call will reach back into this // PendingQueries list to remove Q. - for (auto &Q : MII->second.PendingQueries) + for (auto &Q : MII->second.pendingQueries()) FailedQueries.insert(Q); - for (auto &Q : FailedQueries) - Q->detach(); + MIIsToRemove.push_back(std::move(MII)); + } + + // Detach failed queries. + for (auto &Q : FailedQueries) + Q->detach(); - assert(MII->second.PendingQueries.empty() && + // Remove the MaterializingInfos. + for (auto &MII : MIIsToRemove) { + assert(!MII->second.hasQueriesPending() && "Queries remain after symbol was failed"); MaterializingInfos.erase(MII); @@ -1052,9 +1058,11 @@ void JITDylib::notifyFailed(const SymbolNameSet &FailedSymbols) { void JITDylib::setSearchOrder(JITDylibSearchList NewSearchOrder, bool SearchThisJITDylibFirst, bool MatchNonExportedInThisDylib) { - if (SearchThisJITDylibFirst && NewSearchOrder.front().first != this) - NewSearchOrder.insert(NewSearchOrder.begin(), - {this, MatchNonExportedInThisDylib}); + if (SearchThisJITDylibFirst) { + if (NewSearchOrder.empty() || NewSearchOrder.front().first != this) + NewSearchOrder.insert(NewSearchOrder.begin(), + {this, MatchNonExportedInThisDylib}); + } ES.runSessionLocked([&]() { SearchOrder = std::move(NewSearchOrder); }); } @@ -1092,7 +1100,7 @@ void JITDylib::removeFromSearchOrder(JITDylib &JD) { Error JITDylib::remove(const SymbolNameSet &Names) { return ES.runSessionLocked([&]() -> Error { using SymbolMaterializerItrPair = - std::pair; + std::pair; std::vector SymbolsToRemove; SymbolNameSet Missing; SymbolNameSet Materializing; @@ -1107,13 +1115,14 @@ Error JITDylib::remove(const SymbolNameSet &Names) { } // Note symbol materializing. - if (I->second.getFlags().isMaterializing()) { + if (I->second.isInMaterializationPhase()) { Materializing.insert(Name); continue; } - auto UMII = I->second.getFlags().isLazy() ? UnmaterializedInfos.find(Name) - : UnmaterializedInfos.end(); + auto UMII = I->second.hasMaterializerAttached() + ? UnmaterializedInfos.find(Name) + : UnmaterializedInfos.end(); SymbolsToRemove.push_back(std::make_pair(I, UMII)); } @@ -1143,16 +1152,23 @@ Error JITDylib::remove(const SymbolNameSet &Names) { }); } -SymbolFlagsMap JITDylib::lookupFlags(const SymbolNameSet &Names) { - return ES.runSessionLocked([&, this]() { +Expected JITDylib::lookupFlags(const SymbolNameSet &Names) { + return ES.runSessionLocked([&, this]() -> Expected { SymbolFlagsMap Result; auto Unresolved = lookupFlagsImpl(Result, Names); - if (DefGenerator && !Unresolved.empty()) { - auto NewDefs = DefGenerator(*this, Unresolved); - if (!NewDefs.empty()) { - auto Unresolved2 = lookupFlagsImpl(Result, NewDefs); + if (!Unresolved) + return Unresolved.takeError(); + + if (DefGenerator && !Unresolved->empty()) { + auto NewDefs = DefGenerator(*this, *Unresolved); + if (!NewDefs) + return NewDefs.takeError(); + if (!NewDefs->empty()) { + auto Unresolved2 = lookupFlagsImpl(Result, *NewDefs); + if (!Unresolved2) + return Unresolved2.takeError(); (void)Unresolved2; - assert(Unresolved2.empty() && + assert(Unresolved2->empty() && "All fallback defs should have been found by lookupFlagsImpl"); } }; @@ -1160,41 +1176,42 @@ SymbolFlagsMap JITDylib::lookupFlags(const SymbolNameSet &Names) { }); } -SymbolNameSet JITDylib::lookupFlagsImpl(SymbolFlagsMap &Flags, - const SymbolNameSet &Names) { +Expected JITDylib::lookupFlagsImpl(SymbolFlagsMap &Flags, + const SymbolNameSet &Names) { SymbolNameSet Unresolved; for (auto &Name : Names) { auto I = Symbols.find(Name); - - if (I == Symbols.end()) { + if (I != Symbols.end()) { + assert(!Flags.count(Name) && "Symbol already present in Flags map"); + Flags[Name] = I->second.getFlags(); + } else Unresolved.insert(Name); - continue; - } - - assert(!Flags.count(Name) && "Symbol already present in Flags map"); - Flags[Name] = JITSymbolFlags::stripTransientFlags(I->second.getFlags()); } return Unresolved; } -void JITDylib::lodgeQuery(std::shared_ptr &Q, - SymbolNameSet &Unresolved, bool MatchNonExported, - MaterializationUnitList &MUs) { +Error JITDylib::lodgeQuery(std::shared_ptr &Q, + SymbolNameSet &Unresolved, bool MatchNonExported, + MaterializationUnitList &MUs) { assert(Q && "Query can not be null"); lodgeQueryImpl(Q, Unresolved, MatchNonExported, MUs); if (DefGenerator && !Unresolved.empty()) { auto NewDefs = DefGenerator(*this, Unresolved); - if (!NewDefs.empty()) { - for (auto &D : NewDefs) + if (!NewDefs) + return NewDefs.takeError(); + if (!NewDefs->empty()) { + for (auto &D : *NewDefs) Unresolved.erase(D); - lodgeQueryImpl(Q, NewDefs, MatchNonExported, MUs); - assert(NewDefs.empty() && + lodgeQueryImpl(Q, *NewDefs, MatchNonExported, MUs); + assert(NewDefs->empty() && "All fallback defs should have been found by lookupImpl"); } } + + return Error::success(); } void JITDylib::lodgeQueryImpl( @@ -1204,6 +1221,7 @@ void JITDylib::lodgeQueryImpl( std::vector ToRemove; for (auto Name : Unresolved) { + // Search for the name in Symbols. Skip it if not found. auto SymI = Symbols.find(Name); if (SymI == Symbols.end()) @@ -1213,20 +1231,22 @@ void JITDylib::lodgeQueryImpl( if (!SymI->second.getFlags().isExported() && !MatchNonExported) continue; - // If we matched against Name in JD, mark it to be removed from the Unresolved - // set. + // If we matched against Name in JD, mark it to be removed from the + // Unresolved set. ToRemove.push_back(Name); - // If the symbol has an address then resolve it. - if (SymI->second.getAddress() != 0) - Q->resolve(Name, SymI->second); + // If this symbol already meets the required state for then notify the + // query and continue. + if (SymI->second.getState() >= Q->getRequiredState()) { + Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); + continue; + } - // If the symbol is lazy, get the MaterialiaztionUnit for it. - if (SymI->second.getFlags().isLazy()) { + // Otherwise this symbol does not yet meet the required state. Check whether + // it has a materializer attached, and if so prepare to run it. + if (SymI->second.hasMaterializerAttached()) { assert(SymI->second.getAddress() == 0 && - "Lazy symbol should not have a resolved address"); - assert(!SymI->second.getFlags().isMaterializing() && - "Materializing and lazy should not both be set"); + "Symbol not resolved but already has address?"); auto UMII = UnmaterializedInfos.find(Name); assert(UMII != UnmaterializedInfos.end() && "Lazy symbol should have UnmaterializedInfo"); @@ -1237,27 +1257,20 @@ void JITDylib::lodgeQueryImpl( // materializing state. for (auto &KV : MU->getSymbols()) { auto SymK = Symbols.find(KV.first); - auto Flags = SymK->second.getFlags(); - Flags &= ~JITSymbolFlags::Lazy; - Flags |= JITSymbolFlags::Materializing; - SymK->second.setFlags(Flags); + SymK->second.setMaterializerAttached(false); + SymK->second.setState(SymbolState::Materializing); UnmaterializedInfos.erase(KV.first); } // Add MU to the list of MaterializationUnits to be materialized. MUs.push_back(std::move(MU)); - } else if (!SymI->second.getFlags().isMaterializing()) { - // The symbol is neither lazy nor materializing, so it must be - // ready. Notify the query and continue. - Q->notifySymbolReady(); - continue; } // Add the query to the PendingQueries list. - assert(SymI->second.getFlags().isMaterializing() && + assert(SymI->second.isInMaterializationPhase() && "By this line the symbol should be materializing"); auto &MI = MaterializingInfos[Name]; - MI.PendingQueries.push_back(Q); + MI.addQuery(Q); Q->addQueryDependence(*this, Name); } @@ -1266,40 +1279,43 @@ void JITDylib::lodgeQueryImpl( Unresolved.erase(Name); } -SymbolNameSet JITDylib::legacyLookup(std::shared_ptr Q, - SymbolNameSet Names) { +Expected +JITDylib::legacyLookup(std::shared_ptr Q, + SymbolNameSet Names) { assert(Q && "Query can not be null"); ES.runOutstandingMUs(); - LookupImplActionFlags ActionFlags = None; + bool QueryComplete = false; std::vector> MUs; SymbolNameSet Unresolved = std::move(Names); - ES.runSessionLocked([&, this]() { - ActionFlags = lookupImpl(Q, MUs, Unresolved); + auto Err = ES.runSessionLocked([&, this]() -> Error { + QueryComplete = lookupImpl(Q, MUs, Unresolved); if (DefGenerator && !Unresolved.empty()) { - assert(ActionFlags == None && - "ActionFlags set but unresolved symbols remain?"); + assert(!QueryComplete && "query complete but unresolved symbols remain?"); auto NewDefs = DefGenerator(*this, Unresolved); - if (!NewDefs.empty()) { - for (auto &D : NewDefs) + if (!NewDefs) + return NewDefs.takeError(); + if (!NewDefs->empty()) { + for (auto &D : *NewDefs) Unresolved.erase(D); - ActionFlags = lookupImpl(Q, MUs, NewDefs); - assert(NewDefs.empty() && + QueryComplete = lookupImpl(Q, MUs, *NewDefs); + assert(NewDefs->empty() && "All fallback defs should have been found by lookupImpl"); } } + return Error::success(); }); - assert((MUs.empty() || ActionFlags == None) && - "If action flags are set, there should be no work to do (so no MUs)"); + if (Err) + return std::move(Err); - if (ActionFlags & NotifyFullyResolved) - Q->handleFullyResolved(); + assert((MUs.empty() || !QueryComplete) && + "If action flags are set, there should be no work to do (so no MUs)"); - if (ActionFlags & NotifyFullyReady) - Q->handleFullyReady(); + if (QueryComplete) + Q->handleComplete(); // FIXME: Swap back to the old code below once RuntimeDyld works with // callbacks from asynchronous queries. @@ -1318,13 +1334,13 @@ SymbolNameSet JITDylib::legacyLookup(std::shared_ptr Q, return Unresolved; } -JITDylib::LookupImplActionFlags -JITDylib::lookupImpl(std::shared_ptr &Q, - std::vector> &MUs, - SymbolNameSet &Unresolved) { - LookupImplActionFlags ActionFlags = None; - std::vector ToRemove; +bool JITDylib::lookupImpl( + std::shared_ptr &Q, + std::vector> &MUs, + SymbolNameSet &Unresolved) { + bool QueryComplete = false; + std::vector ToRemove; for (auto Name : Unresolved) { // Search for the name in Symbols. Skip it if not found. @@ -1335,19 +1351,17 @@ JITDylib::lookupImpl(std::shared_ptr &Q, // If we found Name, mark it to be removed from the Unresolved set. ToRemove.push_back(Name); - // If the symbol has an address then resolve it. - if (SymI->second.getAddress() != 0) { - Q->resolve(Name, SymI->second); - if (Q->isFullyResolved()) - ActionFlags |= NotifyFullyResolved; + if (SymI->second.getState() >= Q->getRequiredState()) { + Q->notifySymbolMetRequiredState(Name, SymI->second.getSymbol()); + if (Q->isComplete()) + QueryComplete = true; + continue; } // If the symbol is lazy, get the MaterialiaztionUnit for it. - if (SymI->second.getFlags().isLazy()) { + if (SymI->second.hasMaterializerAttached()) { assert(SymI->second.getAddress() == 0 && "Lazy symbol should not have a resolved address"); - assert(!SymI->second.getFlags().isMaterializing() && - "Materializing and lazy should not both be set"); auto UMII = UnmaterializedInfos.find(Name); assert(UMII != UnmaterializedInfos.end() && "Lazy symbol should have UnmaterializedInfo"); @@ -1358,29 +1372,21 @@ JITDylib::lookupImpl(std::shared_ptr &Q, // materializing state. for (auto &KV : MU->getSymbols()) { auto SymK = Symbols.find(KV.first); - auto Flags = SymK->second.getFlags(); - Flags &= ~JITSymbolFlags::Lazy; - Flags |= JITSymbolFlags::Materializing; - SymK->second.setFlags(Flags); + assert(SymK != Symbols.end() && "Missing symbol table entry"); + SymK->second.setState(SymbolState::Materializing); + SymK->second.setMaterializerAttached(false); UnmaterializedInfos.erase(KV.first); } // Add MU to the list of MaterializationUnits to be materialized. MUs.push_back(std::move(MU)); - } else if (!SymI->second.getFlags().isMaterializing()) { - // The symbol is neither lazy nor materializing, so it must be ready. - // Notify the query and continue. - Q->notifySymbolReady(); - if (Q->isFullyReady()) - ActionFlags |= NotifyFullyReady; - continue; } // Add the query to the PendingQueries list. - assert(SymI->second.getFlags().isMaterializing() && + assert(SymI->second.isInMaterializationPhase() && "By this line the symbol should be materializing"); auto &MI = MaterializingInfos[Name]; - MI.PendingQueries.push_back(Q); + MI.addQuery(Q); Q->addQueryDependence(*this, Name); } @@ -1388,7 +1394,7 @@ JITDylib::lookupImpl(std::shared_ptr &Q, for (auto &Name : ToRemove) Unresolved.erase(Name); - return ActionFlags; + return QueryComplete; } void JITDylib::dump(raw_ostream &OS) { @@ -1405,21 +1411,19 @@ void JITDylib::dump(raw_ostream &OS) { for (auto &KV : Symbols) { OS << " \"" << *KV.first << "\": "; if (auto Addr = KV.second.getAddress()) - OS << format("0x%016" PRIx64, Addr) << ", " << KV.second.getFlags(); + OS << format("0x%016" PRIx64, Addr) << ", " << KV.second.getFlags() + << " "; else - OS << ""; - if (KV.second.getFlags().isLazy() || - KV.second.getFlags().isMaterializing()) { - OS << " ("; - if (KV.second.getFlags().isLazy()) { - auto I = UnmaterializedInfos.find(KV.first); - assert(I != UnmaterializedInfos.end() && - "Lazy symbol should have UnmaterializedInfo"); - OS << " Lazy (MU=" << I->second->MU.get() << ")"; - } - if (KV.second.getFlags().isMaterializing()) - OS << " Materializing"; - OS << ", " << KV.second.getFlags() << " )\n"; + OS << " "; + + OS << KV.second.getState(); + + if (KV.second.hasMaterializerAttached()) { + OS << " (Materializer "; + auto I = UnmaterializedInfos.find(KV.first); + assert(I != UnmaterializedInfos.end() && + "Lazy symbol should have UnmaterializedInfo"); + OS << I->second->MU.get() << ")\n"; } else OS << "\n"; } @@ -1430,10 +1434,10 @@ void JITDylib::dump(raw_ostream &OS) { OS << " \"" << *KV.first << "\":\n" << " IsEmitted = " << (KV.second.IsEmitted ? "true" : "false") << "\n" - << " " << KV.second.PendingQueries.size() + << " " << KV.second.pendingQueries().size() << " pending queries: { "; - for (auto &Q : KV.second.PendingQueries) - OS << Q.get() << " "; + for (const auto &Q : KV.second.pendingQueries()) + OS << Q.get() << " (" << Q->getRequiredState() << ") "; OS << "}\n Dependants:\n"; for (auto &KV2 : KV.second.Dependants) OS << " " << KV2.first->getName() << ": " << KV2.second << "\n"; @@ -1444,6 +1448,51 @@ void JITDylib::dump(raw_ostream &OS) { }); } +void JITDylib::MaterializingInfo::addQuery( + std::shared_ptr Q) { + + auto I = std::lower_bound( + PendingQueries.rbegin(), PendingQueries.rend(), Q->getRequiredState(), + [](const std::shared_ptr &V, SymbolState S) { + return V->getRequiredState() <= S; + }); + PendingQueries.insert(I.base(), std::move(Q)); +} + +void JITDylib::MaterializingInfo::removeQuery( + const AsynchronousSymbolQuery &Q) { + // FIXME: Implement 'find_as' for shared_ptr/T*. + auto I = + std::find_if(PendingQueries.begin(), PendingQueries.end(), + [&Q](const std::shared_ptr &V) { + return V.get() == &Q; + }); + assert(I != PendingQueries.end() && + "Query is not attached to this MaterializingInfo"); + PendingQueries.erase(I); +} + +JITDylib::AsynchronousSymbolQueryList +JITDylib::MaterializingInfo::takeQueriesMeeting(SymbolState RequiredState) { + AsynchronousSymbolQueryList Result; + while (!PendingQueries.empty()) { + if (PendingQueries.back()->getRequiredState() > RequiredState) + break; + + Result.push_back(std::move(PendingQueries.back())); + PendingQueries.pop_back(); + } + + return Result; +} + +JITDylib::AsynchronousSymbolQueryList +JITDylib::MaterializingInfo::takeAllQueries() { + AsynchronousSymbolQueryList Result; + std::swap(Result, PendingQueries); + return Result; +} + JITDylib::JITDylib(ExecutionSession &ES, std::string Name) : ES(ES), JITDylibName(std::move(Name)) { SearchOrder.push_back({this, true}); @@ -1451,77 +1500,52 @@ JITDylib::JITDylib(ExecutionSession &ES, std::string Name) Error JITDylib::defineImpl(MaterializationUnit &MU) { SymbolNameSet Duplicates; - SymbolNameSet MUDefsOverridden; - - struct ExistingDefOverriddenEntry { - SymbolMap::iterator ExistingDefItr; - JITSymbolFlags NewFlags; - }; - std::vector ExistingDefsOverridden; - - for (auto &KV : MU.getSymbols()) { - assert(!KV.second.isLazy() && "Lazy flag should be managed internally."); - assert(!KV.second.isMaterializing() && - "Materializing flags should be managed internally."); + std::vector ExistingDefsOverridden; + std::vector MUDefsOverridden; - SymbolMap::iterator EntryItr; - bool Added; + for (const auto &KV : MU.getSymbols()) { + auto I = Symbols.find(KV.first); - auto NewFlags = KV.second; - NewFlags |= JITSymbolFlags::Lazy; - - std::tie(EntryItr, Added) = Symbols.insert( - std::make_pair(KV.first, JITEvaluatedSymbol(0, NewFlags))); - - if (!Added) { + if (I != Symbols.end()) { if (KV.second.isStrong()) { - if (EntryItr->second.getFlags().isStrong() || - (EntryItr->second.getFlags() & JITSymbolFlags::Materializing)) + if (I->second.getFlags().isStrong() || + I->second.getState() > SymbolState::NeverSearched) Duplicates.insert(KV.first); - else - ExistingDefsOverridden.push_back({EntryItr, NewFlags}); + else { + assert(I->second.getState() == SymbolState::NeverSearched && + "Overridden existing def should be in the never-searched " + "state"); + ExistingDefsOverridden.push_back(KV.first); + } } else - MUDefsOverridden.insert(KV.first); + MUDefsOverridden.push_back(KV.first); } } - if (!Duplicates.empty()) { - // We need to remove the symbols we added. - for (auto &KV : MU.getSymbols()) { - if (Duplicates.count(KV.first)) - continue; - - bool Found = false; - for (const auto &EDO : ExistingDefsOverridden) - if (EDO.ExistingDefItr->first == KV.first) - Found = true; - - if (!Found) - Symbols.erase(KV.first); - } - - // FIXME: Return all duplicates. + // If there were any duplicate definitions then bail out. + if (!Duplicates.empty()) return make_error(**Duplicates.begin()); - } - // Update flags on existing defs and call discard on their materializers. - for (auto &EDO : ExistingDefsOverridden) { - assert(EDO.ExistingDefItr->second.getFlags().isLazy() && - !EDO.ExistingDefItr->second.getFlags().isMaterializing() && - "Overridden existing def should be in the Lazy state"); + // Discard any overridden defs in this MU. + for (auto &S : MUDefsOverridden) + MU.doDiscard(*this, S); - EDO.ExistingDefItr->second.setFlags(EDO.NewFlags); + // Discard existing overridden defs. + for (auto &S : ExistingDefsOverridden) { - auto UMII = UnmaterializedInfos.find(EDO.ExistingDefItr->first); + auto UMII = UnmaterializedInfos.find(S); assert(UMII != UnmaterializedInfos.end() && "Overridden existing def should have an UnmaterializedInfo"); - - UMII->second->MU->doDiscard(*this, EDO.ExistingDefItr->first); + UMII->second->MU->doDiscard(*this, S); } - // Discard overridden symbols povided by MU. - for (auto &Sym : MUDefsOverridden) - MU.doDiscard(*this, Sym); + // Finally, add the defs from this MU. + for (auto &KV : MU.getSymbols()) { + auto &SymEntry = Symbols[KV.first]; + SymEntry.setFlags(KV.second); + SymEntry.setState(SymbolState::NeverSearched); + SymEntry.setMaterializerAttached(true); + } return Error::success(); } @@ -1532,17 +1556,7 @@ void JITDylib::detachQueryHelper(AsynchronousSymbolQuery &Q, assert(MaterializingInfos.count(QuerySymbol) && "QuerySymbol does not have MaterializingInfo"); auto &MI = MaterializingInfos[QuerySymbol]; - - auto IdenticalQuery = - [&](const std::shared_ptr &R) { - return R.get() == &Q; - }; - - auto I = std::find_if(MI.PendingQueries.begin(), MI.PendingQueries.end(), - IdenticalQuery); - assert(I != MI.PendingQueries.end() && - "Query Q should be in the PendingQueries list for QuerySymbol"); - MI.PendingQueries.erase(I); + MI.removeQuery(Q); } } @@ -1582,8 +1596,18 @@ JITDylib &ExecutionSession::getMainJITDylib() { return runSessionLocked([this]() -> JITDylib & { return *JDs.front(); }); } +JITDylib *ExecutionSession::getJITDylibByName(StringRef Name) { + return runSessionLocked([&, this]() -> JITDylib * { + for (auto &JD : JDs) + if (JD->getName() == Name) + return JD.get(); + return nullptr; + }); +} + JITDylib &ExecutionSession::createJITDylib(std::string Name, bool AddToMainDylibSearchOrder) { + assert(!getJITDylibByName(Name) && "JITDylib with that name already exists"); return runSessionLocked([&, this]() -> JITDylib & { JDs.push_back( std::unique_ptr(new JITDylib(*this, std::move(Name)))); @@ -1610,74 +1634,36 @@ void ExecutionSession::legacyFailQuery(AsynchronousSymbolQuery &Q, Error Err) { Expected ExecutionSession::legacyLookup( LegacyAsyncLookupFunction AsyncLookup, SymbolNameSet Names, - bool WaitUntilReady, RegisterDependenciesFunction RegisterDependencies) { + SymbolState RequiredState, + RegisterDependenciesFunction RegisterDependencies) { #if LLVM_ENABLE_THREADS // In the threaded case we use promises to return the results. std::promise PromisedResult; - std::mutex ErrMutex; Error ResolutionError = Error::success(); - std::promise PromisedReady; - Error ReadyError = Error::success(); - auto OnResolve = [&](Expected R) { + auto NotifyComplete = [&](Expected R) { if (R) PromisedResult.set_value(std::move(*R)); else { - { - ErrorAsOutParameter _(&ResolutionError); - std::lock_guard Lock(ErrMutex); - ResolutionError = R.takeError(); - } + ErrorAsOutParameter _(&ResolutionError); + ResolutionError = R.takeError(); PromisedResult.set_value(SymbolMap()); } }; - - std::function OnReady; - if (WaitUntilReady) { - OnReady = [&](Error Err) { - if (Err) { - ErrorAsOutParameter _(&ReadyError); - std::lock_guard Lock(ErrMutex); - ReadyError = std::move(Err); - } - PromisedReady.set_value(); - }; - } else { - OnReady = [&](Error Err) { - if (Err) - reportError(std::move(Err)); - }; - } - #else SymbolMap Result; Error ResolutionError = Error::success(); - Error ReadyError = Error::success(); - auto OnResolve = [&](Expected R) { + auto NotifyComplete = [&](Expected R) { ErrorAsOutParameter _(&ResolutionError); if (R) Result = std::move(*R); else ResolutionError = R.takeError(); }; - - std::function OnReady; - if (WaitUntilReady) { - OnReady = [&](Error Err) { - ErrorAsOutParameter _(&ReadyError); - if (Err) - ReadyError = std::move(Err); - }; - } else { - OnReady = [&](Error Err) { - if (Err) - reportError(std::move(Err)); - }; - } #endif auto Query = std::make_shared( - Names, std::move(OnResolve), std::move(OnReady)); + Names, RequiredState, std::move(NotifyComplete)); // FIXME: This should be run session locked along with the registration code // and error reporting below. SymbolNameSet UnresolvedSymbols = AsyncLookup(Query, std::move(Names)); @@ -1701,39 +1687,13 @@ Expected ExecutionSession::legacyLookup( #if LLVM_ENABLE_THREADS auto ResultFuture = PromisedResult.get_future(); auto Result = ResultFuture.get(); - - { - std::lock_guard Lock(ErrMutex); - if (ResolutionError) { - // ReadyError will never be assigned. Consume the success value. - cantFail(std::move(ReadyError)); - return std::move(ResolutionError); - } - } - - if (WaitUntilReady) { - auto ReadyFuture = PromisedReady.get_future(); - ReadyFuture.get(); - - { - std::lock_guard Lock(ErrMutex); - if (ReadyError) - return std::move(ReadyError); - } - } else - cantFail(std::move(ReadyError)); - + if (ResolutionError) + return std::move(ResolutionError); return std::move(Result); #else - if (ResolutionError) { - // ReadyError will never be assigned. Consume the success value. - cantFail(std::move(ReadyError)); + if (ResolutionError) return std::move(ResolutionError); - } - - if (ReadyError) - return std::move(ReadyError); return Result; #endif @@ -1741,9 +1701,16 @@ Expected ExecutionSession::legacyLookup( void ExecutionSession::lookup( const JITDylibSearchList &SearchOrder, SymbolNameSet Symbols, - SymbolsResolvedCallback OnResolve, SymbolsReadyCallback OnReady, + SymbolState RequiredState, SymbolsResolvedCallback NotifyComplete, RegisterDependenciesFunction RegisterDependencies) { + LLVM_DEBUG({ + runSessionLocked([&]() { + dbgs() << "Looking up " << Symbols << " in " << SearchOrder + << " (required state: " << RequiredState << ")\n"; + }); + }); + // lookup can be re-entered recursively if running on a single thread. Run any // outstanding MUs in case this query depends on them, otherwise this lookup // will starve waiting for a result from an MU that is stuck in the queue. @@ -1751,38 +1718,32 @@ void ExecutionSession::lookup( auto Unresolved = std::move(Symbols); std::map CollectedMUsMap; - auto Q = std::make_shared( - Unresolved, std::move(OnResolve), std::move(OnReady)); - bool QueryIsFullyResolved = false; - bool QueryIsFullyReady = false; - bool QueryFailed = false; - - runSessionLocked([&]() { - for (auto &KV : SearchOrder) { - assert(KV.first && "JITDylibList entries must not be null"); - assert(!CollectedMUsMap.count(KV.first) && - "JITDylibList should not contain duplicate entries"); - - auto &JD = *KV.first; - auto MatchNonExported = KV.second; - JD.lodgeQuery(Q, Unresolved, MatchNonExported, CollectedMUsMap[&JD]); - } + auto Q = std::make_shared(Unresolved, RequiredState, + std::move(NotifyComplete)); + bool QueryComplete = false; + + auto LodgingErr = runSessionLocked([&]() -> Error { + auto LodgeQuery = [&]() -> Error { + for (auto &KV : SearchOrder) { + assert(KV.first && "JITDylibList entries must not be null"); + assert(!CollectedMUsMap.count(KV.first) && + "JITDylibList should not contain duplicate entries"); + + auto &JD = *KV.first; + auto MatchNonExported = KV.second; + if (auto Err = JD.lodgeQuery(Q, Unresolved, MatchNonExported, + CollectedMUsMap[&JD])) + return Err; + } - if (Unresolved.empty()) { - // Query lodged successfully. + if (!Unresolved.empty()) + return make_error(std::move(Unresolved)); - // Record whether this query is fully ready / resolved. We will use - // this to call handleFullyResolved/handleFullyReady outside the session - // lock. - QueryIsFullyResolved = Q->isFullyResolved(); - QueryIsFullyReady = Q->isFullyReady(); + return Error::success(); + }; - // Call the register dependencies function. - if (RegisterDependencies && !Q->QueryRegistrations.empty()) - RegisterDependencies(Q->QueryRegistrations); - } else { - // Query failed due to unresolved symbols. - QueryFailed = true; + if (auto Err = LodgeQuery()) { + // Query failed. // Disconnect the query from its dependencies. Q->detach(); @@ -1791,19 +1752,32 @@ void ExecutionSession::lookup( for (auto &KV : CollectedMUsMap) for (auto &MU : KV.second) KV.first->replace(std::move(MU)); + + return Err; } + + // Query lodged successfully. + + // Record whether this query is fully ready / resolved. We will use + // this to call handleFullyResolved/handleFullyReady outside the session + // lock. + QueryComplete = Q->isComplete(); + + // Call the register dependencies function. + if (RegisterDependencies && !Q->QueryRegistrations.empty()) + RegisterDependencies(Q->QueryRegistrations); + + return Error::success(); }); - if (QueryFailed) { - Q->handleFailed(make_error(std::move(Unresolved))); + if (LodgingErr) { + Q->handleFailed(std::move(LodgingErr)); return; - } else { - if (QueryIsFullyResolved) - Q->handleFullyResolved(); - if (QueryIsFullyReady) - Q->handleFullyReady(); } + if (QueryComplete) + Q->handleComplete(); + // Move the MUs to the OutstandingMUs list, then materialize. { std::lock_guard Lock(OutstandingMUsMutex); @@ -1816,113 +1790,55 @@ void ExecutionSession::lookup( runOutstandingMUs(); } -Expected ExecutionSession::lookup( - const JITDylibSearchList &SearchOrder, const SymbolNameSet &Symbols, - RegisterDependenciesFunction RegisterDependencies, bool WaitUntilReady) { +Expected +ExecutionSession::lookup(const JITDylibSearchList &SearchOrder, + const SymbolNameSet &Symbols, + SymbolState RequiredState, + RegisterDependenciesFunction RegisterDependencies) { #if LLVM_ENABLE_THREADS // In the threaded case we use promises to return the results. std::promise PromisedResult; - std::mutex ErrMutex; Error ResolutionError = Error::success(); - std::promise PromisedReady; - Error ReadyError = Error::success(); - auto OnResolve = [&](Expected R) { + + auto NotifyComplete = [&](Expected R) { if (R) PromisedResult.set_value(std::move(*R)); else { - { - ErrorAsOutParameter _(&ResolutionError); - std::lock_guard Lock(ErrMutex); - ResolutionError = R.takeError(); - } + ErrorAsOutParameter _(&ResolutionError); + ResolutionError = R.takeError(); PromisedResult.set_value(SymbolMap()); } }; - std::function OnReady; - if (WaitUntilReady) { - OnReady = [&](Error Err) { - if (Err) { - ErrorAsOutParameter _(&ReadyError); - std::lock_guard Lock(ErrMutex); - ReadyError = std::move(Err); - } - PromisedReady.set_value(); - }; - } else { - OnReady = [&](Error Err) { - if (Err) - reportError(std::move(Err)); - }; - } - #else SymbolMap Result; Error ResolutionError = Error::success(); - Error ReadyError = Error::success(); - auto OnResolve = [&](Expected R) { + auto NotifyComplete = [&](Expected R) { ErrorAsOutParameter _(&ResolutionError); if (R) Result = std::move(*R); else ResolutionError = R.takeError(); }; - - std::function OnReady; - if (WaitUntilReady) { - OnReady = [&](Error Err) { - ErrorAsOutParameter _(&ReadyError); - if (Err) - ReadyError = std::move(Err); - }; - } else { - OnReady = [&](Error Err) { - if (Err) - reportError(std::move(Err)); - }; - } #endif // Perform the asynchronous lookup. - lookup(SearchOrder, Symbols, OnResolve, OnReady, RegisterDependencies); + lookup(SearchOrder, Symbols, RequiredState, NotifyComplete, + RegisterDependencies); #if LLVM_ENABLE_THREADS auto ResultFuture = PromisedResult.get_future(); auto Result = ResultFuture.get(); - { - std::lock_guard Lock(ErrMutex); - if (ResolutionError) { - // ReadyError will never be assigned. Consume the success value. - cantFail(std::move(ReadyError)); - return std::move(ResolutionError); - } - } - - if (WaitUntilReady) { - auto ReadyFuture = PromisedReady.get_future(); - ReadyFuture.get(); - - { - std::lock_guard Lock(ErrMutex); - if (ReadyError) - return std::move(ReadyError); - } - } else - cantFail(std::move(ReadyError)); + if (ResolutionError) + return std::move(ResolutionError); return std::move(Result); #else - if (ResolutionError) { - // ReadyError will never be assigned. Consume the success value. - cantFail(std::move(ReadyError)); + if (ResolutionError) return std::move(ResolutionError); - } - - if (ReadyError) - return std::move(ReadyError); return Result; #endif @@ -1933,8 +1849,8 @@ ExecutionSession::lookup(const JITDylibSearchList &SearchOrder, SymbolStringPtr Name) { SymbolNameSet Names({Name}); - if (auto ResultMap = lookup(SearchOrder, std::move(Names), - NoDependenciesToRegister, true)) { + if (auto ResultMap = lookup(SearchOrder, std::move(Names), SymbolState::Ready, + NoDependenciesToRegister)) { assert(ResultMap->size() == 1 && "Unexpected number of results"); assert(ResultMap->count(Name) && "Missing result for symbol"); return std::move(ResultMap->begin()->second); diff --git a/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/lib/ExecutionEngine/Orc/ExecutionUtils.cpp index 7c3c50b4d6e5..f7fc5f8f1797 100644 --- a/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ b/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -1,9 +1,8 @@ //===---- ExecutionUtils.cpp - Utilities for executing functions in Orc ---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -130,8 +129,7 @@ Error CtorDtorRunner::run() { auto &ES = JD.getExecutionSession(); if (auto CtorDtorMap = - ES.lookup(JITDylibSearchList({{&JD, true}}), std::move(Names), - NoDependenciesToRegister, true)) { + ES.lookup(JITDylibSearchList({{&JD, true}}), std::move(Names))) { for (auto &KV : CtorDtorsByPriority) { for (auto &Name : KV.second) { assert(CtorDtorMap->count(Name) && "No entry for Name"); @@ -140,13 +138,10 @@ Error CtorDtorRunner::run() { CtorDtor(); } } + CtorDtorsByPriority.clear(); return Error::success(); } else return CtorDtorMap.takeError(); - - CtorDtorsByPriority.clear(); - - return Error::success(); } void LocalCXXRuntimeOverridesBase::runDestructors() { @@ -179,22 +174,24 @@ Error LocalCXXRuntimeOverrides::enable(JITDylib &JD, } DynamicLibrarySearchGenerator::DynamicLibrarySearchGenerator( - sys::DynamicLibrary Dylib, const DataLayout &DL, SymbolPredicate Allow) + sys::DynamicLibrary Dylib, char GlobalPrefix, SymbolPredicate Allow) : Dylib(std::move(Dylib)), Allow(std::move(Allow)), - GlobalPrefix(DL.getGlobalPrefix()) {} + GlobalPrefix(GlobalPrefix) {} Expected -DynamicLibrarySearchGenerator::Load(const char *FileName, const DataLayout &DL, +DynamicLibrarySearchGenerator::Load(const char *FileName, char GlobalPrefix, SymbolPredicate Allow) { std::string ErrMsg; auto Lib = sys::DynamicLibrary::getPermanentLibrary(FileName, &ErrMsg); if (!Lib.isValid()) return make_error(std::move(ErrMsg), inconvertibleErrorCode()); - return DynamicLibrarySearchGenerator(std::move(Lib), DL, std::move(Allow)); + return DynamicLibrarySearchGenerator(std::move(Lib), GlobalPrefix, + std::move(Allow)); } -SymbolNameSet DynamicLibrarySearchGenerator:: -operator()(JITDylib &JD, const SymbolNameSet &Names) { +Expected +DynamicLibrarySearchGenerator::operator()(JITDylib &JD, + const SymbolNameSet &Names) { orc::SymbolNameSet Added; orc::SymbolMap NewSymbols; @@ -210,7 +207,8 @@ operator()(JITDylib &JD, const SymbolNameSet &Names) { if (HasGlobalPrefix && (*Name).front() != GlobalPrefix) continue; - std::string Tmp((*Name).data() + (HasGlobalPrefix ? 1 : 0), (*Name).size()); + std::string Tmp((*Name).data() + HasGlobalPrefix, + (*Name).size() - HasGlobalPrefix); if (void *Addr = Dylib.getAddressOfSymbol(Tmp.c_str())) { Added.insert(Name); NewSymbols[Name] = JITEvaluatedSymbol( diff --git a/lib/ExecutionEngine/Orc/IRCompileLayer.cpp b/lib/ExecutionEngine/Orc/IRCompileLayer.cpp index d952d1be70da..81dfc02f55b2 100644 --- a/lib/ExecutionEngine/Orc/IRCompileLayer.cpp +++ b/lib/ExecutionEngine/Orc/IRCompileLayer.cpp @@ -1,9 +1,8 @@ //===--------------- IRCompileLayer.cpp - IR Compiling Layer --------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/ExecutionEngine/Orc/IRTransformLayer.cpp b/lib/ExecutionEngine/Orc/IRTransformLayer.cpp index 7bc0d696e3ac..e3519284613e 100644 --- a/lib/ExecutionEngine/Orc/IRTransformLayer.cpp +++ b/lib/ExecutionEngine/Orc/IRTransformLayer.cpp @@ -1,9 +1,8 @@ //===-------------- IRTransformLayer.cpp - IR Transform Layer -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index 82000ec5b32b..cc3656fe5dc5 100644 --- a/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -1,9 +1,8 @@ //===---- IndirectionUtils.cpp - Utilities for call indirection in Orc ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -38,8 +37,8 @@ private: void materialize(MaterializationResponsibility R) override { SymbolMap Result; Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported); - R.resolve(Result); - R.emit(); + R.notifyResolved(Result); + R.notifyEmitted(); } void discard(const JITDylib &JD, const SymbolStringPtr &Name) override { @@ -238,11 +237,11 @@ void makeStub(Function &F, Value &ImplPointer) { Module &M = *F.getParent(); BasicBlock *EntryBlock = BasicBlock::Create(M.getContext(), "entry", &F); IRBuilder<> Builder(EntryBlock); - LoadInst *ImplAddr = Builder.CreateLoad(&ImplPointer); + LoadInst *ImplAddr = Builder.CreateLoad(F.getType(), &ImplPointer); std::vector CallArgs; for (auto &A : F.args()) CallArgs.push_back(&A); - CallInst *Call = Builder.CreateCall(ImplAddr, CallArgs); + CallInst *Call = Builder.CreateCall(F.getFunctionType(), ImplAddr, CallArgs); Call->setTailCall(); Call->setAttributes(F.getAttributes()); if (F.getReturnType()->isVoidTy()) diff --git a/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp b/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp index 4af09d196ff9..df23547a9de3 100644 --- a/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp +++ b/lib/ExecutionEngine/Orc/JITTargetMachineBuilder.cpp @@ -1,9 +1,8 @@ //===----- JITTargetMachineBuilder.cpp - Build TargetMachines for JIT -----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/ExecutionEngine/Orc/LLJIT.cpp b/lib/ExecutionEngine/Orc/LLJIT.cpp index e2089f9106bd..b120691faf07 100644 --- a/lib/ExecutionEngine/Orc/LLJIT.cpp +++ b/lib/ExecutionEngine/Orc/LLJIT.cpp @@ -1,58 +1,37 @@ //===--------- LLJIT.cpp - An ORC-based JIT for compiling LLVM IR ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/Orc/LLJIT.h" #include "llvm/ExecutionEngine/Orc/OrcError.h" +#include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h" #include "llvm/ExecutionEngine/SectionMemoryManager.h" #include "llvm/IR/Mangler.h" -namespace { +namespace llvm { +namespace orc { - // A SimpleCompiler that owns its TargetMachine. - class TMOwningSimpleCompiler : public llvm::orc::SimpleCompiler { - public: - TMOwningSimpleCompiler(std::unique_ptr TM) - : llvm::orc::SimpleCompiler(*TM), TM(std::move(TM)) {} - private: - // FIXME: shared because std::functions (and thus - // IRCompileLayer::CompileFunction) are not moveable. - std::shared_ptr TM; - }; +Error LLJITBuilderState::prepareForConstruction() { -} // end anonymous namespace + if (!JTMB) { + if (auto JTMBOrErr = JITTargetMachineBuilder::detectHost()) + JTMB = std::move(*JTMBOrErr); + else + return JTMBOrErr.takeError(); + } -namespace llvm { -namespace orc { + return Error::success(); +} LLJIT::~LLJIT() { if (CompileThreads) CompileThreads->wait(); } -Expected> -LLJIT::Create(JITTargetMachineBuilder JTMB, DataLayout DL, - unsigned NumCompileThreads) { - - if (NumCompileThreads == 0) { - // If NumCompileThreads == 0 then create a single-threaded LLJIT instance. - auto TM = JTMB.createTargetMachine(); - if (!TM) - return TM.takeError(); - return std::unique_ptr(new LLJIT(llvm::make_unique(), - std::move(*TM), std::move(DL))); - } - - return std::unique_ptr(new LLJIT(llvm::make_unique(), - std::move(JTMB), std::move(DL), - NumCompileThreads)); -} - Error LLJIT::defineAbsolute(StringRef Name, JITEvaluatedSymbol Sym) { auto InternedName = ES->intern(Name); SymbolMap Symbols({{InternedName, Sym}}); @@ -65,13 +44,13 @@ Error LLJIT::addIRModule(JITDylib &JD, ThreadSafeModule TSM) { if (auto Err = applyDataLayout(*TSM.getModule())) return Err; - return CompileLayer.add(JD, std::move(TSM), ES->allocateVModule()); + return CompileLayer->add(JD, std::move(TSM), ES->allocateVModule()); } Error LLJIT::addObjectFile(JITDylib &JD, std::unique_ptr Obj) { assert(Obj && "Can not add null object"); - return ObjLinkingLayer.add(JD, std::move(Obj), ES->allocateVModule()); + return ObjLinkingLayer->add(JD, std::move(Obj), ES->allocateVModule()); } Expected LLJIT::lookupLinkerMangled(JITDylib &JD, @@ -79,42 +58,76 @@ Expected LLJIT::lookupLinkerMangled(JITDylib &JD, return ES->lookup(JITDylibSearchList({{&JD, true}}), ES->intern(Name)); } -LLJIT::LLJIT(std::unique_ptr ES, - std::unique_ptr TM, DataLayout DL) - : ES(std::move(ES)), Main(this->ES->getMainJITDylib()), DL(std::move(DL)), - ObjLinkingLayer( - *this->ES, - []() { return llvm::make_unique(); }), - CompileLayer(*this->ES, ObjLinkingLayer, - TMOwningSimpleCompiler(std::move(TM))), - CtorRunner(Main), DtorRunner(Main) {} - -LLJIT::LLJIT(std::unique_ptr ES, JITTargetMachineBuilder JTMB, - DataLayout DL, unsigned NumCompileThreads) - : ES(std::move(ES)), Main(this->ES->getMainJITDylib()), DL(std::move(DL)), - ObjLinkingLayer( - *this->ES, - []() { return llvm::make_unique(); }), - CompileLayer(*this->ES, ObjLinkingLayer, - ConcurrentIRCompiler(std::move(JTMB))), - CtorRunner(Main), DtorRunner(Main) { - assert(NumCompileThreads != 0 && - "Multithreaded LLJIT instance can not be created with 0 threads"); - - // Move modules to new contexts when they're emitted so that we can compile - // them in parallel. - CompileLayer.setCloneToNewContextOnEmit(true); - - // Create a thread pool to compile on and set the execution session - // dispatcher to use the thread pool. - CompileThreads = llvm::make_unique(NumCompileThreads); - this->ES->setDispatchMaterialization( - [this](JITDylib &JD, std::unique_ptr MU) { - // FIXME: Switch to move capture once we have c++14. - auto SharedMU = std::shared_ptr(std::move(MU)); - auto Work = [SharedMU, &JD]() { SharedMU->doMaterialize(JD); }; - CompileThreads->async(std::move(Work)); - }); +std::unique_ptr +LLJIT::createObjectLinkingLayer(LLJITBuilderState &S, ExecutionSession &ES) { + + // If the config state provided an ObjectLinkingLayer factory then use it. + if (S.CreateObjectLinkingLayer) + return S.CreateObjectLinkingLayer(ES); + + // Otherwise default to creating an RTDyldObjectLinkingLayer that constructs + // a new SectionMemoryManager for each object. + auto GetMemMgr = []() { return llvm::make_unique(); }; + return llvm::make_unique(ES, std::move(GetMemMgr)); +} + +Expected +LLJIT::createCompileFunction(LLJITBuilderState &S, + JITTargetMachineBuilder JTMB) { + + /// If there is a custom compile function creator set then use it. + if (S.CreateCompileFunction) + return S.CreateCompileFunction(std::move(JTMB)); + + // Otherwise default to creating a SimpleCompiler, or ConcurrentIRCompiler, + // depending on the number of threads requested. + if (S.NumCompileThreads > 0) + return ConcurrentIRCompiler(std::move(JTMB)); + + auto TM = JTMB.createTargetMachine(); + if (!TM) + return TM.takeError(); + + return TMOwningSimpleCompiler(std::move(*TM)); +} + +LLJIT::LLJIT(LLJITBuilderState &S, Error &Err) + : ES(S.ES ? std::move(S.ES) : llvm::make_unique()), + Main(this->ES->getMainJITDylib()), DL(""), CtorRunner(Main), + DtorRunner(Main) { + + ErrorAsOutParameter _(&Err); + + ObjLinkingLayer = createObjectLinkingLayer(S, *ES); + + if (auto DLOrErr = S.JTMB->getDefaultDataLayoutForTarget()) + DL = std::move(*DLOrErr); + else { + Err = DLOrErr.takeError(); + return; + } + + { + auto CompileFunction = createCompileFunction(S, std::move(*S.JTMB)); + if (!CompileFunction) { + Err = CompileFunction.takeError(); + return; + } + CompileLayer = llvm::make_unique( + *ES, *ObjLinkingLayer, std::move(*CompileFunction)); + } + + if (S.NumCompileThreads > 0) { + CompileLayer->setCloneToNewContextOnEmit(true); + CompileThreads = llvm::make_unique(S.NumCompileThreads); + ES->setDispatchMaterialization( + [this](JITDylib &JD, std::unique_ptr MU) { + // FIXME: Switch to move capture once we have c++14. + auto SharedMU = std::shared_ptr(std::move(MU)); + auto Work = [SharedMU, &JD]() { SharedMU->doMaterialize(JD); }; + CompileThreads->async(std::move(Work)); + }); + } } std::string LLJIT::mangle(StringRef UnmangledName) { @@ -143,35 +156,11 @@ void LLJIT::recordCtorDtors(Module &M) { DtorRunner.add(getDestructors(M)); } -Expected> -LLLazyJIT::Create(JITTargetMachineBuilder JTMB, DataLayout DL, - JITTargetAddress ErrorAddr, unsigned NumCompileThreads) { - auto ES = llvm::make_unique(); - - const Triple &TT = JTMB.getTargetTriple(); - - auto LCTMgr = createLocalLazyCallThroughManager(TT, *ES, ErrorAddr); - if (!LCTMgr) - return LCTMgr.takeError(); - - auto ISMBuilder = createLocalIndirectStubsManagerBuilder(TT); - if (!ISMBuilder) - return make_error( - std::string("No indirect stubs manager builder for ") + TT.str(), - inconvertibleErrorCode()); - - if (NumCompileThreads == 0) { - auto TM = JTMB.createTargetMachine(); - if (!TM) - return TM.takeError(); - return std::unique_ptr( - new LLLazyJIT(std::move(ES), std::move(*TM), std::move(DL), - std::move(*LCTMgr), std::move(ISMBuilder))); - } - - return std::unique_ptr(new LLLazyJIT( - std::move(ES), std::move(JTMB), std::move(DL), NumCompileThreads, - std::move(*LCTMgr), std::move(ISMBuilder))); +Error LLLazyJITBuilderState::prepareForConstruction() { + if (auto Err = LLJITBuilderState::prepareForConstruction()) + return Err; + TT = JTMB->getTargetTriple(); + return Error::success(); } Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) { @@ -182,28 +171,55 @@ Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) { recordCtorDtors(*TSM.getModule()); - return CODLayer.add(JD, std::move(TSM), ES->allocateVModule()); + return CODLayer->add(JD, std::move(TSM), ES->allocateVModule()); } -LLLazyJIT::LLLazyJIT( - std::unique_ptr ES, std::unique_ptr TM, - DataLayout DL, std::unique_ptr LCTMgr, - std::function()> ISMBuilder) - : LLJIT(std::move(ES), std::move(TM), std::move(DL)), - LCTMgr(std::move(LCTMgr)), TransformLayer(*this->ES, CompileLayer), - CODLayer(*this->ES, TransformLayer, *this->LCTMgr, - std::move(ISMBuilder)) {} - -LLLazyJIT::LLLazyJIT( - std::unique_ptr ES, JITTargetMachineBuilder JTMB, - DataLayout DL, unsigned NumCompileThreads, - std::unique_ptr LCTMgr, - std::function()> ISMBuilder) - : LLJIT(std::move(ES), std::move(JTMB), std::move(DL), NumCompileThreads), - LCTMgr(std::move(LCTMgr)), TransformLayer(*this->ES, CompileLayer), - CODLayer(*this->ES, TransformLayer, *this->LCTMgr, - std::move(ISMBuilder)) { - CODLayer.setCloneToNewContextOnEmit(true); +LLLazyJIT::LLLazyJIT(LLLazyJITBuilderState &S, Error &Err) : LLJIT(S, Err) { + + // If LLJIT construction failed then bail out. + if (Err) + return; + + ErrorAsOutParameter _(&Err); + + /// Take/Create the lazy-compile callthrough manager. + if (S.LCTMgr) + LCTMgr = std::move(S.LCTMgr); + else { + if (auto LCTMgrOrErr = createLocalLazyCallThroughManager( + S.TT, *ES, S.LazyCompileFailureAddr)) + LCTMgr = std::move(*LCTMgrOrErr); + else { + Err = LCTMgrOrErr.takeError(); + return; + } + } + + // Take/Create the indirect stubs manager builder. + auto ISMBuilder = std::move(S.ISMBuilder); + + // If none was provided, try to build one. + if (!ISMBuilder) + ISMBuilder = createLocalIndirectStubsManagerBuilder(S.TT); + + // No luck. Bail out. + if (!ISMBuilder) { + Err = make_error("Could not construct " + "IndirectStubsManagerBuilder for target " + + S.TT.str(), + inconvertibleErrorCode()); + return; + } + + // Create the transform layer. + TransformLayer = llvm::make_unique(*ES, *CompileLayer); + + // Create the COD layer. + CODLayer = llvm::make_unique( + *ES, *TransformLayer, *LCTMgr, std::move(ISMBuilder)); + + if (S.NumCompileThreads > 0) + CODLayer->setCloneToNewContextOnEmit(true); } } // End namespace orc. diff --git a/lib/ExecutionEngine/Orc/Layer.cpp b/lib/ExecutionEngine/Orc/Layer.cpp index 11af76825e9f..3ed2dabf4545 100644 --- a/lib/ExecutionEngine/Orc/Layer.cpp +++ b/lib/ExecutionEngine/Orc/Layer.cpp @@ -1,9 +1,8 @@ //===-------------------- Layer.cpp - Layer interfaces --------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -88,17 +87,15 @@ void BasicIRLayerMaterializationUnit::materialize( #ifndef NDEBUG auto &ES = R.getTargetJITDylib().getExecutionSession(); + auto &N = R.getTargetJITDylib().getName(); #endif // NDEBUG auto Lock = TSM.getContextLock(); - LLVM_DEBUG(ES.runSessionLocked([&]() { - dbgs() << "Emitting, for " << R.getTargetJITDylib().getName() << ", " - << *this << "\n"; - });); + LLVM_DEBUG(ES.runSessionLocked( + [&]() { dbgs() << "Emitting, for " << N << ", " << *this << "\n"; });); L.emit(std::move(R), std::move(TSM)); LLVM_DEBUG(ES.runSessionLocked([&]() { - dbgs() << "Finished emitting, for " << R.getTargetJITDylib().getName() - << ", " << *this << "\n"; + dbgs() << "Finished emitting, for " << N << ", " << *this << "\n"; });); } diff --git a/lib/ExecutionEngine/Orc/LazyReexports.cpp b/lib/ExecutionEngine/Orc/LazyReexports.cpp index 55f4a7c5afce..fc8205845654 100644 --- a/lib/ExecutionEngine/Orc/LazyReexports.cpp +++ b/lib/ExecutionEngine/Orc/LazyReexports.cpp @@ -1,9 +1,8 @@ //===---------- LazyReexports.cpp - Utilities for lazy reexports ----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -52,18 +51,15 @@ LazyCallThroughManager::callThroughToSymbol(JITTargetAddress TrampolineAddr) { SymbolName = I->second.second; } - auto LookupResult = ES.lookup(JITDylibSearchList({{SourceJD, true}}), - {SymbolName}, NoDependenciesToRegister, true); + auto LookupResult = + ES.lookup(JITDylibSearchList({{SourceJD, true}}), SymbolName); if (!LookupResult) { ES.reportError(LookupResult.takeError()); return ErrorHandlerAddr; } - assert(LookupResult->size() == 1 && "Unexpected number of results"); - assert(LookupResult->count(SymbolName) && "Unexpected result"); - - auto ResolvedAddr = LookupResult->begin()->second.getAddress(); + auto ResolvedAddr = LookupResult->getAddress(); std::shared_ptr NotifyResolved = nullptr; { @@ -182,8 +178,8 @@ void LazyReexportsMaterializationUnit::materialize( for (auto &Alias : RequestedAliases) Stubs[Alias.first] = ISManager.findStub(*Alias.first, false); - R.resolve(Stubs); - R.emit(); + R.notifyResolved(Stubs); + R.notifyEmitted(); } void LazyReexportsMaterializationUnit::discard(const JITDylib &JD, diff --git a/lib/ExecutionEngine/Orc/Legacy.cpp b/lib/ExecutionEngine/Orc/Legacy.cpp index ddb72544b770..ce6368b57a89 100644 --- a/lib/ExecutionEngine/Orc/Legacy.cpp +++ b/lib/ExecutionEngine/Orc/Legacy.cpp @@ -1,9 +1,8 @@ //===------- Legacy.cpp - Adapters for ExecutionEngine API interop --------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -37,8 +36,7 @@ void JITSymbolResolverAdapter::lookup(const LookupSet &Symbols, }; auto Q = std::make_shared( - InternedSymbols, OnResolvedWithUnwrap, - [this](Error Err) { ES.reportError(std::move(Err)); }); + InternedSymbols, SymbolState::Resolved, OnResolvedWithUnwrap); auto Unresolved = R.lookup(Q, InternedSymbols); if (Unresolved.empty()) { diff --git a/lib/ExecutionEngine/Orc/NullResolver.cpp b/lib/ExecutionEngine/Orc/NullResolver.cpp index 922fc6f021ce..5b4345b870bb 100644 --- a/lib/ExecutionEngine/Orc/NullResolver.cpp +++ b/lib/ExecutionEngine/Orc/NullResolver.cpp @@ -1,9 +1,8 @@ //===---------- NullResolver.cpp - Reject symbol lookup requests ----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp new file mode 100644 index 000000000000..def0b300eca1 --- /dev/null +++ b/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -0,0 +1,483 @@ +//===------- ObjectLinkingLayer.cpp - JITLink backed ORC ObjectLayer ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" + +#include "llvm/ADT/Optional.h" +#include "llvm/ExecutionEngine/JITLink/EHFrameSupport.h" + +#include + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::orc; + +namespace llvm { +namespace orc { + +class ObjectLinkingLayerJITLinkContext final : public JITLinkContext { +public: + ObjectLinkingLayerJITLinkContext(ObjectLinkingLayer &Layer, + MaterializationResponsibility MR, + std::unique_ptr ObjBuffer) + : Layer(Layer), MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) {} + + JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; } + + MemoryBufferRef getObjectBuffer() const override { + return ObjBuffer->getMemBufferRef(); + } + + void notifyFailed(Error Err) override { + Layer.getExecutionSession().reportError(std::move(Err)); + MR.failMaterialization(); + } + + void lookup(const DenseSet &Symbols, + JITLinkAsyncLookupContinuation LookupContinuation) override { + + JITDylibSearchList SearchOrder; + MR.getTargetJITDylib().withSearchOrderDo( + [&](const JITDylibSearchList &JDs) { SearchOrder = JDs; }); + + auto &ES = Layer.getExecutionSession(); + + SymbolNameSet InternedSymbols; + for (auto &S : Symbols) + InternedSymbols.insert(ES.intern(S)); + + // OnResolve -- De-intern the symbols and pass the result to the linker. + // FIXME: Capture LookupContinuation by move once we have c++14. + auto SharedLookupContinuation = + std::make_shared( + std::move(LookupContinuation)); + auto OnResolve = [SharedLookupContinuation](Expected Result) { + if (!Result) + (*SharedLookupContinuation)(Result.takeError()); + else { + AsyncLookupResult LR; + for (auto &KV : *Result) + LR[*KV.first] = KV.second; + (*SharedLookupContinuation)(std::move(LR)); + } + }; + + ES.lookup(SearchOrder, std::move(InternedSymbols), SymbolState::Resolved, + std::move(OnResolve), [this](const SymbolDependenceMap &Deps) { + registerDependencies(Deps); + }); + } + + void notifyResolved(AtomGraph &G) override { + auto &ES = Layer.getExecutionSession(); + + SymbolFlagsMap ExtraSymbolsToClaim; + bool AutoClaim = Layer.AutoClaimObjectSymbols; + + SymbolMap InternedResult; + for (auto *DA : G.defined_atoms()) + if (DA->hasName() && DA->isGlobal()) { + auto InternedName = ES.intern(DA->getName()); + JITSymbolFlags Flags; + + if (DA->isExported()) + Flags |= JITSymbolFlags::Exported; + if (DA->isWeak()) + Flags |= JITSymbolFlags::Weak; + if (DA->isCallable()) + Flags |= JITSymbolFlags::Callable; + if (DA->isCommon()) + Flags |= JITSymbolFlags::Common; + + InternedResult[InternedName] = + JITEvaluatedSymbol(DA->getAddress(), Flags); + if (AutoClaim && !MR.getSymbols().count(InternedName)) { + assert(!ExtraSymbolsToClaim.count(InternedName) && + "Duplicate symbol to claim?"); + ExtraSymbolsToClaim[InternedName] = Flags; + } + } + + for (auto *A : G.absolute_atoms()) + if (A->hasName()) { + auto InternedName = ES.intern(A->getName()); + JITSymbolFlags Flags; + Flags |= JITSymbolFlags::Absolute; + if (A->isWeak()) + Flags |= JITSymbolFlags::Weak; + if (A->isCallable()) + Flags |= JITSymbolFlags::Callable; + InternedResult[InternedName] = + JITEvaluatedSymbol(A->getAddress(), Flags); + if (AutoClaim && !MR.getSymbols().count(InternedName)) { + assert(!ExtraSymbolsToClaim.count(InternedName) && + "Duplicate symbol to claim?"); + ExtraSymbolsToClaim[InternedName] = Flags; + } + } + + if (!ExtraSymbolsToClaim.empty()) + if (auto Err = MR.defineMaterializing(ExtraSymbolsToClaim)) + return notifyFailed(std::move(Err)); + + MR.notifyResolved(InternedResult); + + Layer.notifyLoaded(MR); + } + + void notifyFinalized( + std::unique_ptr A) override { + + if (auto Err = Layer.notifyEmitted(MR, std::move(A))) { + Layer.getExecutionSession().reportError(std::move(Err)); + MR.failMaterialization(); + + return; + } + MR.notifyEmitted(); + } + + AtomGraphPassFunction getMarkLivePass(const Triple &TT) const override { + return [this](AtomGraph &G) { return markResponsibilitySymbolsLive(G); }; + } + + Error modifyPassConfig(const Triple &TT, PassConfiguration &Config) override { + // Add passes to mark duplicate defs as should-discard, and to walk the + // atom graph to build the symbol dependence graph. + Config.PrePrunePasses.push_back( + [this](AtomGraph &G) { return markSymbolsToDiscard(G); }); + Config.PostPrunePasses.push_back( + [this](AtomGraph &G) { return computeNamedSymbolDependencies(G); }); + + Layer.modifyPassConfig(MR, TT, Config); + + return Error::success(); + } + +private: + using AnonAtomNamedDependenciesMap = + DenseMap; + + Error markSymbolsToDiscard(AtomGraph &G) { + auto &ES = Layer.getExecutionSession(); + for (auto *DA : G.defined_atoms()) + if (DA->isWeak() && DA->hasName()) { + auto S = ES.intern(DA->getName()); + auto I = MR.getSymbols().find(S); + if (I == MR.getSymbols().end()) + DA->setShouldDiscard(true); + } + + for (auto *A : G.absolute_atoms()) + if (A->isWeak() && A->hasName()) { + auto S = ES.intern(A->getName()); + auto I = MR.getSymbols().find(S); + if (I == MR.getSymbols().end()) + A->setShouldDiscard(true); + } + + return Error::success(); + } + + Error markResponsibilitySymbolsLive(AtomGraph &G) const { + auto &ES = Layer.getExecutionSession(); + for (auto *DA : G.defined_atoms()) + if (DA->hasName() && + MR.getSymbols().count(ES.intern(DA->getName()))) + DA->setLive(true); + return Error::success(); + } + + Error computeNamedSymbolDependencies(AtomGraph &G) { + auto &ES = MR.getTargetJITDylib().getExecutionSession(); + auto AnonDeps = computeAnonDeps(G); + + for (auto *DA : G.defined_atoms()) { + + // Skip anonymous and non-global atoms: we do not need dependencies for + // these. + if (!DA->hasName() || !DA->isGlobal()) + continue; + + auto DAName = ES.intern(DA->getName()); + SymbolNameSet &DADeps = NamedSymbolDeps[DAName]; + + for (auto &E : DA->edges()) { + auto &TA = E.getTarget(); + + if (TA.hasName()) + DADeps.insert(ES.intern(TA.getName())); + else { + assert(TA.isDefined() && "Anonymous atoms must be defined"); + auto &DTA = static_cast(TA); + auto I = AnonDeps.find(&DTA); + if (I != AnonDeps.end()) + for (auto &S : I->second) + DADeps.insert(S); + } + } + } + + return Error::success(); + } + + AnonAtomNamedDependenciesMap computeAnonDeps(AtomGraph &G) { + + auto &ES = MR.getTargetJITDylib().getExecutionSession(); + AnonAtomNamedDependenciesMap DepMap; + + // For all anonymous atoms: + // (1) Add their named dependencies. + // (2) Add them to the worklist for further iteration if they have any + // depend on any other anonymous atoms. + struct WorklistEntry { + WorklistEntry(DefinedAtom *DA, DenseSet DAAnonDeps) + : DA(DA), DAAnonDeps(std::move(DAAnonDeps)) {} + + DefinedAtom *DA = nullptr; + DenseSet DAAnonDeps; + }; + std::vector Worklist; + for (auto *DA : G.defined_atoms()) + if (!DA->hasName()) { + auto &DANamedDeps = DepMap[DA]; + DenseSet DAAnonDeps; + + for (auto &E : DA->edges()) { + auto &TA = E.getTarget(); + if (TA.hasName()) + DANamedDeps.insert(ES.intern(TA.getName())); + else { + assert(TA.isDefined() && "Anonymous atoms must be defined"); + DAAnonDeps.insert(static_cast(&TA)); + } + } + + if (!DAAnonDeps.empty()) + Worklist.push_back(WorklistEntry(DA, std::move(DAAnonDeps))); + } + + // Loop over all anonymous atoms with anonymous dependencies, propagating + // their respective *named* dependencies. Iterate until we hit a stable + // state. + bool Changed; + do { + Changed = false; + for (auto &WLEntry : Worklist) { + auto *DA = WLEntry.DA; + auto &DANamedDeps = DepMap[DA]; + auto &DAAnonDeps = WLEntry.DAAnonDeps; + + for (auto *TA : DAAnonDeps) { + auto I = DepMap.find(TA); + if (I != DepMap.end()) + for (const auto &S : I->second) + Changed |= DANamedDeps.insert(S).second; + } + } + } while (Changed); + + return DepMap; + } + + void registerDependencies(const SymbolDependenceMap &QueryDeps) { + for (auto &NamedDepsEntry : NamedSymbolDeps) { + auto &Name = NamedDepsEntry.first; + auto &NameDeps = NamedDepsEntry.second; + SymbolDependenceMap SymbolDeps; + + for (const auto &QueryDepsEntry : QueryDeps) { + JITDylib &SourceJD = *QueryDepsEntry.first; + const SymbolNameSet &Symbols = QueryDepsEntry.second; + auto &DepsForJD = SymbolDeps[&SourceJD]; + + for (const auto &S : Symbols) + if (NameDeps.count(S)) + DepsForJD.insert(S); + + if (DepsForJD.empty()) + SymbolDeps.erase(&SourceJD); + } + + MR.addDependencies(Name, SymbolDeps); + } + } + + ObjectLinkingLayer &Layer; + MaterializationResponsibility MR; + std::unique_ptr ObjBuffer; + DenseMap NamedSymbolDeps; +}; + +ObjectLinkingLayer::Plugin::~Plugin() {} + +ObjectLinkingLayer::ObjectLinkingLayer(ExecutionSession &ES, + JITLinkMemoryManager &MemMgr) + : ObjectLayer(ES), MemMgr(MemMgr) {} + +ObjectLinkingLayer::~ObjectLinkingLayer() { + if (auto Err = removeAllModules()) + getExecutionSession().reportError(std::move(Err)); +} + +void ObjectLinkingLayer::emit(MaterializationResponsibility R, + std::unique_ptr O) { + assert(O && "Object must not be null"); + jitLink(llvm::make_unique( + *this, std::move(R), std::move(O))); +} + +void ObjectLinkingLayer::modifyPassConfig(MaterializationResponsibility &MR, + const Triple &TT, + PassConfiguration &PassConfig) { + for (auto &P : Plugins) + P->modifyPassConfig(MR, TT, PassConfig); +} + +void ObjectLinkingLayer::notifyLoaded(MaterializationResponsibility &MR) { + for (auto &P : Plugins) + P->notifyLoaded(MR); +} + +Error ObjectLinkingLayer::notifyEmitted(MaterializationResponsibility &MR, + AllocPtr Alloc) { + Error Err = Error::success(); + for (auto &P : Plugins) + Err = joinErrors(std::move(Err), P->notifyEmitted(MR)); + + if (Err) + return Err; + + { + std::lock_guard Lock(LayerMutex); + UntrackedAllocs.push_back(std::move(Alloc)); + } + + return Error::success(); +} + +Error ObjectLinkingLayer::removeModule(VModuleKey K) { + Error Err = Error::success(); + + for (auto &P : Plugins) + Err = joinErrors(std::move(Err), P->notifyRemovingModule(K)); + + AllocPtr Alloc; + + { + std::lock_guard Lock(LayerMutex); + auto AllocItr = TrackedAllocs.find(K); + Alloc = std::move(AllocItr->second); + TrackedAllocs.erase(AllocItr); + } + + assert(Alloc && "No allocation for key K"); + + return joinErrors(std::move(Err), Alloc->deallocate()); +} + +Error ObjectLinkingLayer::removeAllModules() { + + Error Err = Error::success(); + + for (auto &P : Plugins) + Err = joinErrors(std::move(Err), P->notifyRemovingAllModules()); + + std::vector Allocs; + { + std::lock_guard Lock(LayerMutex); + Allocs = std::move(UntrackedAllocs); + + for (auto &KV : TrackedAllocs) + Allocs.push_back(std::move(KV.second)); + + TrackedAllocs.clear(); + } + + while (!Allocs.empty()) { + Err = joinErrors(std::move(Err), Allocs.back()->deallocate()); + Allocs.pop_back(); + } + + return Err; +} + +EHFrameRegistrationPlugin::EHFrameRegistrationPlugin( + jitlink::EHFrameRegistrar &Registrar) + : Registrar(Registrar) {} + +void EHFrameRegistrationPlugin::modifyPassConfig( + MaterializationResponsibility &MR, const Triple &TT, + PassConfiguration &PassConfig) { + assert(!InProcessLinks.count(&MR) && "Link for MR already being tracked?"); + + PassConfig.PostFixupPasses.push_back( + createEHFrameRecorderPass(TT, [this, &MR](JITTargetAddress Addr) { + if (Addr) + InProcessLinks[&MR] = Addr; + })); +} + +Error EHFrameRegistrationPlugin::notifyEmitted( + MaterializationResponsibility &MR) { + + auto EHFrameAddrItr = InProcessLinks.find(&MR); + if (EHFrameAddrItr == InProcessLinks.end()) + return Error::success(); + + auto EHFrameAddr = EHFrameAddrItr->second; + assert(EHFrameAddr && "eh-frame addr to register can not be null"); + + InProcessLinks.erase(EHFrameAddrItr); + if (auto Key = MR.getVModuleKey()) + TrackedEHFrameAddrs[Key] = EHFrameAddr; + else + UntrackedEHFrameAddrs.push_back(EHFrameAddr); + + return Registrar.registerEHFrames(EHFrameAddr); +} + +Error EHFrameRegistrationPlugin::notifyRemovingModule(VModuleKey K) { + auto EHFrameAddrItr = TrackedEHFrameAddrs.find(K); + if (EHFrameAddrItr == TrackedEHFrameAddrs.end()) + return Error::success(); + + auto EHFrameAddr = EHFrameAddrItr->second; + assert(EHFrameAddr && "Tracked eh-frame addr must not be null"); + + TrackedEHFrameAddrs.erase(EHFrameAddrItr); + + return Registrar.deregisterEHFrames(EHFrameAddr); +} + +Error EHFrameRegistrationPlugin::notifyRemovingAllModules() { + + std::vector EHFrameAddrs = std::move(UntrackedEHFrameAddrs); + EHFrameAddrs.reserve(EHFrameAddrs.size() + TrackedEHFrameAddrs.size()); + + for (auto &KV : TrackedEHFrameAddrs) + EHFrameAddrs.push_back(KV.second); + + TrackedEHFrameAddrs.clear(); + + Error Err = Error::success(); + + while (!EHFrameAddrs.empty()) { + auto EHFrameAddr = EHFrameAddrs.back(); + assert(EHFrameAddr && "Untracked eh-frame addr must not be null"); + EHFrameAddrs.pop_back(); + Err = joinErrors(std::move(Err), Registrar.deregisterEHFrames(EHFrameAddr)); + } + + return Err; +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp b/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp index 825f53204736..815517321b76 100644 --- a/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp +++ b/lib/ExecutionEngine/Orc/ObjectTransformLayer.cpp @@ -1,9 +1,8 @@ //===---------- ObjectTransformLayer.cpp - Object Transform Layer ---------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/ExecutionEngine/Orc/OrcABISupport.cpp b/lib/ExecutionEngine/Orc/OrcABISupport.cpp index aa4055542426..8ed23de419d1 100644 --- a/lib/ExecutionEngine/Orc/OrcABISupport.cpp +++ b/lib/ExecutionEngine/Orc/OrcABISupport.cpp @@ -1,9 +1,8 @@ //===------------- OrcABISupport.cpp - ABI specific support code ----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -148,7 +147,7 @@ Error OrcAArch64::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, const unsigned StubSize = IndirectStubsInfo::StubSize; // Emit at least MinStubs, rounded up to fill the pages allocated. - unsigned PageSize = sys::Process::getPageSize(); + static const unsigned PageSize = sys::Process::getPageSizeEstimate(); unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; unsigned NumStubs = (NumPages * PageSize) / StubSize; @@ -230,7 +229,7 @@ Error OrcX86_64_Base::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, const unsigned StubSize = IndirectStubsInfo::StubSize; // Emit at least MinStubs, rounded up to fill the pages allocated. - unsigned PageSize = sys::Process::getPageSize(); + static const unsigned PageSize = sys::Process::getPageSizeEstimate(); unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; unsigned NumStubs = (NumPages * PageSize) / StubSize; @@ -498,7 +497,7 @@ Error OrcI386::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, const unsigned StubSize = IndirectStubsInfo::StubSize; // Emit at least MinStubs, rounded up to fill the pages allocated. - unsigned PageSize = sys::Process::getPageSize(); + static const unsigned PageSize = sys::Process::getPageSizeEstimate(); unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; unsigned NumStubs = (NumPages * PageSize) / StubSize; @@ -684,7 +683,7 @@ Error OrcMips32_Base::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, const unsigned StubSize = IndirectStubsInfo::StubSize; // Emit at least MinStubs, rounded up to fill the pages allocated. - unsigned PageSize = sys::Process::getPageSize(); + static const unsigned PageSize = sys::Process::getPageSizeEstimate(); unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; unsigned NumStubs = (NumPages * PageSize) / StubSize; @@ -930,7 +929,7 @@ Error OrcMips64::emitIndirectStubsBlock(IndirectStubsInfo &StubsInfo, const unsigned StubSize = IndirectStubsInfo::StubSize; // Emit at least MinStubs, rounded up to fill the pages allocated. - unsigned PageSize = sys::Process::getPageSize(); + static const unsigned PageSize = sys::Process::getPageSizeEstimate(); unsigned NumPages = ((MinStubs * StubSize) + (PageSize - 1)) / PageSize; unsigned NumStubs = (NumPages * PageSize) / StubSize; diff --git a/lib/ExecutionEngine/Orc/OrcCBindings.cpp b/lib/ExecutionEngine/Orc/OrcCBindings.cpp index 6dea64a6e78f..28c8479abba4 100644 --- a/lib/ExecutionEngine/Orc/OrcCBindings.cpp +++ b/lib/ExecutionEngine/Orc/OrcCBindings.cpp @@ -1,9 +1,8 @@ //===----------- OrcCBindings.cpp - C bindings for the Orc APIs -----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/ExecutionEngine/Orc/OrcCBindingsStack.h b/lib/ExecutionEngine/Orc/OrcCBindingsStack.h index 817a4b89bfb0..98129e1690d2 100644 --- a/lib/ExecutionEngine/Orc/OrcCBindingsStack.h +++ b/lib/ExecutionEngine/Orc/OrcCBindingsStack.h @@ -1,9 +1,8 @@ //===- OrcCBindingsStack.h - Orc JIT stack for C bindings -----*- C++ -*---===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -154,8 +153,8 @@ private: for (auto &S : Symbols) { if (auto Sym = findSymbol(*S)) { if (auto Addr = Sym.getAddress()) { - Query->resolve(S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); - Query->notifySymbolReady(); + Query->notifySymbolMetRequiredState( + S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); } else { Stack.ES.legacyFailQuery(*Query, Addr.takeError()); return orc::SymbolNameSet(); @@ -167,11 +166,8 @@ private: UnresolvedSymbols.insert(S); } - if (Query->isFullyResolved()) - Query->handleFullyResolved(); - - if (Query->isFullyReady()) - Query->handleFullyReady(); + if (Query->isComplete()) + Query->handleComplete(); return UnresolvedSymbols; } @@ -215,28 +211,31 @@ public: IndirectStubsManagerBuilder IndirectStubsMgrBuilder) : CCMgr(createCompileCallbackManager(TM, ES)), DL(TM.createDataLayout()), IndirectStubsMgr(IndirectStubsMgrBuilder()), - ObjectLayer(ES, - [this](orc::VModuleKey K) { - auto ResolverI = Resolvers.find(K); - assert(ResolverI != Resolvers.end() && - "No resolver for module K"); - auto Resolver = std::move(ResolverI->second); - Resolvers.erase(ResolverI); - return ObjLayerT::Resources{ - std::make_shared(), Resolver}; - }, - nullptr, - [this](orc::VModuleKey K, const object::ObjectFile &Obj, - const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) { - this->notifyFinalized(K, Obj, LoadedObjInfo); - }, - [this](orc::VModuleKey K, const object::ObjectFile &Obj) { - this->notifyFreed(K, Obj); - }), - CompileLayer(ObjectLayer, orc::SimpleCompiler(TM)), + ObjectLayer( + AcknowledgeORCv1Deprecation, ES, + [this](orc::VModuleKey K) { + auto ResolverI = Resolvers.find(K); + assert(ResolverI != Resolvers.end() && + "No resolver for module K"); + auto Resolver = std::move(ResolverI->second); + Resolvers.erase(ResolverI); + return ObjLayerT::Resources{ + std::make_shared(), Resolver}; + }, + nullptr, + [this](orc::VModuleKey K, const object::ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) { + this->notifyFinalized(K, Obj, LoadedObjInfo); + }, + [this](orc::VModuleKey K, const object::ObjectFile &Obj) { + this->notifyFreed(K, Obj); + }), + CompileLayer(AcknowledgeORCv1Deprecation, ObjectLayer, + orc::SimpleCompiler(TM)), CODLayer(createCODLayer(ES, CompileLayer, CCMgr.get(), std::move(IndirectStubsMgrBuilder), Resolvers)), CXXRuntimeOverrides( + AcknowledgeORCv1Deprecation, [this](const std::string &S) { return mangle(S); }) {} Error shutdown() { @@ -312,7 +311,8 @@ public: // Run the static constructors, and save the static destructor runner for // execution when the JIT is torn down. - orc::LegacyCtorDtorRunner CtorRunner(std::move(CtorNames), K); + orc::LegacyCtorDtorRunner CtorRunner( + AcknowledgeORCv1Deprecation, std::move(CtorNames), K); if (auto Err = CtorRunner.runViaLayer(*this)) return std::move(Err); @@ -469,7 +469,7 @@ private: return nullptr; return llvm::make_unique( - ES, CompileLayer, + AcknowledgeORCv1Deprecation, ES, CompileLayer, [&Resolvers](orc::VModuleKey K) { auto ResolverI = Resolvers.find(K); assert(ResolverI != Resolvers.end() && "No resolver for module K"); diff --git a/lib/ExecutionEngine/Orc/OrcError.cpp b/lib/ExecutionEngine/Orc/OrcError.cpp index f4102b359a6b..e6e9a095319c 100644 --- a/lib/ExecutionEngine/Orc/OrcError.cpp +++ b/lib/ExecutionEngine/Orc/OrcError.cpp @@ -1,9 +1,8 @@ //===---------------- OrcError.cpp - Error codes for ORC ------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp index 617bc2fc64b5..772a9c2c4ab2 100644 --- a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp +++ b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.cpp @@ -1,9 +1,8 @@ //===-------- OrcMCJITReplacement.cpp - Orc-based MCJIT replacement -------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -128,7 +127,8 @@ void OrcMCJITReplacement::runStaticConstructorsDestructors(bool isDtors) { auto &CtorDtorsMap = isDtors ? UnexecutedDestructors : UnexecutedConstructors; for (auto &KV : CtorDtorsMap) - cantFail(LegacyCtorDtorRunner(std::move(KV.second), KV.first) + cantFail(LegacyCtorDtorRunner( + AcknowledgeORCv1Deprecation, std::move(KV.second), KV.first) .runViaLayer(LazyEmitLayer)); CtorDtorsMap.clear(); diff --git a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h index 36e7e83a8bab..169dc8f1d02b 100644 --- a/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h +++ b/lib/ExecutionEngine/Orc/OrcMCJITReplacement.h @@ -1,9 +1,8 @@ //===- OrcMCJITReplacement.h - Orc based MCJIT replacement ------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -177,8 +176,8 @@ class OrcMCJITReplacement : public ExecutionEngine { for (auto &S : Symbols) { if (auto Sym = M.findMangledSymbol(*S)) { if (auto Addr = Sym.getAddress()) { - Query->resolve(S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); - Query->notifySymbolReady(); + Query->notifySymbolMetRequiredState( + S, JITEvaluatedSymbol(*Addr, Sym.getFlags())); NewSymbolsResolved = true; } else { M.ES.legacyFailQuery(*Query, Addr.takeError()); @@ -190,8 +189,8 @@ class OrcMCJITReplacement : public ExecutionEngine { } else { if (auto Sym2 = M.ClientResolver->findSymbol(*S)) { if (auto Addr = Sym2.getAddress()) { - Query->resolve(S, JITEvaluatedSymbol(*Addr, Sym2.getFlags())); - Query->notifySymbolReady(); + Query->notifySymbolMetRequiredState( + S, JITEvaluatedSymbol(*Addr, Sym2.getFlags())); NewSymbolsResolved = true; } else { M.ES.legacyFailQuery(*Query, Addr.takeError()); @@ -205,11 +204,8 @@ class OrcMCJITReplacement : public ExecutionEngine { } } - if (NewSymbolsResolved && Query->isFullyResolved()) - Query->handleFullyResolved(); - - if (NewSymbolsResolved && Query->isFullyReady()) - Query->handleFullyReady(); + if (NewSymbolsResolved && Query->isComplete()) + Query->handleComplete(); return UnresolvedSymbols; } @@ -236,24 +232,24 @@ public: OrcMCJITReplacement(std::shared_ptr MemMgr, std::shared_ptr ClientResolver, std::unique_ptr TM) - : ExecutionEngine(TM->createDataLayout()), - TM(std::move(TM)), + : ExecutionEngine(TM->createDataLayout()), TM(std::move(TM)), MemMgr( std::make_shared(*this, std::move(MemMgr))), Resolver(std::make_shared(*this)), ClientResolver(std::move(ClientResolver)), NotifyObjectLoaded(*this), NotifyFinalized(*this), ObjectLayer( - ES, + AcknowledgeORCv1Deprecation, ES, [this](VModuleKey K) { return ObjectLayerT::Resources{this->MemMgr, this->Resolver}; }, NotifyObjectLoaded, NotifyFinalized), - CompileLayer(ObjectLayer, SimpleCompiler(*this->TM), + CompileLayer(AcknowledgeORCv1Deprecation, ObjectLayer, + SimpleCompiler(*this->TM), [this](VModuleKey K, std::unique_ptr M) { Modules.push_back(std::move(M)); }), - LazyEmitLayer(CompileLayer) {} + LazyEmitLayer(AcknowledgeORCv1Deprecation, CompileLayer) {} static void Register() { OrcMCJITReplacementCtor = createOrcMCJITReplacement; diff --git a/lib/ExecutionEngine/Orc/RPCUtils.cpp b/lib/ExecutionEngine/Orc/RPCUtils.cpp index 2a7ab5ca8180..367b3639f841 100644 --- a/lib/ExecutionEngine/Orc/RPCUtils.cpp +++ b/lib/ExecutionEngine/Orc/RPCUtils.cpp @@ -1,9 +1,8 @@ //===--------------- RPCUtils.cpp - RPCUtils implementation ---------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp b/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp index 299d76183cd4..b22ecd5f80a1 100644 --- a/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp +++ b/lib/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.cpp @@ -1,9 +1,8 @@ //===-- RTDyldObjectLinkingLayer.cpp - RuntimeDyld backed ORC ObjectLayer -===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -42,9 +41,6 @@ public: OnResolved(Result); }; - // We're not waiting for symbols to be ready. Just log any errors. - auto OnReady = [&ES](Error Err) { ES.reportError(std::move(Err)); }; - // Register dependencies for all symbols contained in this set. auto RegisterDependencies = [&](const SymbolDependenceMap &Deps) { MR.addDependenciesForAll(Deps); @@ -53,8 +49,8 @@ public: JITDylibSearchList SearchOrder; MR.getTargetJITDylib().withSearchOrderDo( [&](const JITDylibSearchList &JDs) { SearchOrder = JDs; }); - ES.lookup(SearchOrder, InternedSymbols, OnResolvedWithUnwrap, OnReady, - RegisterDependencies); + ES.lookup(SearchOrder, InternedSymbols, SymbolState::Resolved, + OnResolvedWithUnwrap, RegisterDependencies); } Expected getResponsibilitySet(const LookupSet &Symbols) { @@ -78,11 +74,8 @@ namespace llvm { namespace orc { RTDyldObjectLinkingLayer::RTDyldObjectLinkingLayer( - ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager, - NotifyLoadedFunction NotifyLoaded, NotifyEmittedFunction NotifyEmitted) - : ObjectLayer(ES), GetMemoryManager(GetMemoryManager), - NotifyLoaded(std::move(NotifyLoaded)), - NotifyEmitted(std::move(NotifyEmitted)) {} + ExecutionSession &ES, GetMemoryManagerFunction GetMemoryManager) + : ObjectLayer(ES), GetMemoryManager(GetMemoryManager) {} void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, std::unique_ptr O) { @@ -96,7 +89,13 @@ void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, auto &ES = getExecutionSession(); - auto Obj = object::ObjectFile::createObjectFile(*O); + // Create a MemoryBufferRef backed MemoryBuffer (i.e. shallow) copy of the + // the underlying buffer to pass into RuntimeDyld. This allows us to hold + // ownership of the real underlying buffer and return it to the user once + // the object has been emitted. + auto ObjBuffer = MemoryBuffer::getMemBuffer(O->getMemBufferRef(), false); + + auto Obj = object::ObjectFile::createObjectFile(*ObjBuffer); if (!Obj) { getExecutionSession().reportError(Obj.takeError()); @@ -134,13 +133,8 @@ void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, JITDylibSearchOrderResolver Resolver(*SharedR); - /* Thoughts on proper cross-dylib weak symbol handling: - * - * Change selection of canonical defs to be a manually triggered process, and - * add a 'canonical' bit to symbol definitions. When canonical def selection - * is triggered, sweep the JITDylibs to mark defs as canonical, discard - * duplicate defs. - */ + // FIXME: Switch to move-capture for the 'O' buffer once we have c++14. + MemoryBuffer *UnownedObjBuffer = O.release(); jitLinkForORC( **Obj, std::move(O), *MemMgr, Resolver, ProcessAllSections, [this, K, SharedR, &Obj, InternalSymbols]( @@ -149,8 +143,9 @@ void RTDyldObjectLinkingLayer::emit(MaterializationResponsibility R, return onObjLoad(K, *SharedR, **Obj, std::move(LoadedObjInfo), ResolvedSymbols, *InternalSymbols); }, - [this, K, SharedR](Error Err) { - onObjEmit(K, *SharedR, std::move(Err)); + [this, K, SharedR, UnownedObjBuffer](Error Err) { + std::unique_ptr ObjBuffer(UnownedObjBuffer); + onObjEmit(K, std::move(ObjBuffer), *SharedR, std::move(Err)); }); } @@ -177,7 +172,7 @@ Error RTDyldObjectLinkingLayer::onObjLoad( auto I = R.getSymbols().find(InternedName); if (OverrideObjectFlags && I != R.getSymbols().end()) - Flags = JITSymbolFlags::stripTransientFlags(I->second); + Flags = I->second; else if (AutoClaimObjectSymbols && I == R.getSymbols().end()) ExtraSymbolsToClaim[InternedName] = Flags; } @@ -189,7 +184,7 @@ Error RTDyldObjectLinkingLayer::onObjLoad( if (auto Err = R.defineMaterializing(ExtraSymbolsToClaim)) return Err; - R.resolve(Symbols); + R.notifyResolved(Symbols); if (NotifyLoaded) NotifyLoaded(K, Obj, *LoadedObjInfo); @@ -197,20 +192,29 @@ Error RTDyldObjectLinkingLayer::onObjLoad( return Error::success(); } -void RTDyldObjectLinkingLayer::onObjEmit(VModuleKey K, - MaterializationResponsibility &R, - Error Err) { +void RTDyldObjectLinkingLayer::onObjEmit( + VModuleKey K, std::unique_ptr ObjBuffer, + MaterializationResponsibility &R, Error Err) { if (Err) { getExecutionSession().reportError(std::move(Err)); R.failMaterialization(); return; } - R.emit(); + R.notifyEmitted(); if (NotifyEmitted) - NotifyEmitted(K); + NotifyEmitted(K, std::move(ObjBuffer)); } +LegacyRTDyldObjectLinkingLayer::LegacyRTDyldObjectLinkingLayer( + ExecutionSession &ES, ResourcesGetter GetResources, + NotifyLoadedFtor NotifyLoaded, NotifyFinalizedFtor NotifyFinalized, + NotifyFreedFtor NotifyFreed) + : ES(ES), GetResources(std::move(GetResources)), + NotifyLoaded(std::move(NotifyLoaded)), + NotifyFinalized(std::move(NotifyFinalized)), + NotifyFreed(std::move(NotifyFreed)), ProcessAllSections(false) {} + } // End namespace orc. } // End namespace llvm. diff --git a/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp b/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp index 9525b168fbd3..4cb7376758a7 100644 --- a/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp +++ b/lib/ExecutionEngine/Orc/ThreadSafeModule.cpp @@ -1,10 +1,9 @@ //===-- ThreadSafeModule.cpp - Thread safe Module, Context, and Utilities //h-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp b/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp index f195d0282998..5606421a3cb0 100644 --- a/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp +++ b/lib/ExecutionEngine/PerfJITEvents/PerfJITEventListener.cpp @@ -1,9 +1,8 @@ //===-- PerfJITEventListener.cpp - Tell Linux's perf about JITted code ----===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -266,16 +265,22 @@ void PerfJITEventListener::notifyObjectLoaded( consumeError(AddrOrErr.takeError()); continue; } - uint64_t Addr = *AddrOrErr; uint64_t Size = P.second; + object::SectionedAddress Address; + Address.Address = *AddrOrErr; + + uint64_t SectionIndex = object::SectionedAddress::UndefSection; + if (auto SectOrErr = Sym.getSection()) + if (*SectOrErr != Obj.section_end()) + SectionIndex = SectOrErr.get()->getIndex(); // According to spec debugging info has to come before loading the // corresonding code load. DILineInfoTable Lines = Context->getLineInfoForAddressRange( - Addr, Size, FileLineInfoKind::AbsoluteFilePath); + {*AddrOrErr, SectionIndex}, Size, FileLineInfoKind::AbsoluteFilePath); - NotifyDebug(Addr, Lines); - NotifyCode(Name, Addr, Size); + NotifyDebug(*AddrOrErr, Lines); + NotifyCode(Name, *AddrOrErr, Size); } Dumpstream->flush(); @@ -336,8 +341,8 @@ bool PerfJITEventListener::OpenMarker() { // // Mapping must be PROT_EXEC to ensure it is captured by perf record // even when not using -d option. - MarkerAddr = ::mmap(NULL, sys::Process::getPageSize(), PROT_READ | PROT_EXEC, - MAP_PRIVATE, DumpFd, 0); + MarkerAddr = ::mmap(NULL, sys::Process::getPageSizeEstimate(), + PROT_READ | PROT_EXEC, MAP_PRIVATE, DumpFd, 0); if (MarkerAddr == MAP_FAILED) { errs() << "could not mmap JIT marker\n"; @@ -350,7 +355,7 @@ void PerfJITEventListener::CloseMarker() { if (!MarkerAddr) return; - munmap(MarkerAddr, sys::Process::getPageSize()); + munmap(MarkerAddr, sys::Process::getPageSizeEstimate()); MarkerAddr = nullptr; } diff --git a/lib/ExecutionEngine/RuntimeDyld/JITSymbol.cpp b/lib/ExecutionEngine/RuntimeDyld/JITSymbol.cpp index 0553c217c2a2..4e2d0f422f39 100644 --- a/lib/ExecutionEngine/RuntimeDyld/JITSymbol.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/JITSymbol.cpp @@ -1,9 +1,8 @@ //===----------- JITSymbol.cpp - JITSymbol class implementation -----------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp b/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp index 75d4c2b5134e..46604ff4000c 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp @@ -1,9 +1,8 @@ //===-- RTDyldMemoryManager.cpp - Memory manager for MC-JIT -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -33,8 +32,9 @@ namespace llvm { RTDyldMemoryManager::~RTDyldMemoryManager() {} // Determine whether we can register EH tables. -#if (defined(__GNUC__) && !defined(__ARM_EABI__) && !defined(__ia64__) && \ - !defined(__SEH__) && !defined(__USING_SJLJ_EXCEPTIONS__)) +#if (defined(__GNUC__) && !defined(__ARM_EABI__) && !defined(__ia64__) && \ + !(defined(_AIX) && defined(__ibmxl__)) && !defined(__SEH__) && \ + !defined(__USING_SJLJ_EXCEPTIONS__)) #define HAVE_EHTABLE_SUPPORT 1 #else #define HAVE_EHTABLE_SUPPORT 0 @@ -48,7 +48,7 @@ extern "C" void __deregister_frame(void *); // it may be found at runtime in a dynamically-loaded library. // For example, this happens when building LLVM with Visual C++ // but using the MingW runtime. -void __register_frame(void *p) { +static void __register_frame(void *p) { static bool Searched = false; static void((*rf)(void *)) = 0; @@ -61,7 +61,7 @@ void __register_frame(void *p) { rf(p); } -void __deregister_frame(void *p) { +static void __deregister_frame(void *p) { static bool Searched = false; static void((*df)(void *)) = 0; diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp index 53cb782c55c4..e26e6ce45db4 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyld.cpp @@ -1,9 +1,8 @@ //===-- RuntimeDyld.cpp - Run-time dynamic linker for MC-JIT ----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -13,7 +12,6 @@ #include "llvm/ExecutionEngine/RuntimeDyld.h" #include "RuntimeDyldCOFF.h" -#include "RuntimeDyldCheckerImpl.h" #include "RuntimeDyldELF.h" #include "RuntimeDyldImpl.h" #include "RuntimeDyldMachO.h" @@ -376,10 +374,55 @@ RuntimeDyldImpl::loadObjectImpl(const object::ObjectFile &Obj) { else return IOrErr.takeError(); - // If there is an attached checker, notify it about the stubs for this - // section so that they can be verified. - if (Checker) - Checker->registerStubMap(Obj.getFileName(), SectionID, Stubs); + // If there is a NotifyStubEmitted callback set, call it to register any + // stubs created for this section. + if (NotifyStubEmitted) { + StringRef FileName = Obj.getFileName(); + StringRef SectionName = Sections[SectionID].getName(); + for (auto &KV : Stubs) { + + auto &VR = KV.first; + uint64_t StubAddr = KV.second; + + // If this is a named stub, just call NotifyStubEmitted. + if (VR.SymbolName) { + NotifyStubEmitted(FileName, SectionName, VR.SymbolName, SectionID, + StubAddr); + continue; + } + + // Otherwise we will have to try a reverse lookup on the globla symbol table. + for (auto &GSTMapEntry : GlobalSymbolTable) { + StringRef SymbolName = GSTMapEntry.first(); + auto &GSTEntry = GSTMapEntry.second; + if (GSTEntry.getSectionID() == VR.SectionID && + GSTEntry.getOffset() == VR.Offset) { + NotifyStubEmitted(FileName, SectionName, SymbolName, SectionID, + StubAddr); + break; + } + } + } + } + } + + // Process remaining sections + if (ProcessAllSections) { + LLVM_DEBUG(dbgs() << "Process remaining sections:\n"); + for (section_iterator SI = Obj.section_begin(), SE = Obj.section_end(); + SI != SE; ++SI) { + + /* Ignore already loaded sections */ + if (LocalSections.find(*SI) != LocalSections.end()) + continue; + + bool IsCode = SI->isText(); + if (auto SectionIDOrErr = + findOrEmitSection(Obj, *SI, IsCode, LocalSections)) + LLVM_DEBUG(dbgs() << "\tSectionID: " << (*SectionIDOrErr) << "\n"); + else + return SectionIDOrErr.takeError(); + } } // Give the subclasses a chance to tie-up any loose ends. @@ -497,7 +540,14 @@ Error RuntimeDyldImpl::computeTotalAllocSize(const ObjectFile &Obj, return errorCodeToError(EC); uint64_t StubBufSize = computeSectionStubBufSize(Obj, Section); - uint64_t SectionSize = DataSize + StubBufSize; + + uint64_t PaddingSize = 0; + if (Name == ".eh_frame") + PaddingSize += 4; + if (StubBufSize != 0) + PaddingSize += getStubAlignment() - 1; + + uint64_t SectionSize = DataSize + PaddingSize + StubBufSize; // The .eh_frame section (at least on Linux) needs an extra four bytes // padded @@ -703,9 +753,6 @@ Error RuntimeDyldImpl::emitCommonSymbols(const ObjectFile &Obj, Addr += Size; } - if (Checker) - Checker->registerSection(Obj.getFileName(), SectionID); - return Error::success(); } @@ -725,6 +772,11 @@ RuntimeDyldImpl::emitSection(const ObjectFile &Obj, bool IsReadOnly = isReadOnlyData(Section); uint64_t DataSize = Section.getSize(); + // An alignment of 0 (at least with ELF) is identical to an alignment of 1, + // while being more "polite". Other formats do not support 0-aligned sections + // anyway, so we should guarantee that the alignment is always at least 1. + Alignment = std::max(1u, Alignment); + StringRef Name; if (auto EC = Section.getName(Name)) return errorCodeToError(EC); @@ -747,18 +799,19 @@ RuntimeDyldImpl::emitSection(const ObjectFile &Obj, if (!IsVirtual && !IsZeroInit) { // In either case, set the location of the unrelocated section in memory, // since we still process relocations for it even if we're not applying them. - if (auto EC = Section.getContents(data)) - return errorCodeToError(EC); + if (Expected E = Section.getContents()) + data = *E; + else + return E.takeError(); pData = data.data(); } - // Code section alignment needs to be at least as high as stub alignment or - // padding calculations may by incorrect when the section is remapped to a - // higher alignment. - if (IsCode) { + // If there are any stubs then the section alignment needs to be at least as + // high as stub alignment or padding calculations may by incorrect when the + // section is remapped. + if (StubBufSize != 0) { Alignment = std::max(Alignment, getStubAlignment()); - if (StubBufSize > 0) - PaddingSize += getStubAlignment() - 1; + PaddingSize += getStubAlignment() - 1; } // Some sections, such as debug info, don't need to be loaded for execution. @@ -789,7 +842,7 @@ RuntimeDyldImpl::emitSection(const ObjectFile &Obj, // Align DataSize to stub alignment if we have any stubs (PaddingSize will // have been increased above to account for this). if (StubBufSize > 0) - DataSize &= ~(getStubAlignment() - 1); + DataSize &= -(uint64_t)getStubAlignment(); } LLVM_DEBUG(dbgs() << "emitSection SectionID: " << SectionID << " Name: " @@ -817,9 +870,6 @@ RuntimeDyldImpl::emitSection(const ObjectFile &Obj, if (!IsRequired) Sections.back().setLoadAddress(0); - if (Checker) - Checker->registerSection(Obj.getFileName(), SectionID); - return SectionID; } @@ -1202,42 +1252,43 @@ RuntimeDyld::RuntimeDyld(RuntimeDyld::MemoryManager &MemMgr, // permissions are applied. Dyld = nullptr; ProcessAllSections = false; - Checker = nullptr; } RuntimeDyld::~RuntimeDyld() {} static std::unique_ptr -createRuntimeDyldCOFF(Triple::ArchType Arch, RuntimeDyld::MemoryManager &MM, - JITSymbolResolver &Resolver, bool ProcessAllSections, - RuntimeDyldCheckerImpl *Checker) { +createRuntimeDyldCOFF( + Triple::ArchType Arch, RuntimeDyld::MemoryManager &MM, + JITSymbolResolver &Resolver, bool ProcessAllSections, + RuntimeDyld::NotifyStubEmittedFunction NotifyStubEmitted) { std::unique_ptr Dyld = RuntimeDyldCOFF::create(Arch, MM, Resolver); Dyld->setProcessAllSections(ProcessAllSections); - Dyld->setRuntimeDyldChecker(Checker); + Dyld->setNotifyStubEmitted(std::move(NotifyStubEmitted)); return Dyld; } static std::unique_ptr createRuntimeDyldELF(Triple::ArchType Arch, RuntimeDyld::MemoryManager &MM, JITSymbolResolver &Resolver, bool ProcessAllSections, - RuntimeDyldCheckerImpl *Checker) { + RuntimeDyld::NotifyStubEmittedFunction NotifyStubEmitted) { std::unique_ptr Dyld = RuntimeDyldELF::create(Arch, MM, Resolver); Dyld->setProcessAllSections(ProcessAllSections); - Dyld->setRuntimeDyldChecker(Checker); + Dyld->setNotifyStubEmitted(std::move(NotifyStubEmitted)); return Dyld; } static std::unique_ptr -createRuntimeDyldMachO(Triple::ArchType Arch, RuntimeDyld::MemoryManager &MM, - JITSymbolResolver &Resolver, - bool ProcessAllSections, - RuntimeDyldCheckerImpl *Checker) { +createRuntimeDyldMachO( + Triple::ArchType Arch, RuntimeDyld::MemoryManager &MM, + JITSymbolResolver &Resolver, + bool ProcessAllSections, + RuntimeDyld::NotifyStubEmittedFunction NotifyStubEmitted) { std::unique_ptr Dyld = RuntimeDyldMachO::create(Arch, MM, Resolver); Dyld->setProcessAllSections(ProcessAllSections); - Dyld->setRuntimeDyldChecker(Checker); + Dyld->setNotifyStubEmitted(std::move(NotifyStubEmitted)); return Dyld; } @@ -1247,15 +1298,16 @@ RuntimeDyld::loadObject(const ObjectFile &Obj) { if (Obj.isELF()) Dyld = createRuntimeDyldELF(static_cast(Obj.getArch()), - MemMgr, Resolver, ProcessAllSections, Checker); + MemMgr, Resolver, ProcessAllSections, + std::move(NotifyStubEmitted)); else if (Obj.isMachO()) Dyld = createRuntimeDyldMachO( static_cast(Obj.getArch()), MemMgr, Resolver, - ProcessAllSections, Checker); + ProcessAllSections, std::move(NotifyStubEmitted)); else if (Obj.isCOFF()) Dyld = createRuntimeDyldCOFF( static_cast(Obj.getArch()), MemMgr, Resolver, - ProcessAllSections, Checker); + ProcessAllSections, std::move(NotifyStubEmitted)); else report_fatal_error("Incompatible object format!"); } @@ -1274,6 +1326,11 @@ void *RuntimeDyld::getSymbolLocalAddress(StringRef Name) const { return Dyld->getSymbolLocalAddress(Name); } +unsigned RuntimeDyld::getSymbolSectionID(StringRef Name) const { + assert(Dyld && "No RuntimeDyld instance attached"); + return Dyld->getSymbolSectionID(Name); +} + JITEvaluatedSymbol RuntimeDyld::getSymbol(StringRef Name) const { if (!Dyld) return nullptr; @@ -1312,6 +1369,16 @@ void RuntimeDyld::finalizeWithMemoryManagerLocking() { } } +StringRef RuntimeDyld::getSectionContent(unsigned SectionID) const { + assert(Dyld && "No Dyld instance attached"); + return Dyld->getSectionContent(SectionID); +} + +uint64_t RuntimeDyld::getSectionLoadAddress(unsigned SectionID) const { + assert(Dyld && "No Dyld instance attached"); + return Dyld->getSectionLoadAddress(SectionID); +} + void RuntimeDyld::registerEHFrames() { if (Dyld) Dyld->registerEHFrames(); diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp index 340ddaab186d..d4e3b0ba7670 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.cpp @@ -1,9 +1,8 @@ //===-- RuntimeDyldCOFF.cpp - Run-time dynamic linker for MC-JIT -*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h index 729a358fa0ea..4efd18a2e6c5 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCOFF.h @@ -1,9 +1,8 @@ //===-- RuntimeDyldCOFF.h - Run-time dynamic linker for MC-JIT ---*- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp index 6eb6256080ff..ec31ea4e573c 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldChecker.cpp @@ -1,23 +1,21 @@ //===--- RuntimeDyldChecker.cpp - RuntimeDyld tester framework --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "llvm/ExecutionEngine/RuntimeDyldChecker.h" #include "RuntimeDyldCheckerImpl.h" -#include "RuntimeDyldImpl.h" #include "llvm/ADT/STLExtras.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCDisassembler/MCDisassembler.h" #include "llvm/MC/MCInst.h" +#include "llvm/Support/Endian.h" #include "llvm/Support/MSVCErrorWorkarounds.h" #include "llvm/Support/Path.h" #include -#include #include #include @@ -321,22 +319,22 @@ private: return std::make_pair(EvalResult(NextPC), RemainingExpr); } - // Evaluate a call to stub_addr. + // Evaluate a call to stub_addr/got_addr. // Look up and return the address of the stub for the given // (,
, ) tuple. // On success, returns a pair containing the stub address, plus the expression // remaining to be evaluated. - std::pair evalStubAddr(StringRef Expr, - ParseContext PCtx) const { + std::pair + evalStubOrGOTAddr(StringRef Expr, ParseContext PCtx, bool IsStubAddr) const { if (!Expr.startswith("(")) return std::make_pair(unexpectedToken(Expr, Expr, "expected '('"), ""); StringRef RemainingExpr = Expr.substr(1).ltrim(); // Handle file-name specially, as it may contain characters that aren't // legal for symbols. - StringRef FileName; + StringRef StubContainerName; size_t ComaIdx = RemainingExpr.find(','); - FileName = RemainingExpr.substr(0, ComaIdx).rtrim(); + StubContainerName = RemainingExpr.substr(0, ComaIdx).rtrim(); RemainingExpr = RemainingExpr.substr(ComaIdx).ltrim(); if (!RemainingExpr.startswith(",")) @@ -344,14 +342,6 @@ private: unexpectedToken(RemainingExpr, Expr, "expected ','"), ""); RemainingExpr = RemainingExpr.substr(1).ltrim(); - StringRef SectionName; - std::tie(SectionName, RemainingExpr) = parseSymbol(RemainingExpr); - - if (!RemainingExpr.startswith(",")) - return std::make_pair( - unexpectedToken(RemainingExpr, Expr, "expected ','"), ""); - RemainingExpr = RemainingExpr.substr(1).ltrim(); - StringRef Symbol; std::tie(Symbol, RemainingExpr) = parseSymbol(RemainingExpr); @@ -362,8 +352,8 @@ private: uint64_t StubAddr; std::string ErrorMsg = ""; - std::tie(StubAddr, ErrorMsg) = Checker.getStubAddrFor( - FileName, SectionName, Symbol, PCtx.IsInsideLoad); + std::tie(StubAddr, ErrorMsg) = Checker.getStubOrGOTAddrFor( + StubContainerName, Symbol, PCtx.IsInsideLoad, IsStubAddr); if (ErrorMsg != "") return std::make_pair(EvalResult(ErrorMsg), ""); @@ -423,7 +413,9 @@ private: else if (Symbol == "next_pc") return evalNextPC(RemainingExpr, PCtx); else if (Symbol == "stub_addr") - return evalStubAddr(RemainingExpr, PCtx); + return evalStubOrGOTAddr(RemainingExpr, PCtx, true); + else if (Symbol == "got_addr") + return evalStubOrGOTAddr(RemainingExpr, PCtx, false); else if (Symbol == "section_addr") return evalSectionAddr(RemainingExpr, PCtx); @@ -534,6 +526,11 @@ private: uint64_t LoadAddr = LoadAddrExprResult.getValue(); + // If there is no error but the content pointer is null then this is a + // zero-fill symbol/section. + if (LoadAddr == 0) + return std::make_pair(0, RemainingExpr); + return std::make_pair( EvalResult(Checker.readMemoryAtAddr(LoadAddr, ReadSize)), RemainingExpr); @@ -666,27 +663,29 @@ private: bool decodeInst(StringRef Symbol, MCInst &Inst, uint64_t &Size) const { MCDisassembler *Dis = Checker.Disassembler; - StringRef SectionMem = Checker.getSubsectionStartingAt(Symbol); - ArrayRef SectionBytes( - reinterpret_cast(SectionMem.data()), - SectionMem.size()); + StringRef SymbolMem = Checker.getSymbolContent(Symbol); + ArrayRef SymbolBytes(SymbolMem.bytes_begin(), SymbolMem.size()); MCDisassembler::DecodeStatus S = - Dis->getInstruction(Inst, Size, SectionBytes, 0, nulls(), nulls()); + Dis->getInstruction(Inst, Size, SymbolBytes, 0, nulls(), nulls()); return (S == MCDisassembler::Success); } }; } -RuntimeDyldCheckerImpl::RuntimeDyldCheckerImpl(RuntimeDyld &RTDyld, - MCDisassembler *Disassembler, - MCInstPrinter *InstPrinter, - raw_ostream &ErrStream) - : RTDyld(RTDyld), Disassembler(Disassembler), InstPrinter(InstPrinter), - ErrStream(ErrStream) { - RTDyld.Checker = this; -} +RuntimeDyldCheckerImpl::RuntimeDyldCheckerImpl( + IsSymbolValidFunction IsSymbolValid, GetSymbolInfoFunction GetSymbolInfo, + GetSectionInfoFunction GetSectionInfo, GetStubInfoFunction GetStubInfo, + GetGOTInfoFunction GetGOTInfo, support::endianness Endianness, + MCDisassembler *Disassembler, MCInstPrinter *InstPrinter, + raw_ostream &ErrStream) + : IsSymbolValid(std::move(IsSymbolValid)), + GetSymbolInfo(std::move(GetSymbolInfo)), + GetSectionInfo(std::move(GetSectionInfo)), + GetStubInfo(std::move(GetStubInfo)), GetGOTInfo(std::move(GetGOTInfo)), + Endianness(Endianness), Disassembler(Disassembler), + InstPrinter(InstPrinter), ErrStream(ErrStream) {} bool RuntimeDyldCheckerImpl::check(StringRef CheckExpr) const { CheckExpr = CheckExpr.trim(); @@ -731,242 +730,134 @@ bool RuntimeDyldCheckerImpl::checkAllRulesInBuffer(StringRef RulePrefix, return DidAllTestsPass && (NumRules != 0); } -Expected RuntimeDyldCheckerImpl::lookup( - const JITSymbolResolver::LookupSet &Symbols) const { - -#ifdef _MSC_VER - using ExpectedLookupResult = MSVCPExpected; -#else - using ExpectedLookupResult = Expected; -#endif - - auto ResultP = std::make_shared>(); - auto ResultF = ResultP->get_future(); - - getRTDyld().Resolver.lookup( - Symbols, [=](Expected Result) { - ResultP->set_value(std::move(Result)); - }); - return ResultF.get(); -} - bool RuntimeDyldCheckerImpl::isSymbolValid(StringRef Symbol) const { - if (getRTDyld().getSymbol(Symbol)) - return true; - auto Result = lookup({Symbol}); + return IsSymbolValid(Symbol); +} - if (!Result) { - logAllUnhandledErrors(Result.takeError(), errs(), "RTDyldChecker: "); - return false; +uint64_t RuntimeDyldCheckerImpl::getSymbolLocalAddr(StringRef Symbol) const { + auto SymInfo = GetSymbolInfo(Symbol); + if (!SymInfo) { + logAllUnhandledErrors(SymInfo.takeError(), errs(), "RTDyldChecker: "); + return 0; } - assert(Result->count(Symbol) && "Missing symbol result"); - return true; -} + if (SymInfo->isZeroFill()) + return 0; -uint64_t RuntimeDyldCheckerImpl::getSymbolLocalAddr(StringRef Symbol) const { return static_cast( - reinterpret_cast(getRTDyld().getSymbolLocalAddress(Symbol))); + reinterpret_cast(SymInfo->getContent().data())); } uint64_t RuntimeDyldCheckerImpl::getSymbolRemoteAddr(StringRef Symbol) const { - if (auto InternalSymbol = getRTDyld().getSymbol(Symbol)) - return InternalSymbol.getAddress(); - - auto Result = lookup({Symbol}); - if (!Result) { - logAllUnhandledErrors(Result.takeError(), errs(), "RTDyldChecker: "); + auto SymInfo = GetSymbolInfo(Symbol); + if (!SymInfo) { + logAllUnhandledErrors(SymInfo.takeError(), errs(), "RTDyldChecker: "); return 0; } - auto I = Result->find(Symbol); - assert(I != Result->end() && "Missing symbol result"); - return I->second.getAddress(); + + return SymInfo->getTargetAddress(); } uint64_t RuntimeDyldCheckerImpl::readMemoryAtAddr(uint64_t SrcAddr, unsigned Size) const { uintptr_t PtrSizedAddr = static_cast(SrcAddr); assert(PtrSizedAddr == SrcAddr && "Linker memory pointer out-of-range."); - uint8_t *Src = reinterpret_cast(PtrSizedAddr); - return getRTDyld().readBytesUnaligned(Src, Size); + void *Ptr = reinterpret_cast(PtrSizedAddr); + + switch (Size) { + case 1: + return support::endian::read(Ptr, Endianness); + case 2: + return support::endian::read(Ptr, Endianness); + case 4: + return support::endian::read(Ptr, Endianness); + case 8: + return support::endian::read(Ptr, Endianness); + } + llvm_unreachable("Unsupported read size"); } - -std::pair -RuntimeDyldCheckerImpl::findSectionAddrInfo(StringRef FileName, - StringRef SectionName) const { - - auto SectionMapItr = Stubs.find(FileName); - if (SectionMapItr == Stubs.end()) { - std::string ErrorMsg = "File '"; - ErrorMsg += FileName; - ErrorMsg += "' not found. "; - if (Stubs.empty()) - ErrorMsg += "No stubs registered."; - else { - ErrorMsg += "Available files are:"; - for (const auto& StubEntry : Stubs) { - ErrorMsg += " '"; - ErrorMsg += StubEntry.first; - ErrorMsg += "'"; - } - } - ErrorMsg += "\n"; - return std::make_pair(nullptr, ErrorMsg); +StringRef RuntimeDyldCheckerImpl::getSymbolContent(StringRef Symbol) const { + auto SymInfo = GetSymbolInfo(Symbol); + if (!SymInfo) { + logAllUnhandledErrors(SymInfo.takeError(), errs(), "RTDyldChecker: "); + return StringRef(); } - - auto SectionInfoItr = SectionMapItr->second.find(SectionName); - if (SectionInfoItr == SectionMapItr->second.end()) - return std::make_pair(nullptr, - ("Section '" + SectionName + "' not found in file '" + - FileName + "'\n").str()); - - return std::make_pair(&SectionInfoItr->second, std::string("")); + return SymInfo->getContent(); } std::pair RuntimeDyldCheckerImpl::getSectionAddr( StringRef FileName, StringRef SectionName, bool IsInsideLoad) const { - const SectionAddressInfo *SectionInfo = nullptr; - { - std::string ErrorMsg; - std::tie(SectionInfo, ErrorMsg) = - findSectionAddrInfo(FileName, SectionName); - if (ErrorMsg != "") - return std::make_pair(0, ErrorMsg); - } - - unsigned SectionID = SectionInfo->SectionID; - uint64_t Addr; - if (IsInsideLoad) - Addr = static_cast(reinterpret_cast( - getRTDyld().Sections[SectionID].getAddress())); - else - Addr = getRTDyld().Sections[SectionID].getLoadAddress(); - - return std::make_pair(Addr, std::string("")); -} - -std::pair RuntimeDyldCheckerImpl::getStubAddrFor( - StringRef FileName, StringRef SectionName, StringRef SymbolName, - bool IsInsideLoad) const { - - const SectionAddressInfo *SectionInfo = nullptr; - { - std::string ErrorMsg; - std::tie(SectionInfo, ErrorMsg) = - findSectionAddrInfo(FileName, SectionName); - if (ErrorMsg != "") - return std::make_pair(0, ErrorMsg); + auto SecInfo = GetSectionInfo(FileName, SectionName); + if (!SecInfo) { + std::string ErrMsg; + { + raw_string_ostream ErrMsgStream(ErrMsg); + logAllUnhandledErrors(SecInfo.takeError(), ErrMsgStream, + "RTDyldChecker: "); + } + return std::make_pair(0, std::move(ErrMsg)); } - unsigned SectionID = SectionInfo->SectionID; - const StubOffsetsMap &SymbolStubs = SectionInfo->StubOffsets; - auto StubOffsetItr = SymbolStubs.find(SymbolName); - if (StubOffsetItr == SymbolStubs.end()) - return std::make_pair(0, - ("Stub for symbol '" + SymbolName + "' not found. " - "If '" + SymbolName + "' is an internal symbol this " - "may indicate that the stub target offset is being " - "computed incorrectly.\n").str()); + // If this address is being looked up in "load" mode, return the content + // pointer, otherwise return the target address. - uint64_t StubOffset = StubOffsetItr->second; + uint64_t Addr = 0; - uint64_t Addr; if (IsInsideLoad) { - uintptr_t SectionBase = reinterpret_cast( - getRTDyld().Sections[SectionID].getAddress()); - Addr = static_cast(SectionBase) + StubOffset; - } else { - uint64_t SectionBase = getRTDyld().Sections[SectionID].getLoadAddress(); - Addr = SectionBase + StubOffset; - } - - return std::make_pair(Addr, std::string("")); -} - -StringRef -RuntimeDyldCheckerImpl::getSubsectionStartingAt(StringRef Name) const { - RTDyldSymbolTable::const_iterator pos = - getRTDyld().GlobalSymbolTable.find(Name); - if (pos == getRTDyld().GlobalSymbolTable.end()) - return StringRef(); - const auto &SymInfo = pos->second; - uint8_t *SectionAddr = getRTDyld().getSectionAddress(SymInfo.getSectionID()); - return StringRef(reinterpret_cast(SectionAddr) + - SymInfo.getOffset(), - getRTDyld().Sections[SymInfo.getSectionID()].getSize() - - SymInfo.getOffset()); -} - -Optional -RuntimeDyldCheckerImpl::getSectionLoadAddress(void *LocalAddress) const { - for (auto &S : getRTDyld().Sections) { - if (S.getAddress() == LocalAddress) - return S.getLoadAddress(); - } - return Optional(); -} - -void RuntimeDyldCheckerImpl::registerSection( - StringRef FilePath, unsigned SectionID) { - StringRef FileName = sys::path::filename(FilePath); - const SectionEntry &Section = getRTDyld().Sections[SectionID]; - StringRef SectionName = Section.getName(); + if (SecInfo->isZeroFill()) + Addr = 0; + else + Addr = pointerToJITTargetAddress(SecInfo->getContent().data()); + } else + Addr = SecInfo->getTargetAddress(); - Stubs[FileName][SectionName].SectionID = SectionID; + return std::make_pair(Addr, ""); } -void RuntimeDyldCheckerImpl::registerStubMap( - StringRef FilePath, unsigned SectionID, - const RuntimeDyldImpl::StubMap &RTDyldStubs) { - StringRef FileName = sys::path::filename(FilePath); - const SectionEntry &Section = getRTDyld().Sections[SectionID]; - StringRef SectionName = Section.getName(); - - Stubs[FileName][SectionName].SectionID = SectionID; +std::pair RuntimeDyldCheckerImpl::getStubOrGOTAddrFor( + StringRef StubContainerName, StringRef SymbolName, bool IsInsideLoad, + bool IsStubAddr) const { - for (auto &StubMapEntry : RTDyldStubs) { - std::string SymbolName = ""; + auto StubInfo = IsStubAddr ? GetStubInfo(StubContainerName, SymbolName) + : GetGOTInfo(StubContainerName, SymbolName); - if (StubMapEntry.first.SymbolName) - SymbolName = StubMapEntry.first.SymbolName; - else { - // If this is a (Section, Offset) pair, do a reverse lookup in the - // global symbol table to find the name. - for (auto &GSTEntry : getRTDyld().GlobalSymbolTable) { - const auto &SymInfo = GSTEntry.second; - if (SymInfo.getSectionID() == StubMapEntry.first.SectionID && - SymInfo.getOffset() == - static_cast(StubMapEntry.first.Offset)) { - SymbolName = GSTEntry.first(); - break; - } - } + if (!StubInfo) { + std::string ErrMsg; + { + raw_string_ostream ErrMsgStream(ErrMsg); + logAllUnhandledErrors(StubInfo.takeError(), ErrMsgStream, + "RTDyldChecker: "); } - - if (SymbolName != "") - Stubs[FileName][SectionName].StubOffsets[SymbolName] = - StubMapEntry.second; + return std::make_pair((uint64_t)0, std::move(ErrMsg)); } -} -RuntimeDyldChecker::RuntimeDyldChecker(RuntimeDyld &RTDyld, - MCDisassembler *Disassembler, - MCInstPrinter *InstPrinter, - raw_ostream &ErrStream) - : Impl(make_unique(RTDyld, Disassembler, - InstPrinter, ErrStream)) {} + uint64_t Addr = 0; -RuntimeDyldChecker::~RuntimeDyldChecker() {} + if (IsInsideLoad) { + if (StubInfo->isZeroFill()) + return std::make_pair((uint64_t)0, "Detected zero-filled stub/GOT entry"); + Addr = pointerToJITTargetAddress(StubInfo->getContent().data()); + } else + Addr = StubInfo->getTargetAddress(); -RuntimeDyld& RuntimeDyldChecker::getRTDyld() { - return Impl->RTDyld; + return std::make_pair(Addr, ""); } -const RuntimeDyld& RuntimeDyldChecker::getRTDyld() const { - return Impl->RTDyld; -} +RuntimeDyldChecker::RuntimeDyldChecker( + IsSymbolValidFunction IsSymbolValid, GetSymbolInfoFunction GetSymbolInfo, + GetSectionInfoFunction GetSectionInfo, GetStubInfoFunction GetStubInfo, + GetGOTInfoFunction GetGOTInfo, support::endianness Endianness, + MCDisassembler *Disassembler, MCInstPrinter *InstPrinter, + raw_ostream &ErrStream) + : Impl(::llvm::make_unique( + std::move(IsSymbolValid), std::move(GetSymbolInfo), + std::move(GetSectionInfo), std::move(GetStubInfo), + std::move(GetGOTInfo), Endianness, Disassembler, InstPrinter, + ErrStream)) {} + +RuntimeDyldChecker::~RuntimeDyldChecker() {} bool RuntimeDyldChecker::check(StringRef CheckExpr) const { return Impl->check(CheckExpr); @@ -982,8 +873,3 @@ RuntimeDyldChecker::getSectionAddr(StringRef FileName, StringRef SectionName, bool LocalAddress) { return Impl->getSectionAddr(FileName, SectionName, LocalAddress); } - -Optional -RuntimeDyldChecker::getSectionLoadAddress(void *LocalAddress) const { - return Impl->getSectionLoadAddress(LocalAddress); -} diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h index 6da1a68d06d6..ac9d4d460217 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldCheckerImpl.h @@ -1,9 +1,8 @@ //===-- RuntimeDyldCheckerImpl.h -- RuntimeDyld test framework --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -16,14 +15,22 @@ namespace llvm { class RuntimeDyldCheckerImpl { friend class RuntimeDyldChecker; - friend class RuntimeDyldImpl; friend class RuntimeDyldCheckerExprEval; - friend class RuntimeDyldELF; + + using IsSymbolValidFunction = + RuntimeDyldChecker::IsSymbolValidFunction; + using GetSymbolInfoFunction = RuntimeDyldChecker::GetSymbolInfoFunction; + using GetSectionInfoFunction = RuntimeDyldChecker::GetSectionInfoFunction; + using GetStubInfoFunction = RuntimeDyldChecker::GetStubInfoFunction; + using GetGOTInfoFunction = RuntimeDyldChecker::GetGOTInfoFunction; public: - RuntimeDyldCheckerImpl(RuntimeDyld &RTDyld, MCDisassembler *Disassembler, - MCInstPrinter *InstPrinter, - llvm::raw_ostream &ErrStream); + RuntimeDyldCheckerImpl( + IsSymbolValidFunction IsSymbolValid, GetSymbolInfoFunction GetSymbolInfo, + GetSectionInfoFunction GetSectionInfo, GetStubInfoFunction GetStubInfo, + GetGOTInfoFunction GetGOTInfo, support::endianness Endianness, + MCDisassembler *Disassembler, MCInstPrinter *InstPrinter, + llvm::raw_ostream &ErrStream); bool check(StringRef CheckExpr) const; bool checkAllRulesInBuffer(StringRef RulePrefix, MemoryBuffer *MemBuf) const; @@ -31,15 +38,6 @@ public: private: // StubMap typedefs. - typedef std::map StubOffsetsMap; - struct SectionAddressInfo { - uint64_t SectionID; - StubOffsetsMap StubOffsets; - }; - typedef std::map SectionMap; - typedef std::map StubMap; - - RuntimeDyldImpl &getRTDyld() const { return *RTDyld.Dyld; } Expected lookup(const JITSymbolResolver::LookupSet &Symbols) const; @@ -49,32 +47,27 @@ private: uint64_t getSymbolRemoteAddr(StringRef Symbol) const; uint64_t readMemoryAtAddr(uint64_t Addr, unsigned Size) const; - std::pair findSectionAddrInfo( - StringRef FileName, - StringRef SectionName) const; + StringRef getSymbolContent(StringRef Symbol) const; std::pair getSectionAddr(StringRef FileName, StringRef SectionName, bool IsInsideLoad) const; - std::pair getStubAddrFor(StringRef FileName, - StringRef SectionName, - StringRef Symbol, - bool IsInsideLoad) const; - StringRef getSubsectionStartingAt(StringRef Name) const; + std::pair + getStubOrGOTAddrFor(StringRef StubContainerName, StringRef Symbol, + bool IsInsideLoad, bool IsStubAddr) const; Optional getSectionLoadAddress(void *LocalAddr) const; - void registerSection(StringRef FilePath, unsigned SectionID); - void registerStubMap(StringRef FilePath, unsigned SectionID, - const RuntimeDyldImpl::StubMap &RTDyldStubs); - - RuntimeDyld &RTDyld; + IsSymbolValidFunction IsSymbolValid; + GetSymbolInfoFunction GetSymbolInfo; + GetSectionInfoFunction GetSectionInfo; + GetStubInfoFunction GetStubInfo; + GetGOTInfoFunction GetGOTInfo; + support::endianness Endianness; MCDisassembler *Disassembler; MCInstPrinter *InstPrinter; llvm::raw_ostream &ErrStream; - - StubMap Stubs; }; } diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp index 226ee715e18b..60041a45e2b8 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.cpp @@ -1,9 +1,8 @@ //===-- RuntimeDyldELF.cpp - Run-time dynamic linker for MC-JIT -*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -1857,9 +1856,6 @@ Error RuntimeDyldELF::finalizeLoad(const ObjectFile &Obj, Sections[GOTSectionID] = SectionEntry(".got", Addr, TotalSize, TotalSize, 0); - if (Checker) - Checker->registerSection(Obj.getFileName(), GOTSectionID); - // For now, initialize all GOT entries to zero. We'll fill them in as // needed when GOT-based relocations are applied. memset(Addr, 0, TotalSize); diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h index f37bd0bbaea6..ef0784e2273b 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldELF.h @@ -1,9 +1,8 @@ //===-- RuntimeDyldELF.h - Run-time dynamic linker for MC-JIT ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -61,7 +60,7 @@ class RuntimeDyldELF : public RuntimeDyldImpl { void resolveBPFRelocation(const SectionEntry &Section, uint64_t Offset, uint64_t Value, uint32_t Type, int64_t Addend); - unsigned getMaxStubSize() override { + unsigned getMaxStubSize() const override { if (Arch == Triple::aarch64 || Arch == Triple::aarch64_be) return 20; // movz; movk; movk; movk; br if (Arch == Triple::arm || Arch == Triple::thumb) diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h index 4c650e09ac1f..68b3468fbc9d 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldImpl.h @@ -1,9 +1,8 @@ //===-- RuntimeDyldImpl.h - Run-time dynamic linker for MC-JIT --*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -241,7 +240,6 @@ typedef StringMap RTDyldSymbolTable; class RuntimeDyldImpl { friend class RuntimeDyld::LoadedObjectInfo; - friend class RuntimeDyldCheckerImpl; protected: static const unsigned AbsoluteSymbolSection = ~0U; @@ -251,9 +249,6 @@ protected: // The symbol resolver to use for external symbols. JITSymbolResolver &Resolver; - // Attached RuntimeDyldChecker instance. Null if no instance attached. - RuntimeDyldCheckerImpl *Checker; - // A list of all sections emitted by the dynamic linker. These sections are // referenced in the code by means of their index in this list - SectionID. typedef SmallVector SectionList; @@ -313,20 +308,16 @@ protected: // the end of the list while the list is being processed. sys::Mutex lock; - virtual unsigned getMaxStubSize() = 0; + using NotifyStubEmittedFunction = + RuntimeDyld::NotifyStubEmittedFunction; + NotifyStubEmittedFunction NotifyStubEmitted; + + virtual unsigned getMaxStubSize() const = 0; virtual unsigned getStubAlignment() = 0; bool HasError; std::string ErrorStr; - uint64_t getSectionLoadAddress(unsigned SectionID) const { - return Sections[SectionID].getLoadAddress(); - } - - uint8_t *getSectionAddress(unsigned SectionID) const { - return Sections[SectionID].getAddress(); - } - void writeInt16BE(uint8_t *Addr, uint16_t Value) { if (IsTargetLittleEndian) sys::swapByteOrder(Value); @@ -472,7 +463,7 @@ protected: public: RuntimeDyldImpl(RuntimeDyld::MemoryManager &MemMgr, JITSymbolResolver &Resolver) - : MemMgr(MemMgr), Resolver(Resolver), Checker(nullptr), + : MemMgr(MemMgr), Resolver(Resolver), ProcessAllSections(false), HasError(false) { } @@ -482,13 +473,22 @@ public: this->ProcessAllSections = ProcessAllSections; } - void setRuntimeDyldChecker(RuntimeDyldCheckerImpl *Checker) { - this->Checker = Checker; - } - virtual std::unique_ptr loadObject(const object::ObjectFile &Obj) = 0; + uint64_t getSectionLoadAddress(unsigned SectionID) const { + return Sections[SectionID].getLoadAddress(); + } + + uint8_t *getSectionAddress(unsigned SectionID) const { + return Sections[SectionID].getAddress(); + } + + StringRef getSectionContent(unsigned SectionID) const { + return StringRef(reinterpret_cast(Sections[SectionID].getAddress()), + Sections[SectionID].getStubOffset() + getMaxStubSize()); + } + uint8_t* getSymbolLocalAddress(StringRef Name) const { // FIXME: Just look up as a function for now. Overly simple of course. // Work in progress. @@ -502,6 +502,13 @@ public: return getSectionAddress(SymInfo.getSectionID()) + SymInfo.getOffset(); } + unsigned getSymbolSectionID(StringRef Name) const { + auto GSTItr = GlobalSymbolTable.find(Name); + if (GSTItr == GlobalSymbolTable.end()) + return ~0U; + return GSTItr->second.getSectionID(); + } + JITEvaluatedSymbol getSymbol(StringRef Name) const { // FIXME: Just look up as a function for now. Overly simple of course. // Work in progress. @@ -560,6 +567,10 @@ public: virtual bool isCompatibleFile(const ObjectFile &Obj) const = 0; + void setNotifyStubEmitted(NotifyStubEmittedFunction NotifyStubEmitted) { + this->NotifyStubEmitted = std::move(NotifyStubEmitted); + } + virtual void registerEHFrames(); void deregisterEHFrames(); diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp index d47fcd45be88..202c3ca1c507 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.cpp @@ -1,9 +1,8 @@ //===-- RuntimeDyldMachO.cpp - Run-time dynamic linker for MC-JIT -*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h index d71ca4e54953..650e7b79fbb8 100644 --- a/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h +++ b/lib/ExecutionEngine/RuntimeDyld/RuntimeDyldMachO.h @@ -1,9 +1,8 @@ //===-- RuntimeDyldMachO.h - Run-time dynamic linker for MC-JIT ---*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFI386.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFI386.h index dd65051edad7..40910bea0c36 100644 --- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFI386.h +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFI386.h @@ -1,9 +1,8 @@ //===--- RuntimeDyldCOFFI386.h --- COFF/X86_64 specific code ---*- C++ --*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -28,16 +27,16 @@ public: JITSymbolResolver &Resolver) : RuntimeDyldCOFF(MM, Resolver) {} - unsigned getMaxStubSize() override { + unsigned getMaxStubSize() const override { return 8; // 2-byte jmp instruction + 32-bit relative address + 2 byte pad } unsigned getStubAlignment() override { return 1; } - Expected + Expected processRelocationRef(unsigned SectionID, - relocation_iterator RelI, - const ObjectFile &Obj, + object::relocation_iterator RelI, + const object::ObjectFile &Obj, ObjSectionToIDMap &ObjSectionToID, StubMap &Stubs) override { diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h index 8723dd0fd0ea..bb2e9626e0b0 100644 --- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFThumb.h @@ -1,9 +1,8 @@ //===--- RuntimeDyldCOFFThumb.h --- COFF/Thumb specific code ---*- C++ --*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -22,9 +21,10 @@ namespace llvm { -static bool isThumbFunc(symbol_iterator Symbol, const ObjectFile &Obj, - section_iterator Section) { - Expected SymTypeOrErr = Symbol->getType(); +static bool isThumbFunc(object::symbol_iterator Symbol, + const object::ObjectFile &Obj, + object::section_iterator Section) { + Expected SymTypeOrErr = Symbol->getType(); if (!SymTypeOrErr) { std::string Buf; raw_string_ostream OS(Buf); @@ -33,12 +33,14 @@ static bool isThumbFunc(symbol_iterator Symbol, const ObjectFile &Obj, report_fatal_error(Buf); } - if (*SymTypeOrErr != SymbolRef::ST_Function) + if (*SymTypeOrErr != object::SymbolRef::ST_Function) return false; // We check the IMAGE_SCN_MEM_16BIT flag in the section of the symbol to tell // if it's thumb or not - return cast(Obj).getCOFFSection(*Section)->Characteristics & + return cast(Obj) + .getCOFFSection(*Section) + ->Characteristics & COFF::IMAGE_SCN_MEM_16BIT; } @@ -48,16 +50,16 @@ public: JITSymbolResolver &Resolver) : RuntimeDyldCOFF(MM, Resolver) {} - unsigned getMaxStubSize() override { + unsigned getMaxStubSize() const override { return 16; // 8-byte load instructions, 4-byte jump, 4-byte padding } unsigned getStubAlignment() override { return 1; } - Expected + Expected processRelocationRef(unsigned SectionID, - relocation_iterator RelI, - const ObjectFile &Obj, + object::relocation_iterator RelI, + const object::ObjectFile &Obj, ObjSectionToIDMap &ObjSectionToID, StubMap &Stubs) override { auto Symbol = RelI->getSymbol(); diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h index aee5f6dc3746..d2d74534cf90 100644 --- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldCOFFX86_64.h @@ -1,9 +1,8 @@ //===-- RuntimeDyldCOFFX86_64.h --- COFF/X86_64 specific code ---*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -62,7 +61,7 @@ public: unsigned getStubAlignment() override { return 1; } // 2-byte jmp instruction + 32-bit relative address + 64-bit absolute jump - unsigned getMaxStubSize() override { return 14; } + unsigned getMaxStubSize() const override { return 14; } // The target location for the relocation is described by RE.SectionID and // RE.Offset. RE.SectionID can be used to find the SectionEntry. Each @@ -187,21 +186,21 @@ public: return std::make_tuple(Offset, RelType, Addend); } - Expected + Expected processRelocationRef(unsigned SectionID, - relocation_iterator RelI, - const ObjectFile &Obj, + object::relocation_iterator RelI, + const object::ObjectFile &Obj, ObjSectionToIDMap &ObjSectionToID, StubMap &Stubs) override { // If possible, find the symbol referred to in the relocation, // and the section that contains it. - symbol_iterator Symbol = RelI->getSymbol(); + object::symbol_iterator Symbol = RelI->getSymbol(); if (Symbol == Obj.symbol_end()) report_fatal_error("Unknown symbol in relocation"); auto SectionOrError = Symbol->getSection(); if (!SectionOrError) return SectionOrError.takeError(); - section_iterator SecI = *SectionOrError; + object::section_iterator SecI = *SectionOrError; // If there is no section, this must be an external reference. const bool IsExtern = SecI == Obj.section_end(); @@ -280,11 +279,11 @@ public: UnregisteredEHFrameSections.clear(); } - Error finalizeLoad(const ObjectFile &Obj, + Error finalizeLoad(const object::ObjectFile &Obj, ObjSectionToIDMap &SectionMap) override { // Look for and record the EH frame section IDs. for (const auto &SectionPair : SectionMap) { - const SectionRef &Section = SectionPair.first; + const object::SectionRef &Section = SectionPair.first; StringRef Name; if (auto EC = Section.getName(Name)) return errorCodeToError(EC); diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.cpp b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.cpp index 3a166b40af2d..17cbe612fb43 100644 --- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.cpp +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.cpp @@ -1,9 +1,8 @@ //===-- RuntimeDyldELFMips.cpp ---- ELF/Mips specific code. -----*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.h index f53b9e6bd75a..14fb36f070f8 100644 --- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.h +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldELFMips.h @@ -1,9 +1,8 @@ //===-- RuntimeDyldELFMips.h ---- ELF/Mips specific code. -------*- C++ -*-===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h index 2a619c549cfa..f2ee1b06d494 100644 --- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOAArch64.h @@ -1,9 +1,8 @@ //===-- RuntimeDyldMachOAArch64.h -- MachO/AArch64 specific code. -*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -27,7 +26,7 @@ public: JITSymbolResolver &Resolver) : RuntimeDyldMachOCRTPBase(MM, Resolver) {} - unsigned getMaxStubSize() override { return 8; } + unsigned getMaxStubSize() const override { return 8; } unsigned getStubAlignment() override { return 8; } diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h index ab7cd2bdae15..3bec8b979f7d 100644 --- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOARM.h @@ -1,9 +1,8 @@ //===----- RuntimeDyldMachOARM.h ---- MachO/ARM specific code. ----*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -30,7 +29,7 @@ public: JITSymbolResolver &Resolver) : RuntimeDyldMachOCRTPBase(MM, Resolver) {} - unsigned getMaxStubSize() override { return 8; } + unsigned getMaxStubSize() const override { return 8; } unsigned getStubAlignment() override { return 4; } @@ -225,7 +224,7 @@ public: HighInsn = (HighInsn & 0xf800) | ((Value >> 12) & 0x7ff); uint16_t LowInsn = readBytesUnaligned(LocalAddress + 2, 2); - assert((LowInsn & 0xf800) != 0xf8000 && + assert((LowInsn & 0xf800) == 0xf800 && "Unrecognized thumb branch encoding (BR22 low bits)"); LowInsn = (LowInsn & 0xf800) | ((Value >> 1) & 0x7ff); diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h index d384d70b8b0f..f0de27ba14bb 100644 --- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOI386.h @@ -1,9 +1,8 @@ //===---- RuntimeDyldMachOI386.h ---- MachO/I386 specific code. ---*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -27,7 +26,7 @@ public: JITSymbolResolver &Resolver) : RuntimeDyldMachOCRTPBase(MM, Resolver) {} - unsigned getMaxStubSize() override { return 0; } + unsigned getMaxStubSize() const override { return 0; } unsigned getStubAlignment() override { return 1; } diff --git a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h index 9732ea6a0cd2..28febbdb948c 100644 --- a/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h +++ b/lib/ExecutionEngine/RuntimeDyld/Targets/RuntimeDyldMachOX86_64.h @@ -1,9 +1,8 @@ //===-- RuntimeDyldMachOX86_64.h ---- MachO/X86_64 specific code. -*- C++ -*-=// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -27,9 +26,9 @@ public: JITSymbolResolver &Resolver) : RuntimeDyldMachOCRTPBase(MM, Resolver) {} - unsigned getMaxStubSize() override { return 8; } + unsigned getMaxStubSize() const override { return 8; } - unsigned getStubAlignment() override { return 1; } + unsigned getStubAlignment() override { return 8; } Expected processRelocationRef(unsigned SectionID, relocation_iterator RelI, diff --git a/lib/ExecutionEngine/SectionMemoryManager.cpp b/lib/ExecutionEngine/SectionMemoryManager.cpp index 05ab4a074e37..925049b2a1b4 100644 --- a/lib/ExecutionEngine/SectionMemoryManager.cpp +++ b/lib/ExecutionEngine/SectionMemoryManager.cpp @@ -1,9 +1,8 @@ //===- SectionMemoryManager.cpp - Memory manager for MCJIT/RtDyld *- C++ -*-==// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -65,9 +64,9 @@ uint8_t *SectionMemoryManager::allocateSection( // Look in the list of free memory regions and use a block there if one // is available. for (FreeMemBlock &FreeMB : MemGroup.FreeMem) { - if (FreeMB.Free.size() >= RequiredSize) { + if (FreeMB.Free.allocatedSize() >= RequiredSize) { Addr = (uintptr_t)FreeMB.Free.base(); - uintptr_t EndOfBlock = Addr + FreeMB.Free.size(); + uintptr_t EndOfBlock = Addr + FreeMB.Free.allocatedSize(); // Align the address. Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); @@ -116,7 +115,7 @@ uint8_t *SectionMemoryManager::allocateSection( // Remember that we allocated this memory MemGroup.AllocatedMem.push_back(MB); Addr = (uintptr_t)MB.base(); - uintptr_t EndOfBlock = Addr + MB.size(); + uintptr_t EndOfBlock = Addr + MB.allocatedSize(); // Align the address. Addr = (Addr + Alignment - 1) & ~(uintptr_t)(Alignment - 1); @@ -173,12 +172,12 @@ bool SectionMemoryManager::finalizeMemory(std::string *ErrMsg) { } static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) { - static const size_t PageSize = sys::Process::getPageSize(); + static const size_t PageSize = sys::Process::getPageSizeEstimate(); size_t StartOverlap = (PageSize - ((uintptr_t)M.base() % PageSize)) % PageSize; - size_t TrimmedSize = M.size(); + size_t TrimmedSize = M.allocatedSize(); TrimmedSize -= StartOverlap; TrimmedSize -= TrimmedSize % PageSize; @@ -186,8 +185,9 @@ static sys::MemoryBlock trimBlockToPageSize(sys::MemoryBlock M) { TrimmedSize); assert(((uintptr_t)Trimmed.base() % PageSize) == 0); - assert((Trimmed.size() % PageSize) == 0); - assert(M.base() <= Trimmed.base() && Trimmed.size() <= M.size()); + assert((Trimmed.allocatedSize() % PageSize) == 0); + assert(M.base() <= Trimmed.base() && + Trimmed.allocatedSize() <= M.allocatedSize()); return Trimmed; } @@ -210,17 +210,19 @@ SectionMemoryManager::applyMemoryGroupPermissions(MemoryGroup &MemGroup, } // Remove all blocks which are now empty - MemGroup.FreeMem.erase( - remove_if(MemGroup.FreeMem, - [](FreeMemBlock &FreeMB) { return FreeMB.Free.size() == 0; }), - MemGroup.FreeMem.end()); + MemGroup.FreeMem.erase(remove_if(MemGroup.FreeMem, + [](FreeMemBlock &FreeMB) { + return FreeMB.Free.allocatedSize() == 0; + }), + MemGroup.FreeMem.end()); return std::error_code(); } void SectionMemoryManager::invalidateInstructionCache() { for (sys::MemoryBlock &Block : CodeMem.PendingMem) - sys::Memory::InvalidateInstructionCache(Block.base(), Block.size()); + sys::Memory::InvalidateInstructionCache(Block.base(), + Block.allocatedSize()); } SectionMemoryManager::~SectionMemoryManager() { diff --git a/lib/ExecutionEngine/TargetSelect.cpp b/lib/ExecutionEngine/TargetSelect.cpp index 9626b8d3ffa3..0d9c6cfa0908 100644 --- a/lib/ExecutionEngine/TargetSelect.cpp +++ b/lib/ExecutionEngine/TargetSelect.cpp @@ -1,9 +1,8 @@ //===-- TargetSelect.cpp - Target Chooser Code ----------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // diff --git a/lib/FuzzMutate/FuzzerCLI.cpp b/lib/FuzzMutate/FuzzerCLI.cpp index a70dad37dfcf..63d31c035390 100644 --- a/lib/FuzzMutate/FuzzerCLI.cpp +++ b/lib/FuzzMutate/FuzzerCLI.cpp @@ -1,9 +1,8 @@ //===-- FuzzerCLI.cpp -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/FuzzMutate/IRMutator.cpp b/lib/FuzzMutate/IRMutator.cpp index 40e402cdadef..2fc65981f1db 100644 --- a/lib/FuzzMutate/IRMutator.cpp +++ b/lib/FuzzMutate/IRMutator.cpp @@ -1,9 +1,8 @@ //===-- IRMutator.cpp -----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/FuzzMutate/OpDescriptor.cpp b/lib/FuzzMutate/OpDescriptor.cpp index 1c5d8f606aea..67d44be8b699 100644 --- a/lib/FuzzMutate/OpDescriptor.cpp +++ b/lib/FuzzMutate/OpDescriptor.cpp @@ -1,9 +1,8 @@ //===-- OpDescriptor.cpp --------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/FuzzMutate/Operations.cpp b/lib/FuzzMutate/Operations.cpp index b842f6d64fb1..cf55d09caf7e 100644 --- a/lib/FuzzMutate/Operations.cpp +++ b/lib/FuzzMutate/Operations.cpp @@ -1,9 +1,8 @@ //===-- Operations.cpp ----------------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/FuzzMutate/RandomIRBuilder.cpp b/lib/FuzzMutate/RandomIRBuilder.cpp index 337184535558..1295714839e8 100644 --- a/lib/FuzzMutate/RandomIRBuilder.cpp +++ b/lib/FuzzMutate/RandomIRBuilder.cpp @@ -1,9 +1,8 @@ //===-- RandomIRBuilder.cpp -----------------------------------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// @@ -54,7 +53,8 @@ Value *RandomIRBuilder::newSource(BasicBlock &BB, ArrayRef Insts, IP = ++I->getIterator(); assert(IP != BB.end() && "guaranteed by the findPointer"); } - auto *NewLoad = new LoadInst(Ptr, "L", &*IP); + auto *NewLoad = new LoadInst( + cast(Ptr->getType())->getElementType(), Ptr, "L", &*IP); // Only sample this load if it really matches the descriptor if (Pred.matches(Srcs, NewLoad)) diff --git a/lib/IR/AbstractCallSite.cpp b/lib/IR/AbstractCallSite.cpp new file mode 100644 index 000000000000..b7a81030f41c --- /dev/null +++ b/lib/IR/AbstractCallSite.cpp @@ -0,0 +1,134 @@ +//===-- AbstractCallSite.cpp - Implementation of abstract call sites ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file implements abstract call sites which unify the interface for +// direct, indirect, and callback call sites. +// +// For more information see: +// https://llvm.org/devmtg/2018-10/talk-abstracts.html#talk20 +// +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/StringSwitch.h" +#include "llvm/IR/CallSite.h" +#include "llvm/Support/Debug.h" + +using namespace llvm; + +#define DEBUG_TYPE "abstract-call-sites" + +STATISTIC(NumCallbackCallSites, "Number of callback call sites created"); +STATISTIC(NumDirectAbstractCallSites, + "Number of direct abstract call sites created"); +STATISTIC(NumInvalidAbstractCallSitesUnknownUse, + "Number of invalid abstract call sites created (unknown use)"); +STATISTIC(NumInvalidAbstractCallSitesUnknownCallee, + "Number of invalid abstract call sites created (unknown callee)"); +STATISTIC(NumInvalidAbstractCallSitesNoCallback, + "Number of invalid abstract call sites created (no callback)"); + +/// Create an abstract call site from a use. +AbstractCallSite::AbstractCallSite(const Use *U) : CS(U->getUser()) { + + // First handle unknown users. + if (!CS) { + + // If the use is actually in a constant cast expression which itself + // has only one use, we look through the constant cast expression. + // This happens by updating the use @p U to the use of the constant + // cast expression and afterwards re-initializing CS accordingly. + if (ConstantExpr *CE = dyn_cast(U->getUser())) + if (CE->getNumUses() == 1 && CE->isCast()) { + U = &*CE->use_begin(); + CS = CallSite(U->getUser()); + } + + if (!CS) { + NumInvalidAbstractCallSitesUnknownUse++; + return; + } + } + + // Then handle direct or indirect calls. Thus, if U is the callee of the + // call site CS it is not a callback and we are done. + if (CS.isCallee(U)) { + NumDirectAbstractCallSites++; + return; + } + + // If we cannot identify the broker function we cannot create a callback and + // invalidate the abstract call site. + Function *Callee = CS.getCalledFunction(); + if (!Callee) { + NumInvalidAbstractCallSitesUnknownCallee++; + CS = CallSite(); + return; + } + + MDNode *CallbackMD = Callee->getMetadata(LLVMContext::MD_callback); + if (!CallbackMD) { + NumInvalidAbstractCallSitesNoCallback++; + CS = CallSite(); + return; + } + + unsigned UseIdx = CS.getArgumentNo(U); + MDNode *CallbackEncMD = nullptr; + for (const MDOperand &Op : CallbackMD->operands()) { + MDNode *OpMD = cast(Op.get()); + auto *CBCalleeIdxAsCM = cast(OpMD->getOperand(0)); + uint64_t CBCalleeIdx = + cast(CBCalleeIdxAsCM->getValue())->getZExtValue(); + if (CBCalleeIdx != UseIdx) + continue; + CallbackEncMD = OpMD; + break; + } + + if (!CallbackEncMD) { + NumInvalidAbstractCallSitesNoCallback++; + CS = CallSite(); + return; + } + + NumCallbackCallSites++; + + assert(CallbackEncMD->getNumOperands() >= 2 && "Incomplete !callback metadata"); + + unsigned NumCallOperands = CS.getNumArgOperands(); + // Skip the var-arg flag at the end when reading the metadata. + for (unsigned u = 0, e = CallbackEncMD->getNumOperands() - 1; u < e; u++) { + Metadata *OpAsM = CallbackEncMD->getOperand(u).get(); + auto *OpAsCM = cast(OpAsM); + assert(OpAsCM->getType()->isIntegerTy(64) && + "Malformed !callback metadata"); + + int64_t Idx = cast(OpAsCM->getValue())->getSExtValue(); + assert(-1 <= Idx && Idx <= NumCallOperands && + "Out-of-bounds !callback metadata index"); + + CI.ParameterEncoding.push_back(Idx); + } + + if (!Callee->isVarArg()) + return; + + Metadata *VarArgFlagAsM = + CallbackEncMD->getOperand(CallbackEncMD->getNumOperands() - 1).get(); + auto *VarArgFlagAsCM = cast(VarArgFlagAsM); + assert(VarArgFlagAsCM->getType()->isIntegerTy(1) && + "Malformed !callback metadata var-arg flag"); + + if (VarArgFlagAsCM->getValue()->isNullValue()) + return; + + // Add all variadic arguments at the end. + for (unsigned u = Callee->arg_size(); u < NumCallOperands; u++) + CI.ParameterEncoding.push_back(u); +} diff --git a/lib/IR/AsmWriter.cpp b/lib/IR/AsmWriter.cpp index a5dc623e1a30..eb5760daecb3 100644 --- a/lib/IR/AsmWriter.cpp +++ b/lib/IR/AsmWriter.cpp @@ -1,9 +1,8 @@ //===- AsmWriter.cpp - Printing LLVM as an assembly file ------------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// // @@ -621,7 +620,10 @@ void TypePrinting::print(Type *Ty, raw_ostream &OS) { } case Type::VectorTyID: { VectorType *PTy = cast(Ty); - OS << "<" << PTy->getNumElements() << " x "; + OS << "<"; + if (PTy->isScalable()) + OS << "vscale x "; + OS << PTy->getNumElements() << " x "; print(PTy->getElementType(), OS); OS << '>'; return; @@ -1038,6 +1040,9 @@ void SlotTracker::processIndex() { TidIter != TheIndex->typeIds().end(); TidIter++) CreateTypeIdSlot(TidIter->second.first); + for (auto &TId : TheIndex->typeIdCompatibleVtableMap()) + CreateGUIDSlot(GlobalValue::getGUID(TId.first)); + ST_DEBUG("end processIndex!\n"); } @@ -2002,6 +2007,19 @@ static void writeDINamespace(raw_ostream &Out, const DINamespace *N, Out << ")"; } +static void writeDICommonBlock(raw_ostream &Out, const DICommonBlock *N, + TypePrinting *TypePrinter, SlotTracker *Machine, + const Module *Context) { + Out << "!DICommonBlock("; + MDFieldPrinter Printer(Out, TypePrinter, Machine, Context); + Printer.printMetadata("scope", N->getRawScope(), false); + Printer.printMetadata("declaration", N->getRawDecl(), false); + Printer.printString("name", N->getName()); + Printer.printMetadata("file", N->getRawFile()); + Printer.printInt("line", N->getLineNo()); + Out << ")"; +} + static void writeDIMacro(raw_ostream &Out, const DIMacro *N, TypePrinting *TypePrinter, SlotTracker *Machine, const Module *Context) { @@ -2124,8 +2142,13 @@ static void writeDIExpression(raw_ostream &Out, const DIExpression *N, assert(!OpStr.empty() && "Expected valid opcode"); Out << FS << OpStr; - for (unsigned A = 0, AE = I->getNumArgs(); A != AE; ++A) - Out << FS << I->getArg(A); + if (I->getOp() == dwarf::DW_OP_LLVM_convert) { + Out << FS << I->getArg(0); + Out << FS << dwarf::AttributeEncodingString(I->getArg(1)); + } else { + for (unsigned A = 0, AE = I->getNumArgs(); A != AE; ++A) + Out << FS << I->getArg(A); + } } } else { for (const auto &I : N->getElements()) @@ -2393,6 +2416,7 @@ public: void printGlobalVarSummary(const GlobalVarSummary *GS); void printFunctionSummary(const FunctionSummary *FS); void printTypeIdSummary(const TypeIdSummary &TIS); + void printTypeIdCompatibleVtableSummary(const TypeIdCompatibleVtableInfo &TI); void printTypeTestResolution(const TypeTestResolution &TTRes); void printArgs(const std::vector &Args); void printWPDRes(const WholeProgramDevirtResolution &WPDRes); @@ -2695,6 +2719,15 @@ void AssemblyWriter::printModuleSummaryIndex() { printTypeIdSummary(TidIter->second.second); Out << ") ; guid = " << TidIter->first << "\n"; } + + // Print the TypeIdCompatibleVtableMap entries. + for (auto &TId : TheIndex->typeIdCompatibleVtableMap()) { + auto GUID = GlobalValue::getGUID(TId.first); + Out << "^" << Machine.getGUIDSlot(GUID) + << " = typeidCompatibleVTable: (name: \"" << TId.first << "\""; + printTypeIdCompatibleVtableSummary(TId.second); + Out << ") ; guid = " << GUID << "\n"; + } } static const char * @@ -2777,6 +2810,19 @@ void AssemblyWriter::printTypeIdSummary(const TypeIdSummary &TIS) { Out << ")"; } +void AssemblyWriter::printTypeIdCompatibleVtableSummary( + const TypeIdCompatibleVtableInfo &TI) { + Out << ", summary: ("; + FieldSeparator FS; + for (auto &P : TI) { + Out << FS; + Out << "(offset: " << P.AddressPointOffset << ", "; + Out << "^" << Machine.getGUIDSlot(P.VTableVI.getGUID()); + Out << ")"; + } + Out << ")"; +} + void AssemblyWriter::printArgs(const std::vector &Args) { Out << "args: ("; FieldSeparator FS; @@ -2845,7 +2891,21 @@ void AssemblyWriter::printAliasSummary(const AliasSummary *AS) { } void AssemblyWriter::printGlobalVarSummary(const GlobalVarSummary *GS) { - Out << ", varFlags: (readonly: " << GS->VarFlags.ReadOnly << ")"; + Out << ", varFlags: (readonly: " << GS->VarFlags.MaybeReadOnly << ", " + << "writeonly: " << GS->VarFlags.MaybeWriteOnly << ")"; + + auto VTableFuncs = GS->vTableFuncs(); + if (!VTableFuncs.empty()) { + Out << ", vTableFuncs: ("; + FieldSeparator FS; + for (auto &P : VTableFuncs) { + Out << FS; + Out << "(virtFunc: ^" << Machine.getGUIDSlot(P.FuncVI.getGUID()) + << ", offset: " << P.VTableOffset; + Out << ")"; + } + Out << ")"; + } } static std::string getLinkageName(GlobalValue::LinkageTypes LT) { @@ -3024,6 +3084,7 @@ void AssemblyWriter::printSummary(const GlobalValueSummary &Summary) { Out << ", notEligibleToImport: " << GVFlags.NotEligibleToImport; Out << ", live: " << GVFlags.Live; Out << ", dsoLocal: " << GVFlags.DSOLocal; + Out << ", canAutoHide: " << GVFlags.CanAutoHide; Out << ")"; if (Summary.getSummaryKind() == GlobalValueSummary::AliasKind) @@ -3041,6 +3102,8 @@ void AssemblyWriter::printSummary(const GlobalValueSummary &Summary) { Out << FS; if (Ref.isReadOnly()) Out << "readonly "; + else if (Ref.isWriteOnly()) + Out << "writeonly "; Out << "^" << Machine.getGUIDSlot(Ref.getGUID()); } Out << ")"; @@ -3229,6 +3292,12 @@ void AssemblyWriter::printGlobal(const GlobalVariable *GV) { printEscapedString(GV->getSection(), Out); Out << '"'; } + if (GV->hasPartition()) { + Out << ", partition \""; + printEscapedString(GV->getPartition(), Out); + Out << '"'; + } + maybePrintComdat(Out, *GV); if (GV->getAlignment()) Out << ", align " << GV->getAlignment(); @@ -3280,6 +3349,12 @@ void AssemblyWriter::printIndirectSymbol(const GlobalIndirectSymbol *GIS) { writeOperand(IS, !isa(IS)); } + if (GIS->hasPartition()) { + Out << ", partition \""; + printEscapedString(GIS->getPartition(), Out); + Out << '"'; + } + printInfoComment(*GIS); Out << '\n'; } @@ -3420,6 +3495,11 @@ void AssemblyWriter::printFunction(const Function *F) { printEscapedString(F->getSection(), Out); Out << '"'; } + if (F->hasPartition()) { + Out << " partition \""; + printEscapedString(F->getPartition(), Out); + Out << '"'; + } maybePrintComdat(Out, *F); if (F->getAlignment()) Out << " align " << F->getAlignment(); @@ -3478,23 +3558,24 @@ void AssemblyWriter::printArgument(const Argument *Arg, AttributeSet Attrs) { /// printBasicBlock - This member is called for each basic block in a method. void AssemblyWriter::printBasicBlock(const BasicBlock *BB) { + bool IsEntryBlock = BB == &BB->getParent()->getEntryBlock(); if (BB->hasName()) { // Print out the label if it exists... Out << "\n"; PrintLLVMName(Out, BB->getName(), LabelPrefix); Out << ':'; - } else if (!BB->use_empty()) { // Don't print block # of no uses... - Out << "\n;