From b915e9e0fc85ba6f398b3fab0db6a81a8913af94 Mon Sep 17 00:00:00 2001 From: Dimitry Andric Date: Mon, 2 Jan 2017 19:17:04 +0000 Subject: Vendor import of llvm trunk r290819: https://llvm.org/svn/llvm-project/llvm/trunk@290819 --- lib/Target/AArch64/AArch64.h | 16 +- lib/Target/AArch64/AArch64.td | 80 +- lib/Target/AArch64/AArch64A53Fix835769.cpp | 11 +- lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp | 8 +- lib/Target/AArch64/AArch64AddressTypePromotion.cpp | 14 +- lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp | 8 +- lib/Target/AArch64/AArch64AsmPrinter.cpp | 153 +- lib/Target/AArch64/AArch64BranchRelaxation.cpp | 520 -- lib/Target/AArch64/AArch64CallLowering.cpp | 329 +- lib/Target/AArch64/AArch64CallLowering.h | 28 +- lib/Target/AArch64/AArch64CallingConvention.td | 7 + .../AArch64/AArch64CleanupLocalDynamicTLSPass.cpp | 13 +- lib/Target/AArch64/AArch64CollectLOH.cpp | 36 +- lib/Target/AArch64/AArch64ConditionOptimizer.cpp | 10 +- lib/Target/AArch64/AArch64ConditionalCompares.cpp | 14 +- .../AArch64/AArch64DeadRegisterDefinitionsPass.cpp | 136 +- lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp | 25 +- lib/Target/AArch64/AArch64FastISel.cpp | 123 +- lib/Target/AArch64/AArch64FrameLowering.cpp | 77 +- lib/Target/AArch64/AArch64GenRegisterBankInfo.def | 173 + lib/Target/AArch64/AArch64ISelDAGToDAG.cpp | 77 +- lib/Target/AArch64/AArch64ISelLowering.cpp | 721 +- lib/Target/AArch64/AArch64ISelLowering.h | 45 +- lib/Target/AArch64/AArch64InstrAtomics.td | 14 +- lib/Target/AArch64/AArch64InstrFormats.td | 23 +- lib/Target/AArch64/AArch64InstrInfo.cpp | 299 +- lib/Target/AArch64/AArch64InstrInfo.h | 61 +- lib/Target/AArch64/AArch64InstrInfo.td | 65 +- lib/Target/AArch64/AArch64InstructionSelector.cpp | 1161 +++ lib/Target/AArch64/AArch64InstructionSelector.h | 47 + lib/Target/AArch64/AArch64LegalizerInfo.cpp | 204 + lib/Target/AArch64/AArch64LegalizerInfo.h | 30 + lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp | 373 +- lib/Target/AArch64/AArch64MCInstLower.cpp | 7 +- lib/Target/AArch64/AArch64MachineFunctionInfo.h | 2 +- lib/Target/AArch64/AArch64PromoteConstant.cpp | 10 +- .../AArch64/AArch64RedundantCopyElimination.cpp | 19 +- lib/Target/AArch64/AArch64RegisterBankInfo.cpp | 445 +- lib/Target/AArch64/AArch64RegisterBankInfo.h | 39 +- lib/Target/AArch64/AArch64RegisterInfo.cpp | 45 +- lib/Target/AArch64/AArch64RegisterInfo.h | 10 +- lib/Target/AArch64/AArch64RegisterInfo.td | 2 +- lib/Target/AArch64/AArch64SchedA57.td | 22 +- lib/Target/AArch64/AArch64SchedA57WriteRes.td | 14 +- lib/Target/AArch64/AArch64SchedFalkor.td | 26 + lib/Target/AArch64/AArch64SchedM1.td | 98 +- lib/Target/AArch64/AArch64SchedVulcan.td | 13 +- lib/Target/AArch64/AArch64StorePairSuppress.cpp | 13 +- lib/Target/AArch64/AArch64Subtarget.cpp | 22 +- lib/Target/AArch64/AArch64Subtarget.h | 26 +- lib/Target/AArch64/AArch64TargetMachine.cpp | 224 +- lib/Target/AArch64/AArch64TargetMachine.h | 2 +- lib/Target/AArch64/AArch64TargetObjectFile.cpp | 13 +- lib/Target/AArch64/AArch64TargetObjectFile.h | 4 +- lib/Target/AArch64/AArch64TargetTransformInfo.cpp | 3 +- lib/Target/AArch64/AArch64TargetTransformInfo.h | 7 - lib/Target/AArch64/AArch64VectorByElementOpt.cpp | 371 + lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp | 534 +- lib/Target/AArch64/CMakeLists.txt | 12 +- .../AArch64/Disassembler/AArch64Disassembler.cpp | 24 +- .../AArch64/Disassembler/AArch64Disassembler.h | 1 - .../AArch64/MCTargetDesc/AArch64AsmBackend.cpp | 33 +- .../MCTargetDesc/AArch64ELFObjectWriter.cpp | 202 +- .../AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp | 3 +- .../AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp | 25 +- .../AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp | 16 +- .../AArch64/MCTargetDesc/AArch64MCTargetDesc.h | 16 +- .../MCTargetDesc/AArch64MachObjectWriter.cpp | 2 +- .../AArch64/TargetInfo/AArch64TargetInfo.cpp | 25 +- lib/Target/AMDGPU/AMDGPU.h | 33 +- lib/Target/AMDGPU/AMDGPU.td | 160 +- lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp | 14 +- lib/Target/AMDGPU/AMDGPUAnnotateKernelFeatures.cpp | 8 +- lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp | 100 +- lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp | 415 +- lib/Target/AMDGPU/AMDGPUAsmPrinter.h | 33 +- lib/Target/AMDGPU/AMDGPUCallLowering.cpp | 8 +- lib/Target/AMDGPU/AMDGPUCallLowering.h | 6 +- lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 301 +- lib/Target/AMDGPU/AMDGPUFrameLowering.cpp | 22 +- lib/Target/AMDGPU/AMDGPUFrameLowering.h | 9 +- lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 310 +- lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 824 +- lib/Target/AMDGPU/AMDGPUISelLowering.h | 72 +- lib/Target/AMDGPU/AMDGPUInstrInfo.cpp | 5 - lib/Target/AMDGPU/AMDGPUInstrInfo.h | 18 +- lib/Target/AMDGPU/AMDGPUInstrInfo.td | 70 +- lib/Target/AMDGPU/AMDGPUInstructions.td | 126 +- lib/Target/AMDGPU/AMDGPUIntrinsics.td | 5 +- lib/Target/AMDGPU/AMDGPUMCInstLower.cpp | 137 +- lib/Target/AMDGPU/AMDGPUMCInstLower.h | 15 +- lib/Target/AMDGPU/AMDGPUMachineFunction.cpp | 44 +- lib/Target/AMDGPU/AMDGPUMachineFunction.h | 47 +- .../AMDGPU/AMDGPUOpenCLImageTypeLoweringPass.cpp | 2 +- lib/Target/AMDGPU/AMDGPUPTNote.h | 42 + lib/Target/AMDGPU/AMDGPUPromoteAlloca.cpp | 31 +- lib/Target/AMDGPU/AMDGPURuntimeMetadata.h | 203 +- lib/Target/AMDGPU/AMDGPUSubtarget.cpp | 181 +- lib/Target/AMDGPU/AMDGPUSubtarget.h | 245 +- lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 169 +- lib/Target/AMDGPU/AMDGPUTargetMachine.h | 23 +- lib/Target/AMDGPU/AMDGPUTargetObjectFile.cpp | 13 +- lib/Target/AMDGPU/AMDGPUTargetObjectFile.h | 3 +- lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp | 3 +- lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h | 9 +- lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp | 149 + lib/Target/AMDGPU/AMDILCFGStructurizer.cpp | 49 +- lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp | 1524 +++- lib/Target/AMDGPU/BUFInstructions.td | 1350 ++++ lib/Target/AMDGPU/CIInstructions.td | 336 +- lib/Target/AMDGPU/CMakeLists.txt | 5 + lib/Target/AMDGPU/CaymanInstructions.td | 139 +- lib/Target/AMDGPU/DSInstructions.td | 906 +++ .../AMDGPU/Disassembler/AMDGPUDisassembler.cpp | 238 +- .../AMDGPU/Disassembler/AMDGPUDisassembler.h | 173 +- lib/Target/AMDGPU/EvergreenInstructions.td | 139 +- lib/Target/AMDGPU/FLATInstructions.td | 530 ++ lib/Target/AMDGPU/GCNHazardRecognizer.cpp | 258 +- lib/Target/AMDGPU/GCNHazardRecognizer.h | 9 + lib/Target/AMDGPU/GCNSchedStrategy.cpp | 312 + lib/Target/AMDGPU/GCNSchedStrategy.h | 54 + .../AMDGPU/InstPrinter/AMDGPUInstPrinter.cpp | 485 +- lib/Target/AMDGPU/InstPrinter/AMDGPUInstPrinter.h | 200 +- .../AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp | 137 +- .../AMDGPU/MCTargetDesc/AMDGPUELFObjectWriter.cpp | 26 +- .../AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.h | 13 + .../AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.cpp | 13 +- .../AMDGPU/MCTargetDesc/AMDGPUMCTargetDesc.h | 11 +- lib/Target/AMDGPU/MCTargetDesc/AMDGPURuntimeMD.cpp | 408 + lib/Target/AMDGPU/MCTargetDesc/AMDGPURuntimeMD.h | 26 + .../AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp | 306 +- .../AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.h | 37 +- lib/Target/AMDGPU/MCTargetDesc/CMakeLists.txt | 1 + lib/Target/AMDGPU/MCTargetDesc/LLVMBuild.txt | 2 +- .../AMDGPU/MCTargetDesc/R600MCCodeEmitter.cpp | 18 +- lib/Target/AMDGPU/MCTargetDesc/SIMCCodeEmitter.cpp | 129 +- lib/Target/AMDGPU/MIMGInstructions.td | 763 ++ lib/Target/AMDGPU/Processors.td | 70 +- lib/Target/AMDGPU/R600ClauseMergePass.cpp | 4 +- lib/Target/AMDGPU/R600ControlFlowFinalizer.cpp | 38 +- lib/Target/AMDGPU/R600EmitClauseMarkers.cpp | 4 +- lib/Target/AMDGPU/R600ExpandSpecialInstrs.cpp | 81 +- lib/Target/AMDGPU/R600FrameLowering.cpp | 3 +- lib/Target/AMDGPU/R600FrameLowering.h | 12 +- lib/Target/AMDGPU/R600ISelLowering.cpp | 389 +- lib/Target/AMDGPU/R600InstrFormats.td | 4 +- lib/Target/AMDGPU/R600InstrInfo.cpp | 66 +- lib/Target/AMDGPU/R600InstrInfo.h | 27 +- lib/Target/AMDGPU/R600Instructions.td | 58 +- lib/Target/AMDGPU/R600MachineFunctionInfo.cpp | 4 - lib/Target/AMDGPU/R600MachineFunctionInfo.h | 7 +- lib/Target/AMDGPU/R600MachineScheduler.h | 20 +- lib/Target/AMDGPU/R600OptimizeVectorRegisters.cpp | 75 +- lib/Target/AMDGPU/R600Packetizer.cpp | 6 +- lib/Target/AMDGPU/SIAnnotateControlFlow.cpp | 11 +- lib/Target/AMDGPU/SIDebuggerInsertNops.cpp | 2 +- lib/Target/AMDGPU/SIDefines.h | 196 +- .../AMDGPU/SIFixControlFlowLiveIntervals.cpp | 4 +- lib/Target/AMDGPU/SIFixSGPRCopies.cpp | 160 +- lib/Target/AMDGPU/SIFoldOperands.cpp | 441 +- lib/Target/AMDGPU/SIFrameLowering.cpp | 405 +- lib/Target/AMDGPU/SIFrameLowering.h | 30 +- lib/Target/AMDGPU/SIISelLowering.cpp | 1912 +++-- lib/Target/AMDGPU/SIISelLowering.h | 41 +- lib/Target/AMDGPU/SIInsertSkips.cpp | 329 + lib/Target/AMDGPU/SIInsertWaits.cpp | 127 +- lib/Target/AMDGPU/SIInstrFormats.td | 716 +- lib/Target/AMDGPU/SIInstrInfo.cpp | 1260 +++- lib/Target/AMDGPU/SIInstrInfo.h | 214 +- lib/Target/AMDGPU/SIInstrInfo.td | 3616 ++------- lib/Target/AMDGPU/SIInstructions.td | 3377 ++------- lib/Target/AMDGPU/SIIntrinsics.td | 29 +- lib/Target/AMDGPU/SILoadStoreOptimizer.cpp | 300 +- lib/Target/AMDGPU/SILowerControlFlow.cpp | 930 +-- lib/Target/AMDGPU/SILowerI1Copies.cpp | 32 +- lib/Target/AMDGPU/SIMachineFunctionInfo.cpp | 44 +- lib/Target/AMDGPU/SIMachineFunctionInfo.h | 125 +- lib/Target/AMDGPU/SIMachineScheduler.cpp | 80 +- lib/Target/AMDGPU/SIMachineScheduler.h | 58 +- lib/Target/AMDGPU/SIOptimizeExecMasking.cpp | 304 + lib/Target/AMDGPU/SIRegisterInfo.cpp | 1227 ++- lib/Target/AMDGPU/SIRegisterInfo.h | 152 +- lib/Target/AMDGPU/SIRegisterInfo.td | 178 +- lib/Target/AMDGPU/SISchedule.td | 6 +- lib/Target/AMDGPU/SIShrinkInstructions.cpp | 172 +- lib/Target/AMDGPU/SITypeRewriter.cpp | 4 +- lib/Target/AMDGPU/SIWholeQuadMode.cpp | 424 +- lib/Target/AMDGPU/SMInstructions.td | 535 ++ lib/Target/AMDGPU/SOPInstructions.td | 1229 +++ lib/Target/AMDGPU/TargetInfo/AMDGPUTargetInfo.cpp | 17 +- lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp | 282 +- lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h | 151 +- lib/Target/AMDGPU/Utils/AMDKernelCodeTInfo.h | 131 +- lib/Target/AMDGPU/Utils/AMDKernelCodeTUtils.cpp | 29 +- lib/Target/AMDGPU/VIInstrFormats.td | 277 - lib/Target/AMDGPU/VIInstructions.td | 150 +- lib/Target/AMDGPU/VOP1Instructions.td | 621 ++ lib/Target/AMDGPU/VOP2Instructions.td | 757 ++ lib/Target/AMDGPU/VOP3Instructions.td | 447 ++ lib/Target/AMDGPU/VOPCInstructions.td | 1144 +++ lib/Target/AMDGPU/VOPInstructions.td | 350 + lib/Target/ARM/A15SDOptimizer.cpp | 4 +- lib/Target/ARM/ARM.h | 5 + lib/Target/ARM/ARM.td | 84 +- lib/Target/ARM/ARMAsmPrinter.cpp | 247 +- lib/Target/ARM/ARMAsmPrinter.h | 30 +- lib/Target/ARM/ARMBaseInstrInfo.cpp | 91 +- lib/Target/ARM/ARMBaseInstrInfo.h | 16 +- lib/Target/ARM/ARMBaseRegisterInfo.cpp | 63 +- lib/Target/ARM/ARMBaseRegisterInfo.h | 3 +- lib/Target/ARM/ARMBasicBlockInfo.h | 110 + lib/Target/ARM/ARMCallLowering.cpp | 203 + lib/Target/ARM/ARMCallLowering.h | 42 + lib/Target/ARM/ARMCallingConv.td | 1 + lib/Target/ARM/ARMComputeBlockSize.cpp | 72 + lib/Target/ARM/ARMConstantIslandPass.cpp | 295 +- lib/Target/ARM/ARMConstantPoolValue.cpp | 33 +- lib/Target/ARM/ARMConstantPoolValue.h | 31 +- lib/Target/ARM/ARMExpandPseudoInsts.cpp | 11 +- lib/Target/ARM/ARMFastISel.cpp | 51 +- lib/Target/ARM/ARMFrameLowering.cpp | 333 +- lib/Target/ARM/ARMISelDAGToDAG.cpp | 362 +- lib/Target/ARM/ARMISelLowering.cpp | 990 ++- lib/Target/ARM/ARMISelLowering.h | 21 +- lib/Target/ARM/ARMInstrFormats.td | 17 + lib/Target/ARM/ARMInstrInfo.cpp | 4 +- lib/Target/ARM/ARMInstrInfo.td | 57 +- lib/Target/ARM/ARMInstrNEON.td | 14 +- lib/Target/ARM/ARMInstrThumb.td | 105 +- lib/Target/ARM/ARMInstrThumb2.td | 525 +- lib/Target/ARM/ARMInstrVFP.td | 8 +- lib/Target/ARM/ARMInstructionSelector.cpp | 109 + lib/Target/ARM/ARMInstructionSelector.h | 39 + lib/Target/ARM/ARMLegalizerInfo.cpp | 44 + lib/Target/ARM/ARMLegalizerInfo.h | 29 + lib/Target/ARM/ARMLoadStoreOptimizer.cpp | 14 +- lib/Target/ARM/ARMMCInstLower.cpp | 113 +- lib/Target/ARM/ARMMachineFunctionInfo.cpp | 12 +- lib/Target/ARM/ARMMachineFunctionInfo.h | 25 +- lib/Target/ARM/ARMOptimizeBarriersPass.cpp | 6 +- lib/Target/ARM/ARMRegisterBankInfo.cpp | 127 + lib/Target/ARM/ARMRegisterBankInfo.h | 41 + lib/Target/ARM/ARMSchedule.td | 1 + lib/Target/ARM/ARMScheduleR52.td | 983 +++ lib/Target/ARM/ARMSubtarget.cpp | 71 +- lib/Target/ARM/ARMSubtarget.h | 44 +- lib/Target/ARM/ARMTargetMachine.cpp | 105 +- lib/Target/ARM/ARMTargetObjectFile.cpp | 45 +- lib/Target/ARM/ARMTargetObjectFile.h | 17 +- lib/Target/ARM/ARMTargetTransformInfo.cpp | 21 +- lib/Target/ARM/ARMTargetTransformInfo.h | 17 +- lib/Target/ARM/AsmParser/ARMAsmParser.cpp | 1384 ++-- lib/Target/ARM/CMakeLists.txt | 17 + lib/Target/ARM/Disassembler/ARMDisassembler.cpp | 14 +- lib/Target/ARM/InstPrinter/ARMInstPrinter.cpp | 6 + lib/Target/ARM/LLVMBuild.txt | 2 +- lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp | 58 +- lib/Target/ARM/MCTargetDesc/ARMELFObjectWriter.cpp | 29 +- lib/Target/ARM/MCTargetDesc/ARMELFStreamer.cpp | 4 +- lib/Target/ARM/MCTargetDesc/ARMMCAsmInfo.cpp | 1 + lib/Target/ARM/MCTargetDesc/ARMMCCodeEmitter.cpp | 11 +- lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.cpp | 21 +- lib/Target/ARM/MCTargetDesc/ARMMCTargetDesc.h | 20 +- .../ARM/MCTargetDesc/ARMMachObjectWriter.cpp | 2 +- lib/Target/ARM/MLxExpansionPass.cpp | 13 +- lib/Target/ARM/README-Thumb.txt | 2 +- lib/Target/ARM/TargetInfo/ARMTargetInfo.cpp | 34 +- lib/Target/ARM/Thumb1FrameLowering.cpp | 323 +- lib/Target/ARM/Thumb1InstrInfo.cpp | 4 +- lib/Target/ARM/Thumb2ITBlockPass.cpp | 4 +- lib/Target/ARM/Thumb2InstrInfo.cpp | 4 +- lib/Target/ARM/Thumb2SizeReduction.cpp | 10 +- lib/Target/ARM/ThumbRegisterInfo.cpp | 16 +- lib/Target/AVR/AVR.h | 6 + lib/Target/AVR/AVR.td | 526 +- lib/Target/AVR/AVRAsmPrinter.cpp | 184 + lib/Target/AVR/AVRCallingConv.td | 11 +- lib/Target/AVR/AVRDevices.td | 491 ++ lib/Target/AVR/AVRExpandPseudoInsts.cpp | 1515 ++++ lib/Target/AVR/AVRFrameLowering.cpp | 538 ++ lib/Target/AVR/AVRISelDAGToDAG.cpp | 565 ++ lib/Target/AVR/AVRISelLowering.cpp | 1937 +++++ lib/Target/AVR/AVRISelLowering.h | 15 +- lib/Target/AVR/AVRInstrFormats.td | 2 + lib/Target/AVR/AVRInstrInfo.cpp | 82 +- lib/Target/AVR/AVRInstrInfo.h | 12 +- lib/Target/AVR/AVRInstrInfo.td | 114 +- lib/Target/AVR/AVRInstrumentFunctions.cpp | 222 + lib/Target/AVR/AVRMCInstLower.cpp | 100 + lib/Target/AVR/AVRMCInstLower.h | 43 + lib/Target/AVR/AVRRegisterInfo.cpp | 20 +- lib/Target/AVR/AVRRegisterInfo.h | 6 +- lib/Target/AVR/AVRRelaxMemOperations.cpp | 149 + lib/Target/AVR/AVRTargetMachine.cpp | 25 +- lib/Target/AVR/AVRTargetObjectFile.cpp | 9 +- lib/Target/AVR/AVRTargetObjectFile.h | 3 +- lib/Target/AVR/AsmParser/AVRAsmParser.cpp | 631 ++ lib/Target/AVR/AsmParser/CMakeLists.txt | 3 + lib/Target/AVR/AsmParser/LLVMBuild.txt | 23 + lib/Target/AVR/CMakeLists.txt | 27 +- lib/Target/AVR/Disassembler/AVRDisassembler.cpp | 156 + lib/Target/AVR/Disassembler/CMakeLists.txt | 4 + lib/Target/AVR/Disassembler/LLVMBuild.txt | 23 + lib/Target/AVR/InstPrinter/AVRInstPrinter.cpp | 171 + lib/Target/AVR/InstPrinter/AVRInstPrinter.h | 54 + lib/Target/AVR/InstPrinter/CMakeLists.txt | 8 + lib/Target/AVR/InstPrinter/LLVMBuild.txt | 23 + lib/Target/AVR/LLVMBuild.txt | 10 +- lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp | 473 ++ lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h | 78 + lib/Target/AVR/MCTargetDesc/AVRELFObjectWriter.cpp | 127 + lib/Target/AVR/MCTargetDesc/AVRFixupKinds.h | 149 + lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp | 304 + lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.h | 115 + lib/Target/AVR/MCTargetDesc/AVRMCExpr.cpp | 189 + lib/Target/AVR/MCTargetDesc/AVRMCExpr.h | 88 + lib/Target/AVR/MCTargetDesc/AVRMCTargetDesc.cpp | 121 + lib/Target/AVR/MCTargetDesc/AVRMCTargetDesc.h | 6 +- lib/Target/AVR/MCTargetDesc/CMakeLists.txt | 5 + lib/Target/AVR/MCTargetDesc/LLVMBuild.txt | 2 +- lib/Target/AVR/README.md | 8 + lib/Target/AVR/TargetInfo/AVRTargetInfo.cpp | 14 +- lib/Target/BPF/BPF.td | 6 + lib/Target/BPF/BPFAsmPrinter.cpp | 8 +- lib/Target/BPF/BPFISelDAGToDAG.cpp | 2 +- lib/Target/BPF/BPFInstrInfo.cpp | 14 +- lib/Target/BPF/BPFInstrInfo.h | 8 +- lib/Target/BPF/BPFInstrInfo.td | 114 +- lib/Target/BPF/BPFRegisterInfo.cpp | 4 +- lib/Target/BPF/BPFTargetMachine.cpp | 6 +- lib/Target/BPF/CMakeLists.txt | 2 + lib/Target/BPF/Disassembler/BPFDisassembler.cpp | 154 + lib/Target/BPF/Disassembler/CMakeLists.txt | 4 + lib/Target/BPF/Disassembler/LLVMBuild.txt | 23 + lib/Target/BPF/InstPrinter/BPFInstPrinter.cpp | 18 +- lib/Target/BPF/LLVMBuild.txt | 3 +- lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp | 6 +- lib/Target/BPF/MCTargetDesc/BPFELFObjectWriter.cpp | 8 +- lib/Target/BPF/MCTargetDesc/BPFMCAsmInfo.h | 9 + lib/Target/BPF/MCTargetDesc/BPFMCCodeEmitter.cpp | 19 +- lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.cpp | 27 +- lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h | 13 +- lib/Target/BPF/TargetInfo/BPFTargetInfo.cpp | 26 +- lib/Target/CMakeLists.txt | 4 +- lib/Target/Hexagon/AsmParser/HexagonAsmParser.cpp | 402 +- lib/Target/Hexagon/BitTracker.cpp | 27 +- lib/Target/Hexagon/BitTracker.h | 3 + lib/Target/Hexagon/CMakeLists.txt | 6 +- .../Hexagon/Disassembler/HexagonDisassembler.cpp | 752 +- lib/Target/Hexagon/Hexagon.td | 20 +- lib/Target/Hexagon/HexagonAsmPrinter.cpp | 73 +- lib/Target/Hexagon/HexagonAsmPrinter.h | 2 +- lib/Target/Hexagon/HexagonBitSimplify.cpp | 539 +- lib/Target/Hexagon/HexagonBitTracker.cpp | 59 +- lib/Target/Hexagon/HexagonBlockRanges.cpp | 74 +- lib/Target/Hexagon/HexagonBlockRanges.h | 49 +- lib/Target/Hexagon/HexagonBranchRelaxation.cpp | 46 +- lib/Target/Hexagon/HexagonCFGOptimizer.cpp | 20 +- lib/Target/Hexagon/HexagonCommonGEP.cpp | 196 +- lib/Target/Hexagon/HexagonConstPropagation.cpp | 3149 ++++++++ lib/Target/Hexagon/HexagonCopyToCombine.cpp | 95 +- lib/Target/Hexagon/HexagonEarlyIfConv.cpp | 266 +- lib/Target/Hexagon/HexagonExpandCondsets.cpp | 390 +- lib/Target/Hexagon/HexagonFixupHwLoops.cpp | 8 +- lib/Target/Hexagon/HexagonFrameLowering.cpp | 689 +- lib/Target/Hexagon/HexagonFrameLowering.h | 17 +- lib/Target/Hexagon/HexagonGenExtract.cpp | 49 +- lib/Target/Hexagon/HexagonGenInsert.cpp | 199 +- lib/Target/Hexagon/HexagonGenMux.cpp | 75 +- lib/Target/Hexagon/HexagonGenPredicate.cpp | 81 +- lib/Target/Hexagon/HexagonHardwareLoops.cpp | 143 +- lib/Target/Hexagon/HexagonHazardRecognizer.cpp | 140 + lib/Target/Hexagon/HexagonHazardRecognizer.h | 78 + lib/Target/Hexagon/HexagonISelDAGToDAG.cpp | 1226 ++- lib/Target/Hexagon/HexagonISelLowering.cpp | 731 +- lib/Target/Hexagon/HexagonISelLowering.h | 50 +- lib/Target/Hexagon/HexagonInstrAlias.td | 64 +- lib/Target/Hexagon/HexagonInstrFormats.td | 5 +- lib/Target/Hexagon/HexagonInstrFormatsV4.td | 4 +- lib/Target/Hexagon/HexagonInstrFormatsV60.td | 4 +- lib/Target/Hexagon/HexagonInstrInfo.cpp | 1448 ++-- lib/Target/Hexagon/HexagonInstrInfo.h | 235 +- lib/Target/Hexagon/HexagonInstrInfo.td | 1167 +-- lib/Target/Hexagon/HexagonInstrInfoV3.td | 72 +- lib/Target/Hexagon/HexagonInstrInfoV4.td | 1289 +--- lib/Target/Hexagon/HexagonInstrInfoV5.td | 568 +- lib/Target/Hexagon/HexagonInstrInfoV60.td | 399 +- lib/Target/Hexagon/HexagonInstrInfoVector.td | 420 +- lib/Target/Hexagon/HexagonIntrinsics.td | 57 +- lib/Target/Hexagon/HexagonIntrinsicsDerived.td | 18 +- lib/Target/Hexagon/HexagonIntrinsicsV4.td | 18 +- lib/Target/Hexagon/HexagonIntrinsicsV60.td | 81 +- lib/Target/Hexagon/HexagonIsetDx.td | 138 +- lib/Target/Hexagon/HexagonMachineFunctionInfo.h | 8 - lib/Target/Hexagon/HexagonMachineScheduler.cpp | 18 +- lib/Target/Hexagon/HexagonMachineScheduler.h | 4 +- lib/Target/Hexagon/HexagonNewValueJump.cpp | 60 +- lib/Target/Hexagon/HexagonOperands.td | 365 +- lib/Target/Hexagon/HexagonOptAddrMode.cpp | 125 +- lib/Target/Hexagon/HexagonOptimizeSZextends.cpp | 7 +- lib/Target/Hexagon/HexagonPatterns.td | 3347 +++++++++ lib/Target/Hexagon/HexagonPeephole.cpp | 19 +- lib/Target/Hexagon/HexagonRDF.cpp | 60 - lib/Target/Hexagon/HexagonRDF.h | 28 - lib/Target/Hexagon/HexagonRDFOpt.cpp | 46 +- lib/Target/Hexagon/HexagonRegisterInfo.cpp | 27 +- lib/Target/Hexagon/HexagonRegisterInfo.h | 9 + lib/Target/Hexagon/HexagonRegisterInfo.td | 33 +- lib/Target/Hexagon/HexagonSelectCCInfo.td | 121 - lib/Target/Hexagon/HexagonSelectionDAGInfo.cpp | 7 +- lib/Target/Hexagon/HexagonSelectionDAGInfo.h | 2 + .../Hexagon/HexagonSplitConst32AndConst64.cpp | 126 +- lib/Target/Hexagon/HexagonSplitDouble.cpp | 120 +- lib/Target/Hexagon/HexagonStoreWidening.cpp | 70 +- lib/Target/Hexagon/HexagonSubtarget.cpp | 72 +- lib/Target/Hexagon/HexagonSubtarget.h | 13 +- lib/Target/Hexagon/HexagonSystemInst.td | 2 +- lib/Target/Hexagon/HexagonTargetMachine.cpp | 38 +- lib/Target/Hexagon/HexagonTargetObjectFile.cpp | 87 +- lib/Target/Hexagon/HexagonTargetObjectFile.h | 18 +- lib/Target/Hexagon/HexagonTargetTransformInfo.cpp | 33 + lib/Target/Hexagon/HexagonTargetTransformInfo.h | 13 +- lib/Target/Hexagon/HexagonVLIWPacketizer.cpp | 446 +- lib/Target/Hexagon/HexagonVLIWPacketizer.h | 44 +- lib/Target/Hexagon/HexagonVectorPrint.cpp | 209 + lib/Target/Hexagon/LLVMBuild.txt | 25 +- .../Hexagon/MCTargetDesc/HexagonAsmBackend.cpp | 7 +- lib/Target/Hexagon/MCTargetDesc/HexagonBaseInfo.h | 9 +- .../Hexagon/MCTargetDesc/HexagonMCAsmInfo.cpp | 1 - .../Hexagon/MCTargetDesc/HexagonMCCodeEmitter.cpp | 4 + .../Hexagon/MCTargetDesc/HexagonMCCodeEmitter.h | 5 + .../Hexagon/MCTargetDesc/HexagonMCCompound.cpp | 6 +- .../Hexagon/MCTargetDesc/HexagonMCDuplexInfo.cpp | 240 +- .../Hexagon/MCTargetDesc/HexagonMCELFStreamer.cpp | 23 +- .../Hexagon/MCTargetDesc/HexagonMCELFStreamer.h | 17 +- .../Hexagon/MCTargetDesc/HexagonMCInstrInfo.cpp | 111 +- .../Hexagon/MCTargetDesc/HexagonMCInstrInfo.h | 3 +- .../Hexagon/MCTargetDesc/HexagonMCTargetDesc.cpp | 58 +- .../Hexagon/MCTargetDesc/HexagonMCTargetDesc.h | 20 +- .../Hexagon/MCTargetDesc/HexagonShuffler.cpp | 10 +- lib/Target/Hexagon/RDFCopy.cpp | 69 +- lib/Target/Hexagon/RDFCopy.h | 1 + lib/Target/Hexagon/RDFGraph.cpp | 675 +- lib/Target/Hexagon/RDFGraph.h | 261 +- lib/Target/Hexagon/RDFLiveness.cpp | 538 +- lib/Target/Hexagon/RDFLiveness.h | 44 +- .../Hexagon/TargetInfo/HexagonTargetInfo.cpp | 8 +- lib/Target/LLVMBuild.txt | 1 + lib/Target/Lanai/AsmParser/LLVMBuild.txt | 2 +- lib/Target/Lanai/AsmParser/LanaiAsmParser.cpp | 6 +- lib/Target/Lanai/Disassembler/LLVMBuild.txt | 2 +- .../Lanai/Disassembler/LanaiDisassembler.cpp | 10 +- lib/Target/Lanai/InstPrinter/LLVMBuild.txt | 2 +- lib/Target/Lanai/LLVMBuild.txt | 2 +- lib/Target/Lanai/Lanai.h | 2 +- lib/Target/Lanai/LanaiAluCode.h | 4 +- lib/Target/Lanai/LanaiAsmPrinter.cpp | 8 +- lib/Target/Lanai/LanaiDelaySlotFiller.cpp | 15 +- lib/Target/Lanai/LanaiFrameLowering.cpp | 32 +- lib/Target/Lanai/LanaiISelDAGToDAG.cpp | 26 +- lib/Target/Lanai/LanaiISelLowering.cpp | 75 +- lib/Target/Lanai/LanaiISelLowering.h | 1 + lib/Target/Lanai/LanaiInstrInfo.cpp | 19 +- lib/Target/Lanai/LanaiInstrInfo.h | 10 +- lib/Target/Lanai/LanaiInstrInfo.td | 8 - lib/Target/Lanai/LanaiMCInstLower.cpp | 1 - lib/Target/Lanai/LanaiMCInstLower.h | 3 +- lib/Target/Lanai/LanaiMemAluCombiner.cpp | 7 +- lib/Target/Lanai/LanaiRegisterInfo.cpp | 8 +- lib/Target/Lanai/LanaiTargetMachine.cpp | 3 +- lib/Target/Lanai/LanaiTargetObjectFile.cpp | 51 +- lib/Target/Lanai/LanaiTargetObjectFile.h | 14 +- lib/Target/Lanai/LanaiTargetTransformInfo.h | 5 - lib/Target/Lanai/MCTargetDesc/CMakeLists.txt | 2 +- lib/Target/Lanai/MCTargetDesc/LLVMBuild.txt | 4 +- lib/Target/Lanai/MCTargetDesc/LanaiAsmBackend.cpp | 8 +- .../Lanai/MCTargetDesc/LanaiMCCodeEmitter.cpp | 17 - .../Lanai/MCTargetDesc/LanaiMCTargetDesc.cpp | 29 +- lib/Target/Lanai/MCTargetDesc/LanaiMCTargetDesc.h | 6 +- lib/Target/Lanai/TargetInfo/LanaiTargetInfo.cpp | 9 +- .../MSP430/MCTargetDesc/MSP430MCTargetDesc.cpp | 11 +- .../MSP430/MCTargetDesc/MSP430MCTargetDesc.h | 2 +- lib/Target/MSP430/MSP430AsmPrinter.cpp | 6 +- lib/Target/MSP430/MSP430BranchSelector.cpp | 308 +- lib/Target/MSP430/MSP430FrameLowering.cpp | 26 +- lib/Target/MSP430/MSP430ISelDAGToDAG.cpp | 2 +- lib/Target/MSP430/MSP430ISelLowering.cpp | 35 +- lib/Target/MSP430/MSP430InstrInfo.cpp | 21 +- lib/Target/MSP430/MSP430InstrInfo.h | 12 +- lib/Target/MSP430/MSP430RegisterInfo.cpp | 4 +- lib/Target/MSP430/MSP430TargetMachine.cpp | 2 +- lib/Target/MSP430/TargetInfo/MSP430TargetInfo.cpp | 11 +- lib/Target/Mips/AsmParser/MipsAsmParser.cpp | 1110 ++- lib/Target/Mips/Disassembler/MipsDisassembler.cpp | 184 +- lib/Target/Mips/InstPrinter/MipsInstPrinter.cpp | 1 + lib/Target/Mips/MCTargetDesc/MipsABIInfo.cpp | 7 +- lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp | 26 +- lib/Target/Mips/MCTargetDesc/MipsBaseInfo.h | 5 +- .../Mips/MCTargetDesc/MipsELFObjectWriter.cpp | 21 +- lib/Target/Mips/MCTargetDesc/MipsFixupKinds.h | 4 + lib/Target/Mips/MCTargetDesc/MipsMCAsmInfo.cpp | 9 + lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.cpp | 40 +- lib/Target/Mips/MCTargetDesc/MipsMCCodeEmitter.h | 7 + lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.cpp | 17 +- lib/Target/Mips/MCTargetDesc/MipsMCTargetDesc.h | 21 +- lib/Target/Mips/MicroMips32r6InstrInfo.td | 579 +- lib/Target/Mips/MicroMips64r6InstrFormats.td | 46 + lib/Target/Mips/MicroMips64r6InstrInfo.td | 117 +- lib/Target/Mips/MicroMipsInstrFPU.td | 21 +- lib/Target/Mips/MicroMipsInstrFormats.td | 24 + lib/Target/Mips/MicroMipsInstrInfo.td | 280 +- lib/Target/Mips/Mips.td | 6 +- lib/Target/Mips/Mips16FrameLowering.cpp | 28 +- lib/Target/Mips/Mips16HardFloat.cpp | 4 +- lib/Target/Mips/Mips16ISelDAGToDAG.cpp | 7 +- lib/Target/Mips/Mips16InstrInfo.cpp | 14 +- lib/Target/Mips/Mips16RegisterInfo.cpp | 4 +- lib/Target/Mips/Mips32r6InstrFormats.td | 16 + lib/Target/Mips/Mips32r6InstrInfo.td | 192 +- lib/Target/Mips/Mips64InstrInfo.td | 81 +- lib/Target/Mips/Mips64r6InstrInfo.td | 60 +- lib/Target/Mips/MipsAsmPrinter.cpp | 33 +- lib/Target/Mips/MipsAsmPrinter.h | 4 +- lib/Target/Mips/MipsConstantIslandPass.cpp | 27 +- lib/Target/Mips/MipsDelaySlotFiller.cpp | 62 +- lib/Target/Mips/MipsEVAInstrFormats.td | 4 +- lib/Target/Mips/MipsFastISel.cpp | 237 +- lib/Target/Mips/MipsFrameLowering.cpp | 26 +- lib/Target/Mips/MipsHazardSchedule.cpp | 75 +- lib/Target/Mips/MipsISelDAGToDAG.cpp | 22 +- lib/Target/Mips/MipsISelDAGToDAG.h | 15 +- lib/Target/Mips/MipsISelLowering.cpp | 159 +- lib/Target/Mips/MipsISelLowering.h | 9 +- lib/Target/Mips/MipsInstrFPU.td | 43 +- lib/Target/Mips/MipsInstrFormats.td | 4 + lib/Target/Mips/MipsInstrInfo.cpp | 85 +- lib/Target/Mips/MipsInstrInfo.h | 12 +- lib/Target/Mips/MipsInstrInfo.td | 322 +- lib/Target/Mips/MipsLongBranch.cpp | 12 +- lib/Target/Mips/MipsMSAInstrInfo.td | 90 +- lib/Target/Mips/MipsMachineFunction.cpp | 8 +- lib/Target/Mips/MipsModuleISelDAGToDAG.cpp | 2 +- lib/Target/Mips/MipsOptimizePICCall.cpp | 2 +- lib/Target/Mips/MipsOs16.cpp | 4 +- lib/Target/Mips/MipsRegisterInfo.cpp | 6 +- lib/Target/Mips/MipsRegisterInfo.td | 6 + lib/Target/Mips/MipsSEFrameLowering.cpp | 62 +- lib/Target/Mips/MipsSEISelDAGToDAG.cpp | 78 +- lib/Target/Mips/MipsSEISelDAGToDAG.h | 19 +- lib/Target/Mips/MipsSEISelLowering.cpp | 358 +- lib/Target/Mips/MipsSEISelLowering.h | 14 + lib/Target/Mips/MipsSEInstrInfo.cpp | 41 +- lib/Target/Mips/MipsSERegisterInfo.cpp | 8 +- lib/Target/Mips/MipsSchedule.td | 83 +- lib/Target/Mips/MipsScheduleGeneric.td | 1048 +++ lib/Target/Mips/MipsScheduleP5600.td | 254 +- lib/Target/Mips/MipsTargetMachine.cpp | 17 +- lib/Target/Mips/MipsTargetObjectFile.cpp | 40 +- lib/Target/Mips/MipsTargetObjectFile.h | 14 +- lib/Target/Mips/TargetInfo/MipsTargetInfo.cpp | 31 +- lib/Target/NVPTX/CMakeLists.txt | 3 +- lib/Target/NVPTX/LLVMBuild.txt | 2 +- lib/Target/NVPTX/MCTargetDesc/NVPTXBaseInfo.h | 58 +- .../NVPTX/MCTargetDesc/NVPTXMCTargetDesc.cpp | 2 +- lib/Target/NVPTX/MCTargetDesc/NVPTXMCTargetDesc.h | 4 +- lib/Target/NVPTX/NVPTX.h | 7 +- lib/Target/NVPTX/NVPTX.td | 9 +- lib/Target/NVPTX/NVPTXAllocaHoisting.cpp | 4 +- lib/Target/NVPTX/NVPTXAsmPrinter.cpp | 70 +- lib/Target/NVPTX/NVPTXAsmPrinter.h | 2 +- .../NVPTX/NVPTXFavorNonGenericAddrSpaces.cpp | 289 - lib/Target/NVPTX/NVPTXFrameLowering.cpp | 2 +- lib/Target/NVPTX/NVPTXGenericToNVVM.cpp | 37 - lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp | 59 +- lib/Target/NVPTX/NVPTXISelDAGToDAG.h | 2 +- lib/Target/NVPTX/NVPTXISelLowering.cpp | 173 +- lib/Target/NVPTX/NVPTXISelLowering.h | 3 +- lib/Target/NVPTX/NVPTXInferAddressSpaces.cpp | 3 - lib/Target/NVPTX/NVPTXInstrInfo.cpp | 28 +- lib/Target/NVPTX/NVPTXInstrInfo.h | 9 +- lib/Target/NVPTX/NVPTXInstrInfo.td | 161 +- lib/Target/NVPTX/NVPTXIntrinsics.td | 196 + lib/Target/NVPTX/NVPTXLowerAggrCopies.cpp | 4 +- lib/Target/NVPTX/NVPTXLowerAlloca.cpp | 8 +- lib/Target/NVPTX/NVPTXLowerArgs.cpp | 253 + lib/Target/NVPTX/NVPTXLowerKernelArgs.cpp | 234 - lib/Target/NVPTX/NVPTXMCExpr.cpp | 4 +- lib/Target/NVPTX/NVPTXPeephole.cpp | 2 +- lib/Target/NVPTX/NVPTXPrologEpilogPass.cpp | 52 +- lib/Target/NVPTX/NVPTXRegisterInfo.cpp | 26 +- lib/Target/NVPTX/NVPTXReplaceImageHandles.cpp | 2 +- lib/Target/NVPTX/NVPTXSubtarget.cpp | 2 - lib/Target/NVPTX/NVPTXSubtarget.h | 8 + lib/Target/NVPTX/NVPTXTargetMachine.cpp | 40 +- lib/Target/NVPTX/NVPTXTargetObjectFile.h | 6 +- lib/Target/NVPTX/NVPTXTargetTransformInfo.cpp | 23 + lib/Target/NVPTX/NVPTXTargetTransformInfo.h | 7 - lib/Target/NVPTX/NVPTXUtilities.cpp | 247 +- lib/Target/NVPTX/NVPTXUtilities.h | 61 +- lib/Target/NVPTX/NVVMIntrRange.cpp | 4 + lib/Target/NVPTX/NVVMReflect.cpp | 1 - lib/Target/NVPTX/TargetInfo/NVPTXTargetInfo.cpp | 14 +- lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp | 369 +- .../PowerPC/Disassembler/PPCDisassembler.cpp | 39 +- lib/Target/PowerPC/InstPrinter/PPCInstPrinter.cpp | 61 +- lib/Target/PowerPC/InstPrinter/PPCInstPrinter.h | 1 + lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp | 3 +- .../PowerPC/MCTargetDesc/PPCMCCodeEmitter.cpp | 22 +- .../PowerPC/MCTargetDesc/PPCMCTargetDesc.cpp | 3 +- lib/Target/PowerPC/MCTargetDesc/PPCMCTargetDesc.h | 10 +- lib/Target/PowerPC/P9InstrResources.td | 808 ++ lib/Target/PowerPC/PPC.td | 63 +- lib/Target/PowerPC/PPCAsmPrinter.cpp | 209 +- lib/Target/PowerPC/PPCBoolRetToInt.cpp | 80 +- lib/Target/PowerPC/PPCBranchSelector.cpp | 81 +- lib/Target/PowerPC/PPCCTRLoops.cpp | 8 +- lib/Target/PowerPC/PPCCallingConv.td | 26 +- lib/Target/PowerPC/PPCEarlyReturn.cpp | 4 +- lib/Target/PowerPC/PPCFastISel.cpp | 85 +- lib/Target/PowerPC/PPCFrameLowering.cpp | 536 +- lib/Target/PowerPC/PPCHazardRecognizers.cpp | 2 +- lib/Target/PowerPC/PPCISelDAGToDAG.cpp | 222 +- lib/Target/PowerPC/PPCISelLowering.cpp | 907 ++- lib/Target/PowerPC/PPCISelLowering.h | 77 +- lib/Target/PowerPC/PPCInstr64Bit.td | 10 - lib/Target/PowerPC/PPCInstrAltivec.td | 142 +- lib/Target/PowerPC/PPCInstrFormats.td | 59 + lib/Target/PowerPC/PPCInstrInfo.cpp | 262 +- lib/Target/PowerPC/PPCInstrInfo.h | 29 +- lib/Target/PowerPC/PPCInstrInfo.td | 135 +- lib/Target/PowerPC/PPCInstrQPX.td | 10 +- lib/Target/PowerPC/PPCInstrVSX.td | 775 +- lib/Target/PowerPC/PPCLoopPreIncPrep.cpp | 38 +- lib/Target/PowerPC/PPCMCInstLower.cpp | 8 +- lib/Target/PowerPC/PPCMIPeephole.cpp | 164 +- lib/Target/PowerPC/PPCQPXLoadSplat.cpp | 2 +- lib/Target/PowerPC/PPCRegisterInfo.cpp | 31 +- lib/Target/PowerPC/PPCRegisterInfo.h | 2 +- lib/Target/PowerPC/PPCRegisterInfo.td | 35 +- lib/Target/PowerPC/PPCSchedule.td | 2 + lib/Target/PowerPC/PPCScheduleP9.td | 335 + lib/Target/PowerPC/PPCSubtarget.cpp | 3 +- lib/Target/PowerPC/PPCSubtarget.h | 9 +- lib/Target/PowerPC/PPCTLSDynamicCall.cpp | 26 +- lib/Target/PowerPC/PPCTargetMachine.cpp | 29 +- lib/Target/PowerPC/PPCTargetObjectFile.cpp | 8 +- lib/Target/PowerPC/PPCTargetObjectFile.h | 3 +- lib/Target/PowerPC/PPCTargetTransformInfo.cpp | 26 +- lib/Target/PowerPC/PPCTargetTransformInfo.h | 7 - lib/Target/PowerPC/PPCVSXCopy.cpp | 42 +- lib/Target/PowerPC/PPCVSXFMAMutate.cpp | 78 +- lib/Target/PowerPC/PPCVSXSwapRemoval.cpp | 3 +- lib/Target/PowerPC/README.txt | 5 + .../PowerPC/TargetInfo/PowerPCTargetInfo.cpp | 25 +- lib/Target/README.txt | 4 +- lib/Target/RISCV/CMakeLists.txt | 14 + lib/Target/RISCV/LLVMBuild.txt | 31 + lib/Target/RISCV/MCTargetDesc/CMakeLists.txt | 7 + lib/Target/RISCV/MCTargetDesc/LLVMBuild.txt | 23 + lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp | 91 + .../RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp | 47 + lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.cpp | 25 + lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.h | 31 + .../RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp | 91 + .../RISCV/MCTargetDesc/RISCVMCTargetDesc.cpp | 59 + lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.h | 58 + lib/Target/RISCV/RISCV.td | 27 + lib/Target/RISCV/RISCVInstrFormats.td | 152 + lib/Target/RISCV/RISCVInstrInfo.td | 55 + lib/Target/RISCV/RISCVRegisterInfo.td | 90 + lib/Target/RISCV/RISCVTargetMachine.cpp | 58 + lib/Target/RISCV/RISCVTargetMachine.h | 40 + lib/Target/RISCV/TargetInfo/CMakeLists.txt | 3 + lib/Target/RISCV/TargetInfo/LLVMBuild.txt | 23 + lib/Target/RISCV/TargetInfo/RISCVTargetInfo.cpp | 30 + lib/Target/Sparc/AsmParser/SparcAsmParser.cpp | 32 +- lib/Target/Sparc/DelaySlotFiller.cpp | 6 +- .../Sparc/Disassembler/SparcDisassembler.cpp | 10 +- lib/Target/Sparc/LeonFeatures.td | 131 +- lib/Target/Sparc/LeonPasses.cpp | 645 +- lib/Target/Sparc/LeonPasses.h | 114 +- lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp | 3 +- .../Sparc/MCTargetDesc/SparcMCCodeEmitter.cpp | 17 +- .../Sparc/MCTargetDesc/SparcMCTargetDesc.cpp | 15 +- lib/Target/Sparc/MCTargetDesc/SparcMCTargetDesc.h | 10 +- lib/Target/Sparc/Sparc.td | 147 +- lib/Target/Sparc/SparcAsmPrinter.cpp | 10 +- lib/Target/Sparc/SparcFrameLowering.cpp | 49 +- lib/Target/Sparc/SparcISelDAGToDAG.cpp | 21 +- lib/Target/Sparc/SparcISelLowering.cpp | 151 +- lib/Target/Sparc/SparcInstrInfo.cpp | 46 +- lib/Target/Sparc/SparcInstrInfo.h | 10 +- lib/Target/Sparc/SparcInstrInfo.td | 24 +- lib/Target/Sparc/SparcRegisterInfo.td | 1 - lib/Target/Sparc/SparcSubtarget.cpp | 9 +- lib/Target/Sparc/SparcSubtarget.h | 20 +- lib/Target/Sparc/SparcTargetMachine.cpp | 57 +- lib/Target/Sparc/SparcTargetObjectFile.cpp | 13 +- lib/Target/Sparc/SparcTargetObjectFile.h | 10 +- lib/Target/Sparc/TargetInfo/SparcTargetInfo.cpp | 21 +- lib/Target/SystemZ/AsmParser/SystemZAsmParser.cpp | 590 +- lib/Target/SystemZ/CMakeLists.txt | 3 + .../SystemZ/Disassembler/SystemZDisassembler.cpp | 45 +- .../SystemZ/InstPrinter/SystemZInstPrinter.cpp | 19 +- .../SystemZ/InstPrinter/SystemZInstPrinter.h | 3 +- .../SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp | 12 +- .../SystemZ/MCTargetDesc/SystemZMCCodeEmitter.cpp | 41 + lib/Target/SystemZ/MCTargetDesc/SystemZMCFixups.h | 4 +- .../SystemZ/MCTargetDesc/SystemZMCObjectWriter.cpp | 4 + .../SystemZ/MCTargetDesc/SystemZMCTargetDesc.cpp | 24 +- .../SystemZ/MCTargetDesc/SystemZMCTargetDesc.h | 7 +- lib/Target/SystemZ/SystemZ.h | 1 + lib/Target/SystemZ/SystemZ.td | 14 +- lib/Target/SystemZ/SystemZAsmPrinter.cpp | 6 +- lib/Target/SystemZ/SystemZAsmPrinter.h | 4 +- lib/Target/SystemZ/SystemZElimCompare.cpp | 92 +- lib/Target/SystemZ/SystemZExpandPseudo.cpp | 153 + lib/Target/SystemZ/SystemZFeatures.td | 171 + lib/Target/SystemZ/SystemZFrameLowering.cpp | 40 +- lib/Target/SystemZ/SystemZHazardRecognizer.cpp | 337 + lib/Target/SystemZ/SystemZHazardRecognizer.h | 128 + lib/Target/SystemZ/SystemZISelDAGToDAG.cpp | 148 +- lib/Target/SystemZ/SystemZISelLowering.cpp | 138 +- lib/Target/SystemZ/SystemZISelLowering.h | 10 +- lib/Target/SystemZ/SystemZInstrBuilder.h | 4 +- lib/Target/SystemZ/SystemZInstrFP.td | 250 +- lib/Target/SystemZ/SystemZInstrFormats.td | 2037 ++++- lib/Target/SystemZ/SystemZInstrInfo.cpp | 341 +- lib/Target/SystemZ/SystemZInstrInfo.h | 54 +- lib/Target/SystemZ/SystemZInstrInfo.td | 1223 +-- lib/Target/SystemZ/SystemZInstrVector.td | 237 +- lib/Target/SystemZ/SystemZLDCleanup.cpp | 2 +- lib/Target/SystemZ/SystemZLongBranch.cpp | 10 +- lib/Target/SystemZ/SystemZMachineScheduler.cpp | 153 + lib/Target/SystemZ/SystemZMachineScheduler.h | 112 + lib/Target/SystemZ/SystemZOperands.td | 47 +- lib/Target/SystemZ/SystemZOperators.td | 24 +- lib/Target/SystemZ/SystemZProcessors.td | 103 +- lib/Target/SystemZ/SystemZRegisterInfo.cpp | 5 + lib/Target/SystemZ/SystemZRegisterInfo.td | 28 +- lib/Target/SystemZ/SystemZSchedule.td | 77 + lib/Target/SystemZ/SystemZScheduleZ13.td | 1064 +++ lib/Target/SystemZ/SystemZScheduleZ196.td | 769 ++ lib/Target/SystemZ/SystemZScheduleZEC12.td | 807 ++ lib/Target/SystemZ/SystemZShortenInst.cpp | 6 +- lib/Target/SystemZ/SystemZSubtarget.cpp | 8 +- lib/Target/SystemZ/SystemZSubtarget.h | 17 + lib/Target/SystemZ/SystemZTargetMachine.cpp | 26 +- lib/Target/SystemZ/SystemZTargetTransformInfo.cpp | 57 + lib/Target/SystemZ/SystemZTargetTransformInfo.h | 9 +- .../SystemZ/TargetInfo/SystemZTargetInfo.cpp | 9 +- lib/Target/TargetIntrinsicInfo.cpp | 2 +- lib/Target/TargetLoweringObjectFile.cpp | 61 +- lib/Target/TargetMachine.cpp | 32 +- lib/Target/TargetMachineC.cpp | 5 +- lib/Target/TargetRecip.cpp | 225 - lib/Target/TargetSubtargetInfo.cpp | 54 - lib/Target/WebAssembly/CMakeLists.txt | 9 +- .../Disassembler/WebAssemblyDisassembler.cpp | 9 +- .../InstPrinter/WebAssemblyInstPrinter.cpp | 60 +- .../InstPrinter/WebAssemblyInstPrinter.h | 2 + lib/Target/WebAssembly/LLVMBuild.txt | 2 +- .../MCTargetDesc/WebAssemblyAsmBackend.cpp | 6 +- .../MCTargetDesc/WebAssemblyMCCodeEmitter.cpp | 58 +- .../MCTargetDesc/WebAssemblyMCTargetDesc.cpp | 16 +- .../MCTargetDesc/WebAssemblyMCTargetDesc.h | 84 +- .../MCTargetDesc/WebAssemblyTargetStreamer.cpp | 56 +- .../MCTargetDesc/WebAssemblyTargetStreamer.h | 19 +- lib/Target/WebAssembly/README.txt | 52 +- .../TargetInfo/WebAssemblyTargetInfo.cpp | 14 +- lib/Target/WebAssembly/WebAssembly.h | 6 + lib/Target/WebAssembly/WebAssembly.td | 2 +- lib/Target/WebAssembly/WebAssemblyArgumentMove.cpp | 25 +- lib/Target/WebAssembly/WebAssemblyAsmPrinter.cpp | 92 +- lib/Target/WebAssembly/WebAssemblyCFGStackify.cpp | 148 +- .../WebAssembly/WebAssemblyCallIndirectFixup.cpp | 120 + .../WebAssembly/WebAssemblyExplicitLocals.cpp | 308 + lib/Target/WebAssembly/WebAssemblyFastISel.cpp | 161 +- .../WebAssemblyFixIrreducibleControlFlow.cpp | 2 +- .../WebAssembly/WebAssemblyFrameLowering.cpp | 104 +- lib/Target/WebAssembly/WebAssemblyFrameLowering.h | 1 + lib/Target/WebAssembly/WebAssemblyISelDAGToDAG.cpp | 2 +- lib/Target/WebAssembly/WebAssemblyISelLowering.cpp | 43 +- lib/Target/WebAssembly/WebAssemblyInstrCall.td | 76 +- lib/Target/WebAssembly/WebAssemblyInstrControl.td | 49 +- lib/Target/WebAssembly/WebAssemblyInstrConv.td | 54 +- lib/Target/WebAssembly/WebAssemblyInstrFloat.td | 44 +- lib/Target/WebAssembly/WebAssemblyInstrFormats.td | 67 +- lib/Target/WebAssembly/WebAssemblyInstrInfo.cpp | 26 +- lib/Target/WebAssembly/WebAssemblyInstrInfo.h | 10 +- lib/Target/WebAssembly/WebAssemblyInstrInfo.td | 98 +- lib/Target/WebAssembly/WebAssemblyInstrInteger.td | 64 +- lib/Target/WebAssembly/WebAssemblyInstrMemory.td | 627 +- lib/Target/WebAssembly/WebAssemblyInstrSIMD.td | 7 +- .../WebAssembly/WebAssemblyLowerBrUnless.cpp | 6 +- .../WebAssemblyLowerEmscriptenEHSjLj.cpp | 1184 +++ .../WebAssembly/WebAssemblyMachineFunctionInfo.cpp | 37 + .../WebAssembly/WebAssemblyMachineFunctionInfo.h | 26 + .../WebAssemblyOptimizeLiveIntervals.cpp | 2 +- .../WebAssembly/WebAssemblyOptimizeReturned.cpp | 2 +- lib/Target/WebAssembly/WebAssemblyPeephole.cpp | 53 +- .../WebAssemblyPrepareForLiveIntervals.cpp | 28 +- lib/Target/WebAssembly/WebAssemblyRegColoring.cpp | 2 +- lib/Target/WebAssembly/WebAssemblyRegNumbering.cpp | 22 +- lib/Target/WebAssembly/WebAssemblyRegStackify.cpp | 139 +- lib/Target/WebAssembly/WebAssemblyRegisterInfo.cpp | 22 +- lib/Target/WebAssembly/WebAssemblyRegisterInfo.td | 10 +- .../WebAssembly/WebAssemblyReplacePhysRegs.cpp | 6 +- .../WebAssembly/WebAssemblySetP2AlignOperands.cpp | 47 +- lib/Target/WebAssembly/WebAssemblyStoreResults.cpp | 26 +- .../WebAssembly/WebAssemblyTargetMachine.cpp | 55 +- .../WebAssembly/WebAssemblyTargetTransformInfo.h | 7 - lib/Target/WebAssembly/WebAssemblyUtilities.cpp | 71 + lib/Target/WebAssembly/WebAssemblyUtilities.h | 34 + lib/Target/WebAssembly/known_gcc_test_failures.txt | 1 - lib/Target/X86/AsmParser/X86AsmParser.cpp | 585 +- lib/Target/X86/AsmParser/X86Operand.h | 4 +- lib/Target/X86/CMakeLists.txt | 18 + lib/Target/X86/Disassembler/X86Disassembler.cpp | 24 +- .../X86/Disassembler/X86DisassemblerDecoder.cpp | 12 +- .../X86/Disassembler/X86DisassemblerDecoder.h | 2 +- lib/Target/X86/InstPrinter/X86ATTInstPrinter.cpp | 3 + lib/Target/X86/InstPrinter/X86InstComments.cpp | 254 +- lib/Target/X86/InstPrinter/X86InstComments.h | 5 + lib/Target/X86/InstPrinter/X86IntelInstPrinter.cpp | 3 + lib/Target/X86/LLVMBuild.txt | 2 +- lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp | 18 +- lib/Target/X86/MCTargetDesc/X86BaseInfo.h | 244 +- lib/Target/X86/MCTargetDesc/X86MCAsmInfo.cpp | 3 +- lib/Target/X86/MCTargetDesc/X86MCCodeEmitter.cpp | 195 +- lib/Target/X86/MCTargetDesc/X86MCTargetDesc.cpp | 6 +- lib/Target/X86/MCTargetDesc/X86MCTargetDesc.h | 10 +- lib/Target/X86/TargetInfo/X86TargetInfo.cpp | 17 +- lib/Target/X86/Utils/X86ShuffleDecode.cpp | 26 +- lib/Target/X86/Utils/X86ShuffleDecode.h | 2 + lib/Target/X86/X86.h | 7 + lib/Target/X86/X86.td | 34 +- lib/Target/X86/X86AsmPrinter.cpp | 16 +- lib/Target/X86/X86AsmPrinter.h | 27 +- lib/Target/X86/X86CallFrameOptimization.cpp | 14 +- lib/Target/X86/X86CallLowering.cpp | 46 + lib/Target/X86/X86CallLowering.h | 39 + lib/Target/X86/X86CallingConv.cpp | 208 + lib/Target/X86/X86CallingConv.h | 38 +- lib/Target/X86/X86CallingConv.td | 276 +- lib/Target/X86/X86EvexToVex.cpp | 213 + lib/Target/X86/X86ExpandPseudo.cpp | 45 +- lib/Target/X86/X86FastISel.cpp | 331 +- lib/Target/X86/X86FixupBWInsts.cpp | 10 +- lib/Target/X86/X86FixupLEAs.cpp | 4 +- lib/Target/X86/X86FixupSetCC.cpp | 5 +- lib/Target/X86/X86FloatingPoint.cpp | 51 +- lib/Target/X86/X86FrameLowering.cpp | 251 +- lib/Target/X86/X86FrameLowering.h | 30 +- lib/Target/X86/X86ISelDAGToDAG.cpp | 130 +- lib/Target/X86/X86ISelLowering.cpp | 7874 +++++++++++++------- lib/Target/X86/X86ISelLowering.h | 209 +- lib/Target/X86/X86InstrAVX512.td | 3078 +++++--- lib/Target/X86/X86InstrArithmetic.td | 2 +- lib/Target/X86/X86InstrBuilder.h | 74 +- lib/Target/X86/X86InstrCompiler.td | 74 +- lib/Target/X86/X86InstrControl.td | 53 +- lib/Target/X86/X86InstrFMA.td | 176 +- lib/Target/X86/X86InstrFMA3Info.cpp | 285 + lib/Target/X86/X86InstrFMA3Info.h | 315 + lib/Target/X86/X86InstrFPStack.td | 12 +- lib/Target/X86/X86InstrFormats.td | 149 +- lib/Target/X86/X86InstrFragmentsSIMD.td | 311 +- lib/Target/X86/X86InstrInfo.cpp | 3791 +++++++--- lib/Target/X86/X86InstrInfo.h | 83 +- lib/Target/X86/X86InstrInfo.td | 47 +- lib/Target/X86/X86InstrMMX.td | 9 +- lib/Target/X86/X86InstrSSE.td | 1902 +++-- lib/Target/X86/X86InstrShiftRotate.td | 37 +- lib/Target/X86/X86InstrSystem.td | 9 +- lib/Target/X86/X86InstrTablesInfo.h | 1148 +++ lib/Target/X86/X86InstrXOP.td | 87 +- lib/Target/X86/X86InterleavedAccess.cpp | 221 + lib/Target/X86/X86IntrinsicsInfo.h | 756 +- lib/Target/X86/X86MCInstLower.cpp | 309 +- lib/Target/X86/X86OptimizeLEAs.cpp | 18 +- lib/Target/X86/X86PadShortFunction.cpp | 4 +- lib/Target/X86/X86RegisterInfo.cpp | 133 +- lib/Target/X86/X86RegisterInfo.h | 5 +- lib/Target/X86/X86RegisterInfo.td | 2 + lib/Target/X86/X86SelectionDAGInfo.cpp | 4 +- lib/Target/X86/X86ShuffleDecodeConstantPool.cpp | 403 +- lib/Target/X86/X86ShuffleDecodeConstantPool.h | 4 +- lib/Target/X86/X86Subtarget.cpp | 31 + lib/Target/X86/X86Subtarget.h | 36 +- lib/Target/X86/X86TargetMachine.cpp | 105 +- lib/Target/X86/X86TargetMachine.h | 2 - lib/Target/X86/X86TargetObjectFile.cpp | 34 +- lib/Target/X86/X86TargetObjectFile.h | 24 +- lib/Target/X86/X86TargetTransformInfo.cpp | 754 +- lib/Target/X86/X86TargetTransformInfo.h | 16 +- lib/Target/X86/X86VZeroUpper.cpp | 4 +- lib/Target/X86/X86WinAllocaExpander.cpp | 3 +- lib/Target/X86/X86WinEHState.cpp | 2 +- .../XCore/Disassembler/XCoreDisassembler.cpp | 4 +- .../XCore/MCTargetDesc/XCoreMCTargetDesc.cpp | 16 +- lib/Target/XCore/MCTargetDesc/XCoreMCTargetDesc.h | 3 +- lib/Target/XCore/TargetInfo/XCoreTargetInfo.cpp | 7 +- lib/Target/XCore/XCoreAsmPrinter.cpp | 16 +- lib/Target/XCore/XCoreFrameLowering.cpp | 110 +- lib/Target/XCore/XCoreFrameToArgsOffsetElim.cpp | 12 +- lib/Target/XCore/XCoreISelDAGToDAG.cpp | 2 +- lib/Target/XCore/XCoreISelLowering.cpp | 20 +- lib/Target/XCore/XCoreInstrInfo.cpp | 32 +- lib/Target/XCore/XCoreInstrInfo.h | 10 +- lib/Target/XCore/XCoreInstrInfo.td | 8 - lib/Target/XCore/XCoreMCInstLower.cpp | 7 +- lib/Target/XCore/XCoreMCInstLower.h | 3 +- lib/Target/XCore/XCoreMachineFunctionInfo.cpp | 18 +- lib/Target/XCore/XCoreRegisterInfo.cpp | 4 +- lib/Target/XCore/XCoreSelectionDAGInfo.cpp | 2 +- lib/Target/XCore/XCoreTargetMachine.cpp | 2 +- lib/Target/XCore/XCoreTargetObjectFile.cpp | 20 +- lib/Target/XCore/XCoreTargetObjectFile.h | 6 +- lib/Target/XCore/XCoreTargetTransformInfo.h | 7 - 920 files changed, 98889 insertions(+), 43513 deletions(-) delete mode 100644 lib/Target/AArch64/AArch64BranchRelaxation.cpp create mode 100644 lib/Target/AArch64/AArch64GenRegisterBankInfo.def create mode 100644 lib/Target/AArch64/AArch64InstructionSelector.cpp create mode 100644 lib/Target/AArch64/AArch64InstructionSelector.h create mode 100644 lib/Target/AArch64/AArch64LegalizerInfo.cpp create mode 100644 lib/Target/AArch64/AArch64LegalizerInfo.h create mode 100644 lib/Target/AArch64/AArch64SchedFalkor.td create mode 100644 lib/Target/AArch64/AArch64VectorByElementOpt.cpp create mode 100644 lib/Target/AMDGPU/AMDGPUPTNote.h create mode 100644 lib/Target/AMDGPU/AMDGPUUnifyMetadata.cpp create mode 100644 lib/Target/AMDGPU/BUFInstructions.td create mode 100644 lib/Target/AMDGPU/DSInstructions.td create mode 100644 lib/Target/AMDGPU/FLATInstructions.td create mode 100644 lib/Target/AMDGPU/GCNSchedStrategy.cpp create mode 100644 lib/Target/AMDGPU/GCNSchedStrategy.h create mode 100644 lib/Target/AMDGPU/MCTargetDesc/AMDGPURuntimeMD.cpp create mode 100644 lib/Target/AMDGPU/MCTargetDesc/AMDGPURuntimeMD.h create mode 100644 lib/Target/AMDGPU/MIMGInstructions.td create mode 100644 lib/Target/AMDGPU/SIInsertSkips.cpp create mode 100644 lib/Target/AMDGPU/SIOptimizeExecMasking.cpp create mode 100644 lib/Target/AMDGPU/SMInstructions.td create mode 100644 lib/Target/AMDGPU/SOPInstructions.td create mode 100644 lib/Target/AMDGPU/VOP1Instructions.td create mode 100644 lib/Target/AMDGPU/VOP2Instructions.td create mode 100644 lib/Target/AMDGPU/VOP3Instructions.td create mode 100644 lib/Target/AMDGPU/VOPCInstructions.td create mode 100644 lib/Target/AMDGPU/VOPInstructions.td create mode 100644 lib/Target/ARM/ARMBasicBlockInfo.h create mode 100644 lib/Target/ARM/ARMCallLowering.cpp create mode 100644 lib/Target/ARM/ARMCallLowering.h create mode 100644 lib/Target/ARM/ARMComputeBlockSize.cpp create mode 100644 lib/Target/ARM/ARMInstructionSelector.cpp create mode 100644 lib/Target/ARM/ARMInstructionSelector.h create mode 100644 lib/Target/ARM/ARMLegalizerInfo.cpp create mode 100644 lib/Target/ARM/ARMLegalizerInfo.h create mode 100644 lib/Target/ARM/ARMRegisterBankInfo.cpp create mode 100644 lib/Target/ARM/ARMRegisterBankInfo.h create mode 100644 lib/Target/ARM/ARMScheduleR52.td create mode 100644 lib/Target/AVR/AVRAsmPrinter.cpp create mode 100644 lib/Target/AVR/AVRDevices.td create mode 100644 lib/Target/AVR/AVRExpandPseudoInsts.cpp create mode 100644 lib/Target/AVR/AVRFrameLowering.cpp create mode 100644 lib/Target/AVR/AVRISelDAGToDAG.cpp create mode 100644 lib/Target/AVR/AVRISelLowering.cpp create mode 100644 lib/Target/AVR/AVRInstrumentFunctions.cpp create mode 100644 lib/Target/AVR/AVRMCInstLower.cpp create mode 100644 lib/Target/AVR/AVRMCInstLower.h create mode 100644 lib/Target/AVR/AVRRelaxMemOperations.cpp create mode 100644 lib/Target/AVR/AsmParser/AVRAsmParser.cpp create mode 100644 lib/Target/AVR/AsmParser/CMakeLists.txt create mode 100644 lib/Target/AVR/AsmParser/LLVMBuild.txt create mode 100644 lib/Target/AVR/Disassembler/AVRDisassembler.cpp create mode 100644 lib/Target/AVR/Disassembler/CMakeLists.txt create mode 100644 lib/Target/AVR/Disassembler/LLVMBuild.txt create mode 100644 lib/Target/AVR/InstPrinter/AVRInstPrinter.cpp create mode 100644 lib/Target/AVR/InstPrinter/AVRInstPrinter.h create mode 100644 lib/Target/AVR/InstPrinter/CMakeLists.txt create mode 100644 lib/Target/AVR/InstPrinter/LLVMBuild.txt create mode 100644 lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp create mode 100644 lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h create mode 100644 lib/Target/AVR/MCTargetDesc/AVRELFObjectWriter.cpp create mode 100644 lib/Target/AVR/MCTargetDesc/AVRFixupKinds.h create mode 100644 lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.cpp create mode 100644 lib/Target/AVR/MCTargetDesc/AVRMCCodeEmitter.h create mode 100644 lib/Target/AVR/MCTargetDesc/AVRMCExpr.cpp create mode 100644 lib/Target/AVR/MCTargetDesc/AVRMCExpr.h create mode 100644 lib/Target/AVR/MCTargetDesc/AVRMCTargetDesc.cpp create mode 100644 lib/Target/AVR/README.md create mode 100644 lib/Target/BPF/Disassembler/BPFDisassembler.cpp create mode 100644 lib/Target/BPF/Disassembler/CMakeLists.txt create mode 100644 lib/Target/BPF/Disassembler/LLVMBuild.txt create mode 100644 lib/Target/Hexagon/HexagonConstPropagation.cpp create mode 100644 lib/Target/Hexagon/HexagonHazardRecognizer.cpp create mode 100644 lib/Target/Hexagon/HexagonHazardRecognizer.h create mode 100644 lib/Target/Hexagon/HexagonPatterns.td delete mode 100644 lib/Target/Hexagon/HexagonRDF.cpp delete mode 100644 lib/Target/Hexagon/HexagonRDF.h delete mode 100644 lib/Target/Hexagon/HexagonSelectCCInfo.td create mode 100644 lib/Target/Hexagon/HexagonVectorPrint.cpp create mode 100644 lib/Target/Mips/MipsScheduleGeneric.td delete mode 100644 lib/Target/NVPTX/NVPTXFavorNonGenericAddrSpaces.cpp create mode 100644 lib/Target/NVPTX/NVPTXLowerArgs.cpp delete mode 100644 lib/Target/NVPTX/NVPTXLowerKernelArgs.cpp create mode 100644 lib/Target/PowerPC/P9InstrResources.td create mode 100644 lib/Target/PowerPC/PPCScheduleP9.td create mode 100644 lib/Target/RISCV/CMakeLists.txt create mode 100644 lib/Target/RISCV/LLVMBuild.txt create mode 100644 lib/Target/RISCV/MCTargetDesc/CMakeLists.txt create mode 100644 lib/Target/RISCV/MCTargetDesc/LLVMBuild.txt create mode 100644 lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp create mode 100644 lib/Target/RISCV/MCTargetDesc/RISCVELFObjectWriter.cpp create mode 100644 lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.cpp create mode 100644 lib/Target/RISCV/MCTargetDesc/RISCVMCAsmInfo.h create mode 100644 lib/Target/RISCV/MCTargetDesc/RISCVMCCodeEmitter.cpp create mode 100644 lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.cpp create mode 100644 lib/Target/RISCV/MCTargetDesc/RISCVMCTargetDesc.h create mode 100644 lib/Target/RISCV/RISCV.td create mode 100644 lib/Target/RISCV/RISCVInstrFormats.td create mode 100644 lib/Target/RISCV/RISCVInstrInfo.td create mode 100644 lib/Target/RISCV/RISCVRegisterInfo.td create mode 100644 lib/Target/RISCV/RISCVTargetMachine.cpp create mode 100644 lib/Target/RISCV/RISCVTargetMachine.h create mode 100644 lib/Target/RISCV/TargetInfo/CMakeLists.txt create mode 100644 lib/Target/RISCV/TargetInfo/LLVMBuild.txt create mode 100644 lib/Target/RISCV/TargetInfo/RISCVTargetInfo.cpp create mode 100644 lib/Target/SystemZ/SystemZExpandPseudo.cpp create mode 100644 lib/Target/SystemZ/SystemZFeatures.td create mode 100644 lib/Target/SystemZ/SystemZHazardRecognizer.cpp create mode 100644 lib/Target/SystemZ/SystemZHazardRecognizer.h create mode 100644 lib/Target/SystemZ/SystemZMachineScheduler.cpp create mode 100644 lib/Target/SystemZ/SystemZMachineScheduler.h create mode 100644 lib/Target/SystemZ/SystemZSchedule.td create mode 100644 lib/Target/SystemZ/SystemZScheduleZ13.td create mode 100644 lib/Target/SystemZ/SystemZScheduleZ196.td create mode 100644 lib/Target/SystemZ/SystemZScheduleZEC12.td delete mode 100644 lib/Target/TargetRecip.cpp delete mode 100644 lib/Target/TargetSubtargetInfo.cpp create mode 100644 lib/Target/WebAssembly/WebAssemblyCallIndirectFixup.cpp create mode 100644 lib/Target/WebAssembly/WebAssemblyExplicitLocals.cpp create mode 100644 lib/Target/WebAssembly/WebAssemblyLowerEmscriptenEHSjLj.cpp create mode 100644 lib/Target/WebAssembly/WebAssemblyUtilities.cpp create mode 100644 lib/Target/WebAssembly/WebAssemblyUtilities.h create mode 100644 lib/Target/X86/X86CallLowering.cpp create mode 100644 lib/Target/X86/X86CallLowering.h create mode 100644 lib/Target/X86/X86CallingConv.cpp create mode 100755 lib/Target/X86/X86EvexToVex.cpp create mode 100644 lib/Target/X86/X86InstrFMA3Info.cpp create mode 100644 lib/Target/X86/X86InstrFMA3Info.h create mode 100755 lib/Target/X86/X86InstrTablesInfo.h create mode 100644 lib/Target/X86/X86InterleavedAccess.cpp (limited to 'lib/Target') diff --git a/lib/Target/AArch64/AArch64.h b/lib/Target/AArch64/AArch64.h index c767c75fce57..fd106a8d9b0b 100644 --- a/lib/Target/AArch64/AArch64.h +++ b/lib/Target/AArch64/AArch64.h @@ -30,12 +30,12 @@ FunctionPass *createAArch64DeadRegisterDefinitions(); FunctionPass *createAArch64RedundantCopyEliminationPass(); FunctionPass *createAArch64ConditionalCompares(); FunctionPass *createAArch64AdvSIMDScalar(); -FunctionPass *createAArch64BranchRelaxation(); FunctionPass *createAArch64ISelDag(AArch64TargetMachine &TM, CodeGenOpt::Level OptLevel); FunctionPass *createAArch64StorePairSuppressPass(); FunctionPass *createAArch64ExpandPseudoPass(); FunctionPass *createAArch64LoadStoreOptimizationPass(); +FunctionPass *createAArch64VectorByElementOptPass(); ModulePass *createAArch64PromoteConstantPass(); FunctionPass *createAArch64ConditionOptimizerPass(); FunctionPass *createAArch64AddressTypePromotionPass(); @@ -46,7 +46,21 @@ FunctionPass *createAArch64CleanupLocalDynamicTLSPass(); FunctionPass *createAArch64CollectLOHPass(); +void initializeAArch64A53Fix835769Pass(PassRegistry&); +void initializeAArch64A57FPLoadBalancingPass(PassRegistry&); +void initializeAArch64AddressTypePromotionPass(PassRegistry&); +void initializeAArch64AdvSIMDScalarPass(PassRegistry&); +void initializeAArch64CollectLOHPass(PassRegistry&); +void initializeAArch64ConditionalComparesPass(PassRegistry&); +void initializeAArch64ConditionOptimizerPass(PassRegistry&); +void initializeAArch64DeadRegisterDefinitionsPass(PassRegistry&); void initializeAArch64ExpandPseudoPass(PassRegistry&); +void initializeAArch64LoadStoreOptPass(PassRegistry&); +void initializeAArch64VectorByElementOptPass(PassRegistry&); +void initializeAArch64PromoteConstantPass(PassRegistry&); +void initializeAArch64RedundantCopyEliminationPass(PassRegistry&); +void initializeAArch64StorePairSuppressPass(PassRegistry&); +void initializeLDTLSCleanupPass(PassRegistry&); } // end namespace llvm #endif diff --git a/lib/Target/AArch64/AArch64.td b/lib/Target/AArch64/AArch64.td index b97a0f155dc2..c40391d5ad9d 100644 --- a/lib/Target/AArch64/AArch64.td +++ b/lib/Target/AArch64/AArch64.td @@ -35,6 +35,9 @@ def FeatureCRC : SubtargetFeature<"crc", "HasCRC", "true", def FeatureRAS : SubtargetFeature<"ras", "HasRAS", "true", "Enable ARMv8 Reliability, Availability and Serviceability Extensions">; +def FeatureLSE : SubtargetFeature<"lse", "HasLSE", "true", + "Enable ARMv8.1 Large System Extension (LSE) atomic instructions">; + def FeaturePerfMon : SubtargetFeature<"perfmon", "HasPerfMon", "true", "Enable ARMv8 PMUv3 Performance Monitors extension">; @@ -61,10 +64,6 @@ def FeatureReserveX18 : SubtargetFeature<"reserve-x18", "ReserveX18", "true", "Reserve X18, making it unavailable " "as a GPR">; -def FeatureMergeNarrowLd : SubtargetFeature<"merge-narrow-ld", - "MergeNarrowLoads", "true", - "Merge narrow load instructions">; - def FeatureUseAA : SubtargetFeature<"use-aa", "UseAA", "true", "Use alias analysis during codegen">; @@ -94,23 +93,28 @@ def FeatureAlternateSExtLoadCVTF32Pattern : SubtargetFeature< "alternate-sextload-cvt-f32-pattern", "UseAlternateSExtLoadCVTF32Pattern", "true", "Use alternative pattern for sextload convert to f32">; -def FeatureMacroOpFusion : SubtargetFeature< - "macroop-fusion", "HasMacroOpFusion", "true", - "CPU supports macro op fusion">; +def FeatureArithmeticBccFusion : SubtargetFeature< + "arith-bcc-fusion", "HasArithmeticBccFusion", "true", + "CPU fuses arithmetic+bcc operations">; + +def FeatureArithmeticCbzFusion : SubtargetFeature< + "arith-cbz-fusion", "HasArithmeticCbzFusion", "true", + "CPU fuses arithmetic + cbz/cbnz operations">; def FeatureDisableLatencySchedHeuristic : SubtargetFeature< "disable-latency-sched-heuristic", "DisableLatencySchedHeuristic", "true", "Disable latency scheduling heuristic">; def FeatureUseRSqrt : SubtargetFeature< - "use-reverse-square-root", "UseRSqrt", "true", "Use reverse square root">; + "use-reciprocal-square-root", "UseRSqrt", "true", + "Use the reciprocal square root approximation">; //===----------------------------------------------------------------------===// // Architectures. // def HasV8_1aOps : SubtargetFeature<"v8.1a", "HasV8_1aOps", "true", - "Support ARM v8.1a instructions", [FeatureCRC]>; + "Support ARM v8.1a instructions", [FeatureCRC, FeatureLSE]>; def HasV8_2aOps : SubtargetFeature<"v8.2a", "HasV8_2aOps", "true", "Support ARM v8.2a instructions", [HasV8_1aOps, FeatureRAS]>; @@ -143,8 +147,9 @@ include "AArch64SystemOperands.td" include "AArch64SchedA53.td" include "AArch64SchedA57.td" include "AArch64SchedCyclone.td" -include "AArch64SchedM1.td" +include "AArch64SchedFalkor.td" include "AArch64SchedKryo.td" +include "AArch64SchedM1.td" include "AArch64SchedVulcan.td" def ProcA35 : SubtargetFeature<"a35", "ARMProcFamily", "CortexA35", @@ -176,7 +181,6 @@ def ProcA57 : SubtargetFeature<"a57", "ARMProcFamily", "CortexA57", FeatureCrypto, FeatureCustomCheapAsMoveHandling, FeatureFPARMv8, - FeatureMergeNarrowLd, FeatureNEON, FeaturePerfMon, FeaturePostRAScheduler, @@ -207,7 +211,8 @@ def ProcCyclone : SubtargetFeature<"cyclone", "ARMProcFamily", "Cyclone", FeatureCrypto, FeatureDisableLatencySchedHeuristic, FeatureFPARMv8, - FeatureMacroOpFusion, + FeatureArithmeticBccFusion, + FeatureArithmeticCbzFusion, FeatureNEON, FeaturePerfMon, FeatureSlowMisaligned128Store, @@ -216,17 +221,31 @@ def ProcCyclone : SubtargetFeature<"cyclone", "ARMProcFamily", "Cyclone", ]>; def ProcExynosM1 : SubtargetFeature<"exynosm1", "ARMProcFamily", "ExynosM1", - "Samsung Exynos-M1 processors", [ - FeatureAvoidQuadLdStPairs, - FeatureCRC, - FeatureCrypto, - FeatureCustomCheapAsMoveHandling, - FeatureFPARMv8, - FeatureNEON, - FeaturePerfMon, - FeaturePostRAScheduler, - FeatureUseRSqrt - ]>; + "Samsung Exynos-M1 processors", + [FeatureAvoidQuadLdStPairs, + FeatureCRC, + FeatureCrypto, + FeatureCustomCheapAsMoveHandling, + FeatureFPARMv8, + FeatureNEON, + FeaturePerfMon, + FeaturePostRAScheduler, + FeatureSlowMisaligned128Store, + FeatureUseRSqrt, + FeatureZCZeroing]>; + +def ProcExynosM2 : SubtargetFeature<"exynosm2", "ARMProcFamily", "ExynosM1", + "Samsung Exynos-M2/M3 processors", + [FeatureAvoidQuadLdStPairs, + FeatureCRC, + FeatureCrypto, + FeatureCustomCheapAsMoveHandling, + FeatureFPARMv8, + FeatureNEON, + FeaturePerfMon, + FeaturePostRAScheduler, + FeatureSlowMisaligned128Store, + FeatureZCZeroing]>; def ProcKryo : SubtargetFeature<"kryo", "ARMProcFamily", "Kryo", "Qualcomm Kryo processors", [ @@ -234,7 +253,6 @@ def ProcKryo : SubtargetFeature<"kryo", "ARMProcFamily", "Kryo", FeatureCrypto, FeatureCustomCheapAsMoveHandling, FeatureFPARMv8, - FeatureMergeNarrowLd, FeatureNEON, FeaturePerfMon, FeaturePostRAScheduler, @@ -242,12 +260,21 @@ def ProcKryo : SubtargetFeature<"kryo", "ARMProcFamily", "Kryo", FeatureZCZeroing ]>; +def ProcFalkor : SubtargetFeature<"falkor", "ARMProcFamily", "Falkor", + "Qualcomm Falkor processors", [ + FeatureCRC, + FeatureCrypto, + FeatureFPARMv8, + FeatureNEON, + FeaturePerfMon + ]>; + def ProcVulcan : SubtargetFeature<"vulcan", "ARMProcFamily", "Vulcan", "Broadcom Vulcan processors", [ FeatureCRC, FeatureCrypto, FeatureFPARMv8, - FeatureMacroOpFusion, + FeatureArithmeticBccFusion, FeatureNEON, FeaturePostRAScheduler, FeaturePredictableSelectIsExpensive, @@ -270,6 +297,9 @@ def : ProcessorModel<"cortex-a72", CortexA57Model, [ProcA72]>; def : ProcessorModel<"cortex-a73", CortexA57Model, [ProcA73]>; def : ProcessorModel<"cyclone", CycloneModel, [ProcCyclone]>; def : ProcessorModel<"exynos-m1", ExynosM1Model, [ProcExynosM1]>; +def : ProcessorModel<"exynos-m2", ExynosM1Model, [ProcExynosM2]>; +def : ProcessorModel<"exynos-m3", ExynosM1Model, [ProcExynosM2]>; +def : ProcessorModel<"falkor", FalkorModel, [ProcFalkor]>; def : ProcessorModel<"kryo", KryoModel, [ProcKryo]>; def : ProcessorModel<"vulcan", VulcanModel, [ProcVulcan]>; diff --git a/lib/Target/AArch64/AArch64A53Fix835769.cpp b/lib/Target/AArch64/AArch64A53Fix835769.cpp index c2cca63f4977..e6afb42440a7 100644 --- a/lib/Target/AArch64/AArch64A53Fix835769.cpp +++ b/lib/Target/AArch64/AArch64A53Fix835769.cpp @@ -82,16 +82,18 @@ class AArch64A53Fix835769 : public MachineFunctionPass { public: static char ID; - explicit AArch64A53Fix835769() : MachineFunctionPass(ID) {} + explicit AArch64A53Fix835769() : MachineFunctionPass(ID) { + initializeAArch64A53Fix835769Pass(*PassRegistry::getPassRegistry()); + } bool runOnMachineFunction(MachineFunction &F) override; MachineFunctionProperties getRequiredProperties() const override { return MachineFunctionProperties().set( - MachineFunctionProperties::Property::AllVRegsAllocated); + MachineFunctionProperties::Property::NoVRegs); } - const char *getPassName() const override { + StringRef getPassName() const override { return "Workaround A53 erratum 835769 pass"; } @@ -107,6 +109,9 @@ char AArch64A53Fix835769::ID = 0; } // end anonymous namespace +INITIALIZE_PASS(AArch64A53Fix835769, "aarch64-fix-cortex-a53-835769-pass", + "AArch64 fix for A53 erratum 835769", false, false) + //===----------------------------------------------------------------------===// bool diff --git a/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp b/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp index 0465e59dc54a..0aa597bcdc56 100644 --- a/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp +++ b/lib/Target/AArch64/AArch64A57FPLoadBalancing.cpp @@ -95,10 +95,6 @@ static bool isMla(MachineInstr *MI) { } } -namespace llvm { -static void initializeAArch64A57FPLoadBalancingPass(PassRegistry &); -} - //===----------------------------------------------------------------------===// namespace { @@ -126,10 +122,10 @@ public: MachineFunctionProperties getRequiredProperties() const override { return MachineFunctionProperties().set( - MachineFunctionProperties::Property::AllVRegsAllocated); + MachineFunctionProperties::Property::NoVRegs); } - const char *getPassName() const override { + StringRef getPassName() const override { return "A57 FP Anti-dependency breaker"; } diff --git a/lib/Target/AArch64/AArch64AddressTypePromotion.cpp b/lib/Target/AArch64/AArch64AddressTypePromotion.cpp index 4846ef08c983..0cbb2db1134a 100644 --- a/lib/Target/AArch64/AArch64AddressTypePromotion.cpp +++ b/lib/Target/AArch64/AArch64AddressTypePromotion.cpp @@ -46,10 +46,6 @@ using namespace llvm; #define DEBUG_TYPE "aarch64-type-promotion" -static cl::opt -EnableAddressTypePromotion("aarch64-type-promotion", cl::Hidden, - cl::desc("Enable the type promotion pass"), - cl::init(true)); static cl::opt EnableMerge("aarch64-type-promotion-merge", cl::Hidden, cl::desc("Enable merging of redundant sexts when one is dominating" @@ -62,10 +58,6 @@ EnableMerge("aarch64-type-promotion-merge", cl::Hidden, // AArch64AddressTypePromotion //===----------------------------------------------------------------------===// -namespace llvm { -void initializeAArch64AddressTypePromotionPass(PassRegistry &); -} - namespace { class AArch64AddressTypePromotion : public FunctionPass { @@ -76,9 +68,7 @@ public: initializeAArch64AddressTypePromotionPass(*PassRegistry::getPassRegistry()); } - const char *getPassName() const override { - return AARCH64_TYPE_PROMO_NAME; - } + StringRef getPassName() const override { return AARCH64_TYPE_PROMO_NAME; } /// Iterate over the functions and promote the computation of interesting // sext instructions. @@ -481,7 +471,7 @@ bool AArch64AddressTypePromotion::runOnFunction(Function &F) { if (skipFunction(F)) return false; - if (!EnableAddressTypePromotion || F.isDeclaration()) + if (F.isDeclaration()) return false; Func = &F; ConsideredSExtType = Type::getInt64Ty(Func->getContext()); diff --git a/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp b/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp index d0a2dd3fa1fc..bc2320dd20b3 100644 --- a/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp +++ b/lib/Target/AArch64/AArch64AdvSIMDScalarPass.cpp @@ -61,10 +61,6 @@ STATISTIC(NumScalarInsnsUsed, "Number of scalar instructions used"); STATISTIC(NumCopiesDeleted, "Number of cross-class copies deleted"); STATISTIC(NumCopiesInserted, "Number of cross-class copies inserted"); -namespace llvm { -void initializeAArch64AdvSIMDScalarPass(PassRegistry &); -} - #define AARCH64_ADVSIMD_NAME "AdvSIMD Scalar Operation Optimization" namespace { @@ -94,9 +90,7 @@ public: bool runOnMachineFunction(MachineFunction &F) override; - const char *getPassName() const override { - return AARCH64_ADVSIMD_NAME; - } + StringRef getPassName() const override { return AARCH64_ADVSIMD_NAME; } void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesCFG(); diff --git a/lib/Target/AArch64/AArch64AsmPrinter.cpp b/lib/Target/AArch64/AArch64AsmPrinter.cpp index 22374f754603..b2d96a32fd3a 100644 --- a/lib/Target/AArch64/AArch64AsmPrinter.cpp +++ b/lib/Target/AArch64/AArch64AsmPrinter.cpp @@ -37,6 +37,9 @@ #include "llvm/MC/MCLinkerOptimizationHint.h" #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSymbol.h" +#include "llvm/MC/MCSymbolELF.h" +#include "llvm/MC/MCSectionELF.h" +#include "llvm/MC/MCSectionMachO.h" #include "llvm/Support/Debug.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Support/raw_ostream.h" @@ -56,9 +59,7 @@ public: : AsmPrinter(TM, std::move(Streamer)), MCInstLowering(OutContext, *this), SM(*this), AArch64FI(nullptr) {} - const char *getPassName() const override { - return "AArch64 Assembly Printer"; - } + StringRef getPassName() const override { return "AArch64 Assembly Printer"; } /// \brief Wrapper for MCInstLowering.lowerOperand() for the /// tblgen'erated pseudo lowering. @@ -70,6 +71,14 @@ public: const MachineInstr &MI); void LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM, const MachineInstr &MI); + + void LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI); + void LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI); + void LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI); + + void EmitXRayTable(); + void EmitSled(const MachineInstr &MI, SledKind Kind); + /// \brief tblgen'erated driver function for lowering simple MI->MC /// pseudo instructions. bool emitPseudoExpansionLowering(MCStreamer &OutStreamer, @@ -85,7 +94,9 @@ public: bool runOnMachineFunction(MachineFunction &F) override { AArch64FI = F.getInfo(); STI = static_cast(&F.getSubtarget()); - return AsmPrinter::runOnMachineFunction(F); + bool Result = AsmPrinter::runOnMachineFunction(F); + EmitXRayTable(); + return Result; } private: @@ -124,6 +135,114 @@ private: //===----------------------------------------------------------------------===// +void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_ENTER(const MachineInstr &MI) +{ + EmitSled(MI, SledKind::FUNCTION_ENTER); +} + +void AArch64AsmPrinter::LowerPATCHABLE_FUNCTION_EXIT(const MachineInstr &MI) +{ + EmitSled(MI, SledKind::FUNCTION_EXIT); +} + +void AArch64AsmPrinter::LowerPATCHABLE_TAIL_CALL(const MachineInstr &MI) +{ + EmitSled(MI, SledKind::TAIL_CALL); +} + +void AArch64AsmPrinter::EmitXRayTable() +{ + //TODO: merge the logic for ELF XRay sleds at a higher level, so to avoid + // code duplication as it is now for x86_64, ARM32 and AArch64. + if (Sleds.empty()) + return; + + auto PrevSection = OutStreamer->getCurrentSectionOnly(); + auto Fn = MF->getFunction(); + MCSection *Section; + + if (STI->isTargetELF()) { + if (Fn->hasComdat()) + Section = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS, + ELF::SHF_ALLOC | ELF::SHF_GROUP, 0, + Fn->getComdat()->getName()); + else + Section = OutContext.getELFSection("xray_instr_map", ELF::SHT_PROGBITS, + ELF::SHF_ALLOC); + } else if (STI->isTargetMachO()) { + Section = OutContext.getMachOSection("__DATA", "xray_instr_map", 0, + SectionKind::getReadOnlyWithRel()); + } else { + llvm_unreachable("Unsupported target"); + } + + // Before we switch over, we force a reference to a label inside the + // xray_instr_map section. Since EmitXRayTable() is always called just + // before the function's end, we assume that this is happening after the + // last return instruction. + // + // We then align the reference to 16 byte boundaries, which we determined + // experimentally to be beneficial to avoid causing decoder stalls. + MCSymbol *Tmp = OutContext.createTempSymbol("xray_synthetic_", true); + OutStreamer->EmitCodeAlignment(16); + OutStreamer->EmitSymbolValue(Tmp, 8, false); + OutStreamer->SwitchSection(Section); + OutStreamer->EmitLabel(Tmp); + for (const auto &Sled : Sleds) { + OutStreamer->EmitSymbolValue(Sled.Sled, 8); + OutStreamer->EmitSymbolValue(CurrentFnSym, 8); + auto Kind = static_cast(Sled.Kind); + OutStreamer->EmitBytes( + StringRef(reinterpret_cast(&Kind), 1)); + OutStreamer->EmitBytes( + StringRef(reinterpret_cast(&Sled.AlwaysInstrument), 1)); + OutStreamer->EmitZeros(14); + } + OutStreamer->SwitchSection(PrevSection); + + Sleds.clear(); +} + +void AArch64AsmPrinter::EmitSled(const MachineInstr &MI, SledKind Kind) +{ + static const int8_t NoopsInSledCount = 7; + // We want to emit the following pattern: + // + // .Lxray_sled_N: + // ALIGN + // B #32 + // ; 7 NOP instructions (28 bytes) + // .tmpN + // + // We need the 28 bytes (7 instructions) because at runtime, we'd be patching + // over the full 32 bytes (8 instructions) with the following pattern: + // + // STP X0, X30, [SP, #-16]! ; push X0 and the link register to the stack + // LDR W0, #12 ; W0 := function ID + // LDR X16,#12 ; X16 := addr of __xray_FunctionEntry or __xray_FunctionExit + // BLR X16 ; call the tracing trampoline + // ;DATA: 32 bits of function ID + // ;DATA: lower 32 bits of the address of the trampoline + // ;DATA: higher 32 bits of the address of the trampoline + // LDP X0, X30, [SP], #16 ; pop X0 and the link register from the stack + // + OutStreamer->EmitCodeAlignment(4); + auto CurSled = OutContext.createTempSymbol("xray_sled_", true); + OutStreamer->EmitLabel(CurSled); + auto Target = OutContext.createTempSymbol(); + + // Emit "B #32" instruction, which jumps over the next 28 bytes. + // The operand has to be the number of 4-byte instructions to jump over, + // including the current instruction. + EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::B).addImm(8)); + + for (int8_t I = 0; I < NoopsInSledCount; I++) + EmitToStreamer(*OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0)); + + OutStreamer->EmitLabel(Target); + recordSled(CurSled, MI, Kind); +} + void AArch64AsmPrinter::EmitEndOfAsmFile(Module &M) { const Triple &TT = TM.getTargetTriple(); if (TT.isOSBinFormatMachO()) { @@ -162,7 +281,7 @@ MCSymbol *AArch64AsmPrinter::GetCPISymbol(unsigned CPID) const { // Darwin uses a linker-private symbol name for constant-pools (to // avoid addends on the relocation?), ELF has no such concept and // uses a normal private symbol. - if (getDataLayout().getLinkerPrivateGlobalPrefix()[0]) + if (!getDataLayout().getLinkerPrivateGlobalPrefix().empty()) return OutContext.getOrCreateSymbol( Twine(getDataLayout().getLinkerPrivateGlobalPrefix()) + "CPI" + Twine(getFunctionNumber()) + "_" + Twine(CPID)); @@ -354,7 +473,7 @@ void AArch64AsmPrinter::PrintDebugValueComment(const MachineInstr *MI, void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM, const MachineInstr &MI) { - unsigned NumNOPBytes = MI.getOperand(1).getImm(); + unsigned NumNOPBytes = StackMapOpers(&MI).getNumPatchBytes(); SM.recordStackMap(MI); assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!"); @@ -386,7 +505,7 @@ void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM, PatchPointOpers Opers(&MI); - int64_t CallTarget = Opers.getMetaOper(PatchPointOpers::TargetPos).getImm(); + int64_t CallTarget = Opers.getCallTarget().getImm(); unsigned EncodedBytes = 0; if (CallTarget) { assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget && @@ -411,7 +530,7 @@ void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM, EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg)); } // Emit padding. - unsigned NumBytes = Opers.getMetaOper(PatchPointOpers::NBytesPos).getImm(); + unsigned NumBytes = Opers.getNumPatchBytes(); assert(NumBytes >= EncodedBytes && "Patchpoint can't request size less than the length of a call."); assert((NumBytes - EncodedBytes) % 4 == 0 && @@ -569,6 +688,18 @@ void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) { case TargetOpcode::PATCHPOINT: return LowerPATCHPOINT(*OutStreamer, SM, *MI); + + case TargetOpcode::PATCHABLE_FUNCTION_ENTER: + LowerPATCHABLE_FUNCTION_ENTER(*MI); + return; + + case TargetOpcode::PATCHABLE_FUNCTION_EXIT: + LowerPATCHABLE_FUNCTION_EXIT(*MI); + return; + + case TargetOpcode::PATCHABLE_TAIL_CALL: + LowerPATCHABLE_TAIL_CALL(*MI); + return; } // Finally, do the automated lowerings for everything else. @@ -579,7 +710,7 @@ void AArch64AsmPrinter::EmitInstruction(const MachineInstr *MI) { // Force static initialization. extern "C" void LLVMInitializeAArch64AsmPrinter() { - RegisterAsmPrinter X(TheAArch64leTarget); - RegisterAsmPrinter Y(TheAArch64beTarget); - RegisterAsmPrinter Z(TheARM64Target); + RegisterAsmPrinter X(getTheAArch64leTarget()); + RegisterAsmPrinter Y(getTheAArch64beTarget()); + RegisterAsmPrinter Z(getTheARM64Target()); } diff --git a/lib/Target/AArch64/AArch64BranchRelaxation.cpp b/lib/Target/AArch64/AArch64BranchRelaxation.cpp deleted file mode 100644 index 9ec6ae4118a4..000000000000 --- a/lib/Target/AArch64/AArch64BranchRelaxation.cpp +++ /dev/null @@ -1,520 +0,0 @@ -//===-- AArch64BranchRelaxation.cpp - AArch64 branch relaxation -----------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -//===----------------------------------------------------------------------===// - -#include "AArch64.h" -#include "AArch64InstrInfo.h" -#include "AArch64MachineFunctionInfo.h" -#include "AArch64Subtarget.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/MachineInstrBuilder.h" -#include "llvm/Support/CommandLine.h" -#include "llvm/Support/Debug.h" -#include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/Format.h" -#include "llvm/Support/raw_ostream.h" -using namespace llvm; - -#define DEBUG_TYPE "aarch64-branch-relax" - -static cl::opt -BranchRelaxation("aarch64-branch-relax", cl::Hidden, cl::init(true), - cl::desc("Relax out of range conditional branches")); - -static cl::opt -TBZDisplacementBits("aarch64-tbz-offset-bits", cl::Hidden, cl::init(14), - cl::desc("Restrict range of TB[N]Z instructions (DEBUG)")); - -static cl::opt -CBZDisplacementBits("aarch64-cbz-offset-bits", cl::Hidden, cl::init(19), - cl::desc("Restrict range of CB[N]Z instructions (DEBUG)")); - -static cl::opt -BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19), - cl::desc("Restrict range of Bcc instructions (DEBUG)")); - -STATISTIC(NumSplit, "Number of basic blocks split"); -STATISTIC(NumRelaxed, "Number of conditional branches relaxed"); - -namespace llvm { -void initializeAArch64BranchRelaxationPass(PassRegistry &); -} - -#define AARCH64_BR_RELAX_NAME "AArch64 branch relaxation pass" - -namespace { -class AArch64BranchRelaxation : public MachineFunctionPass { - /// BasicBlockInfo - Information about the offset and size of a single - /// basic block. - struct BasicBlockInfo { - /// Offset - Distance from the beginning of the function to the beginning - /// of this basic block. - /// - /// The offset is always aligned as required by the basic block. - unsigned Offset; - - /// Size - Size of the basic block in bytes. If the block contains - /// inline assembly, this is a worst case estimate. - /// - /// The size does not include any alignment padding whether from the - /// beginning of the block, or from an aligned jump table at the end. - unsigned Size; - - BasicBlockInfo() : Offset(0), Size(0) {} - - /// Compute the offset immediately following this block. If LogAlign is - /// specified, return the offset the successor block will get if it has - /// this alignment. - unsigned postOffset(unsigned LogAlign = 0) const { - unsigned PO = Offset + Size; - unsigned Align = 1 << LogAlign; - return (PO + Align - 1) / Align * Align; - } - }; - - SmallVector BlockInfo; - - MachineFunction *MF; - const AArch64InstrInfo *TII; - - bool relaxBranchInstructions(); - void scanFunction(); - MachineBasicBlock *splitBlockBeforeInstr(MachineInstr *MI); - void adjustBlockOffsets(MachineBasicBlock &MBB); - bool isBlockInRange(MachineInstr *MI, MachineBasicBlock *BB, unsigned Disp); - bool fixupConditionalBranch(MachineInstr *MI); - void computeBlockSize(const MachineBasicBlock &MBB); - unsigned getInstrOffset(MachineInstr *MI) const; - void dumpBBs(); - void verify(); - -public: - static char ID; - AArch64BranchRelaxation() : MachineFunctionPass(ID) { - initializeAArch64BranchRelaxationPass(*PassRegistry::getPassRegistry()); - } - - bool runOnMachineFunction(MachineFunction &MF) override; - - const char *getPassName() const override { - return AARCH64_BR_RELAX_NAME; - } -}; -char AArch64BranchRelaxation::ID = 0; -} - -INITIALIZE_PASS(AArch64BranchRelaxation, "aarch64-branch-relax", - AARCH64_BR_RELAX_NAME, false, false) - -/// verify - check BBOffsets, BBSizes, alignment of islands -void AArch64BranchRelaxation::verify() { -#ifndef NDEBUG - unsigned PrevNum = MF->begin()->getNumber(); - for (MachineBasicBlock &MBB : *MF) { - unsigned Align = MBB.getAlignment(); - unsigned Num = MBB.getNumber(); - assert(BlockInfo[Num].Offset % (1u << Align) == 0); - assert(!Num || BlockInfo[PrevNum].postOffset() <= BlockInfo[Num].Offset); - PrevNum = Num; - } -#endif -} - -/// print block size and offset information - debugging -void AArch64BranchRelaxation::dumpBBs() { - for (auto &MBB : *MF) { - const BasicBlockInfo &BBI = BlockInfo[MBB.getNumber()]; - dbgs() << format("BB#%u\toffset=%08x\t", MBB.getNumber(), BBI.Offset) - << format("size=%#x\n", BBI.Size); - } -} - -/// BBHasFallthrough - Return true if the specified basic block can fallthrough -/// into the block immediately after it. -static bool BBHasFallthrough(MachineBasicBlock *MBB) { - // Get the next machine basic block in the function. - MachineFunction::iterator MBBI(MBB); - // Can't fall off end of function. - auto NextBB = std::next(MBBI); - if (NextBB == MBB->getParent()->end()) - return false; - - for (MachineBasicBlock *S : MBB->successors()) - if (S == &*NextBB) - return true; - - return false; -} - -/// scanFunction - Do the initial scan of the function, building up -/// information about each block. -void AArch64BranchRelaxation::scanFunction() { - BlockInfo.clear(); - BlockInfo.resize(MF->getNumBlockIDs()); - - // First thing, compute the size of all basic blocks, and see if the function - // has any inline assembly in it. If so, we have to be conservative about - // alignment assumptions, as we don't know for sure the size of any - // instructions in the inline assembly. - for (MachineBasicBlock &MBB : *MF) - computeBlockSize(MBB); - - // Compute block offsets and known bits. - adjustBlockOffsets(*MF->begin()); -} - -/// computeBlockSize - Compute the size for MBB. -/// This function updates BlockInfo directly. -void AArch64BranchRelaxation::computeBlockSize(const MachineBasicBlock &MBB) { - unsigned Size = 0; - for (const MachineInstr &MI : MBB) - Size += TII->GetInstSizeInBytes(MI); - BlockInfo[MBB.getNumber()].Size = Size; -} - -/// getInstrOffset - Return the current offset of the specified machine -/// instruction from the start of the function. This offset changes as stuff is -/// moved around inside the function. -unsigned AArch64BranchRelaxation::getInstrOffset(MachineInstr *MI) const { - MachineBasicBlock *MBB = MI->getParent(); - - // The offset is composed of two things: the sum of the sizes of all MBB's - // before this instruction's block, and the offset from the start of the block - // it is in. - unsigned Offset = BlockInfo[MBB->getNumber()].Offset; - - // Sum instructions before MI in MBB. - for (MachineBasicBlock::iterator I = MBB->begin(); &*I != MI; ++I) { - assert(I != MBB->end() && "Didn't find MI in its own basic block?"); - Offset += TII->GetInstSizeInBytes(*I); - } - return Offset; -} - -void AArch64BranchRelaxation::adjustBlockOffsets(MachineBasicBlock &Start) { - unsigned PrevNum = Start.getNumber(); - for (auto &MBB : make_range(MachineFunction::iterator(Start), MF->end())) { - unsigned Num = MBB.getNumber(); - if (!Num) // block zero is never changed from offset zero. - continue; - // Get the offset and known bits at the end of the layout predecessor. - // Include the alignment of the current block. - unsigned LogAlign = MBB.getAlignment(); - BlockInfo[Num].Offset = BlockInfo[PrevNum].postOffset(LogAlign); - PrevNum = Num; - } -} - -/// Split the basic block containing MI into two blocks, which are joined by -/// an unconditional branch. Update data structures and renumber blocks to -/// account for this change and returns the newly created block. -/// NOTE: Successor list of the original BB is out of date after this function, -/// and must be updated by the caller! Other transforms follow using this -/// utility function, so no point updating now rather than waiting. -MachineBasicBlock * -AArch64BranchRelaxation::splitBlockBeforeInstr(MachineInstr *MI) { - MachineBasicBlock *OrigBB = MI->getParent(); - - // Create a new MBB for the code after the OrigBB. - MachineBasicBlock *NewBB = - MF->CreateMachineBasicBlock(OrigBB->getBasicBlock()); - MF->insert(++OrigBB->getIterator(), NewBB); - - // Splice the instructions starting with MI over to NewBB. - NewBB->splice(NewBB->end(), OrigBB, MI, OrigBB->end()); - - // Add an unconditional branch from OrigBB to NewBB. - // Note the new unconditional branch is not being recorded. - // There doesn't seem to be meaningful DebugInfo available; this doesn't - // correspond to anything in the source. - BuildMI(OrigBB, DebugLoc(), TII->get(AArch64::B)).addMBB(NewBB); - - // Insert an entry into BlockInfo to align it properly with the block numbers. - BlockInfo.insert(BlockInfo.begin() + NewBB->getNumber(), BasicBlockInfo()); - - // Figure out how large the OrigBB is. As the first half of the original - // block, it cannot contain a tablejump. The size includes - // the new jump we added. (It should be possible to do this without - // recounting everything, but it's very confusing, and this is rarely - // executed.) - computeBlockSize(*OrigBB); - - // Figure out how large the NewMBB is. As the second half of the original - // block, it may contain a tablejump. - computeBlockSize(*NewBB); - - // All BBOffsets following these blocks must be modified. - adjustBlockOffsets(*OrigBB); - - ++NumSplit; - - return NewBB; -} - -/// isBlockInRange - Returns true if the distance between specific MI and -/// specific BB can fit in MI's displacement field. -bool AArch64BranchRelaxation::isBlockInRange(MachineInstr *MI, - MachineBasicBlock *DestBB, - unsigned Bits) { - unsigned MaxOffs = ((1 << (Bits - 1)) - 1) << 2; - unsigned BrOffset = getInstrOffset(MI); - unsigned DestOffset = BlockInfo[DestBB->getNumber()].Offset; - - DEBUG(dbgs() << "Branch of destination BB#" << DestBB->getNumber() - << " from BB#" << MI->getParent()->getNumber() - << " max delta=" << MaxOffs << " from " << getInstrOffset(MI) - << " to " << DestOffset << " offset " - << int(DestOffset - BrOffset) << "\t" << *MI); - - // Branch before the Dest. - if (BrOffset <= DestOffset) - return (DestOffset - BrOffset <= MaxOffs); - return (BrOffset - DestOffset <= MaxOffs); -} - -static bool isConditionalBranch(unsigned Opc) { - switch (Opc) { - default: - return false; - case AArch64::TBZW: - case AArch64::TBNZW: - case AArch64::TBZX: - case AArch64::TBNZX: - case AArch64::CBZW: - case AArch64::CBNZW: - case AArch64::CBZX: - case AArch64::CBNZX: - case AArch64::Bcc: - return true; - } -} - -static MachineBasicBlock *getDestBlock(MachineInstr *MI) { - switch (MI->getOpcode()) { - default: - llvm_unreachable("unexpected opcode!"); - case AArch64::TBZW: - case AArch64::TBNZW: - case AArch64::TBZX: - case AArch64::TBNZX: - return MI->getOperand(2).getMBB(); - case AArch64::CBZW: - case AArch64::CBNZW: - case AArch64::CBZX: - case AArch64::CBNZX: - case AArch64::Bcc: - return MI->getOperand(1).getMBB(); - } -} - -static unsigned getOppositeConditionOpcode(unsigned Opc) { - switch (Opc) { - default: - llvm_unreachable("unexpected opcode!"); - case AArch64::TBNZW: return AArch64::TBZW; - case AArch64::TBNZX: return AArch64::TBZX; - case AArch64::TBZW: return AArch64::TBNZW; - case AArch64::TBZX: return AArch64::TBNZX; - case AArch64::CBNZW: return AArch64::CBZW; - case AArch64::CBNZX: return AArch64::CBZX; - case AArch64::CBZW: return AArch64::CBNZW; - case AArch64::CBZX: return AArch64::CBNZX; - case AArch64::Bcc: return AArch64::Bcc; // Condition is an operand for Bcc. - } -} - -static unsigned getBranchDisplacementBits(unsigned Opc) { - switch (Opc) { - default: - llvm_unreachable("unexpected opcode!"); - case AArch64::TBNZW: - case AArch64::TBZW: - case AArch64::TBNZX: - case AArch64::TBZX: - return TBZDisplacementBits; - case AArch64::CBNZW: - case AArch64::CBZW: - case AArch64::CBNZX: - case AArch64::CBZX: - return CBZDisplacementBits; - case AArch64::Bcc: - return BCCDisplacementBits; - } -} - -static inline void invertBccCondition(MachineInstr *MI) { - assert(MI->getOpcode() == AArch64::Bcc && "Unexpected opcode!"); - AArch64CC::CondCode CC = (AArch64CC::CondCode)MI->getOperand(0).getImm(); - CC = AArch64CC::getInvertedCondCode(CC); - MI->getOperand(0).setImm((int64_t)CC); -} - -/// fixupConditionalBranch - Fix up a conditional branch whose destination is -/// too far away to fit in its displacement field. It is converted to an inverse -/// conditional branch + an unconditional branch to the destination. -bool AArch64BranchRelaxation::fixupConditionalBranch(MachineInstr *MI) { - MachineBasicBlock *DestBB = getDestBlock(MI); - - // Add an unconditional branch to the destination and invert the branch - // condition to jump over it: - // tbz L1 - // => - // tbnz L2 - // b L1 - // L2: - - // If the branch is at the end of its MBB and that has a fall-through block, - // direct the updated conditional branch to the fall-through block. Otherwise, - // split the MBB before the next instruction. - MachineBasicBlock *MBB = MI->getParent(); - MachineInstr *BMI = &MBB->back(); - bool NeedSplit = (BMI != MI) || !BBHasFallthrough(MBB); - - if (BMI != MI) { - if (std::next(MachineBasicBlock::iterator(MI)) == - std::prev(MBB->getLastNonDebugInstr()) && - BMI->getOpcode() == AArch64::B) { - // Last MI in the BB is an unconditional branch. Can we simply invert the - // condition and swap destinations: - // beq L1 - // b L2 - // => - // bne L2 - // b L1 - MachineBasicBlock *NewDest = BMI->getOperand(0).getMBB(); - if (isBlockInRange(MI, NewDest, - getBranchDisplacementBits(MI->getOpcode()))) { - DEBUG(dbgs() << " Invert condition and swap its destination with " - << *BMI); - BMI->getOperand(0).setMBB(DestBB); - unsigned OpNum = (MI->getOpcode() == AArch64::TBZW || - MI->getOpcode() == AArch64::TBNZW || - MI->getOpcode() == AArch64::TBZX || - MI->getOpcode() == AArch64::TBNZX) - ? 2 - : 1; - MI->getOperand(OpNum).setMBB(NewDest); - MI->setDesc(TII->get(getOppositeConditionOpcode(MI->getOpcode()))); - if (MI->getOpcode() == AArch64::Bcc) - invertBccCondition(MI); - return true; - } - } - } - - if (NeedSplit) { - // Analyze the branch so we know how to update the successor lists. - MachineBasicBlock *TBB, *FBB; - SmallVector Cond; - TII->analyzeBranch(*MBB, TBB, FBB, Cond, false); - - MachineBasicBlock *NewBB = splitBlockBeforeInstr(MI); - // No need for the branch to the next block. We're adding an unconditional - // branch to the destination. - int delta = TII->GetInstSizeInBytes(MBB->back()); - BlockInfo[MBB->getNumber()].Size -= delta; - MBB->back().eraseFromParent(); - // BlockInfo[SplitBB].Offset is wrong temporarily, fixed below - - // Update the successor lists according to the transformation to follow. - // Do it here since if there's no split, no update is needed. - MBB->replaceSuccessor(FBB, NewBB); - NewBB->addSuccessor(FBB); - } - MachineBasicBlock *NextBB = &*std::next(MachineFunction::iterator(MBB)); - - DEBUG(dbgs() << " Insert B to BB#" << DestBB->getNumber() - << ", invert condition and change dest. to BB#" - << NextBB->getNumber() << "\n"); - - // Insert a new conditional branch and a new unconditional branch. - MachineInstrBuilder MIB = BuildMI( - MBB, DebugLoc(), TII->get(getOppositeConditionOpcode(MI->getOpcode()))) - .addOperand(MI->getOperand(0)); - if (MI->getOpcode() == AArch64::TBZW || MI->getOpcode() == AArch64::TBNZW || - MI->getOpcode() == AArch64::TBZX || MI->getOpcode() == AArch64::TBNZX) - MIB.addOperand(MI->getOperand(1)); - if (MI->getOpcode() == AArch64::Bcc) - invertBccCondition(MIB); - MIB.addMBB(NextBB); - BlockInfo[MBB->getNumber()].Size += TII->GetInstSizeInBytes(MBB->back()); - BuildMI(MBB, DebugLoc(), TII->get(AArch64::B)).addMBB(DestBB); - BlockInfo[MBB->getNumber()].Size += TII->GetInstSizeInBytes(MBB->back()); - - // Remove the old conditional branch. It may or may not still be in MBB. - BlockInfo[MI->getParent()->getNumber()].Size -= TII->GetInstSizeInBytes(*MI); - MI->eraseFromParent(); - - // Finally, keep the block offsets up to date. - adjustBlockOffsets(*MBB); - return true; -} - -bool AArch64BranchRelaxation::relaxBranchInstructions() { - bool Changed = false; - // Relaxing branches involves creating new basic blocks, so re-eval - // end() for termination. - for (MachineFunction::iterator I = MF->begin(); I != MF->end(); ++I) { - MachineBasicBlock &MBB = *I; - MachineInstr &MI = *MBB.getFirstTerminator(); - if (isConditionalBranch(MI.getOpcode()) && - !isBlockInRange(&MI, getDestBlock(&MI), - getBranchDisplacementBits(MI.getOpcode()))) { - fixupConditionalBranch(&MI); - ++NumRelaxed; - Changed = true; - } - } - return Changed; -} - -bool AArch64BranchRelaxation::runOnMachineFunction(MachineFunction &mf) { - MF = &mf; - - // If the pass is disabled, just bail early. - if (!BranchRelaxation) - return false; - - DEBUG(dbgs() << "***** AArch64BranchRelaxation *****\n"); - - TII = (const AArch64InstrInfo *)MF->getSubtarget().getInstrInfo(); - - // Renumber all of the machine basic blocks in the function, guaranteeing that - // the numbers agree with the position of the block in the function. - MF->RenumberBlocks(); - - // Do the initial scan of the function, building up information about the - // sizes of each block. - scanFunction(); - - DEBUG(dbgs() << " Basic blocks before relaxation\n"); - DEBUG(dumpBBs()); - - bool MadeChange = false; - while (relaxBranchInstructions()) - MadeChange = true; - - // After a while, this might be made debug-only, but it is not expensive. - verify(); - - DEBUG(dbgs() << " Basic blocks after relaxation\n"); - DEBUG(dbgs() << '\n'; dumpBBs()); - - BlockInfo.clear(); - - return MadeChange; -} - -/// Returns an instance of the AArch64 Branch Relaxation pass. -FunctionPass *llvm::createAArch64BranchRelaxation() { - return new AArch64BranchRelaxation(); -} diff --git a/lib/Target/AArch64/AArch64CallLowering.cpp b/lib/Target/AArch64/AArch64CallLowering.cpp index e3522e63c21c..a4950af32097 100644 --- a/lib/Target/AArch64/AArch64CallLowering.cpp +++ b/lib/Target/AArch64/AArch64CallLowering.cpp @@ -16,9 +16,14 @@ #include "AArch64CallLowering.h" #include "AArch64ISelLowering.h" +#include "llvm/CodeGen/Analysis.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h" +#include "llvm/CodeGen/GlobalISel/Utils.h" #include "llvm/CodeGen/MachineInstrBuilder.h" - +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/Target/TargetRegisterInfo.h" +#include "llvm/Target/TargetSubtargetInfo.h" using namespace llvm; #ifndef LLVM_BUILD_GLOBAL_ISEL @@ -29,76 +34,284 @@ AArch64CallLowering::AArch64CallLowering(const AArch64TargetLowering &TLI) : CallLowering(&TLI) { } +struct IncomingArgHandler : public CallLowering::ValueHandler { + IncomingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI) + : ValueHandler(MIRBuilder, MRI) {} + + unsigned getStackAddress(uint64_t Size, int64_t Offset, + MachinePointerInfo &MPO) override { + auto &MFI = MIRBuilder.getMF().getFrameInfo(); + int FI = MFI.CreateFixedObject(Size, Offset, true); + MPO = MachinePointerInfo::getFixedStack(MIRBuilder.getMF(), FI); + unsigned AddrReg = MRI.createGenericVirtualRegister(LLT::pointer(0, 64)); + MIRBuilder.buildFrameIndex(AddrReg, FI); + return AddrReg; + } + + void assignValueToReg(unsigned ValVReg, unsigned PhysReg, + CCValAssign &VA) override { + markPhysRegUsed(PhysReg); + MIRBuilder.buildCopy(ValVReg, PhysReg); + // FIXME: assert extension + } + + void assignValueToAddress(unsigned ValVReg, unsigned Addr, uint64_t Size, + MachinePointerInfo &MPO, CCValAssign &VA) override { + auto MMO = MIRBuilder.getMF().getMachineMemOperand( + MPO, MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, Size, + 0); + MIRBuilder.buildLoad(ValVReg, Addr, *MMO); + } + + /// How the physical register gets marked varies between formal + /// parameters (it's a basic-block live-in), and a call instruction + /// (it's an implicit-def of the BL). + virtual void markPhysRegUsed(unsigned PhysReg) = 0; +}; + +struct FormalArgHandler : public IncomingArgHandler { + FormalArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI) + : IncomingArgHandler(MIRBuilder, MRI) {} + + void markPhysRegUsed(unsigned PhysReg) override { + MIRBuilder.getMBB().addLiveIn(PhysReg); + } +}; + +struct CallReturnHandler : public IncomingArgHandler { + CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, + MachineInstrBuilder MIB) + : IncomingArgHandler(MIRBuilder, MRI), MIB(MIB) {} + + void markPhysRegUsed(unsigned PhysReg) override { + MIB.addDef(PhysReg, RegState::Implicit); + } + + MachineInstrBuilder MIB; +}; + +struct OutgoingArgHandler : public CallLowering::ValueHandler { + OutgoingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI, + MachineInstrBuilder MIB) + : ValueHandler(MIRBuilder, MRI), MIB(MIB) {} + + unsigned getStackAddress(uint64_t Size, int64_t Offset, + MachinePointerInfo &MPO) override { + LLT p0 = LLT::pointer(0, 64); + LLT s64 = LLT::scalar(64); + unsigned SPReg = MRI.createGenericVirtualRegister(p0); + MIRBuilder.buildCopy(SPReg, AArch64::SP); + + unsigned OffsetReg = MRI.createGenericVirtualRegister(s64); + MIRBuilder.buildConstant(OffsetReg, Offset); + + unsigned AddrReg = MRI.createGenericVirtualRegister(p0); + MIRBuilder.buildGEP(AddrReg, SPReg, OffsetReg); + + MPO = MachinePointerInfo::getStack(MIRBuilder.getMF(), Offset); + return AddrReg; + } + + void assignValueToReg(unsigned ValVReg, unsigned PhysReg, + CCValAssign &VA) override { + MIB.addUse(PhysReg, RegState::Implicit); + unsigned ExtReg = extendRegister(ValVReg, VA); + MIRBuilder.buildCopy(PhysReg, ExtReg); + } + + void assignValueToAddress(unsigned ValVReg, unsigned Addr, uint64_t Size, + MachinePointerInfo &MPO, CCValAssign &VA) override { + auto MMO = MIRBuilder.getMF().getMachineMemOperand( + MPO, MachineMemOperand::MOStore, Size, 0); + MIRBuilder.buildStore(ValVReg, Addr, *MMO); + } + + MachineInstrBuilder MIB; +}; + +void AArch64CallLowering::splitToValueTypes(const ArgInfo &OrigArg, + SmallVectorImpl &SplitArgs, + const DataLayout &DL, + MachineRegisterInfo &MRI, + SplitArgTy PerformArgSplit) const { + const AArch64TargetLowering &TLI = *getTLI(); + LLVMContext &Ctx = OrigArg.Ty->getContext(); + + SmallVector SplitVTs; + SmallVector Offsets; + ComputeValueVTs(TLI, DL, OrigArg.Ty, SplitVTs, &Offsets, 0); + + if (SplitVTs.size() == 1) { + // No splitting to do, but we want to replace the original type (e.g. [1 x + // double] -> double). + SplitArgs.emplace_back(OrigArg.Reg, SplitVTs[0].getTypeForEVT(Ctx), + OrigArg.Flags); + return; + } + + unsigned FirstRegIdx = SplitArgs.size(); + for (auto SplitVT : SplitVTs) { + // FIXME: set split flags if they're actually used (e.g. i128 on AAPCS). + Type *SplitTy = SplitVT.getTypeForEVT(Ctx); + SplitArgs.push_back( + ArgInfo{MRI.createGenericVirtualRegister(LLT{*SplitTy, DL}), SplitTy, + OrigArg.Flags}); + } + + SmallVector BitOffsets; + for (auto Offset : Offsets) + BitOffsets.push_back(Offset * 8); + + SmallVector SplitRegs; + for (auto I = &SplitArgs[FirstRegIdx]; I != SplitArgs.end(); ++I) + SplitRegs.push_back(I->Reg); + + PerformArgSplit(SplitRegs, BitOffsets); +} + bool AArch64CallLowering::lowerReturn(MachineIRBuilder &MIRBuilder, - const Value *Val, unsigned VReg) const { - MachineInstr *Return = MIRBuilder.buildInstr(AArch64::RET_ReallyLR); - assert(Return && "Unable to build a return instruction?!"); + const Value *Val, unsigned VReg) const { + MachineFunction &MF = MIRBuilder.getMF(); + const Function &F = *MF.getFunction(); + auto MIB = MIRBuilder.buildInstrNoInsert(AArch64::RET_ReallyLR); assert(((Val && VReg) || (!Val && !VReg)) && "Return value without a vreg"); + bool Success = true; if (VReg) { - assert(Val->getType()->isIntegerTy() && "Type not supported yet"); - unsigned Size = Val->getType()->getPrimitiveSizeInBits(); - assert((Size == 64 || Size == 32) && "Size not supported yet"); - unsigned ResReg = (Size == 32) ? AArch64::W0 : AArch64::X0; - // Set the insertion point to be right before Return. - MIRBuilder.setInstr(*Return, /* Before */ true); - MachineInstr *Copy = - MIRBuilder.buildInstr(TargetOpcode::COPY, ResReg, VReg); - (void)Copy; - assert(Copy->getNextNode() == Return && - "The insertion did not happen where we expected"); - MachineInstrBuilder(MIRBuilder.getMF(), Return) - .addReg(ResReg, RegState::Implicit); + const AArch64TargetLowering &TLI = *getTLI(); + CCAssignFn *AssignFn = TLI.CCAssignFnForReturn(F.getCallingConv()); + MachineRegisterInfo &MRI = MF.getRegInfo(); + auto &DL = F.getParent()->getDataLayout(); + + ArgInfo OrigArg{VReg, Val->getType()}; + setArgFlags(OrigArg, AttributeSet::ReturnIndex, DL, F); + + SmallVector SplitArgs; + splitToValueTypes(OrigArg, SplitArgs, DL, MRI, + [&](ArrayRef Regs, ArrayRef Offsets) { + MIRBuilder.buildExtract(Regs, Offsets, VReg); + }); + + OutgoingArgHandler Handler(MIRBuilder, MRI, MIB); + Success = handleAssignments(MIRBuilder, AssignFn, SplitArgs, Handler); } + + MIRBuilder.insertInstr(MIB); + return Success; +} + +bool AArch64CallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder, + const Function &F, + ArrayRef VRegs) const { + auto &Args = F.getArgumentList(); + MachineFunction &MF = MIRBuilder.getMF(); + MachineBasicBlock &MBB = MIRBuilder.getMBB(); + MachineRegisterInfo &MRI = MF.getRegInfo(); + auto &DL = F.getParent()->getDataLayout(); + + SmallVector SplitArgs; + unsigned i = 0; + for (auto &Arg : Args) { + ArgInfo OrigArg{VRegs[i], Arg.getType()}; + setArgFlags(OrigArg, i + 1, DL, F); + splitToValueTypes(OrigArg, SplitArgs, DL, MRI, + [&](ArrayRef Regs, ArrayRef Offsets) { + MIRBuilder.buildSequence(VRegs[i], Regs, Offsets); + }); + ++i; + } + + if (!MBB.empty()) + MIRBuilder.setInstr(*MBB.begin()); + + const AArch64TargetLowering &TLI = *getTLI(); + CCAssignFn *AssignFn = + TLI.CCAssignFnForCall(F.getCallingConv(), /*IsVarArg=*/false); + + FormalArgHandler Handler(MIRBuilder, MRI); + if (!handleAssignments(MIRBuilder, AssignFn, SplitArgs, Handler)) + return false; + + // Move back to the end of the basic block. + MIRBuilder.setMBB(MBB); + return true; } -bool AArch64CallLowering::lowerFormalArguments( - MachineIRBuilder &MIRBuilder, const Function::ArgumentListType &Args, - const SmallVectorImpl &VRegs) const { +bool AArch64CallLowering::lowerCall(MachineIRBuilder &MIRBuilder, + const MachineOperand &Callee, + const ArgInfo &OrigRet, + ArrayRef OrigArgs) const { MachineFunction &MF = MIRBuilder.getMF(); const Function &F = *MF.getFunction(); + MachineRegisterInfo &MRI = MF.getRegInfo(); + auto &DL = F.getParent()->getDataLayout(); - SmallVector ArgLocs; - CCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs, F.getContext()); + SmallVector SplitArgs; + for (auto &OrigArg : OrigArgs) { + splitToValueTypes(OrigArg, SplitArgs, DL, MRI, + [&](ArrayRef Regs, ArrayRef Offsets) { + MIRBuilder.buildExtract(Regs, Offsets, OrigArg.Reg); + }); + } - unsigned NumArgs = Args.size(); - Function::const_arg_iterator CurOrigArg = Args.begin(); + // Find out which ABI gets to decide where things go. const AArch64TargetLowering &TLI = *getTLI(); - for (unsigned i = 0; i != NumArgs; ++i, ++CurOrigArg) { - MVT ValVT = MVT::getVT(CurOrigArg->getType()); - CCAssignFn *AssignFn = - TLI.CCAssignFnForCall(F.getCallingConv(), /*IsVarArg=*/false); - bool Res = - AssignFn(i, ValVT, ValVT, CCValAssign::Full, ISD::ArgFlagsTy(), CCInfo); - assert(!Res && "Call operand has unhandled type"); - (void)Res; - } - assert(ArgLocs.size() == Args.size() && - "We have a different number of location and args?!"); - for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) { - CCValAssign &VA = ArgLocs[i]; - - assert(VA.isRegLoc() && "Not yet implemented"); - // Transform the arguments in physical registers into virtual ones. - MIRBuilder.getMBB().addLiveIn(VA.getLocReg()); - MIRBuilder.buildInstr(TargetOpcode::COPY, VRegs[i], VA.getLocReg()); - - switch (VA.getLocInfo()) { - default: - llvm_unreachable("Unknown loc info!"); - case CCValAssign::Full: - break; - case CCValAssign::BCvt: - // We don't care about bitcast. - break; - case CCValAssign::AExt: - case CCValAssign::SExt: - case CCValAssign::ZExt: - // Zero/Sign extend the register. - assert(0 && "Not yet implemented"); - break; - } + CCAssignFn *CallAssignFn = + TLI.CCAssignFnForCall(F.getCallingConv(), /*IsVarArg=*/false); + + // Create a temporarily-floating call instruction so we can add the implicit + // uses of arg registers. + auto MIB = MIRBuilder.buildInstrNoInsert(Callee.isReg() ? AArch64::BLR + : AArch64::BL); + MIB.addOperand(Callee); + + // Tell the call which registers are clobbered. + auto TRI = MF.getSubtarget().getRegisterInfo(); + MIB.addRegMask(TRI->getCallPreservedMask(MF, F.getCallingConv())); + + // Do the actual argument marshalling. + SmallVector PhysRegs; + OutgoingArgHandler Handler(MIRBuilder, MRI, MIB); + if (!handleAssignments(MIRBuilder, CallAssignFn, SplitArgs, Handler)) + return false; + + // Now we can add the actual call instruction to the correct basic block. + MIRBuilder.insertInstr(MIB); + + // If Callee is a reg, since it is used by a target specific + // instruction, it must have a register class matching the + // constraint of that instruction. + if (Callee.isReg()) + MIB->getOperand(0).setReg(constrainOperandRegClass( + MF, *TRI, MRI, *MF.getSubtarget().getInstrInfo(), + *MF.getSubtarget().getRegBankInfo(), *MIB, MIB->getDesc(), + Callee.getReg(), 0)); + + // Finally we can copy the returned value back into its virtual-register. In + // symmetry with the arugments, the physical register must be an + // implicit-define of the call instruction. + CCAssignFn *RetAssignFn = TLI.CCAssignFnForReturn(F.getCallingConv()); + if (OrigRet.Reg) { + SplitArgs.clear(); + + SmallVector RegOffsets; + SmallVector SplitRegs; + splitToValueTypes(OrigRet, SplitArgs, DL, MRI, + [&](ArrayRef Regs, ArrayRef Offsets) { + std::copy(Offsets.begin(), Offsets.end(), + std::back_inserter(RegOffsets)); + std::copy(Regs.begin(), Regs.end(), + std::back_inserter(SplitRegs)); + }); + + CallReturnHandler Handler(MIRBuilder, MRI, MIB); + if (!handleAssignments(MIRBuilder, RetAssignFn, SplitArgs, Handler)) + return false; + + if (!RegOffsets.empty()) + MIRBuilder.buildSequence(OrigRet.Reg, SplitRegs, RegOffsets); } + return true; } diff --git a/lib/Target/AArch64/AArch64CallLowering.h b/lib/Target/AArch64/AArch64CallLowering.h index 411622803461..ce6676249df6 100644 --- a/lib/Target/AArch64/AArch64CallLowering.h +++ b/lib/Target/AArch64/AArch64CallLowering.h @@ -16,6 +16,7 @@ #define LLVM_LIB_TARGET_AARCH64_AARCH64CALLLOWERING #include "llvm/CodeGen/GlobalISel/CallLowering.h" +#include "llvm/CodeGen/ValueTypes.h" namespace llvm { @@ -27,10 +28,29 @@ class AArch64CallLowering: public CallLowering { bool lowerReturn(MachineIRBuilder &MIRBuiler, const Value *Val, unsigned VReg) const override; - bool - lowerFormalArguments(MachineIRBuilder &MIRBuilder, - const Function::ArgumentListType &Args, - const SmallVectorImpl &VRegs) const override; + + bool lowerFormalArguments(MachineIRBuilder &MIRBuilder, const Function &F, + ArrayRef VRegs) const override; + + bool lowerCall(MachineIRBuilder &MIRBuilder, const MachineOperand &Callee, + const ArgInfo &OrigRet, + ArrayRef OrigArgs) const override; + +private: + typedef std::function + RegHandler; + + typedef std::function + MemHandler; + + typedef std::function, ArrayRef)> + SplitArgTy; + + void splitToValueTypes(const ArgInfo &OrigArgInfo, + SmallVectorImpl &SplitArgs, + const DataLayout &DL, MachineRegisterInfo &MRI, + SplitArgTy SplitArg) const; }; } // End of namespace llvm; #endif diff --git a/lib/Target/AArch64/AArch64CallingConvention.td b/lib/Target/AArch64/AArch64CallingConvention.td index 178e3971640e..9058617768dd 100644 --- a/lib/Target/AArch64/AArch64CallingConvention.td +++ b/lib/Target/AArch64/AArch64CallingConvention.td @@ -23,6 +23,7 @@ class CCIfBigEndian : //===----------------------------------------------------------------------===// def CC_AArch64_AAPCS : CallingConv<[ + CCIfType<[iPTR], CCBitConvertToType>, CCIfType<[v2f32], CCBitConvertToType>, CCIfType<[v2f64, v4f32], CCBitConvertToType>, @@ -86,6 +87,7 @@ def CC_AArch64_AAPCS : CallingConv<[ ]>; def RetCC_AArch64_AAPCS : CallingConv<[ + CCIfType<[iPTR], CCBitConvertToType>, CCIfType<[v2f32], CCBitConvertToType>, CCIfType<[v2f64, v4f32], CCBitConvertToType>, @@ -98,6 +100,7 @@ def RetCC_AArch64_AAPCS : CallingConv<[ CCIfBigEndian>>, + CCIfType<[i1, i8, i16], CCPromoteToType>, CCIfType<[i32], CCAssignToRegWithShadow<[W0, W1, W2, W3, W4, W5, W6, W7], [X0, X1, X2, X3, X4, X5, X6, X7]>>, CCIfType<[i64], CCAssignToRegWithShadow<[X0, X1, X2, X3, X4, X5, X6, X7], @@ -121,6 +124,7 @@ def RetCC_AArch64_AAPCS : CallingConv<[ // + i128s (i.e. split i64s) don't need even registers. // + Stack slots are sized as needed rather than being at least 64-bit. def CC_AArch64_DarwinPCS : CallingConv<[ + CCIfType<[iPTR], CCBitConvertToType>, CCIfType<[v2f32], CCBitConvertToType>, CCIfType<[v2f64, v4f32, f128], CCBitConvertToType>, @@ -176,6 +180,7 @@ def CC_AArch64_DarwinPCS : CallingConv<[ ]>; def CC_AArch64_DarwinPCS_VarArg : CallingConv<[ + CCIfType<[iPTR], CCBitConvertToType>, CCIfType<[v2f32], CCBitConvertToType>, CCIfType<[v2f64, v4f32, f128], CCBitConvertToType>, @@ -243,6 +248,8 @@ def RetCC_AArch64_WebKit_JS : CallingConv<[ // register mapping". def CC_AArch64_GHC : CallingConv<[ + CCIfType<[iPTR], CCBitConvertToType>, + // Handle all vector types as either f64 or v2f64. CCIfType<[v1i64, v2i32, v4i16, v8i8, v2f32], CCBitConvertToType>, CCIfType<[v2i64, v4i32, v8i16, v16i8, v4f32, f128], CCBitConvertToType>, diff --git a/lib/Target/AArch64/AArch64CleanupLocalDynamicTLSPass.cpp b/lib/Target/AArch64/AArch64CleanupLocalDynamicTLSPass.cpp index 011a03622ba5..6f8dd3e3ac0c 100644 --- a/lib/Target/AArch64/AArch64CleanupLocalDynamicTLSPass.cpp +++ b/lib/Target/AArch64/AArch64CleanupLocalDynamicTLSPass.cpp @@ -33,10 +33,14 @@ #include "llvm/CodeGen/MachineRegisterInfo.h" using namespace llvm; +#define TLSCLEANUP_PASS_NAME "AArch64 Local Dynamic TLS Access Clean-up" + namespace { struct LDTLSCleanup : public MachineFunctionPass { static char ID; - LDTLSCleanup() : MachineFunctionPass(ID) {} + LDTLSCleanup() : MachineFunctionPass(ID) { + initializeLDTLSCleanupPass(*PassRegistry::getPassRegistry()); + } bool runOnMachineFunction(MachineFunction &MF) override { if (skipFunction(*MF.getFunction())) @@ -128,9 +132,7 @@ struct LDTLSCleanup : public MachineFunctionPass { return Copy; } - const char *getPassName() const override { - return "Local Dynamic TLS Access Clean-up"; - } + StringRef getPassName() const override { return TLSCLEANUP_PASS_NAME; } void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesCFG(); @@ -140,6 +142,9 @@ struct LDTLSCleanup : public MachineFunctionPass { }; } +INITIALIZE_PASS(LDTLSCleanup, "aarch64-local-dynamic-tls-cleanup", + TLSCLEANUP_PASS_NAME, false, false) + char LDTLSCleanup::ID = 0; FunctionPass *llvm::createAArch64CleanupLocalDynamicTLSPass() { return new LDTLSCleanup(); diff --git a/lib/Target/AArch64/AArch64CollectLOH.cpp b/lib/Target/AArch64/AArch64CollectLOH.cpp index 5eecb3a86856..7666011f75b6 100644 --- a/lib/Target/AArch64/AArch64CollectLOH.cpp +++ b/lib/Target/AArch64/AArch64CollectLOH.cpp @@ -138,6 +138,7 @@ BasicBlockScopeOnly("aarch64-collect-loh-bb-only", cl::Hidden, STATISTIC(NumADRPSimpleCandidate, "Number of simplifiable ADRP dominate by another"); +#ifndef NDEBUG STATISTIC(NumADRPComplexCandidate2, "Number of simplifiable ADRP reachable by 2 defs"); STATISTIC(NumADRPComplexCandidate3, @@ -156,18 +157,17 @@ STATISTIC(NumLDRToLDRWithImm, "Number of simplifiable LDR with imm reachable by LDR"); STATISTIC(NumADDToLDR, "Number of simplifiable LDR reachable by ADD"); STATISTIC(NumLDRToLDR, "Number of simplifiable LDR reachable by LDR"); +#endif // NDEBUG STATISTIC(NumADRPToLDR, "Number of simplifiable LDR reachable by ADRP"); +#ifndef NDEBUG STATISTIC(NumCplxLvl1, "Number of complex case of level 1"); STATISTIC(NumTooCplxLvl1, "Number of too complex case of level 1"); STATISTIC(NumCplxLvl2, "Number of complex case of level 2"); STATISTIC(NumTooCplxLvl2, "Number of too complex case of level 2"); +#endif // NDEBUG STATISTIC(NumADRSimpleCandidate, "Number of simplifiable ADRP + ADD"); STATISTIC(NumADRComplexCandidate, "Number of too complex ADRP + ADD"); -namespace llvm { -void initializeAArch64CollectLOHPass(PassRegistry &); -} - #define AARCH64_COLLECT_LOH_NAME "AArch64 Collect Linker Optimization Hint (LOH)" namespace { @@ -181,12 +181,10 @@ struct AArch64CollectLOH : public MachineFunctionPass { MachineFunctionProperties getRequiredProperties() const override { return MachineFunctionProperties().set( - MachineFunctionProperties::Property::AllVRegsAllocated); + MachineFunctionProperties::Property::NoVRegs); } - const char *getPassName() const override { - return AARCH64_COLLECT_LOH_NAME; - } + StringRef getPassName() const override { return AARCH64_COLLECT_LOH_NAME; } void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesAll(); @@ -631,7 +629,7 @@ static void computeADRP(const InstrToInstrs &UseToDefs, AArch64FI.addLOHDirective(MCLOH_AdrpAdrp, {L2, L1}); ++NumADRPSimpleCandidate; } -#ifdef DEBUG +#ifndef NDEBUG else if (Size == 2) ++NumADRPComplexCandidate2; else if (Size == 3) @@ -775,10 +773,10 @@ static void computeOthers(const InstrToInstrs &UseToDefs, AArch64FunctionInfo &AArch64FI, const MapRegToId &RegToId, const MachineDominatorTree *MDT) { SetOfMachineInstr *InvolvedInLOHs = nullptr; -#ifdef DEBUG +#ifndef NDEBUG SetOfMachineInstr InvolvedInLOHsStorage; InvolvedInLOHs = &InvolvedInLOHsStorage; -#endif // DEBUG +#endif // NDEBUG DEBUG(dbgs() << "*** Compute LOH for Others\n"); // ADRP -> ADD/LDR -> LDR/STR pattern. // Fall back to ADRP -> ADD pattern if we fail to catch the bigger pattern. @@ -819,7 +817,7 @@ static void computeOthers(const InstrToInstrs &UseToDefs, // PotentialCandidates are result of a chain ADRP -> ADD/LDR -> // A potential candidate becomes a candidate, if its current immediate // operand is zero and all nodes of the chain have respectively only one user -#ifdef DEBUG +#ifndef NDEBUG SetOfMachineInstr DefsOfPotentialCandidates; #endif for (const MachineInstr *Candidate : PotentialCandidates) { @@ -835,7 +833,7 @@ static void computeOthers(const InstrToInstrs &UseToDefs, getUses(DefsPerColorToUses, RegToId.find(Def->getOperand(0).getReg())->second, *Def); if (Users->size() > 1) { -#ifdef DEBUG +#ifndef NDEBUG // if all the uses of this def are in potential candidate, this is // a complex candidate of level 2. bool IsLevel2 = true; @@ -848,7 +846,7 @@ static void computeOthers(const InstrToInstrs &UseToDefs, } if (IsLevel2) ++NumCplxLvl2; -#endif // DEBUG +#endif // NDEBUG PotentialADROpportunities.insert(Def); continue; } @@ -863,7 +861,7 @@ static void computeOthers(const InstrToInstrs &UseToDefs, getUses(DefsPerColorToUses, RegToId.find(Def->getOperand(0).getReg())->second, *Def); if (Users->size() > 1) { -#ifdef DEBUG +#ifndef NDEBUG // if all the uses of this def are in the defs of the potential candidate, // this is a complex candidate of level 1 if (DefsOfPotentialCandidates.empty()) { @@ -885,7 +883,7 @@ static void computeOthers(const InstrToInstrs &UseToDefs, } if (!Found) ++NumCplxLvl1; -#endif // DEBUG +#endif // NDEBUG continue; } @@ -932,7 +930,7 @@ static void computeOthers(const InstrToInstrs &UseToDefs, "L2 already involved in LOH."); assert((!InvolvedInLOHs || InvolvedInLOHs->insert(Candidate)) && "Candidate already involved in LOH."); -#ifdef DEBUG +#ifndef NDEBUG // get the immediate of the load if (Candidate->getOperand(2).getImm() == 0) if (ImmediateDefOpc == AArch64::ADDXri) @@ -943,7 +941,7 @@ static void computeOthers(const InstrToInstrs &UseToDefs, ++NumADDToLDRWithImm; else ++NumLDRToLDRWithImm; -#endif // DEBUG +#endif // NDEBUG } } else { if (ImmediateDefOpc == AArch64::ADRP) @@ -966,7 +964,7 @@ static void computeOthers(const InstrToInstrs &UseToDefs, "L2 already involved in LOH."); assert((!InvolvedInLOHs || InvolvedInLOHs->insert(Candidate)) && "Candidate already involved in LOH."); -#ifdef DEBUG +#ifndef NDEBUG // get the immediate of the store if (Candidate->getOperand(2).getImm() == 0) if (ImmediateDefOpc == AArch64::ADDXri) diff --git a/lib/Target/AArch64/AArch64ConditionOptimizer.cpp b/lib/Target/AArch64/AArch64ConditionOptimizer.cpp index 8fff381d391e..8b186328d125 100644 --- a/lib/Target/AArch64/AArch64ConditionOptimizer.cpp +++ b/lib/Target/AArch64/AArch64ConditionOptimizer.cpp @@ -95,7 +95,9 @@ public: typedef std::tuple CmpInfo; static char ID; - AArch64ConditionOptimizer() : MachineFunctionPass(ID) {} + AArch64ConditionOptimizer() : MachineFunctionPass(ID) { + initializeAArch64ConditionOptimizerPass(*PassRegistry::getPassRegistry()); + } void getAnalysisUsage(AnalysisUsage &AU) const override; MachineInstr *findSuitableCompare(MachineBasicBlock *MBB); CmpInfo adjustCmp(MachineInstr *CmpMI, AArch64CC::CondCode Cmp); @@ -103,7 +105,7 @@ public: bool adjustTo(MachineInstr *CmpMI, AArch64CC::CondCode Cmp, MachineInstr *To, int ToImm); bool runOnMachineFunction(MachineFunction &MF) override; - const char *getPassName() const override { + StringRef getPassName() const override { return "AArch64 Condition Optimizer"; } }; @@ -111,10 +113,6 @@ public: char AArch64ConditionOptimizer::ID = 0; -namespace llvm { -void initializeAArch64ConditionOptimizerPass(PassRegistry &); -} - INITIALIZE_PASS_BEGIN(AArch64ConditionOptimizer, "aarch64-condopt", "AArch64 CondOpt Pass", false, false) INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) diff --git a/lib/Target/AArch64/AArch64ConditionalCompares.cpp b/lib/Target/AArch64/AArch64ConditionalCompares.cpp index e1b0dc724b39..da09b36cac9c 100644 --- a/lib/Target/AArch64/AArch64ConditionalCompares.cpp +++ b/lib/Target/AArch64/AArch64ConditionalCompares.cpp @@ -329,7 +329,7 @@ MachineInstr *SSACCmpConv::findConvertibleCompare(MachineBasicBlock *MBB) { ++NumImmRangeRejs; return nullptr; } - // Fall through. + LLVM_FALLTHROUGH; case AArch64::SUBSWrr: case AArch64::SUBSXrr: case AArch64::ADDSWrr: @@ -568,7 +568,7 @@ void SSACCmpConv::convert(SmallVectorImpl &RemovedBlocks) { CmpBB->removeSuccessor(Tail, true); Head->transferSuccessorsAndUpdatePHIs(CmpBB); DebugLoc TermDL = Head->getFirstTerminator()->getDebugLoc(); - TII->RemoveBranch(*Head); + TII->removeBranch(*Head); // If the Head terminator was one of the cbz / tbz branches with built-in // compare, we need to insert an explicit compare instruction in its place. @@ -732,10 +732,12 @@ class AArch64ConditionalCompares : public MachineFunctionPass { public: static char ID; - AArch64ConditionalCompares() : MachineFunctionPass(ID) {} + AArch64ConditionalCompares() : MachineFunctionPass(ID) { + initializeAArch64ConditionalComparesPass(*PassRegistry::getPassRegistry()); + } void getAnalysisUsage(AnalysisUsage &AU) const override; bool runOnMachineFunction(MachineFunction &MF) override; - const char *getPassName() const override { + StringRef getPassName() const override { return "AArch64 Conditional Compares"; } @@ -750,10 +752,6 @@ private: char AArch64ConditionalCompares::ID = 0; -namespace llvm { -void initializeAArch64ConditionalComparesPass(PassRegistry &); -} - INITIALIZE_PASS_BEGIN(AArch64ConditionalCompares, "aarch64-ccmp", "AArch64 CCMP Pass", false, false) INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) diff --git a/lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp b/lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp index 7a6f7669db5f..30e2b2310456 100644 --- a/lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp +++ b/lib/Target/AArch64/AArch64DeadRegisterDefinitionsPass.cpp @@ -6,9 +6,9 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// -// When allowed by the instruction, replace a dead definition of a GPR with -// the zero register. This makes the code a bit friendlier towards the -// hardware's register renamer. +/// \file When allowed by the instruction, replace a dead definition of a GPR +/// with the zero register. This makes the code a bit friendlier towards the +/// hardware's register renamer. //===----------------------------------------------------------------------===// #include "AArch64.h" @@ -17,43 +17,37 @@ #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Target/TargetSubtargetInfo.h" +#include "llvm/Target/TargetInstrInfo.h" using namespace llvm; #define DEBUG_TYPE "aarch64-dead-defs" STATISTIC(NumDeadDefsReplaced, "Number of dead definitions replaced"); -namespace llvm { -void initializeAArch64DeadRegisterDefinitionsPass(PassRegistry &); -} - #define AARCH64_DEAD_REG_DEF_NAME "AArch64 Dead register definitions" namespace { class AArch64DeadRegisterDefinitions : public MachineFunctionPass { private: const TargetRegisterInfo *TRI; - bool implicitlyDefinesOverlappingReg(unsigned Reg, const MachineInstr &MI); - bool processMachineBasicBlock(MachineBasicBlock &MBB); - bool usesFrameIndex(const MachineInstr &MI); + const MachineRegisterInfo *MRI; + const TargetInstrInfo *TII; + bool Changed; + void processMachineBasicBlock(MachineBasicBlock &MBB); public: static char ID; // Pass identification, replacement for typeid. - explicit AArch64DeadRegisterDefinitions() : MachineFunctionPass(ID) { + AArch64DeadRegisterDefinitions() : MachineFunctionPass(ID) { initializeAArch64DeadRegisterDefinitionsPass( *PassRegistry::getPassRegistry()); } bool runOnMachineFunction(MachineFunction &F) override; - MachineFunctionProperties getRequiredProperties() const override { - return MachineFunctionProperties().set( - MachineFunctionProperties::Property::AllVRegsAllocated); - } - - const char *getPassName() const override { return AARCH64_DEAD_REG_DEF_NAME; } + StringRef getPassName() const override { return AARCH64_DEAD_REG_DEF_NAME; } void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesCFG(); @@ -66,25 +60,16 @@ char AArch64DeadRegisterDefinitions::ID = 0; INITIALIZE_PASS(AArch64DeadRegisterDefinitions, "aarch64-dead-defs", AARCH64_DEAD_REG_DEF_NAME, false, false) -bool AArch64DeadRegisterDefinitions::implicitlyDefinesOverlappingReg( - unsigned Reg, const MachineInstr &MI) { - for (const MachineOperand &MO : MI.implicit_operands()) - if (MO.isReg() && MO.isDef()) - if (TRI->regsOverlap(Reg, MO.getReg())) - return true; - return false; -} - -bool AArch64DeadRegisterDefinitions::usesFrameIndex(const MachineInstr &MI) { - for (const MachineOperand &Op : MI.uses()) - if (Op.isFI()) +static bool usesFrameIndex(const MachineInstr &MI) { + for (const MachineOperand &MO : MI.uses()) + if (MO.isFI()) return true; return false; } -bool AArch64DeadRegisterDefinitions::processMachineBasicBlock( +void AArch64DeadRegisterDefinitions::processMachineBasicBlock( MachineBasicBlock &MBB) { - bool Changed = false; + const MachineFunction &MF = *MBB.getParent(); for (MachineInstr &MI : MBB) { if (usesFrameIndex(MI)) { // We need to skip this instruction because while it appears to have a @@ -99,62 +84,63 @@ bool AArch64DeadRegisterDefinitions::processMachineBasicBlock( DEBUG(dbgs() << " Ignoring, XZR or WZR already used by the instruction\n"); continue; } - for (int i = 0, e = MI.getDesc().getNumDefs(); i != e; ++i) { - MachineOperand &MO = MI.getOperand(i); - if (MO.isReg() && MO.isDead() && MO.isDef()) { - assert(!MO.isImplicit() && "Unexpected implicit def!"); - DEBUG(dbgs() << " Dead def operand #" << i << " in:\n "; - MI.print(dbgs())); - // Be careful not to change the register if it's a tied operand. - if (MI.isRegTiedToUseOperand(i)) { - DEBUG(dbgs() << " Ignoring, def is tied operand.\n"); - continue; - } - // Don't change the register if there's an implicit def of a subreg or - // superreg. - if (implicitlyDefinesOverlappingReg(MO.getReg(), MI)) { - DEBUG(dbgs() << " Ignoring, implicitly defines overlap reg.\n"); - continue; - } - // Make sure the instruction take a register class that contains - // the zero register and replace it if so. - unsigned NewReg; - switch (MI.getDesc().OpInfo[i].RegClass) { - default: - DEBUG(dbgs() << " Ignoring, register is not a GPR.\n"); - continue; - case AArch64::GPR32RegClassID: - NewReg = AArch64::WZR; - break; - case AArch64::GPR64RegClassID: - NewReg = AArch64::XZR; - break; - } - DEBUG(dbgs() << " Replacing with zero register. New:\n "); - MO.setReg(NewReg); - DEBUG(MI.print(dbgs())); - ++NumDeadDefsReplaced; - // Only replace one dead register, see check for zero register above. - break; + const MCInstrDesc &Desc = MI.getDesc(); + for (int I = 0, E = Desc.getNumDefs(); I != E; ++I) { + MachineOperand &MO = MI.getOperand(I); + if (!MO.isReg() || !MO.isDef()) + continue; + // We should not have any relevant physreg defs that are replacable by + // zero before register allocation. So we just check for dead vreg defs. + unsigned Reg = MO.getReg(); + if (!TargetRegisterInfo::isVirtualRegister(Reg) || + (!MO.isDead() && !MRI->use_nodbg_empty(Reg))) + continue; + assert(!MO.isImplicit() && "Unexpected implicit def!"); + DEBUG(dbgs() << " Dead def operand #" << I << " in:\n "; + MI.print(dbgs())); + // Be careful not to change the register if it's a tied operand. + if (MI.isRegTiedToUseOperand(I)) { + DEBUG(dbgs() << " Ignoring, def is tied operand.\n"); + continue; + } + const TargetRegisterClass *RC = TII->getRegClass(Desc, I, TRI, MF); + unsigned NewReg; + if (RC == nullptr) { + DEBUG(dbgs() << " Ignoring, register is not a GPR.\n"); + continue; + } else if (RC->contains(AArch64::WZR)) + NewReg = AArch64::WZR; + else if (RC->contains(AArch64::XZR)) + NewReg = AArch64::XZR; + else { + DEBUG(dbgs() << " Ignoring, register is not a GPR.\n"); + continue; } + DEBUG(dbgs() << " Replacing with zero register. New:\n "); + MO.setReg(NewReg); + MO.setIsDead(); + DEBUG(MI.print(dbgs())); + ++NumDeadDefsReplaced; + Changed = true; + // Only replace one dead register, see check for zero register above. + break; } } - return Changed; } // Scan the function for instructions that have a dead definition of a // register. Replace that register with the zero register when possible. bool AArch64DeadRegisterDefinitions::runOnMachineFunction(MachineFunction &MF) { - TRI = MF.getSubtarget().getRegisterInfo(); - bool Changed = false; - DEBUG(dbgs() << "***** AArch64DeadRegisterDefinitions *****\n"); - if (skipFunction(*MF.getFunction())) return false; + TRI = MF.getSubtarget().getRegisterInfo(); + TII = MF.getSubtarget().getInstrInfo(); + MRI = &MF.getRegInfo(); + DEBUG(dbgs() << "***** AArch64DeadRegisterDefinitions *****\n"); + Changed = false; for (auto &MBB : MF) - if (processMachineBasicBlock(MBB)) - Changed = true; + processMachineBasicBlock(MBB); return Changed; } diff --git a/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp index a1c98251cec4..fe1c0beee0eb 100644 --- a/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp +++ b/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp @@ -23,10 +23,6 @@ #include "llvm/Support/MathExtras.h" using namespace llvm; -namespace llvm { -void initializeAArch64ExpandPseudoPass(PassRegistry &); -} - #define AARCH64_EXPAND_PSEUDO_NAME "AArch64 pseudo instruction expansion pass" namespace { @@ -41,9 +37,7 @@ public: bool runOnMachineFunction(MachineFunction &Fn) override; - const char *getPassName() const override { - return AARCH64_EXPAND_PSEUDO_NAME; - } + StringRef getPassName() const override { return AARCH64_EXPAND_PSEUDO_NAME; } private: bool expandMBB(MachineBasicBlock &MBB); @@ -719,19 +713,19 @@ bool AArch64ExpandPseudo::expandCMP_SWAP_128( .addOperand(DesiredLo) .addImm(0); BuildMI(LoadCmpBB, DL, TII->get(AArch64::CSINCWr), StatusReg) - .addReg(AArch64::WZR) - .addReg(AArch64::WZR) + .addUse(AArch64::WZR) + .addUse(AArch64::WZR) .addImm(AArch64CC::EQ); BuildMI(LoadCmpBB, DL, TII->get(AArch64::SUBSXrs), AArch64::XZR) .addReg(DestHi.getReg(), getKillRegState(DestHi.isDead())) .addOperand(DesiredHi) .addImm(0); BuildMI(LoadCmpBB, DL, TII->get(AArch64::CSINCWr), StatusReg) - .addReg(StatusReg, RegState::Kill) - .addReg(StatusReg, RegState::Kill) + .addUse(StatusReg, RegState::Kill) + .addUse(StatusReg, RegState::Kill) .addImm(AArch64CC::EQ); BuildMI(LoadCmpBB, DL, TII->get(AArch64::CBNZW)) - .addReg(StatusReg, RegState::Kill) + .addUse(StatusReg, RegState::Kill) .addMBB(DoneBB); LoadCmpBB->addSuccessor(DoneBB); LoadCmpBB->addSuccessor(StoreBB); @@ -903,9 +897,14 @@ bool AArch64ExpandPseudo::expandMI(MachineBasicBlock &MBB, case AArch64::MOVi64imm: return expandMOVImm(MBB, MBBI, 64); case AArch64::RET_ReallyLR: { + // Hiding the LR use with RET_ReallyLR may lead to extra kills in the + // function and missing live-ins. We are fine in practice because callee + // saved register handling ensures the register value is restored before + // RET, but we need the undef flag here to appease the MachineVerifier + // liveness checks. MachineInstrBuilder MIB = BuildMI(MBB, MBBI, MI.getDebugLoc(), TII->get(AArch64::RET)) - .addReg(AArch64::LR); + .addReg(AArch64::LR, RegState::Undef); transferImpOps(MI, MIB, MIB); MI.eraseFromParent(); return true; diff --git a/lib/Target/AArch64/AArch64FastISel.cpp b/lib/Target/AArch64/AArch64FastISel.cpp index e2ab7ab79be1..fe2c2d4550a7 100644 --- a/lib/Target/AArch64/AArch64FastISel.cpp +++ b/lib/Target/AArch64/AArch64FastISel.cpp @@ -134,6 +134,7 @@ private: bool selectFRem(const Instruction *I); bool selectSDiv(const Instruction *I); bool selectGetElementPtr(const Instruction *I); + bool selectAtomicCmpXchg(const AtomicCmpXchgInst *I); // Utility helper routines. bool isTypeLegal(Type *Ty, MVT &VT); @@ -185,6 +186,8 @@ private: MachineMemOperand *MMO = nullptr); bool emitStore(MVT VT, unsigned SrcReg, Address Addr, MachineMemOperand *MMO = nullptr); + bool emitStoreRelease(MVT VT, unsigned SrcReg, unsigned AddrReg, + MachineMemOperand *MMO = nullptr); unsigned emitIntExt(MVT SrcVT, unsigned SrcReg, MVT DestVT, bool isZExt); unsigned emiti1Ext(unsigned SrcReg, MVT DestVT, bool isZExt); unsigned emitAdd(MVT RetVT, const Value *LHS, const Value *RHS, @@ -554,7 +557,7 @@ bool AArch64FastISel::computeAddress(const Value *Obj, Address &Addr, Type *Ty) for (gep_type_iterator GTI = gep_type_begin(U), E = gep_type_end(U); GTI != E; ++GTI) { const Value *Op = GTI.getOperand(); - if (StructType *STy = dyn_cast(*GTI)) { + if (StructType *STy = GTI.getStructTypeOrNull()) { const StructLayout *SL = DL.getStructLayout(STy); unsigned Idx = cast(Op)->getZExtValue(); TmpOffset += SL->getElementOffset(Idx); @@ -1997,6 +2000,28 @@ bool AArch64FastISel::selectLoad(const Instruction *I) { return true; } +bool AArch64FastISel::emitStoreRelease(MVT VT, unsigned SrcReg, + unsigned AddrReg, + MachineMemOperand *MMO) { + unsigned Opc; + switch (VT.SimpleTy) { + default: return false; + case MVT::i8: Opc = AArch64::STLRB; break; + case MVT::i16: Opc = AArch64::STLRH; break; + case MVT::i32: Opc = AArch64::STLRW; break; + case MVT::i64: Opc = AArch64::STLRX; break; + } + + const MCInstrDesc &II = TII.get(Opc); + SrcReg = constrainOperandRegClass(II, SrcReg, 0); + AddrReg = constrainOperandRegClass(II, AddrReg, 1); + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II) + .addReg(SrcReg) + .addReg(AddrReg) + .addMemOperand(MMO); + return true; +} + bool AArch64FastISel::emitStore(MVT VT, unsigned SrcReg, Address Addr, MachineMemOperand *MMO) { if (!TLI.allowsMisalignedMemoryAccesses(VT)) @@ -2071,8 +2096,7 @@ bool AArch64FastISel::selectStore(const Instruction *I) { // Verify we have a legal type before going any further. Currently, we handle // simple types that will directly fit in a register (i32/f32/i64/f64) or // those that can be sign or zero-extended to a basic operation (i1/i8/i16). - if (!isTypeSupported(Op0->getType(), VT, /*IsVectorAllowed=*/true) || - cast(I)->isAtomic()) + if (!isTypeSupported(Op0->getType(), VT, /*IsVectorAllowed=*/true)) return false; const Value *PtrV = I->getOperand(1); @@ -2109,9 +2133,23 @@ bool AArch64FastISel::selectStore(const Instruction *I) { if (!SrcReg) return false; + auto *SI = cast(I); + + // Try to emit a STLR for seq_cst/release. + if (SI->isAtomic()) { + AtomicOrdering Ord = SI->getOrdering(); + // The non-atomic instructions are sufficient for relaxed stores. + if (isReleaseOrStronger(Ord)) { + // The STLR addressing mode only supports a base reg; pass that directly. + unsigned AddrReg = getRegForValue(PtrV); + return emitStoreRelease(VT, SrcReg, AddrReg, + createMachineMemOperandFor(I)); + } + } + // See if we can handle this address. Address Addr; - if (!computeAddress(I->getOperand(1), Addr, I->getOperand(0)->getType())) + if (!computeAddress(PtrV, Addr, Op0->getType())) return false; if (!emitStore(VT, SrcReg, Addr, createMachineMemOperandFor(I))) @@ -2822,7 +2860,7 @@ bool AArch64FastISel::fastLowerArguments() { return false; CallingConv::ID CC = F->getCallingConv(); - if (CC != CallingConv::C) + if (CC != CallingConv::C && CC != CallingConv::Swift) return false; // Only handle simple cases of up to 8 GPR and FPR each. @@ -3328,8 +3366,8 @@ bool AArch64FastISel::fastLowerIntrinsicCall(const IntrinsicInst *II) { switch (II->getIntrinsicID()) { default: return false; case Intrinsic::frameaddress: { - MachineFrameInfo *MFI = FuncInfo.MF->getFrameInfo(); - MFI->setFrameAddressIsTaken(true); + MachineFrameInfo &MFI = FuncInfo.MF->getFrameInfo(); + MFI.setFrameAddressIsTaken(true); const AArch64RegisterInfo *RegInfo = static_cast(Subtarget->getRegisterInfo()); @@ -4847,7 +4885,7 @@ bool AArch64FastISel::selectGetElementPtr(const Instruction *I) { for (gep_type_iterator GTI = gep_type_begin(I), E = gep_type_end(I); GTI != E; ++GTI) { const Value *Idx = GTI.getOperand(); - if (auto *StTy = dyn_cast(*GTI)) { + if (auto *StTy = GTI.getStructTypeOrNull()) { unsigned Field = cast(Idx)->getZExtValue(); // N = N + Offset if (Field) @@ -4903,6 +4941,73 @@ bool AArch64FastISel::selectGetElementPtr(const Instruction *I) { return true; } +bool AArch64FastISel::selectAtomicCmpXchg(const AtomicCmpXchgInst *I) { + assert(TM.getOptLevel() == CodeGenOpt::None && + "cmpxchg survived AtomicExpand at optlevel > -O0"); + + auto *RetPairTy = cast(I->getType()); + Type *RetTy = RetPairTy->getTypeAtIndex(0U); + assert(RetPairTy->getTypeAtIndex(1U)->isIntegerTy(1) && + "cmpxchg has a non-i1 status result"); + + MVT VT; + if (!isTypeLegal(RetTy, VT)) + return false; + + const TargetRegisterClass *ResRC; + unsigned Opc, CmpOpc; + // This only supports i32/i64, because i8/i16 aren't legal, and the generic + // extractvalue selection doesn't support that. + if (VT == MVT::i32) { + Opc = AArch64::CMP_SWAP_32; + CmpOpc = AArch64::SUBSWrs; + ResRC = &AArch64::GPR32RegClass; + } else if (VT == MVT::i64) { + Opc = AArch64::CMP_SWAP_64; + CmpOpc = AArch64::SUBSXrs; + ResRC = &AArch64::GPR64RegClass; + } else { + return false; + } + + const MCInstrDesc &II = TII.get(Opc); + + const unsigned AddrReg = constrainOperandRegClass( + II, getRegForValue(I->getPointerOperand()), II.getNumDefs()); + const unsigned DesiredReg = constrainOperandRegClass( + II, getRegForValue(I->getCompareOperand()), II.getNumDefs() + 1); + const unsigned NewReg = constrainOperandRegClass( + II, getRegForValue(I->getNewValOperand()), II.getNumDefs() + 2); + + const unsigned ResultReg1 = createResultReg(ResRC); + const unsigned ResultReg2 = createResultReg(&AArch64::GPR32RegClass); + const unsigned ScratchReg = createResultReg(&AArch64::GPR32RegClass); + + // FIXME: MachineMemOperand doesn't support cmpxchg yet. + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, II) + .addDef(ResultReg1) + .addDef(ScratchReg) + .addUse(AddrReg) + .addUse(DesiredReg) + .addUse(NewReg); + + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(CmpOpc)) + .addDef(VT == MVT::i32 ? AArch64::WZR : AArch64::XZR) + .addUse(ResultReg1) + .addUse(DesiredReg) + .addImm(0); + + BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(AArch64::CSINCWr)) + .addDef(ResultReg2) + .addUse(AArch64::WZR) + .addUse(AArch64::WZR) + .addImm(AArch64CC::NE); + + assert((ResultReg1 + 1) == ResultReg2 && "Nonconsecutive result registers."); + updateValueMap(I, ResultReg1, 2); + return true; +} + bool AArch64FastISel::fastSelectInstruction(const Instruction *I) { switch (I->getOpcode()) { default: @@ -4976,6 +5081,8 @@ bool AArch64FastISel::fastSelectInstruction(const Instruction *I) { return selectFRem(I); case Instruction::GetElementPtr: return selectGetElementPtr(I); + case Instruction::AtomicCmpXchg: + return selectAtomicCmpXchg(cast(I)); } // fall-back to target-independent instruction selection. diff --git a/lib/Target/AArch64/AArch64FrameLowering.cpp b/lib/Target/AArch64/AArch64FrameLowering.cpp index 82111e5c7259..f5b8c35375f8 100644 --- a/lib/Target/AArch64/AArch64FrameLowering.cpp +++ b/lib/Target/AArch64/AArch64FrameLowering.cpp @@ -124,23 +124,23 @@ bool AArch64FrameLowering::canUseRedZone(const MachineFunction &MF) const { if (MF.getFunction()->hasFnAttribute(Attribute::NoRedZone)) return false; - const MachineFrameInfo *MFI = MF.getFrameInfo(); + const MachineFrameInfo &MFI = MF.getFrameInfo(); const AArch64FunctionInfo *AFI = MF.getInfo(); unsigned NumBytes = AFI->getLocalStackSize(); - return !(MFI->hasCalls() || hasFP(MF) || NumBytes > 128); + return !(MFI.hasCalls() || hasFP(MF) || NumBytes > 128); } /// hasFP - Return true if the specified function should have a dedicated frame /// pointer register. bool AArch64FrameLowering::hasFP(const MachineFunction &MF) const { - const MachineFrameInfo *MFI = MF.getFrameInfo(); + const MachineFrameInfo &MFI = MF.getFrameInfo(); const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo(); // Retain behavior of always omitting the FP for leaf functions when possible. - return (MFI->hasCalls() && + return (MFI.hasCalls() && MF.getTarget().Options.DisableFramePointerElim(MF)) || - MFI->hasVarSizedObjects() || MFI->isFrameAddressTaken() || - MFI->hasStackMap() || MFI->hasPatchPoint() || + MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken() || + MFI.hasStackMap() || MFI.hasPatchPoint() || RegInfo->needsStackRealignment(MF); } @@ -151,7 +151,7 @@ bool AArch64FrameLowering::hasFP(const MachineFunction &MF) const { /// included as part of the stack frame. bool AArch64FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const { - return !MF.getFrameInfo()->hasVarSizedObjects(); + return !MF.getFrameInfo().hasVarSizedObjects(); } MachineBasicBlock::iterator AArch64FrameLowering::eliminateCallFramePseudoInstr( @@ -203,23 +203,23 @@ MachineBasicBlock::iterator AArch64FrameLowering::eliminateCallFramePseudoInstr( void AArch64FrameLowering::emitCalleeSavedFrameMoves( MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI) const { MachineFunction &MF = *MBB.getParent(); - MachineFrameInfo *MFI = MF.getFrameInfo(); - MachineModuleInfo &MMI = MF.getMMI(); - const MCRegisterInfo *MRI = MMI.getContext().getRegisterInfo(); - const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo(); + MachineFrameInfo &MFI = MF.getFrameInfo(); + const TargetSubtargetInfo &STI = MF.getSubtarget(); + const MCRegisterInfo *MRI = STI.getRegisterInfo(); + const TargetInstrInfo *TII = STI.getInstrInfo(); DebugLoc DL = MBB.findDebugLoc(MBBI); // Add callee saved registers to move list. - const std::vector &CSI = MFI->getCalleeSavedInfo(); + const std::vector &CSI = MFI.getCalleeSavedInfo(); if (CSI.empty()) return; for (const auto &Info : CSI) { unsigned Reg = Info.getReg(); int64_t Offset = - MFI->getObjectOffset(Info.getFrameIdx()) - getOffsetOfLocalArea(); + MFI.getObjectOffset(Info.getFrameIdx()) - getOffsetOfLocalArea(); unsigned DwarfReg = MRI->getDwarfRegNum(Reg, true); - unsigned CFIIndex = MMI.addFrameInst( + unsigned CFIIndex = MF.addFrameInst( MCCFIInstruction::createOffset(nullptr, DwarfReg, Offset)); BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) .addCFIIndex(CFIIndex) @@ -286,7 +286,7 @@ bool AArch64FrameLowering::canUseAsPrologue( bool AArch64FrameLowering::shouldCombineCSRLocalStackBump( MachineFunction &MF, unsigned StackBumpBytes) const { AArch64FunctionInfo *AFI = MF.getInfo(); - const MachineFrameInfo *MFI = MF.getFrameInfo(); + const MachineFrameInfo &MFI = MF.getFrameInfo(); const AArch64Subtarget &Subtarget = MF.getSubtarget(); const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo(); @@ -298,7 +298,7 @@ bool AArch64FrameLowering::shouldCombineCSRLocalStackBump( if (StackBumpBytes >= 512) return false; - if (MFI->hasVarSizedObjects()) + if (MFI.hasVarSizedObjects()) return false; if (RegInfo->needsStackRealignment(MF)) @@ -407,7 +407,7 @@ static void fixupCalleeSaveRestoreStackOffset(MachineInstr &MI, void AArch64FrameLowering::emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const { MachineBasicBlock::iterator MBBI = MBB.begin(); - const MachineFrameInfo *MFI = MF.getFrameInfo(); + const MachineFrameInfo &MFI = MF.getFrameInfo(); const Function *Fn = MF.getFunction(); const AArch64Subtarget &Subtarget = MF.getSubtarget(); const AArch64RegisterInfo *RegInfo = Subtarget.getRegisterInfo(); @@ -426,7 +426,7 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, if (MF.getFunction()->getCallingConv() == CallingConv::GHC) return; - int NumBytes = (int)MFI->getStackSize(); + int NumBytes = (int)MFI.getStackSize(); if (!AFI->hasStackFrame()) { assert(!HasFP && "unexpected function without stack frame but with FP"); @@ -446,7 +446,7 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, // Label used to tie together the PROLOG_LABEL and the MachineMoves. MCSymbol *FrameLabel = MMI.getContext().createTempSymbol(); // Encode the stack size of the leaf function. - unsigned CFIIndex = MMI.addFrameInst( + unsigned CFIIndex = MF.addFrameInst( MCCFIInstruction::createDefCfaOffset(FrameLabel, -NumBytes)); BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) .addCFIIndex(CFIIndex) @@ -513,7 +513,7 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, MachineInstr::FrameSetup); if (NeedsRealignment) { - const unsigned Alignment = MFI->getMaxAlignment(); + const unsigned Alignment = MFI.getMaxAlignment(); const unsigned NrBitsToZero = countTrailingZeros(Alignment); assert(NrBitsToZero > 1); assert(scratchSPReg != AArch64::SP); @@ -621,15 +621,15 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, if (HasFP) { // Define the current CFA rule to use the provided FP. unsigned Reg = RegInfo->getDwarfRegNum(FramePtr, true); - unsigned CFIIndex = MMI.addFrameInst( + unsigned CFIIndex = MF.addFrameInst( MCCFIInstruction::createDefCfa(nullptr, Reg, 2 * StackGrowth)); BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) .addCFIIndex(CFIIndex) .setMIFlags(MachineInstr::FrameSetup); } else { // Encode the stack size of the leaf function. - unsigned CFIIndex = MMI.addFrameInst( - MCCFIInstruction::createDefCfaOffset(nullptr, -MFI->getStackSize())); + unsigned CFIIndex = MF.addFrameInst( + MCCFIInstruction::createDefCfaOffset(nullptr, -MFI.getStackSize())); BuildMI(MBB, MBBI, DL, TII->get(TargetOpcode::CFI_INSTRUCTION)) .addCFIIndex(CFIIndex) .setMIFlags(MachineInstr::FrameSetup); @@ -644,7 +644,7 @@ void AArch64FrameLowering::emitPrologue(MachineFunction &MF, void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const { MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr(); - MachineFrameInfo *MFI = MF.getFrameInfo(); + MachineFrameInfo &MFI = MF.getFrameInfo(); const AArch64Subtarget &Subtarget = MF.getSubtarget(); const TargetInstrInfo *TII = Subtarget.getInstrInfo(); DebugLoc DL; @@ -655,7 +655,7 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, IsTailCallReturn = RetOpcode == AArch64::TCRETURNdi || RetOpcode == AArch64::TCRETURNri; } - int NumBytes = MFI->getStackSize(); + int NumBytes = MFI.getStackSize(); const AArch64FunctionInfo *AFI = MF.getInfo(); // All calls are tail calls in GHC calling conv, and functions have no @@ -762,7 +762,7 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, // FIXME: Rather than doing the math here, we should instead just use // non-post-indexed loads for the restores if we aren't actually going to // be able to save any instructions. - if (MFI->hasVarSizedObjects() || AFI->isStackRealigned()) + if (MFI.hasVarSizedObjects() || AFI->isStackRealigned()) emitFrameOffset(MBB, LastPopI, DL, AArch64::SP, AArch64::FP, -CSStackSize + 16, TII, MachineInstr::FrameDestroy); else if (NumBytes) @@ -790,13 +790,13 @@ int AArch64FrameLowering::getFrameIndexReference(const MachineFunction &MF, int AArch64FrameLowering::resolveFrameIndexReference(const MachineFunction &MF, int FI, unsigned &FrameReg, bool PreferFP) const { - const MachineFrameInfo *MFI = MF.getFrameInfo(); + const MachineFrameInfo &MFI = MF.getFrameInfo(); const AArch64RegisterInfo *RegInfo = static_cast( MF.getSubtarget().getRegisterInfo()); const AArch64FunctionInfo *AFI = MF.getInfo(); - int FPOffset = MFI->getObjectOffset(FI) + 16; - int Offset = MFI->getObjectOffset(FI) + MFI->getStackSize(); - bool isFixed = MFI->isFixedObjectIndex(FI); + int FPOffset = MFI.getObjectOffset(FI) + 16; + int Offset = MFI.getObjectOffset(FI) + MFI.getStackSize(); + bool isFixed = MFI.isFixedObjectIndex(FI); // Use frame pointer to reference fixed objects. Use it for locals if // there are VLAs or a dynamically realigned SP (and thus the SP isn't @@ -821,7 +821,7 @@ int AArch64FrameLowering::resolveFrameIndexReference(const MachineFunction &MF, // using the FP regardless, though, as the SP offset is unknown // and we don't have a base pointer available. If an offset is // available via the FP and the SP, use whichever is closest. - if (PreferFP || MFI->hasVarSizedObjects() || FPOffset >= 0 || + if (PreferFP || MFI.hasVarSizedObjects() || FPOffset >= 0 || (FPOffset >= -256 && Offset > -FPOffset)) UseFP = true; } @@ -869,7 +869,7 @@ static bool produceCompactUnwindFrame(MachineFunction &MF) { Attrs.hasAttrSomewhere(Attribute::SwiftError)); } - +namespace { struct RegPairInfo { RegPairInfo() : Reg1(AArch64::NoRegister), Reg2(AArch64::NoRegister) {} unsigned Reg1; @@ -879,6 +879,7 @@ struct RegPairInfo { bool IsGPR; bool isPaired() const { return Reg2 != AArch64::NoRegister; } }; +} // end anonymous namespace static void computeCalleeSaveRegisterPairs( MachineFunction &MF, const std::vector &CSI, @@ -888,7 +889,7 @@ static void computeCalleeSaveRegisterPairs( return; AArch64FunctionInfo *AFI = MF.getInfo(); - MachineFrameInfo *MFI = MF.getFrameInfo(); + MachineFrameInfo &MFI = MF.getFrameInfo(); CallingConv::ID CC = MF.getFunction()->getCallingConv(); unsigned Count = CSI.size(); (void)CC; @@ -941,8 +942,8 @@ static void computeCalleeSaveRegisterPairs( // Round up size of non-pair to pair size if we need to pad the // callee-save area to ensure 16-byte alignment. Offset -= 16; - assert(MFI->getObjectAlignment(RPI.FrameIdx) <= 16); - MFI->setObjectAlignment(RPI.FrameIdx, 16); + assert(MFI.getObjectAlignment(RPI.FrameIdx) <= 16); + MFI.setObjectAlignment(RPI.FrameIdx, 16); AFI->setCalleeSaveStackHasFreeSpace(true); } else Offset -= RPI.isPaired() ? 16 : 8; @@ -1149,8 +1150,8 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF, // realistically that's not a big deal at this stage of the game. // The CSR spill slots have not been allocated yet, so estimateStackSize // won't include them. - MachineFrameInfo *MFI = MF.getFrameInfo(); - unsigned CFSize = MFI->estimateStackSize(MF) + 8 * NumRegsSpilled; + MachineFrameInfo &MFI = MF.getFrameInfo(); + unsigned CFSize = MFI.estimateStackSize(MF) + 8 * NumRegsSpilled; DEBUG(dbgs() << "Estimated stack frame size: " << CFSize << " bytes.\n"); bool BigStack = (CFSize >= 256); if (BigStack || !CanEliminateFrame || RegInfo->cannotEliminateFrame(MF)) @@ -1180,7 +1181,7 @@ void AArch64FrameLowering::determineCalleeSaves(MachineFunction &MF, // an emergency spill slot. if (!ExtraCSSpill) { const TargetRegisterClass *RC = &AArch64::GPR64RegClass; - int FI = MFI->CreateStackObject(RC->getSize(), RC->getAlignment(), false); + int FI = MFI.CreateStackObject(RC->getSize(), RC->getAlignment(), false); RS->addScavengingFrameIndex(FI); DEBUG(dbgs() << "No available CS registers, allocated fi#" << FI << " as the emergency spill slot.\n"); diff --git a/lib/Target/AArch64/AArch64GenRegisterBankInfo.def b/lib/Target/AArch64/AArch64GenRegisterBankInfo.def new file mode 100644 index 000000000000..e927d58ad612 --- /dev/null +++ b/lib/Target/AArch64/AArch64GenRegisterBankInfo.def @@ -0,0 +1,173 @@ +//===- AArch64GenRegisterBankInfo.def ----------------------------*- C++ -*-==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// This file defines all the static objects used by AArch64RegisterBankInfo. +/// \todo This should be generated by TableGen. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_BUILD_GLOBAL_ISEL +#error "You shouldn't build this" +#endif + +namespace llvm { +namespace AArch64 { + +RegisterBank GPRRegBank; +RegisterBank FPRRegBank; +RegisterBank CCRRegBank; + +RegisterBank *RegBanks[] = {&GPRRegBank, &FPRRegBank, &CCRRegBank}; + +// PartialMappings. +enum PartialMappingIdx { + PMI_None = -1, + PMI_GPR32 = 1, + PMI_GPR64, + PMI_FPR32, + PMI_FPR64, + PMI_FPR128, + PMI_FPR256, + PMI_FPR512, + PMI_FirstGPR = PMI_GPR32, + PMI_LastGPR = PMI_GPR64, + PMI_FirstFPR = PMI_FPR32, + PMI_LastFPR = PMI_FPR512, + PMI_Min = PMI_FirstGPR, +}; + +static unsigned getRegBankBaseIdxOffset(unsigned Size) { + assert(Size && "0-sized type!!"); + // Make anything smaller than 32 gets 32 + Size = ((Size + 31) / 32) * 32; + // 32 is 0, 64 is 1, 128 is 2, and so on. + return Log2_32(Size) - /*Log2_32(32)=*/ 5; +} + +RegisterBankInfo::PartialMapping PartMappings[] { + /* StartIdx, Length, RegBank */ + // 0: GPR 32-bit value. + {0, 32, GPRRegBank}, + // 1: GPR 64-bit value. + {0, 64, GPRRegBank}, + // 2: FPR 32-bit value. + {0, 32, FPRRegBank}, + // 3: FPR 64-bit value. + {0, 64, FPRRegBank}, + // 4: FPR 128-bit value. + {0, 128, FPRRegBank}, + // 5: FPR 256-bit value. + {0, 256, FPRRegBank}, + // 6: FPR 512-bit value. + {0, 512, FPRRegBank} +}; + +enum ValueMappingIdx { + First3OpsIdx = 0, + Last3OpsIdx = 18, + DistanceBetweenRegBanks = 3, + FirstCrossRegCpyIdx = 21, + LastCrossRegCpyIdx = 27, + DistanceBetweenCrossRegCpy = 2 +}; + +// ValueMappings. +RegisterBankInfo::ValueMapping ValMappings[]{ + /* BreakDown, NumBreakDowns */ + // 3-operands instructions (all binary operations should end up with one of + // those mapping). + // 0: GPR 32-bit value. <-- This must match First3OpsIdx. + {&PartMappings[PMI_GPR32 - PMI_Min], 1}, + {&PartMappings[PMI_GPR32 - PMI_Min], 1}, + {&PartMappings[PMI_GPR32 - PMI_Min], 1}, + // 3: GPR 64-bit value. + {&PartMappings[PMI_GPR64 - PMI_Min], 1}, + {&PartMappings[PMI_GPR64 - PMI_Min], 1}, + {&PartMappings[PMI_GPR64 - PMI_Min], 1}, + // 6: FPR 32-bit value. + {&PartMappings[PMI_FPR32 - PMI_Min], 1}, + {&PartMappings[PMI_FPR32 - PMI_Min], 1}, + {&PartMappings[PMI_FPR32 - PMI_Min], 1}, + // 9: FPR 64-bit value. + {&PartMappings[PMI_FPR64 - PMI_Min], 1}, + {&PartMappings[PMI_FPR64 - PMI_Min], 1}, + {&PartMappings[PMI_FPR64 - PMI_Min], 1}, + // 12: FPR 128-bit value. + {&PartMappings[PMI_FPR128 - PMI_Min], 1}, + {&PartMappings[PMI_FPR128 - PMI_Min], 1}, + {&PartMappings[PMI_FPR128 - PMI_Min], 1}, + // 15: FPR 256-bit value. + {&PartMappings[PMI_FPR256 - PMI_Min], 1}, + {&PartMappings[PMI_FPR256 - PMI_Min], 1}, + {&PartMappings[PMI_FPR256 - PMI_Min], 1}, + // 18: FPR 512-bit value. <-- This must match Last3OpsIdx. + {&PartMappings[PMI_FPR512 - PMI_Min], 1}, + {&PartMappings[PMI_FPR512 - PMI_Min], 1}, + {&PartMappings[PMI_FPR512 - PMI_Min], 1}, + // Cross register bank copies. + // 21: GPR 32-bit value to FPR 32-bit value. <-- This must match + // FirstCrossRegCpyIdx. + {&PartMappings[PMI_GPR32 - PMI_Min], 1}, + {&PartMappings[PMI_FPR32 - PMI_Min], 1}, + // 23: GPR 64-bit value to FPR 64-bit value. + {&PartMappings[PMI_GPR64 - PMI_Min], 1}, + {&PartMappings[PMI_FPR64 - PMI_Min], 1}, + // 25: FPR 32-bit value to GPR 32-bit value. + {&PartMappings[PMI_FPR32 - PMI_Min], 1}, + {&PartMappings[PMI_GPR32 - PMI_Min], 1}, + // 27: FPR 64-bit value to GPR 64-bit value. <-- This must match + // LastCrossRegCpyIdx. + {&PartMappings[PMI_FPR64 - PMI_Min], 1}, + {&PartMappings[PMI_GPR64 - PMI_Min], 1} +}; + +/// Get the pointer to the ValueMapping representing the RegisterBank +/// at \p RBIdx with a size of \p Size. +/// +/// The returned mapping works for instructions with the same kind of +/// operands for up to 3 operands. +/// +/// \pre \p RBIdx != PartialMappingIdx::None +const RegisterBankInfo::ValueMapping * +getValueMapping(PartialMappingIdx RBIdx, unsigned Size) { + assert(RBIdx != PartialMappingIdx::PMI_None && "No mapping needed for that"); + unsigned ValMappingIdx = First3OpsIdx + + (RBIdx - AArch64::PartialMappingIdx::PMI_Min + + getRegBankBaseIdxOffset(Size)) * + ValueMappingIdx::DistanceBetweenRegBanks; + assert(ValMappingIdx >= AArch64::First3OpsIdx && + ValMappingIdx <= AArch64::Last3OpsIdx && "Mapping out of bound"); + + return &ValMappings[ValMappingIdx]; +} + +/// Get the pointer to the ValueMapping of the operands of a copy +/// instruction from a GPR or FPR register to a GPR or FPR register +/// with a size of \p Size. +/// +/// If \p DstIsGPR is true, the destination of the copy is on GPR, +/// otherwise it is on FPR. Same thing for \p SrcIsGPR. +const RegisterBankInfo::ValueMapping * +getCopyMapping(bool DstIsGPR, bool SrcIsGPR, unsigned Size) { + PartialMappingIdx DstRBIdx = DstIsGPR ? PMI_FirstGPR : PMI_FirstFPR; + PartialMappingIdx SrcRBIdx = SrcIsGPR ? PMI_FirstGPR : PMI_FirstFPR; + if (DstRBIdx == SrcRBIdx) + return getValueMapping(DstRBIdx, Size); + assert(Size <= 64 && "GPR cannot handle that size"); + unsigned ValMappingIdx = + FirstCrossRegCpyIdx + + (DstRBIdx - PMI_Min + getRegBankBaseIdxOffset(Size)) * + ValueMappingIdx::DistanceBetweenCrossRegCpy; + assert(ValMappingIdx >= AArch64::FirstCrossRegCpyIdx && + ValMappingIdx <= AArch64::LastCrossRegCpyIdx && + "Mapping out of bound"); + return &ValMappings[ValMappingIdx]; +} + +} // End AArch64 namespace. +} // End llvm namespace. diff --git a/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp index 8d649250f656..3099383e5b32 100644 --- a/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -47,7 +47,7 @@ public: : SelectionDAGISel(tm, OptLevel), Subtarget(nullptr), ForCodeSize(false) {} - const char *getPassName() const override { + StringRef getPassName() const override { return "AArch64 Instruction Selection"; } @@ -349,7 +349,7 @@ bool AArch64DAGToDAGISel::SelectShiftedRegister(SDValue N, bool AllowROR, return false; if (ConstantSDNode *RHS = dyn_cast(N.getOperand(1))) { - unsigned BitSize = N.getValueType().getSizeInBits(); + unsigned BitSize = N.getValueSizeInBits(); unsigned Val = RHS->getZExtValue() & (BitSize - 1); unsigned ShVal = AArch64_AM::getShifterImm(ShType, Val); @@ -586,6 +586,11 @@ bool AArch64DAGToDAGISel::SelectArithExtendedRegister(SDValue N, SDValue &Reg, return false; Reg = N.getOperand(0); + + // Don't match if free 32-bit -> 64-bit zext can be used instead. + if (Ext == AArch64_AM::UXTW && + Reg->getValueType(0).getSizeInBits() == 32 && isDef32(*Reg.getNode())) + return false; } // AArch64 mandates that the RHS of the operation must use the smallest @@ -1149,6 +1154,12 @@ void AArch64DAGToDAGISel::SelectLoad(SDNode *N, unsigned NumVecs, unsigned Opc, CurDAG->getTargetExtractSubreg(SubRegIdx + i, dl, VT, SuperReg)); ReplaceUses(SDValue(N, NumVecs), SDValue(Ld, 1)); + + // Transfer memoperands. + MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); + MemOp[0] = cast(N)->getMemOperand(); + cast(Ld)->setMemRefs(MemOp, MemOp + 1); + CurDAG->RemoveDeadNode(N); } @@ -1197,6 +1208,11 @@ void AArch64DAGToDAGISel::SelectStore(SDNode *N, unsigned NumVecs, SDValue Ops[] = {RegSeq, N->getOperand(NumVecs + 2), N->getOperand(0)}; SDNode *St = CurDAG->getMachineNode(Opc, dl, N->getValueType(0), Ops); + // Transfer memoperands. + MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); + MemOp[0] = cast(N)->getMemOperand(); + cast(St)->setMemRefs(MemOp, MemOp + 1); + ReplaceNode(N, St); } @@ -1266,7 +1282,7 @@ void AArch64DAGToDAGISel::SelectLoadLane(SDNode *N, unsigned NumVecs, SmallVector Regs(N->op_begin() + 2, N->op_begin() + 2 + NumVecs); if (Narrow) - std::transform(Regs.begin(), Regs.end(), Regs.begin(), + transform(Regs, Regs.begin(), WidenVector(*CurDAG)); SDValue RegSeq = createQTuple(Regs); @@ -1305,7 +1321,7 @@ void AArch64DAGToDAGISel::SelectPostLoadLane(SDNode *N, unsigned NumVecs, SmallVector Regs(N->op_begin() + 1, N->op_begin() + 1 + NumVecs); if (Narrow) - std::transform(Regs.begin(), Regs.end(), Regs.begin(), + transform(Regs, Regs.begin(), WidenVector(*CurDAG)); SDValue RegSeq = createQTuple(Regs); @@ -1360,7 +1376,7 @@ void AArch64DAGToDAGISel::SelectStoreLane(SDNode *N, unsigned NumVecs, SmallVector Regs(N->op_begin() + 2, N->op_begin() + 2 + NumVecs); if (Narrow) - std::transform(Regs.begin(), Regs.end(), Regs.begin(), + transform(Regs, Regs.begin(), WidenVector(*CurDAG)); SDValue RegSeq = createQTuple(Regs); @@ -1390,7 +1406,7 @@ void AArch64DAGToDAGISel::SelectPostStoreLane(SDNode *N, unsigned NumVecs, SmallVector Regs(N->op_begin() + 1, N->op_begin() + 1 + NumVecs); if (Narrow) - std::transform(Regs.begin(), Regs.end(), Regs.begin(), + transform(Regs, Regs.begin(), WidenVector(*CurDAG)); SDValue RegSeq = createQTuple(Regs); @@ -1859,23 +1875,52 @@ static void getUsefulBitsFromBFM(SDValue Op, SDValue Orig, APInt &UsefulBits, uint64_t MSB = cast(Op.getOperand(3).getNode())->getZExtValue(); - if (Op.getOperand(1) == Orig) - return getUsefulBitsFromBitfieldMoveOpd(Op, UsefulBits, Imm, MSB, Depth); - APInt OpUsefulBits(UsefulBits); OpUsefulBits = 1; + APInt ResultUsefulBits(UsefulBits.getBitWidth(), 0); + ResultUsefulBits.flipAllBits(); + APInt Mask(UsefulBits.getBitWidth(), 0); + + getUsefulBits(Op, ResultUsefulBits, Depth + 1); + if (MSB >= Imm) { - OpUsefulBits = OpUsefulBits.shl(MSB - Imm + 1); + // The instruction is a BFXIL. + uint64_t Width = MSB - Imm + 1; + uint64_t LSB = Imm; + + OpUsefulBits = OpUsefulBits.shl(Width); --OpUsefulBits; - UsefulBits &= ~OpUsefulBits; - getUsefulBits(Op, UsefulBits, Depth + 1); + + if (Op.getOperand(1) == Orig) { + // Copy the low bits from the result to bits starting from LSB. + Mask = ResultUsefulBits & OpUsefulBits; + Mask = Mask.shl(LSB); + } + + if (Op.getOperand(0) == Orig) + // Bits starting from LSB in the input contribute to the result. + Mask |= (ResultUsefulBits & ~OpUsefulBits); } else { - OpUsefulBits = OpUsefulBits.shl(MSB + 1); + // The instruction is a BFI. + uint64_t Width = MSB + 1; + uint64_t LSB = UsefulBits.getBitWidth() - Imm; + + OpUsefulBits = OpUsefulBits.shl(Width); --OpUsefulBits; - UsefulBits = ~(OpUsefulBits.shl(OpUsefulBits.getBitWidth() - Imm)); - getUsefulBits(Op, UsefulBits, Depth + 1); + OpUsefulBits = OpUsefulBits.shl(LSB); + + if (Op.getOperand(1) == Orig) { + // Copy the bits from the result to the zero bits. + Mask = ResultUsefulBits & OpUsefulBits; + Mask = Mask.lshr(LSB); + } + + if (Op.getOperand(0) == Orig) + Mask |= (ResultUsefulBits & ~OpUsefulBits); } + + UsefulBits &= Mask; } static void getUsefulBitsForUse(SDNode *UserNode, APInt &UsefulBits, @@ -1931,7 +1976,7 @@ static void getUsefulBits(SDValue Op, APInt &UsefulBits, unsigned Depth) { return; // Initialize UsefulBits if (!Depth) { - unsigned Bitwidth = Op.getValueType().getScalarType().getSizeInBits(); + unsigned Bitwidth = Op.getScalarValueSizeInBits(); // At the beginning, assume every produced bits is useful UsefulBits = APInt(Bitwidth, 0); UsefulBits.flipAllBits(); diff --git a/lib/Target/AArch64/AArch64ISelLowering.cpp b/lib/Target/AArch64/AArch64ISelLowering.cpp index 06bfe340e758..4c98253878e4 100644 --- a/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -53,13 +53,6 @@ cl::opt EnableAArch64ELFLocalDynamicTLSGeneration( cl::desc("Allow AArch64 Local Dynamic TLS code generation"), cl::init(false)); -// Disabled for causing self-hosting failures once returned-attribute inference -// was enabled. -static cl::opt -EnableThisRetForwarding("aarch64-this-return-forwarding", cl::Hidden, - cl::desc("Directly forward this return"), - cl::init(false)); - /// Value type used for condition codes. static const MVT MVT_CC = MVT::i32; @@ -520,6 +513,12 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, setPrefFunctionAlignment(STI.getPrefFunctionAlignment()); setPrefLoopAlignment(STI.getPrefLoopAlignment()); + // Only change the limit for entries in a jump table if specified by + // the subtarget, but not at the command line. + unsigned MaxJT = STI.getMaximumJumpTableSize(); + if (MaxJT && getMaximumJumpTableSize() == 0) + setMaximumJumpTableSize(MaxJT); + setHasExtractBitsInsn(true); setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); @@ -764,7 +763,7 @@ void AArch64TargetLowering::computeKnownBitsForTargetNode( case Intrinsic::aarch64_ldxr: { unsigned BitWidth = KnownOne.getBitWidth(); EVT VT = cast(Op)->getMemoryVT(); - unsigned MemBits = VT.getScalarType().getSizeInBits(); + unsigned MemBits = VT.getScalarSizeInBits(); KnownZero |= APInt::getHighBitsSet(BitWidth, BitWidth - MemBits); return; } @@ -960,8 +959,10 @@ const char *AArch64TargetLowering::getTargetNodeName(unsigned Opcode) const { case AArch64ISD::ST4LANEpost: return "AArch64ISD::ST4LANEpost"; case AArch64ISD::SMULL: return "AArch64ISD::SMULL"; case AArch64ISD::UMULL: return "AArch64ISD::UMULL"; - case AArch64ISD::FRSQRTE: return "AArch64ISD::FRSQRTE"; case AArch64ISD::FRECPE: return "AArch64ISD::FRECPE"; + case AArch64ISD::FRECPS: return "AArch64ISD::FRECPS"; + case AArch64ISD::FRSQRTE: return "AArch64ISD::FRSQRTE"; + case AArch64ISD::FRSQRTS: return "AArch64ISD::FRSQRTS"; } return nullptr; } @@ -1186,7 +1187,8 @@ static void changeVectorFPCCToAArch64CC(ISD::CondCode CC, changeFPCCToAArch64CC(CC, CondCode, CondCode2); break; case ISD::SETUO: - Invert = true; // Fallthrough + Invert = true; + LLVM_FALLTHROUGH; case ISD::SETO: CondCode = AArch64CC::MI; CondCode2 = AArch64CC::GE; @@ -2136,7 +2138,7 @@ static bool isExtendedBUILD_VECTOR(SDNode *N, SelectionDAG &DAG, for (const SDValue &Elt : N->op_values()) { if (ConstantSDNode *C = dyn_cast(Elt)) { - unsigned EltSize = VT.getVectorElementType().getSizeInBits(); + unsigned EltSize = VT.getScalarSizeInBits(); unsigned HalfSize = EltSize / 2; if (isSigned) { if (!isIntN(HalfSize, C->getSExtValue())) @@ -2163,7 +2165,7 @@ static SDValue skipExtensionForVectorMULL(SDNode *N, SelectionDAG &DAG) { assert(N->getOpcode() == ISD::BUILD_VECTOR && "expected BUILD_VECTOR"); EVT VT = N->getValueType(0); SDLoc dl(N); - unsigned EltSize = VT.getVectorElementType().getSizeInBits() / 2; + unsigned EltSize = VT.getScalarSizeInBits() / 2; unsigned NumElts = VT.getVectorNumElements(); MVT TruncVT = MVT::getIntegerVT(EltSize); SmallVector Ops; @@ -2435,18 +2437,25 @@ CCAssignFn *AArch64TargetLowering::CCAssignFnForCall(CallingConv::ID CC, case CallingConv::Fast: case CallingConv::PreserveMost: case CallingConv::CXX_FAST_TLS: + case CallingConv::Swift: if (!Subtarget->isTargetDarwin()) return CC_AArch64_AAPCS; return IsVarArg ? CC_AArch64_DarwinPCS_VarArg : CC_AArch64_DarwinPCS; } } +CCAssignFn * +AArch64TargetLowering::CCAssignFnForReturn(CallingConv::ID CC) const { + return CC == CallingConv::WebKit_JS ? RetCC_AArch64_WebKit_JS + : RetCC_AArch64_AAPCS; +} + SDValue AArch64TargetLowering::LowerFormalArguments( SDValue Chain, CallingConv::ID CallConv, bool isVarArg, const SmallVectorImpl &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl &InVals) const { MachineFunction &MF = DAG.getMachineFunction(); - MachineFrameInfo *MFI = MF.getFrameInfo(); + MachineFrameInfo &MFI = MF.getFrameInfo(); // Assign locations to all of the incoming arguments. SmallVector ArgLocs; @@ -2499,7 +2508,7 @@ SDValue AArch64TargetLowering::LowerFormalArguments( // FIXME: This works on big-endian for composite byvals, which are the common // case. It should also work for fundamental types too. unsigned FrameIdx = - MFI->CreateFixedObject(8 * NumRegs, VA.getLocMemOffset(), false); + MFI.CreateFixedObject(8 * NumRegs, VA.getLocMemOffset(), false); SDValue FrameIdxN = DAG.getFrameIndex(FrameIdx, PtrVT); InVals.push_back(FrameIdxN); @@ -2564,7 +2573,7 @@ SDValue AArch64TargetLowering::LowerFormalArguments( !Ins[i].Flags.isInConsecutiveRegs()) BEAlign = 8 - ArgSize; - int FI = MFI->CreateFixedObject(ArgSize, ArgOffset + BEAlign, true); + int FI = MFI.CreateFixedObject(ArgSize, ArgOffset + BEAlign, true); // Create load nodes to retrieve arguments from the stack. SDValue FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout())); @@ -2614,7 +2623,7 @@ SDValue AArch64TargetLowering::LowerFormalArguments( unsigned StackOffset = CCInfo.getNextStackOffset(); // We currently pass all varargs at 8-byte alignment. StackOffset = ((StackOffset + 7) & ~7); - FuncInfo->setVarArgsStackIndex(MFI->CreateFixedObject(4, StackOffset, true)); + FuncInfo->setVarArgsStackIndex(MFI.CreateFixedObject(4, StackOffset, true)); } unsigned StackArgSize = CCInfo.getNextStackOffset(); @@ -2645,7 +2654,7 @@ void AArch64TargetLowering::saveVarArgRegisters(CCState &CCInfo, const SDLoc &DL, SDValue &Chain) const { MachineFunction &MF = DAG.getMachineFunction(); - MachineFrameInfo *MFI = MF.getFrameInfo(); + MachineFrameInfo &MFI = MF.getFrameInfo(); AArch64FunctionInfo *FuncInfo = MF.getInfo(); auto PtrVT = getPointerTy(DAG.getDataLayout()); @@ -2660,7 +2669,7 @@ void AArch64TargetLowering::saveVarArgRegisters(CCState &CCInfo, unsigned GPRSaveSize = 8 * (NumGPRArgRegs - FirstVariadicGPR); int GPRIdx = 0; if (GPRSaveSize != 0) { - GPRIdx = MFI->CreateStackObject(GPRSaveSize, 8, false); + GPRIdx = MFI.CreateStackObject(GPRSaveSize, 8, false); SDValue FIN = DAG.getFrameIndex(GPRIdx, PtrVT); @@ -2688,7 +2697,7 @@ void AArch64TargetLowering::saveVarArgRegisters(CCState &CCInfo, unsigned FPRSaveSize = 16 * (NumFPRArgRegs - FirstVariadicFPR); int FPRIdx = 0; if (FPRSaveSize != 0) { - FPRIdx = MFI->CreateStackObject(FPRSaveSize, 16, false); + FPRIdx = MFI.CreateStackObject(FPRSaveSize, 16, false); SDValue FIN = DAG.getFrameIndex(FPRIdx, PtrVT); @@ -2735,7 +2744,7 @@ SDValue AArch64TargetLowering::LowerCallResult( // Pass 'this' value directly from the argument to return value, to avoid // reg unit interference - if (i == 0 && isThisReturn && EnableThisRetForwarding) { + if (i == 0 && isThisReturn) { assert(!VA.needsCustom() && VA.getLocVT() == MVT::i64 && "unexpected return calling convention register assignment"); InVals.push_back(ThisVal); @@ -2763,15 +2772,29 @@ SDValue AArch64TargetLowering::LowerCallResult( return Chain; } +/// Return true if the calling convention is one that we can guarantee TCO for. +static bool canGuaranteeTCO(CallingConv::ID CC) { + return CC == CallingConv::Fast; +} + +/// Return true if we might ever do TCO for calls with this calling convention. +static bool mayTailCallThisCC(CallingConv::ID CC) { + switch (CC) { + case CallingConv::C: + case CallingConv::PreserveMost: + case CallingConv::Swift: + return true; + default: + return canGuaranteeTCO(CC); + } +} + bool AArch64TargetLowering::isEligibleForTailCallOptimization( SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg, const SmallVectorImpl &Outs, const SmallVectorImpl &OutVals, const SmallVectorImpl &Ins, SelectionDAG &DAG) const { - // For CallingConv::C this function knows whether the ABI needs - // changing. That's not true for other conventions so they will have to opt in - // manually. - if (!IsTailCallConvention(CalleeCC) && CalleeCC != CallingConv::C) + if (!mayTailCallThisCC(CalleeCC)) return false; MachineFunction &MF = DAG.getMachineFunction(); @@ -2788,9 +2811,8 @@ bool AArch64TargetLowering::isEligibleForTailCallOptimization( if (i->hasByValAttr()) return false; - if (getTargetMachine().Options.GuaranteedTailCallOpt) { - return IsTailCallConvention(CalleeCC) && CCMatch; - } + if (getTargetMachine().Options.GuaranteedTailCallOpt) + return canGuaranteeTCO(CalleeCC) && CCMatch; // Externally-defined functions with weak linkage should not be // tail-called on AArch64 when the OS does not support dynamic @@ -2872,11 +2894,11 @@ bool AArch64TargetLowering::isEligibleForTailCallOptimization( SDValue AArch64TargetLowering::addTokenForArgument(SDValue Chain, SelectionDAG &DAG, - MachineFrameInfo *MFI, + MachineFrameInfo &MFI, int ClobberedFI) const { SmallVector ArgChains; - int64_t FirstByte = MFI->getObjectOffset(ClobberedFI); - int64_t LastByte = FirstByte + MFI->getObjectSize(ClobberedFI) - 1; + int64_t FirstByte = MFI.getObjectOffset(ClobberedFI); + int64_t LastByte = FirstByte + MFI.getObjectSize(ClobberedFI) - 1; // Include the original chain at the beginning of the list. When this is // used by target LowerCall hooks, this helps legalize find the @@ -2890,9 +2912,9 @@ SDValue AArch64TargetLowering::addTokenForArgument(SDValue Chain, if (LoadSDNode *L = dyn_cast(*U)) if (FrameIndexSDNode *FI = dyn_cast(L->getBasePtr())) if (FI->getIndex() < 0) { - int64_t InFirstByte = MFI->getObjectOffset(FI->getIndex()); + int64_t InFirstByte = MFI.getObjectOffset(FI->getIndex()); int64_t InLastByte = InFirstByte; - InLastByte += MFI->getObjectSize(FI->getIndex()) - 1; + InLastByte += MFI.getObjectSize(FI->getIndex()) - 1; if ((InFirstByte <= FirstByte && FirstByte <= InLastByte) || (FirstByte <= InFirstByte && InFirstByte <= LastByte)) @@ -2908,11 +2930,6 @@ bool AArch64TargetLowering::DoesCalleeRestoreStack(CallingConv::ID CallCC, return CallCC == CallingConv::Fast && TailCallOpt; } -bool AArch64TargetLowering::IsTailCallConvention(CallingConv::ID CallCC) const { - return CallCC == CallingConv::Fast || - CallCC == CallingConv::PreserveMost; -} - /// LowerCall - Lower a call to a callseq_start + CALL + callseq_end chain, /// and add input and output parameter nodes. SDValue @@ -3119,7 +3136,7 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, if (IsTailCall) { Offset = Offset + FPDiff; - int FI = MF.getFrameInfo()->CreateFixedObject(OpSize, Offset, true); + int FI = MF.getFrameInfo().CreateFixedObject(OpSize, Offset, true); DstAddr = DAG.getFrameIndex(FI, PtrVT); DstInfo = @@ -3253,7 +3270,7 @@ AArch64TargetLowering::LowerCall(CallLoweringInfo &CLI, // If we're doing a tall call, use a TC_RETURN here rather than an // actual call instruction. if (IsTailCall) { - MF.getFrameInfo()->setHasTailCall(); + MF.getFrameInfo().setHasTailCall(); return DAG.getNode(AArch64ISD::TC_RETURN, DL, NodeTys, Ops); } @@ -3444,15 +3461,16 @@ AArch64TargetLowering::LowerDarwinGlobalTLSAddress(SDValue Op, // The first entry in the descriptor is a function pointer that we must call // to obtain the address of the variable. SDValue Chain = DAG.getEntryNode(); - SDValue FuncTLVGet = - DAG.getLoad(MVT::i64, DL, Chain, DescAddr, - MachinePointerInfo::getGOT(DAG.getMachineFunction()), - /* Alignment = */ 8, MachineMemOperand::MONonTemporal | - MachineMemOperand::MOInvariant); + SDValue FuncTLVGet = DAG.getLoad( + MVT::i64, DL, Chain, DescAddr, + MachinePointerInfo::getGOT(DAG.getMachineFunction()), + /* Alignment = */ 8, + MachineMemOperand::MONonTemporal | MachineMemOperand::MOInvariant | + MachineMemOperand::MODereferenceable); Chain = FuncTLVGet.getValue(1); - MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); - MFI->setAdjustsStack(true); + MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); + MFI.setAdjustsStack(true); // TLS calls preserve all registers except those that absolutely must be // trashed: X0 (it takes an argument), LR (it's a call) and NZCV (let's not be @@ -3705,7 +3723,7 @@ SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const { // Don't combine AND since emitComparison converts the AND to an ANDS // (a.k.a. TST) and the test in the test bit and branch instruction // becomes redundant. This would also increase register pressure. - uint64_t Mask = LHS.getValueType().getSizeInBits() - 1; + uint64_t Mask = LHS.getValueSizeInBits() - 1; return DAG.getNode(AArch64ISD::TBNZ, dl, MVT::Other, Chain, LHS, DAG.getConstant(Mask, dl, MVT::i64), Dest); } @@ -3715,7 +3733,7 @@ SDValue AArch64TargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const { // Don't combine AND since emitComparison converts the AND to an ANDS // (a.k.a. TST) and the test in the test bit and branch instruction // becomes redundant. This would also increase register pressure. - uint64_t Mask = LHS.getValueType().getSizeInBits() - 1; + uint64_t Mask = LHS.getValueSizeInBits() - 1; return DAG.getNode(AArch64ISD::TBZ, dl, MVT::Other, Chain, LHS, DAG.getConstant(Mask, dl, MVT::i64), Dest); } @@ -4036,6 +4054,33 @@ SDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS, } } + // Avoid materializing a constant when possible by reusing a known value in + // a register. However, don't perform this optimization if the known value + // is one, zero or negative one in the case of a CSEL. We can always + // materialize these values using CSINC, CSEL and CSINV with wzr/xzr as the + // FVal, respectively. + ConstantSDNode *RHSVal = dyn_cast(RHS); + if (Opcode == AArch64ISD::CSEL && RHSVal && !RHSVal->isOne() && + !RHSVal->isNullValue() && !RHSVal->isAllOnesValue()) { + AArch64CC::CondCode AArch64CC = changeIntCCToAArch64CC(CC); + // Transform "a == C ? C : x" to "a == C ? a : x" and "a != C ? x : C" to + // "a != C ? x : a" to avoid materializing C. + if (CTVal && CTVal == RHSVal && AArch64CC == AArch64CC::EQ) + TVal = LHS; + else if (CFVal && CFVal == RHSVal && AArch64CC == AArch64CC::NE) + FVal = LHS; + } else if (Opcode == AArch64ISD::CSNEG && RHSVal && RHSVal->isOne()) { + assert (CTVal && CFVal && "Expected constant operands for CSNEG."); + // Use a CSINV to transform "a == C ? 1 : -1" to "a == C ? a : -1" to + // avoid materializing C. + AArch64CC::CondCode AArch64CC = changeIntCCToAArch64CC(CC); + if (CTVal == RHSVal && AArch64CC == AArch64CC::EQ) { + Opcode = AArch64ISD::CSINV; + TVal = LHS; + FVal = DAG.getConstant(0, dl, FVal.getValueType()); + } + } + SDValue CCVal; SDValue Cmp = getAArch64Cmp(LHS, RHS, CC, CCVal, DAG, dl); @@ -4053,6 +4098,26 @@ SDValue AArch64TargetLowering::LowerSELECT_CC(ISD::CondCode CC, SDValue LHS, // clean. Some of them require two CSELs to implement. AArch64CC::CondCode CC1, CC2; changeFPCCToAArch64CC(CC, CC1, CC2); + + if (DAG.getTarget().Options.UnsafeFPMath) { + // Transform "a == 0.0 ? 0.0 : x" to "a == 0.0 ? a : x" and + // "a != 0.0 ? x : 0.0" to "a != 0.0 ? x : a" to avoid materializing 0.0. + ConstantFPSDNode *RHSVal = dyn_cast(RHS); + if (RHSVal && RHSVal->isZero()) { + ConstantFPSDNode *CFVal = dyn_cast(FVal); + ConstantFPSDNode *CTVal = dyn_cast(TVal); + + if ((CC == ISD::SETEQ || CC == ISD::SETOEQ || CC == ISD::SETUEQ) && + CTVal && CTVal->isZero() && TVal.getValueType() == LHS.getValueType()) + TVal = LHS; + else if ((CC == ISD::SETNE || CC == ISD::SETONE || CC == ISD::SETUNE) && + CFVal && CFVal->isZero() && + FVal.getValueType() == LHS.getValueType()) + FVal = LHS; + } + } + + // Emit first, and possibly only, CSEL. SDValue CC1Val = DAG.getConstant(CC1, dl, MVT::i32); SDValue CS1 = DAG.getNode(AArch64ISD::CSEL, dl, VT, TVal, FVal, CC1Val, Cmp); @@ -4378,8 +4443,8 @@ SDValue AArch64TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const { SDValue AArch64TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const { - MachineFrameInfo *MFI = DAG.getMachineFunction().getFrameInfo(); - MFI->setFrameAddressIsTaken(true); + MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); + MFI.setFrameAddressIsTaken(true); EVT VT = Op.getValueType(); SDLoc DL(Op); @@ -4408,8 +4473,8 @@ unsigned AArch64TargetLowering::getRegisterByName(const char* RegName, EVT VT, SDValue AArch64TargetLowering::LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const { MachineFunction &MF = DAG.getMachineFunction(); - MachineFrameInfo *MFI = MF.getFrameInfo(); - MFI->setReturnAddressIsTaken(true); + MachineFrameInfo &MFI = MF.getFrameInfo(); + MFI.setReturnAddressIsTaken(true); EVT VT = Op.getValueType(); SDLoc DL(Op); @@ -4559,38 +4624,96 @@ bool AArch64TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { // AArch64 Optimization Hooks //===----------------------------------------------------------------------===// -/// getEstimate - Return the appropriate estimate DAG for either the reciprocal -/// or the reciprocal square root. -static SDValue getEstimate(const AArch64Subtarget &ST, - const AArch64TargetLowering::DAGCombinerInfo &DCI, unsigned Opcode, - const SDValue &Operand, unsigned &ExtraSteps) { - if (!ST.hasNEON()) - return SDValue(); - +static SDValue getEstimate(const AArch64Subtarget *ST, unsigned Opcode, + SDValue Operand, SelectionDAG &DAG, + int &ExtraSteps) { EVT VT = Operand.getValueType(); + if (ST->hasNEON() && + (VT == MVT::f64 || VT == MVT::v1f64 || VT == MVT::v2f64 || + VT == MVT::f32 || VT == MVT::v1f32 || + VT == MVT::v2f32 || VT == MVT::v4f32)) { + if (ExtraSteps == TargetLoweringBase::ReciprocalEstimate::Unspecified) + // For the reciprocal estimates, convergence is quadratic, so the number + // of digits is doubled after each iteration. In ARMv8, the accuracy of + // the initial estimate is 2^-8. Thus the number of extra steps to refine + // the result for float (23 mantissa bits) is 2 and for double (52 + // mantissa bits) is 3. + ExtraSteps = VT == MVT::f64 ? 3 : 2; - std::string RecipOp; - RecipOp = Opcode == (AArch64ISD::FRECPE) ? "div": "sqrt"; - RecipOp = ((VT.isVector()) ? "vec-": "") + RecipOp; - RecipOp += (VT.getScalarType() == MVT::f64) ? "d": "f"; + return DAG.getNode(Opcode, SDLoc(Operand), VT, Operand); + } - TargetRecip Recips = DCI.DAG.getTarget().Options.Reciprocals; - if (!Recips.isEnabled(RecipOp)) - return SDValue(); + return SDValue(); +} + +SDValue AArch64TargetLowering::getSqrtEstimate(SDValue Operand, + SelectionDAG &DAG, int Enabled, + int &ExtraSteps, + bool &UseOneConst, + bool Reciprocal) const { + if (Enabled == ReciprocalEstimate::Enabled || + (Enabled == ReciprocalEstimate::Unspecified && Subtarget->useRSqrt())) + if (SDValue Estimate = getEstimate(Subtarget, AArch64ISD::FRSQRTE, Operand, + DAG, ExtraSteps)) { + SDLoc DL(Operand); + EVT VT = Operand.getValueType(); + + SDNodeFlags Flags; + Flags.setUnsafeAlgebra(true); + + // Newton reciprocal square root iteration: E * 0.5 * (3 - X * E^2) + // AArch64 reciprocal square root iteration instruction: 0.5 * (3 - M * N) + for (int i = ExtraSteps; i > 0; --i) { + SDValue Step = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Estimate, + &Flags); + Step = DAG.getNode(AArch64ISD::FRSQRTS, DL, VT, Operand, Step, &Flags); + Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, &Flags); + } + + if (!Reciprocal) { + EVT CCVT = getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), + VT); + SDValue FPZero = DAG.getConstantFP(0.0, DL, VT); + SDValue Eq = DAG.getSetCC(DL, CCVT, Operand, FPZero, ISD::SETEQ); + + Estimate = DAG.getNode(ISD::FMUL, DL, VT, Operand, Estimate, &Flags); + // Correct the result if the operand is 0.0. + Estimate = DAG.getNode(VT.isVector() ? ISD::VSELECT : ISD::SELECT, DL, + VT, Eq, Operand, Estimate); + } - ExtraSteps = Recips.getRefinementSteps(RecipOp); - return DCI.DAG.getNode(Opcode, SDLoc(Operand), VT, Operand); + ExtraSteps = 0; + return Estimate; + } + + return SDValue(); } SDValue AArch64TargetLowering::getRecipEstimate(SDValue Operand, - DAGCombinerInfo &DCI, unsigned &ExtraSteps) const { - return getEstimate(*Subtarget, DCI, AArch64ISD::FRECPE, Operand, ExtraSteps); -} + SelectionDAG &DAG, int Enabled, + int &ExtraSteps) const { + if (Enabled == ReciprocalEstimate::Enabled) + if (SDValue Estimate = getEstimate(Subtarget, AArch64ISD::FRECPE, Operand, + DAG, ExtraSteps)) { + SDLoc DL(Operand); + EVT VT = Operand.getValueType(); + + SDNodeFlags Flags; + Flags.setUnsafeAlgebra(true); + + // Newton reciprocal iteration: E * (2 - X * E) + // AArch64 reciprocal iteration instruction: (2 - M * N) + for (int i = ExtraSteps; i > 0; --i) { + SDValue Step = DAG.getNode(AArch64ISD::FRECPS, DL, VT, Operand, + Estimate, &Flags); + Estimate = DAG.getNode(ISD::FMUL, DL, VT, Estimate, Step, &Flags); + } -SDValue AArch64TargetLowering::getRsqrtEstimate(SDValue Operand, - DAGCombinerInfo &DCI, unsigned &ExtraSteps, bool &UseOneConst) const { - UseOneConst = true; - return getEstimate(*Subtarget, DCI, AArch64ISD::FRSQRTE, Operand, ExtraSteps); + ExtraSteps = 0; + return Estimate; + } + + return SDValue(); } //===----------------------------------------------------------------------===// @@ -4704,7 +4827,9 @@ AArch64TargetLowering::getRegForInlineAsmConstraint( return std::make_pair(0U, &AArch64::GPR64commonRegClass); return std::make_pair(0U, &AArch64::GPR32commonRegClass); case 'w': - if (VT == MVT::f32) + if (VT.getSizeInBits() == 16) + return std::make_pair(0U, &AArch64::FPR16RegClass); + if (VT.getSizeInBits() == 32) return std::make_pair(0U, &AArch64::FPR32RegClass); if (VT.getSizeInBits() == 64) return std::make_pair(0U, &AArch64::FPR64RegClass); @@ -4971,7 +5096,7 @@ SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op, // Add this element source to the list if it's not already there. SDValue SourceVec = V.getOperand(0); - auto Source = std::find(Sources.begin(), Sources.end(), SourceVec); + auto Source = find(Sources, SourceVec); if (Source == Sources.end()) Source = Sources.insert(Sources.end(), ShuffleSourceInfo(SourceVec)); @@ -4996,7 +5121,7 @@ SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op, } } unsigned ResMultiplier = - VT.getVectorElementType().getSizeInBits() / SmallestEltTy.getSizeInBits(); + VT.getScalarSizeInBits() / SmallestEltTy.getSizeInBits(); NumElts = VT.getSizeInBits() / SmallestEltTy.getSizeInBits(); EVT ShuffleVT = EVT::getVectorVT(*DAG.getContext(), SmallestEltTy, NumElts); @@ -5081,21 +5206,21 @@ SDValue AArch64TargetLowering::ReconstructShuffle(SDValue Op, // The stars all align, our next step is to produce the mask for the shuffle. SmallVector Mask(ShuffleVT.getVectorNumElements(), -1); - int BitsPerShuffleLane = ShuffleVT.getVectorElementType().getSizeInBits(); + int BitsPerShuffleLane = ShuffleVT.getScalarSizeInBits(); for (unsigned i = 0; i < VT.getVectorNumElements(); ++i) { SDValue Entry = Op.getOperand(i); if (Entry.isUndef()) continue; - auto Src = std::find(Sources.begin(), Sources.end(), Entry.getOperand(0)); + auto Src = find(Sources, Entry.getOperand(0)); int EltNo = cast(Entry.getOperand(1))->getSExtValue(); // EXTRACT_VECTOR_ELT performs an implicit any_ext; BUILD_VECTOR an implicit // trunc. So only std::min(SrcBits, DestBits) actually get defined in this // segment. EVT OrigEltTy = Entry.getOperand(0).getValueType().getVectorElementType(); - int BitsDefined = std::min(OrigEltTy.getSizeInBits(), - VT.getVectorElementType().getSizeInBits()); + int BitsDefined = + std::min(OrigEltTy.getSizeInBits(), VT.getScalarSizeInBits()); int LanesDefined = BitsDefined / BitsPerShuffleLane; // This source is expected to fill ResMultiplier lanes of the final shuffle, @@ -5157,8 +5282,7 @@ static bool isSingletonEXTMask(ArrayRef M, EVT VT, unsigned &Imm) { static bool isEXTMask(ArrayRef M, EVT VT, bool &ReverseEXT, unsigned &Imm) { // Look for the first non-undef element. - const int *FirstRealElt = std::find_if(M.begin(), M.end(), - [](int Elt) {return Elt >= 0;}); + const int *FirstRealElt = find_if(M, [](int Elt) { return Elt >= 0; }); // Benefit form APInt to handle overflow when calculating expected element. unsigned NumElts = VT.getVectorNumElements(); @@ -5200,7 +5324,7 @@ static bool isREVMask(ArrayRef M, EVT VT, unsigned BlockSize) { assert((BlockSize == 16 || BlockSize == 32 || BlockSize == 64) && "Only possible block sizes for REV are: 16, 32, 64"); - unsigned EltSz = VT.getVectorElementType().getSizeInBits(); + unsigned EltSz = VT.getScalarSizeInBits(); if (EltSz == 64) return false; @@ -5381,7 +5505,7 @@ static SDValue tryFormConcatFromShuffle(SDValue Op, SelectionDAG &DAG) { VT.getVectorElementType() != V1.getValueType().getVectorElementType()) return SDValue(); - bool SplitV0 = V0.getValueType().getSizeInBits() == 128; + bool SplitV0 = V0.getValueSizeInBits() == 128; if (!isConcatMask(Mask, VT, SplitV0)) return SDValue(); @@ -5392,7 +5516,7 @@ static SDValue tryFormConcatFromShuffle(SDValue Op, SelectionDAG &DAG) { V0 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, CastVT, V0, DAG.getConstant(0, DL, MVT::i64)); } - if (V1.getValueType().getSizeInBits() == 128) { + if (V1.getValueSizeInBits() == 128) { V1 = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, CastVT, V1, DAG.getConstant(0, DL, MVT::i64)); } @@ -5523,7 +5647,7 @@ static SDValue GenerateTBL(SDValue Op, ArrayRef ShuffleMask, MVT IndexVT = MVT::v8i8; unsigned IndexLen = 8; - if (Op.getValueType().getSizeInBits() == 128) { + if (Op.getValueSizeInBits() == 128) { IndexVT = MVT::v16i8; IndexLen = 16; } @@ -5918,7 +6042,7 @@ static SDValue tryLowerToSLI(SDNode *N, SelectionDAG &DAG) { // Is C1 == ~C2, taking into account how much one can shift elements of a // particular size? uint64_t C2 = C2node->getZExtValue(); - unsigned ElemSizeInBits = VT.getVectorElementType().getSizeInBits(); + unsigned ElemSizeInBits = VT.getScalarSizeInBits(); if (C2 > ElemSizeInBits) return SDValue(); unsigned ElemMask = (1 << ElemSizeInBits) - 1; @@ -6351,7 +6475,7 @@ FailedModImm: // DUPLANE works on 128-bit vectors, widen it if necessary. SDValue Lane = Value.getOperand(1); Value = Value.getOperand(0); - if (Value.getValueType().getSizeInBits() == 64) + if (Value.getValueSizeInBits() == 64) Value = WidenVector(Value, DAG); unsigned Opcode = getDUPLANEOp(VT.getVectorElementType()); @@ -6414,7 +6538,7 @@ FailedModImm: if (!isConstant && !usesOnlyOneValue) { SDValue Vec = DAG.getUNDEF(VT); SDValue Op0 = Op.getOperand(0); - unsigned ElemSize = VT.getVectorElementType().getSizeInBits(); + unsigned ElemSize = VT.getScalarSizeInBits(); unsigned i = 0; // For 32 and 64 bit types, use INSERT_SUBREG for lane zero to // a) Avoid a RMW dependency on the full vector register, and @@ -6528,7 +6652,7 @@ SDValue AArch64TargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op, return SDValue(); unsigned Val = Cst->getZExtValue(); - unsigned Size = Op.getValueType().getSizeInBits(); + unsigned Size = Op.getValueSizeInBits(); // This will get lowered to an appropriate EXTRACT_SUBREG in ISel. if (Val == 0) @@ -6536,7 +6660,7 @@ SDValue AArch64TargetLowering::LowerEXTRACT_SUBVECTOR(SDValue Op, // If this is extracting the upper 64-bits of a 128-bit vector, we match // that directly. - if (Size == 64 && Val * VT.getVectorElementType().getSizeInBits() == 64) + if (Size == 64 && Val * VT.getScalarSizeInBits() == 64) return Op; return SDValue(); @@ -6606,7 +6730,7 @@ static bool getVShiftImm(SDValue Op, unsigned ElementBits, int64_t &Cnt) { /// 0 <= Value <= ElementBits for a long left shift. static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) { assert(VT.isVector() && "vector shift count is not a vector type"); - int64_t ElementBits = VT.getVectorElementType().getSizeInBits(); + int64_t ElementBits = VT.getScalarSizeInBits(); if (!getVShiftImm(Op, ElementBits, Cnt)) return false; return (Cnt >= 0 && (isLong ? Cnt - 1 : Cnt) < ElementBits); @@ -6617,7 +6741,7 @@ static bool isVShiftLImm(SDValue Op, EVT VT, bool isLong, int64_t &Cnt) { /// 1 <= Value <= ElementBits for a right shift; or static bool isVShiftRImm(SDValue Op, EVT VT, bool isNarrow, int64_t &Cnt) { assert(VT.isVector() && "vector shift count is not a vector type"); - int64_t ElementBits = VT.getVectorElementType().getSizeInBits(); + int64_t ElementBits = VT.getScalarSizeInBits(); if (!getVShiftImm(Op, ElementBits, Cnt)) return false; return (Cnt >= 1 && Cnt <= (isNarrow ? ElementBits / 2 : ElementBits)); @@ -6631,7 +6755,7 @@ SDValue AArch64TargetLowering::LowerVectorSRA_SRL_SHL(SDValue Op, if (!Op.getOperand(1).getValueType().isVector()) return Op; - unsigned EltSize = VT.getVectorElementType().getSizeInBits(); + unsigned EltSize = VT.getScalarSizeInBits(); switch (Op.getOpcode()) { default: @@ -6716,8 +6840,8 @@ static SDValue EmitVectorComparison(SDValue LHS, SDValue RHS, case AArch64CC::LT: if (!NoNans) return SDValue(); - // If we ignore NaNs then we can use to the MI implementation. - // Fallthrough. + // If we ignore NaNs then we can use to the MI implementation. + LLVM_FALLTHROUGH; case AArch64CC::MI: if (IsZero) return DAG.getNode(AArch64ISD::FCMLTz, dl, VT, LHS); @@ -7033,8 +7157,8 @@ bool AArch64TargetLowering::isExtFreeImpl(const Instruction *Ext) const { case Instruction::GetElementPtr: { gep_type_iterator GTI = gep_type_begin(Instr); auto &DL = Ext->getModule()->getDataLayout(); - std::advance(GTI, U.getOperandNo()); - Type *IdxTy = *GTI; + std::advance(GTI, U.getOperandNo()-1); + Type *IdxTy = GTI.getIndexedType(); // This extension will end up with a shift because of the scaling factor. // 8-bit sized types have a scaling factor of 1, thus a shift amount of 0. // Get the shift amount based on the scaling factor: @@ -7052,7 +7176,7 @@ bool AArch64TargetLowering::isExtFreeImpl(const Instruction *Ext) const { // trunc(sext ty1 to ty2) to ty1. if (Instr->getType() == Ext->getOperand(0)->getType()) continue; - // FALL THROUGH. + LLVM_FALLTHROUGH; default: return false; } @@ -7063,16 +7187,6 @@ bool AArch64TargetLowering::isExtFreeImpl(const Instruction *Ext) const { return true; } -bool AArch64TargetLowering::hasPairedLoad(Type *LoadedType, - unsigned &RequiredAligment) const { - if (!LoadedType->isIntegerTy() && !LoadedType->isFloatTy()) - return false; - // Cyclone supports unaligned accesses. - RequiredAligment = 0; - unsigned NumBits = LoadedType->getPrimitiveSizeInBits(); - return NumBits == 32 || NumBits == 64; -} - bool AArch64TargetLowering::hasPairedLoad(EVT LoadedType, unsigned &RequiredAligment) const { if (!LoadedType.isSimple() || @@ -7167,7 +7281,7 @@ static Constant *getSequentialMask(IRBuilder<> &Builder, unsigned Start, /// /// E.g. Lower an interleaved store (Factor = 3): /// %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1, -/// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11> +/// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11> /// store <12 x i32> %i.vec, <12 x i32>* %ptr /// /// Into: @@ -7178,6 +7292,17 @@ static Constant *getSequentialMask(IRBuilder<> &Builder, unsigned Start, /// /// Note that the new shufflevectors will be removed and we'll only generate one /// st3 instruction in CodeGen. +/// +/// Example for a more general valid mask (Factor 3). Lower: +/// %i.vec = shuffle <32 x i32> %v0, <32 x i32> %v1, +/// <4, 32, 16, 5, 33, 17, 6, 34, 18, 7, 35, 19> +/// store <12 x i32> %i.vec, <12 x i32>* %ptr +/// +/// Into: +/// %sub.v0 = shuffle <32 x i32> %v0, <32 x i32> v1, <4, 5, 6, 7> +/// %sub.v1 = shuffle <32 x i32> %v0, <32 x i32> v1, <32, 33, 34, 35> +/// %sub.v2 = shuffle <32 x i32> %v0, <32 x i32> v1, <16, 17, 18, 19> +/// call void llvm.aarch64.neon.st3(%sub.v0, %sub.v1, %sub.v2, %ptr) bool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const { @@ -7188,9 +7313,9 @@ bool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI, assert(VecTy->getVectorNumElements() % Factor == 0 && "Invalid interleaved store"); - unsigned NumSubElts = VecTy->getVectorNumElements() / Factor; + unsigned LaneLen = VecTy->getVectorNumElements() / Factor; Type *EltTy = VecTy->getVectorElementType(); - VectorType *SubVecTy = VectorType::get(EltTy, NumSubElts); + VectorType *SubVecTy = VectorType::get(EltTy, LaneLen); const DataLayout &DL = SI->getModule()->getDataLayout(); unsigned SubVecSize = DL.getTypeSizeInBits(SubVecTy); @@ -7215,7 +7340,7 @@ bool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI, Op0 = Builder.CreatePtrToInt(Op0, IntVecTy); Op1 = Builder.CreatePtrToInt(Op1, IntVecTy); - SubVecTy = VectorType::get(IntTy, NumSubElts); + SubVecTy = VectorType::get(IntTy, LaneLen); } Type *PtrTy = SubVecTy->getPointerTo(SI->getPointerAddressSpace()); @@ -7229,9 +7354,28 @@ bool AArch64TargetLowering::lowerInterleavedStore(StoreInst *SI, SmallVector Ops; // Split the shufflevector operands into sub vectors for the new stN call. - for (unsigned i = 0; i < Factor; i++) - Ops.push_back(Builder.CreateShuffleVector( - Op0, Op1, getSequentialMask(Builder, NumSubElts * i, NumSubElts))); + auto Mask = SVI->getShuffleMask(); + for (unsigned i = 0; i < Factor; i++) { + if (Mask[i] >= 0) { + Ops.push_back(Builder.CreateShuffleVector( + Op0, Op1, getSequentialMask(Builder, Mask[i], LaneLen))); + } else { + unsigned StartMask = 0; + for (unsigned j = 1; j < LaneLen; j++) { + if (Mask[j*Factor + i] >= 0) { + StartMask = Mask[j*Factor + i] - j; + break; + } + } + // Note: If all elements in a chunk are undefs, StartMask=0! + // Note: Filling undef gaps with random elements is ok, since + // those elements were being written anyway (with undefs). + // In the case of all undefs we're defaulting to using elems from 0 + // Note: StartMask cannot be negative, it's checked in isReInterleaveMask + Ops.push_back(Builder.CreateShuffleVector( + Op0, Op1, getSequentialMask(Builder, StartMask, LaneLen))); + } + } Ops.push_back(Builder.CreateBitCast(SI->getPointerOperand(), PtrTy)); Builder.CreateCall(StNFunc, Ops); @@ -7323,7 +7467,7 @@ bool AArch64TargetLowering::isLegalAddressingMode(const DataLayout &DL, int64_t Offset = AM.BaseOffs; // 9-bit signed offset - if (Offset >= -(1LL << 9) && Offset <= (1LL << 9) - 1) + if (isInt<9>(Offset)) return true; // 12-bit unsigned offset @@ -7337,8 +7481,7 @@ bool AArch64TargetLowering::isLegalAddressingMode(const DataLayout &DL, // Check reg1 + SIZE_IN_BYTES * reg2 and reg1 + reg2 - return !AM.Scale || AM.Scale == 1 || - (AM.Scale > 0 && (uint64_t)AM.Scale == NumBytes); + return AM.Scale == 1 || (AM.Scale > 0 && (uint64_t)AM.Scale == NumBytes); } int AArch64TargetLowering::getScalingFactorCost(const DataLayout &DL, @@ -7544,57 +7687,98 @@ static SDValue performMulCombine(SDNode *N, SelectionDAG &DAG, if (DCI.isBeforeLegalizeOps()) return SDValue(); + // The below optimizations require a constant RHS. + if (!isa(N->getOperand(1))) + return SDValue(); + + ConstantSDNode *C = cast(N->getOperand(1)); + const APInt &ConstValue = C->getAPIntValue(); + // Multiplication of a power of two plus/minus one can be done more // cheaply as as shift+add/sub. For now, this is true unilaterally. If // future CPUs have a cheaper MADD instruction, this may need to be // gated on a subtarget feature. For Cyclone, 32-bit MADD is 4 cycles and // 64-bit is 5 cycles, so this is always a win. - if (ConstantSDNode *C = dyn_cast(N->getOperand(1))) { - const APInt &Value = C->getAPIntValue(); - EVT VT = N->getValueType(0); - SDLoc DL(N); - if (Value.isNonNegative()) { - // (mul x, 2^N + 1) => (add (shl x, N), x) - APInt VM1 = Value - 1; - if (VM1.isPowerOf2()) { - SDValue ShiftedVal = - DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), - DAG.getConstant(VM1.logBase2(), DL, MVT::i64)); - return DAG.getNode(ISD::ADD, DL, VT, ShiftedVal, - N->getOperand(0)); - } - // (mul x, 2^N - 1) => (sub (shl x, N), x) - APInt VP1 = Value + 1; - if (VP1.isPowerOf2()) { - SDValue ShiftedVal = - DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), - DAG.getConstant(VP1.logBase2(), DL, MVT::i64)); - return DAG.getNode(ISD::SUB, DL, VT, ShiftedVal, - N->getOperand(0)); - } - } else { - // (mul x, -(2^N - 1)) => (sub x, (shl x, N)) - APInt VNP1 = -Value + 1; - if (VNP1.isPowerOf2()) { - SDValue ShiftedVal = - DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), - DAG.getConstant(VNP1.logBase2(), DL, MVT::i64)); - return DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), - ShiftedVal); - } - // (mul x, -(2^N + 1)) => - (add (shl x, N), x) - APInt VNM1 = -Value - 1; - if (VNM1.isPowerOf2()) { - SDValue ShiftedVal = - DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0), - DAG.getConstant(VNM1.logBase2(), DL, MVT::i64)); - SDValue Add = - DAG.getNode(ISD::ADD, DL, VT, ShiftedVal, N->getOperand(0)); - return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Add); - } - } + // More aggressively, some multiplications N0 * C can be lowered to + // shift+add+shift if the constant C = A * B where A = 2^N + 1 and B = 2^M, + // e.g. 6=3*2=(2+1)*2. + // TODO: consider lowering more cases, e.g. C = 14, -6, -14 or even 45 + // which equals to (1+2)*16-(1+2). + SDValue N0 = N->getOperand(0); + // TrailingZeroes is used to test if the mul can be lowered to + // shift+add+shift. + unsigned TrailingZeroes = ConstValue.countTrailingZeros(); + if (TrailingZeroes) { + // Conservatively do not lower to shift+add+shift if the mul might be + // folded into smul or umul. + if (N0->hasOneUse() && (isSignExtended(N0.getNode(), DAG) || + isZeroExtended(N0.getNode(), DAG))) + return SDValue(); + // Conservatively do not lower to shift+add+shift if the mul might be + // folded into madd or msub. + if (N->hasOneUse() && (N->use_begin()->getOpcode() == ISD::ADD || + N->use_begin()->getOpcode() == ISD::SUB)) + return SDValue(); } - return SDValue(); + // Use ShiftedConstValue instead of ConstValue to support both shift+add/sub + // and shift+add+shift. + APInt ShiftedConstValue = ConstValue.ashr(TrailingZeroes); + + unsigned ShiftAmt, AddSubOpc; + // Is the shifted value the LHS operand of the add/sub? + bool ShiftValUseIsN0 = true; + // Do we need to negate the result? + bool NegateResult = false; + + if (ConstValue.isNonNegative()) { + // (mul x, 2^N + 1) => (add (shl x, N), x) + // (mul x, 2^N - 1) => (sub (shl x, N), x) + // (mul x, (2^N + 1) * 2^M) => (shl (add (shl x, N), x), M) + APInt SCVMinus1 = ShiftedConstValue - 1; + APInt CVPlus1 = ConstValue + 1; + if (SCVMinus1.isPowerOf2()) { + ShiftAmt = SCVMinus1.logBase2(); + AddSubOpc = ISD::ADD; + } else if (CVPlus1.isPowerOf2()) { + ShiftAmt = CVPlus1.logBase2(); + AddSubOpc = ISD::SUB; + } else + return SDValue(); + } else { + // (mul x, -(2^N - 1)) => (sub x, (shl x, N)) + // (mul x, -(2^N + 1)) => - (add (shl x, N), x) + APInt CVNegPlus1 = -ConstValue + 1; + APInt CVNegMinus1 = -ConstValue - 1; + if (CVNegPlus1.isPowerOf2()) { + ShiftAmt = CVNegPlus1.logBase2(); + AddSubOpc = ISD::SUB; + ShiftValUseIsN0 = false; + } else if (CVNegMinus1.isPowerOf2()) { + ShiftAmt = CVNegMinus1.logBase2(); + AddSubOpc = ISD::ADD; + NegateResult = true; + } else + return SDValue(); + } + + SDLoc DL(N); + EVT VT = N->getValueType(0); + SDValue ShiftedVal = DAG.getNode(ISD::SHL, DL, VT, N0, + DAG.getConstant(ShiftAmt, DL, MVT::i64)); + + SDValue AddSubN0 = ShiftValUseIsN0 ? ShiftedVal : N0; + SDValue AddSubN1 = ShiftValUseIsN0 ? N0 : ShiftedVal; + SDValue Res = DAG.getNode(AddSubOpc, DL, VT, AddSubN0, AddSubN1); + assert(!(NegateResult && TrailingZeroes) && + "NegateResult and TrailingZeroes cannot both be true for now."); + // Negate the result. + if (NegateResult) + return DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Res); + // Shift the result. + if (TrailingZeroes) + return DAG.getNode(ISD::SHL, DL, VT, Res, + DAG.getConstant(TrailingZeroes, DL, MVT::i64)); + return Res; } static SDValue performVectorCompareAndMaskUnaryOpCombine(SDNode *N, @@ -7655,7 +7839,7 @@ static SDValue performIntToFpCombine(SDNode *N, SelectionDAG &DAG, return SDValue(); // Only optimize when the source and destination types have the same width. - if (VT.getSizeInBits() != N->getOperand(0).getValueType().getSizeInBits()) + if (VT.getSizeInBits() != N->getOperand(0).getValueSizeInBits()) return SDValue(); // If the result of an integer load is only used by an integer-to-float @@ -7757,13 +7941,15 @@ static SDValue performFpToIntCombine(SDNode *N, SelectionDAG &DAG, /// Fold a floating-point divide by power of two into fixed-point to /// floating-point conversion. static SDValue performFDivCombine(SDNode *N, SelectionDAG &DAG, + TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget) { if (!Subtarget->hasNEON()) return SDValue(); SDValue Op = N->getOperand(0); unsigned Opc = Op->getOpcode(); - if (!Op.getValueType().isVector() || + if (!Op.getValueType().isVector() || !Op.getValueType().isSimple() || + !Op.getOperand(0).getValueType().isSimple() || (Opc != ISD::SINT_TO_FP && Opc != ISD::UINT_TO_FP)) return SDValue(); @@ -7800,10 +7986,13 @@ static SDValue performFDivCombine(SDNode *N, SelectionDAG &DAG, ResTy = FloatBits == 32 ? MVT::v2i32 : MVT::v2i64; break; case 4: - ResTy = MVT::v4i32; + ResTy = FloatBits == 32 ? MVT::v4i32 : MVT::v4i64; break; } + if (ResTy == MVT::v4i64 && DCI.isBeforeLegalizeOps()) + return SDValue(); + SDLoc DL(N); SDValue ConvInput = Op.getOperand(0); bool IsSigned = Opc == ISD::SINT_TO_FP; @@ -7901,7 +8090,7 @@ static SDValue tryCombineToBSL(SDNode *N, // We only have to look for constant vectors here since the general, variable // case can be handled in TableGen. - unsigned Bits = VT.getVectorElementType().getSizeInBits(); + unsigned Bits = VT.getScalarSizeInBits(); uint64_t BitMask = Bits == 64 ? -1ULL : ((1ULL << Bits) - 1); for (int i = 1; i >= 0; --i) for (int j = 1; j >= 0; --j) { @@ -8090,7 +8279,7 @@ static SDValue performConcatVectorsCombine(SDNode *N, // splat. The indexed instructions are going to be expecting a DUPLANE64, so // canonicalise to that. if (N0 == N1 && VT.getVectorNumElements() == 2) { - assert(VT.getVectorElementType().getSizeInBits() == 64); + assert(VT.getScalarSizeInBits() == 64); return DAG.getNode(AArch64ISD::DUPLANE64, dl, VT, WidenVector(N0, DAG), DAG.getConstant(0, dl, MVT::i64)); } @@ -8153,7 +8342,7 @@ static SDValue tryCombineFixedPointConvert(SDNode *N, // The vector width should be 128 bits by the time we get here, even // if it started as 64 bits (the extract_vector handling will have // done so). - assert(Vec.getValueType().getSizeInBits() == 128 && + assert(Vec.getValueSizeInBits() == 128 && "unexpected vector size on extract_vector_elt!"); if (Vec.getValueType() == MVT::v4i32) VecResTy = MVT::v4f32; @@ -8655,7 +8844,7 @@ static SDValue performExtendCombine(SDNode *N, if (SrcVT.getSizeInBits() != 64) return SDValue(); - unsigned SrcEltSize = SrcVT.getVectorElementType().getSizeInBits(); + unsigned SrcEltSize = SrcVT.getScalarSizeInBits(); unsigned ElementCount = SrcVT.getVectorNumElements(); SrcVT = MVT::getVectorVT(MVT::getIntegerVT(SrcEltSize * 2), ElementCount); SDLoc DL(N); @@ -8684,13 +8873,100 @@ static SDValue performExtendCombine(SDNode *N, return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi); } +static SDValue splitStoreSplat(SelectionDAG &DAG, StoreSDNode &St, + SDValue SplatVal, unsigned NumVecElts) { + unsigned OrigAlignment = St.getAlignment(); + unsigned EltOffset = SplatVal.getValueType().getSizeInBits() / 8; + + // Create scalar stores. This is at least as good as the code sequence for a + // split unaligned store which is a dup.s, ext.b, and two stores. + // Most of the time the three stores should be replaced by store pair + // instructions (stp). + SDLoc DL(&St); + SDValue BasePtr = St.getBasePtr(); + SDValue NewST1 = + DAG.getStore(St.getChain(), DL, SplatVal, BasePtr, St.getPointerInfo(), + OrigAlignment, St.getMemOperand()->getFlags()); + + unsigned Offset = EltOffset; + while (--NumVecElts) { + unsigned Alignment = MinAlign(OrigAlignment, Offset); + SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr, + DAG.getConstant(Offset, DL, MVT::i64)); + NewST1 = DAG.getStore(NewST1.getValue(0), DL, SplatVal, OffsetPtr, + St.getPointerInfo(), Alignment, + St.getMemOperand()->getFlags()); + Offset += EltOffset; + } + return NewST1; +} + +/// Replace a splat of zeros to a vector store by scalar stores of WZR/XZR. The +/// load store optimizer pass will merge them to store pair stores. This should +/// be better than a movi to create the vector zero followed by a vector store +/// if the zero constant is not re-used, since one instructions and one register +/// live range will be removed. +/// +/// For example, the final generated code should be: +/// +/// stp xzr, xzr, [x0] +/// +/// instead of: +/// +/// movi v0.2d, #0 +/// str q0, [x0] +/// +static SDValue replaceZeroVectorStore(SelectionDAG &DAG, StoreSDNode &St) { + SDValue StVal = St.getValue(); + EVT VT = StVal.getValueType(); + + // It is beneficial to scalarize a zero splat store for 2 or 3 i64 elements or + // 2, 3 or 4 i32 elements. + int NumVecElts = VT.getVectorNumElements(); + if (!(((NumVecElts == 2 || NumVecElts == 3) && + VT.getVectorElementType().getSizeInBits() == 64) || + ((NumVecElts == 2 || NumVecElts == 3 || NumVecElts == 4) && + VT.getVectorElementType().getSizeInBits() == 32))) + return SDValue(); + + if (StVal.getOpcode() != ISD::BUILD_VECTOR) + return SDValue(); + + // If the zero constant has more than one use then the vector store could be + // better since the constant mov will be amortized and stp q instructions + // should be able to be formed. + if (!StVal.hasOneUse()) + return SDValue(); + + // If the immediate offset of the address operand is too large for the stp + // instruction, then bail out. + if (DAG.isBaseWithConstantOffset(St.getBasePtr())) { + int64_t Offset = St.getBasePtr()->getConstantOperandVal(1); + if (Offset < -512 || Offset > 504) + return SDValue(); + } + + for (int I = 0; I < NumVecElts; ++I) { + SDValue EltVal = StVal.getOperand(I); + if (!isNullConstant(EltVal) && !isNullFPConstant(EltVal)) + return SDValue(); + } + + // Use WZR/XZR here to prevent DAGCombiner::MergeConsecutiveStores from + // undoing this transformation. + SDValue SplatVal = VT.getVectorElementType().getSizeInBits() == 32 + ? DAG.getRegister(AArch64::WZR, MVT::i32) + : DAG.getRegister(AArch64::XZR, MVT::i64); + return splitStoreSplat(DAG, St, SplatVal, NumVecElts); +} + /// Replace a splat of a scalar to a vector store by scalar stores of the scalar /// value. The load store optimizer pass will merge them to store pair stores. /// This has better performance than a splat of the scalar followed by a split /// vector store. Even if the stores are not merged it is four stores vs a dup, /// followed by an ext.b and two stores. -static SDValue replaceSplatVectorStore(SelectionDAG &DAG, StoreSDNode *St) { - SDValue StVal = St->getValue(); +static SDValue replaceSplatVectorStore(SelectionDAG &DAG, StoreSDNode &St) { + SDValue StVal = St.getValue(); EVT VT = StVal.getValueType(); // Don't replace floating point stores, they possibly won't be transformed to @@ -8698,55 +8974,48 @@ static SDValue replaceSplatVectorStore(SelectionDAG &DAG, StoreSDNode *St) { if (VT.isFloatingPoint()) return SDValue(); - // Check for insert vector elements. - if (StVal.getOpcode() != ISD::INSERT_VECTOR_ELT) - return SDValue(); - // We can express a splat as store pair(s) for 2 or 4 elements. unsigned NumVecElts = VT.getVectorNumElements(); if (NumVecElts != 4 && NumVecElts != 2) return SDValue(); - SDValue SplatVal = StVal.getOperand(1); - unsigned RemainInsertElts = NumVecElts - 1; // Check that this is a splat. - while (--RemainInsertElts) { - SDValue NextInsertElt = StVal.getOperand(0); - if (NextInsertElt.getOpcode() != ISD::INSERT_VECTOR_ELT) + // Make sure that each of the relevant vector element locations are inserted + // to, i.e. 0 and 1 for v2i64 and 0, 1, 2, 3 for v4i32. + std::bitset<4> IndexNotInserted((1 << NumVecElts) - 1); + SDValue SplatVal; + for (unsigned I = 0; I < NumVecElts; ++I) { + // Check for insert vector elements. + if (StVal.getOpcode() != ISD::INSERT_VECTOR_ELT) return SDValue(); - if (NextInsertElt.getOperand(1) != SplatVal) + + // Check that same value is inserted at each vector element. + if (I == 0) + SplatVal = StVal.getOperand(1); + else if (StVal.getOperand(1) != SplatVal) return SDValue(); - StVal = NextInsertElt; - } - unsigned OrigAlignment = St->getAlignment(); - unsigned EltOffset = NumVecElts == 4 ? 4 : 8; - unsigned Alignment = std::min(OrigAlignment, EltOffset); - // Create scalar stores. This is at least as good as the code sequence for a - // split unaligned store which is a dup.s, ext.b, and two stores. - // Most of the time the three stores should be replaced by store pair - // instructions (stp). - SDLoc DL(St); - SDValue BasePtr = St->getBasePtr(); - SDValue NewST1 = - DAG.getStore(St->getChain(), DL, SplatVal, BasePtr, St->getPointerInfo(), - St->getAlignment(), St->getMemOperand()->getFlags()); + // Check insert element index. + ConstantSDNode *CIndex = dyn_cast(StVal.getOperand(2)); + if (!CIndex) + return SDValue(); + uint64_t IndexVal = CIndex->getZExtValue(); + if (IndexVal >= NumVecElts) + return SDValue(); + IndexNotInserted.reset(IndexVal); - unsigned Offset = EltOffset; - while (--NumVecElts) { - SDValue OffsetPtr = DAG.getNode(ISD::ADD, DL, MVT::i64, BasePtr, - DAG.getConstant(Offset, DL, MVT::i64)); - NewST1 = DAG.getStore(NewST1.getValue(0), DL, SplatVal, OffsetPtr, - St->getPointerInfo(), Alignment, - St->getMemOperand()->getFlags()); - Offset += EltOffset; + StVal = StVal.getOperand(0); } - return NewST1; + // Check that all vector element locations were inserted to. + if (IndexNotInserted.any()) + return SDValue(); + + return splitStoreSplat(DAG, St, SplatVal, NumVecElts); } -static SDValue split16BStores(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, - SelectionDAG &DAG, - const AArch64Subtarget *Subtarget) { +static SDValue splitStores(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, + SelectionDAG &DAG, + const AArch64Subtarget *Subtarget) { if (!DCI.isBeforeLegalize()) return SDValue(); @@ -8754,6 +9023,17 @@ static SDValue split16BStores(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, if (S->isVolatile()) return SDValue(); + SDValue StVal = S->getValue(); + EVT VT = StVal.getValueType(); + if (!VT.isVector()) + return SDValue(); + + // If we get a splat of zeros, convert this vector store to a store of + // scalars. They will be merged into store pairs of xzr thereby removing one + // instruction and one register. + if (SDValue ReplacedZeroSplat = replaceZeroVectorStore(DAG, *S)) + return ReplacedZeroSplat; + // FIXME: The logic for deciding if an unaligned store should be split should // be included in TLI.allowsMisalignedMemoryAccesses(), and there should be // a call to that function here. @@ -8765,12 +9045,9 @@ static SDValue split16BStores(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, if (DAG.getMachineFunction().getFunction()->optForMinSize()) return SDValue(); - SDValue StVal = S->getValue(); - EVT VT = StVal.getValueType(); - // Don't split v2i64 vectors. Memcpy lowering produces those and splitting // those up regresses performance on micro-benchmarks and olden/bh. - if (!VT.isVector() || VT.getVectorNumElements() < 2 || VT == MVT::v2i64) + if (VT.getVectorNumElements() < 2 || VT == MVT::v2i64) return SDValue(); // Split unaligned 16B stores. They are terrible for performance. @@ -8785,7 +9062,7 @@ static SDValue split16BStores(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, // If we get a splat of a scalar convert this vector store to a store of // scalars. They will be merged into store pairs thereby removing two // instructions. - if (SDValue ReplacedSplat = replaceSplatVectorStore(DAG, S)) + if (SDValue ReplacedSplat = replaceSplatVectorStore(DAG, *S)) return ReplacedSplat; SDLoc DL(S); @@ -8928,7 +9205,7 @@ static SDValue performSTORECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG, const AArch64Subtarget *Subtarget) { - if (SDValue Split = split16BStores(N, DCI, DAG, Subtarget)) + if (SDValue Split = splitStores(N, DCI, DAG, Subtarget)) return Split; if (Subtarget->supportsAddressTopByteIgnored() && @@ -9862,7 +10139,7 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N, case ISD::FP_TO_UINT: return performFpToIntCombine(N, DAG, DCI, Subtarget); case ISD::FDIV: - return performFDivCombine(N, DAG, Subtarget); + return performFDivCombine(N, DAG, DCI, Subtarget); case ISD::OR: return performORCombine(N, DCI, Subtarget); case ISD::SRL: @@ -9995,8 +10272,10 @@ bool AArch64TargetLowering::getIndexedAddressParts(SDNode *Op, SDValue &Base, // All of the indexed addressing mode instructions take a signed // 9 bit immediate offset. if (ConstantSDNode *RHS = dyn_cast(Op->getOperand(1))) { - int64_t RHSC = (int64_t)RHS->getZExtValue(); - if (RHSC >= 256 || RHSC <= -256) + int64_t RHSC = RHS->getSExtValue(); + if (Op->getOpcode() == ISD::SUB) + RHSC = -(uint64_t)RHSC; + if (!isInt<9>(RHSC)) return false; IsInc = (Op->getOpcode() == ISD::ADD); Offset = Op->getOperand(1); diff --git a/lib/Target/AArch64/AArch64ISelLowering.h b/lib/Target/AArch64/AArch64ISelLowering.h index c87cfed1f892..054ccc31674f 100644 --- a/lib/Target/AArch64/AArch64ISelLowering.h +++ b/lib/Target/AArch64/AArch64ISelLowering.h @@ -187,9 +187,9 @@ enum NodeType : unsigned { SMULL, UMULL, - // Reciprocal estimates. - FRECPE, - FRSQRTE, + // Reciprocal estimates and steps. + FRECPE, FRECPS, + FRSQRTE, FRSQRTS, // NEON Load/Store with post-increment base updates LD2post = ISD::FIRST_TARGET_MEMORY_OPCODE, @@ -219,6 +219,21 @@ enum NodeType : unsigned { } // end namespace AArch64ISD +namespace { + +// Any instruction that defines a 32-bit result zeros out the high half of the +// register. Truncate can be lowered to EXTRACT_SUBREG. CopyFromReg may +// be copying from a truncate. But any other 32-bit operation will zero-extend +// up to 64 bits. +// FIXME: X86 also checks for CMOV here. Do we need something similar? +static inline bool isDef32(const SDNode &N) { + unsigned Opc = N.getOpcode(); + return Opc != ISD::TRUNCATE && Opc != TargetOpcode::EXTRACT_SUBREG && + Opc != ISD::CopyFromReg; +} + +} // end anonymous namespace + class AArch64Subtarget; class AArch64TargetMachine; @@ -230,6 +245,9 @@ public: /// Selects the correct CCAssignFn for a given CallingConvention value. CCAssignFn *CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg) const; + /// Selects the correct CCAssignFn for a given CallingConvention value. + CCAssignFn *CCAssignFnForReturn(CallingConv::ID CC) const; + /// Determine which of the bits specified in Mask are known to be either zero /// or one and return them in the KnownZero/KnownOne bitsets. void computeKnownBitsForTargetNode(const SDValue Op, APInt &KnownZero, @@ -295,8 +313,6 @@ public: bool isZExtFree(EVT VT1, EVT VT2) const override; bool isZExtFree(SDValue Val, EVT VT2) const override; - bool hasPairedLoad(Type *LoadedType, - unsigned &RequiredAligment) const override; bool hasPairedLoad(EVT LoadedType, unsigned &RequiredAligment) const override; unsigned getMaxSupportedInterleaveFactor() const override { return 4; } @@ -396,6 +412,11 @@ public: return true; } + bool hasAndNotCompare(SDValue) const override { + // 'bics' + return true; + } + bool hasBitPreservingFPLogic(EVT VT) const override { // FIXME: Is this always true? It should be true for vectors at least. return VT == MVT::f32 || VT == MVT::f64; @@ -453,12 +474,10 @@ private: /// object and incorporates their load into the current chain. This prevents /// an upcoming store from clobbering the stack argument before it's used. SDValue addTokenForArgument(SDValue Chain, SelectionDAG &DAG, - MachineFrameInfo *MFI, int ClobberedFI) const; + MachineFrameInfo &MFI, int ClobberedFI) const; bool DoesCalleeRestoreStack(CallingConv::ID CallCC, bool TailCallOpt) const; - bool IsTailCallConvention(CallingConv::ID CallCC) const; - void saveVarArgRegisters(CCState &CCInfo, SelectionDAG &DAG, const SDLoc &DL, SDValue &Chain) const; @@ -520,11 +539,11 @@ private: SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG, std::vector *Created) const override; - SDValue getRsqrtEstimate(SDValue Operand, DAGCombinerInfo &DCI, - unsigned &RefinementSteps, - bool &UseOneConstNR) const override; - SDValue getRecipEstimate(SDValue Operand, DAGCombinerInfo &DCI, - unsigned &RefinementSteps) const override; + SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, + int &ExtraSteps, bool &UseOneConst, + bool Reciprocal) const override; + SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, + int &ExtraSteps) const override; unsigned combineRepeatedFPDivisors() const override; ConstraintType getConstraintType(StringRef Constraint) const override; diff --git a/lib/Target/AArch64/AArch64InstrAtomics.td b/lib/Target/AArch64/AArch64InstrAtomics.td index 59de62ad2877..867074c3c374 100644 --- a/lib/Target/AArch64/AArch64InstrAtomics.td +++ b/lib/Target/AArch64/AArch64InstrAtomics.td @@ -377,28 +377,28 @@ def : Pat<(int_aarch64_clrex), (CLREX 0xf)>; // significantly more naive than the standard expansion: we conservatively // assume seq_cst, strong cmpxchg and omit clrex on failure. -let Constraints = "@earlyclobber $Rd,@earlyclobber $status", +let Constraints = "@earlyclobber $Rd,@earlyclobber $scratch", mayLoad = 1, mayStore = 1 in { -def CMP_SWAP_8 : Pseudo<(outs GPR32:$Rd, GPR32:$status), +def CMP_SWAP_8 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch), (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>, Sched<[WriteAtomic]>; -def CMP_SWAP_16 : Pseudo<(outs GPR32:$Rd, GPR32:$status), +def CMP_SWAP_16 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch), (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>, Sched<[WriteAtomic]>; -def CMP_SWAP_32 : Pseudo<(outs GPR32:$Rd, GPR32:$status), +def CMP_SWAP_32 : Pseudo<(outs GPR32:$Rd, GPR32:$scratch), (ins GPR64:$addr, GPR32:$desired, GPR32:$new), []>, Sched<[WriteAtomic]>; -def CMP_SWAP_64 : Pseudo<(outs GPR64:$Rd, GPR32:$status), +def CMP_SWAP_64 : Pseudo<(outs GPR64:$Rd, GPR32:$scratch), (ins GPR64:$addr, GPR64:$desired, GPR64:$new), []>, Sched<[WriteAtomic]>; } -let Constraints = "@earlyclobber $RdLo,@earlyclobber $RdHi,@earlyclobber $status", +let Constraints = "@earlyclobber $RdLo,@earlyclobber $RdHi,@earlyclobber $scratch", mayLoad = 1, mayStore = 1 in -def CMP_SWAP_128 : Pseudo<(outs GPR64:$RdLo, GPR64:$RdHi, GPR32:$status), +def CMP_SWAP_128 : Pseudo<(outs GPR64:$RdLo, GPR64:$RdHi, GPR32:$scratch), (ins GPR64:$addr, GPR64:$desiredLo, GPR64:$desiredHi, GPR64:$newLo, GPR64:$newHi), []>, Sched<[WriteAtomic]>; diff --git a/lib/Target/AArch64/AArch64InstrFormats.td b/lib/Target/AArch64/AArch64InstrFormats.td index 34d35e961210..cefdf51b50d2 100644 --- a/lib/Target/AArch64/AArch64InstrFormats.td +++ b/lib/Target/AArch64/AArch64InstrFormats.td @@ -3936,27 +3936,27 @@ class BaseFPConversion type, bits<2> opcode, RegisterClass dstType, multiclass FPConversion { // Double-precision to Half-precision def HDr : BaseFPConversion<0b01, 0b11, FPR16, FPR64, asm, - [(set FPR16:$Rd, (fround FPR64:$Rn))]>; + [(set FPR16:$Rd, (fpround FPR64:$Rn))]>; // Double-precision to Single-precision def SDr : BaseFPConversion<0b01, 0b00, FPR32, FPR64, asm, - [(set FPR32:$Rd, (fround FPR64:$Rn))]>; + [(set FPR32:$Rd, (fpround FPR64:$Rn))]>; // Half-precision to Double-precision def DHr : BaseFPConversion<0b11, 0b01, FPR64, FPR16, asm, - [(set FPR64:$Rd, (fextend FPR16:$Rn))]>; + [(set FPR64:$Rd, (fpextend FPR16:$Rn))]>; // Half-precision to Single-precision def SHr : BaseFPConversion<0b11, 0b00, FPR32, FPR16, asm, - [(set FPR32:$Rd, (fextend FPR16:$Rn))]>; + [(set FPR32:$Rd, (fpextend FPR16:$Rn))]>; // Single-precision to Double-precision def DSr : BaseFPConversion<0b00, 0b01, FPR64, FPR32, asm, - [(set FPR64:$Rd, (fextend FPR32:$Rn))]>; + [(set FPR64:$Rd, (fpextend FPR32:$Rn))]>; // Single-precision to Half-precision def HSr : BaseFPConversion<0b00, 0b11, FPR16, FPR32, asm, - [(set FPR16:$Rd, (fround FPR32:$Rn))]>; + [(set FPR16:$Rd, (fpround FPR32:$Rn))]>; } //--- @@ -9348,7 +9348,7 @@ class SHAInstSS opc, string asm, Intrinsic OpNode> // ST{}[] , [] // ST{} , [] -let Predicates = [HasV8_1a], mayLoad = 1, mayStore = 1, hasSideEffects = 1 in +let Predicates = [HasLSE], mayLoad = 1, mayStore = 1, hasSideEffects = 1 in class BaseCASEncoding pattern> : I { @@ -9369,6 +9369,7 @@ class BaseCASEncoding @@ -9401,7 +9402,7 @@ multiclass CompareAndSwapPair Acq, bits<1> Rel, string order> { def d : BaseCASP; } -let Predicates = [HasV8_1a] in +let Predicates = [HasLSE] in class BaseSWP : I<(outs RC:$Rt),(ins RC:$Rs, GPR64sp:$Rn), "swp" # order # size, "\t$Rs, $Rt, [$Rn]","",[]>, @@ -9424,6 +9425,7 @@ class BaseSWP let Inst{11-10} = 0b00; let Inst{9-5} = Rn; let Inst{4-0} = Rt; + let Predicates = [HasLSE]; } multiclass Swap Acq, bits<1> Rel, string order> { @@ -9433,7 +9435,7 @@ multiclass Swap Acq, bits<1> Rel, string order> { let Sz = 0b11, Acq = Acq, Rel = Rel in def d : BaseSWP; } -let Predicates = [HasV8_1a], mayLoad = 1, mayStore = 1, hasSideEffects = 1 in +let Predicates = [HasLSE], mayLoad = 1, mayStore = 1, hasSideEffects = 1 in class BaseLDOPregister : I<(outs RC:$Rt),(ins RC:$Rs, GPR64sp:$Rn), "ld" # op # order # size, "\t$Rs, $Rt, [$Rn]","",[]>, @@ -9456,6 +9458,7 @@ class BaseLDOPregister let Inst{11-10} = 0b00; let Inst{9-5} = Rn; let Inst{4-0} = Rt; + let Predicates = [HasLSE]; } multiclass LDOPregister opc, string op, bits<1> Acq, bits<1> Rel, @@ -9470,7 +9473,7 @@ multiclass LDOPregister opc, string op, bits<1> Acq, bits<1> Rel, def d : BaseLDOPregister; } -let Predicates = [HasV8_1a] in +let Predicates = [HasLSE] in class BaseSTOPregister : InstAlias; diff --git a/lib/Target/AArch64/AArch64InstrInfo.cpp b/lib/Target/AArch64/AArch64InstrInfo.cpp index 0aa4708f35ac..626c934f236e 100644 --- a/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -19,6 +19,7 @@ #include "llvm/CodeGen/MachineMemOperand.h" #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/PseudoSourceValue.h" +#include "llvm/CodeGen/StackMaps.h" #include "llvm/MC/MCInst.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/TargetRegistry.h" @@ -29,16 +30,28 @@ using namespace llvm; #define GET_INSTRINFO_CTOR_DTOR #include "AArch64GenInstrInfo.inc" -static LLVM_CONSTEXPR MachineMemOperand::Flags MOSuppressPair = +static const MachineMemOperand::Flags MOSuppressPair = MachineMemOperand::MOTargetFlag1; +static cl::opt +TBZDisplacementBits("aarch64-tbz-offset-bits", cl::Hidden, cl::init(14), + cl::desc("Restrict range of TB[N]Z instructions (DEBUG)")); + +static cl::opt +CBZDisplacementBits("aarch64-cbz-offset-bits", cl::Hidden, cl::init(19), + cl::desc("Restrict range of CB[N]Z instructions (DEBUG)")); + +static cl::opt +BCCDisplacementBits("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19), + cl::desc("Restrict range of Bcc instructions (DEBUG)")); + AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI) : AArch64GenInstrInfo(AArch64::ADJCALLSTACKDOWN, AArch64::ADJCALLSTACKUP), RI(STI.getTargetTriple()), Subtarget(STI) {} /// GetInstSize - Return the number of bytes of code the specified /// instruction may be. This returns the maximum number of bytes. -unsigned AArch64InstrInfo::GetInstSizeInBytes(const MachineInstr &MI) const { +unsigned AArch64InstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { const MachineBasicBlock &MBB = *MI.getParent(); const MachineFunction *MF = MBB.getParent(); const MCAsmInfo *MAI = MF->getTarget().getMCAsmInfo(); @@ -46,19 +59,38 @@ unsigned AArch64InstrInfo::GetInstSizeInBytes(const MachineInstr &MI) const { if (MI.getOpcode() == AArch64::INLINEASM) return getInlineAsmLength(MI.getOperand(0).getSymbolName(), *MAI); + // FIXME: We currently only handle pseudoinstructions that don't get expanded + // before the assembly printer. + unsigned NumBytes = 0; const MCInstrDesc &Desc = MI.getDesc(); switch (Desc.getOpcode()) { default: - // Anything not explicitly designated otherwise is a nomal 4-byte insn. - return 4; + // Anything not explicitly designated otherwise is a normal 4-byte insn. + NumBytes = 4; + break; case TargetOpcode::DBG_VALUE: case TargetOpcode::EH_LABEL: case TargetOpcode::IMPLICIT_DEF: case TargetOpcode::KILL: - return 0; + NumBytes = 0; + break; + case TargetOpcode::STACKMAP: + // The upper bound for a stackmap intrinsic is the full length of its shadow + NumBytes = StackMapOpers(&MI).getNumPatchBytes(); + assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!"); + break; + case TargetOpcode::PATCHPOINT: + // The size of the patchpoint intrinsic is the number of bytes requested + NumBytes = PatchPointOpers(&MI).getNumPatchBytes(); + assert(NumBytes % 4 == 0 && "Invalid number of NOP bytes requested!"); + break; + case AArch64::TLSDESC_CALLSEQ: + // This gets lowered to an instruction sequence which takes 16 bytes + NumBytes = 16; + break; } - llvm_unreachable("GetInstSizeInBytes()- Unable to determin insn size"); + return NumBytes; } static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target, @@ -92,6 +124,56 @@ static void parseCondBranch(MachineInstr *LastInst, MachineBasicBlock *&Target, } } +static unsigned getBranchDisplacementBits(unsigned Opc) { + switch (Opc) { + default: + llvm_unreachable("unexpected opcode!"); + case AArch64::B: + return 64; + case AArch64::TBNZW: + case AArch64::TBZW: + case AArch64::TBNZX: + case AArch64::TBZX: + return TBZDisplacementBits; + case AArch64::CBNZW: + case AArch64::CBZW: + case AArch64::CBNZX: + case AArch64::CBZX: + return CBZDisplacementBits; + case AArch64::Bcc: + return BCCDisplacementBits; + } +} + +bool AArch64InstrInfo::isBranchOffsetInRange(unsigned BranchOp, + int64_t BrOffset) const { + unsigned Bits = getBranchDisplacementBits(BranchOp); + assert(Bits >= 3 && "max branch displacement must be enough to jump" + "over conditional branch expansion"); + return isIntN(Bits, BrOffset / 4); +} + +MachineBasicBlock *AArch64InstrInfo::getBranchDestBlock( + const MachineInstr &MI) const { + switch (MI.getOpcode()) { + default: + llvm_unreachable("unexpected opcode!"); + case AArch64::B: + return MI.getOperand(0).getMBB(); + case AArch64::TBZW: + case AArch64::TBNZW: + case AArch64::TBZX: + case AArch64::TBNZX: + return MI.getOperand(2).getMBB(); + case AArch64::CBZW: + case AArch64::CBNZW: + case AArch64::CBZX: + case AArch64::CBNZX: + case AArch64::Bcc: + return MI.getOperand(1).getMBB(); + } +} + // Branch analysis. bool AArch64InstrInfo::analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, @@ -180,7 +262,7 @@ bool AArch64InstrInfo::analyzeBranch(MachineBasicBlock &MBB, return true; } -bool AArch64InstrInfo::ReverseBranchCondition( +bool AArch64InstrInfo::reverseBranchCondition( SmallVectorImpl &Cond) const { if (Cond[0].getImm() != -1) { // Regular Bcc @@ -221,7 +303,8 @@ bool AArch64InstrInfo::ReverseBranchCondition( return false; } -unsigned AArch64InstrInfo::RemoveBranch(MachineBasicBlock &MBB) const { +unsigned AArch64InstrInfo::removeBranch(MachineBasicBlock &MBB, + int *BytesRemoved) const { MachineBasicBlock::iterator I = MBB.getLastNonDebugInstr(); if (I == MBB.end()) return 0; @@ -235,14 +318,23 @@ unsigned AArch64InstrInfo::RemoveBranch(MachineBasicBlock &MBB) const { I = MBB.end(); - if (I == MBB.begin()) + if (I == MBB.begin()) { + if (BytesRemoved) + *BytesRemoved = 4; return 1; + } --I; - if (!isCondBranchOpcode(I->getOpcode())) + if (!isCondBranchOpcode(I->getOpcode())) { + if (BytesRemoved) + *BytesRemoved = 4; return 1; + } // Remove the branch. I->eraseFromParent(); + if (BytesRemoved) + *BytesRemoved = 8; + return 2; } @@ -263,25 +355,34 @@ void AArch64InstrInfo::instantiateCondBranch( } } -unsigned AArch64InstrInfo::InsertBranch(MachineBasicBlock &MBB, +unsigned AArch64InstrInfo::insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef Cond, - const DebugLoc &DL) const { + const DebugLoc &DL, + int *BytesAdded) const { // Shouldn't be a fall through. - assert(TBB && "InsertBranch must not be told to insert a fallthrough"); + assert(TBB && "insertBranch must not be told to insert a fallthrough"); if (!FBB) { if (Cond.empty()) // Unconditional branch? BuildMI(&MBB, DL, get(AArch64::B)).addMBB(TBB); else instantiateCondBranch(MBB, DL, TBB, Cond); + + if (BytesAdded) + *BytesAdded = 4; + return 1; } // Two-way conditional branch. instantiateCondBranch(MBB, DL, TBB, Cond); BuildMI(&MBB, DL, get(AArch64::B)).addMBB(FBB); + + if (BytesAdded) + *BytesAdded = 8; + return 2; } @@ -315,7 +416,8 @@ static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg, // if NZCV is used, do not fold. if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) == -1) return 0; - // fall-through to ADDXri and ADDWri. + // fall-through to ADDXri and ADDWri. + LLVM_FALLTHROUGH; case AArch64::ADDXri: case AArch64::ADDWri: // add x, 1 -> csinc. @@ -342,7 +444,8 @@ static unsigned canFoldIntoCSel(const MachineRegisterInfo &MRI, unsigned VReg, // if NZCV is used, do not fold. if (DefMI->findRegisterDefOperandIdx(AArch64::NZCV, true) == -1) return 0; - // fall-through to SUBXrr and SUBWrr. + // fall-through to SUBXrr and SUBWrr. + LLVM_FALLTHROUGH; case AArch64::SUBXrr: case AArch64::SUBWrr: { // neg x -> csneg, represented as sub dst, xzr, src. @@ -858,9 +961,9 @@ static bool areCFlagsAccessedBetweenInstrs( return true; // From must be above To. - assert(std::find_if(MachineBasicBlock::reverse_iterator(To), - To->getParent()->rend(), [From](MachineInstr &MI) { - return MachineBasicBlock::iterator(MI) == From; + assert(std::find_if(++To.getReverse(), To->getParent()->rend(), + [From](MachineInstr &MI) { + return MI.getIterator() == From; }) != To->getParent()->rend()); // We iterate backward starting \p To until we hit \p From. @@ -968,6 +1071,7 @@ static bool areCFlagsAliveInSuccessors(MachineBasicBlock *MBB) { return false; } +namespace { struct UsedNZCV { bool N; bool Z; @@ -982,6 +1086,7 @@ struct UsedNZCV { return *this; } }; +} // end anonymous namespace /// Find a condition code used by the instruction. /// Returns AArch64CC::Invalid if either the instruction does not use condition @@ -1526,7 +1631,6 @@ bool AArch64InstrInfo::isCandidateToMergeOrPair(MachineInstr &MI) const { switch (MI.getOpcode()) { default: break; - case AArch64::LDURQi: case AArch64::STURQi: case AArch64::LDRQui: @@ -1541,36 +1645,8 @@ bool AArch64InstrInfo::isCandidateToMergeOrPair(MachineInstr &MI) const { bool AArch64InstrInfo::getMemOpBaseRegImmOfs( MachineInstr &LdSt, unsigned &BaseReg, int64_t &Offset, const TargetRegisterInfo *TRI) const { - switch (LdSt.getOpcode()) { - default: - return false; - // Scaled instructions. - case AArch64::STRSui: - case AArch64::STRDui: - case AArch64::STRQui: - case AArch64::STRXui: - case AArch64::STRWui: - case AArch64::LDRSui: - case AArch64::LDRDui: - case AArch64::LDRQui: - case AArch64::LDRXui: - case AArch64::LDRWui: - case AArch64::LDRSWui: - // Unscaled instructions. - case AArch64::STURSi: - case AArch64::STURDi: - case AArch64::STURQi: - case AArch64::STURXi: - case AArch64::STURWi: - case AArch64::LDURSi: - case AArch64::LDURDi: - case AArch64::LDURQi: - case AArch64::LDURWi: - case AArch64::LDURXi: - case AArch64::LDURSWi: - unsigned Width; - return getMemOpBaseRegImmOfsWidth(LdSt, BaseReg, Offset, Width, TRI); - }; + unsigned Width; + return getMemOpBaseRegImmOfsWidth(LdSt, BaseReg, Offset, Width, TRI); } bool AArch64InstrInfo::getMemOpBaseRegImmOfsWidth( @@ -1769,6 +1845,9 @@ bool AArch64InstrInfo::shouldClusterMemOps(MachineInstr &FirstLdSt, if (NumLoads > 1) return false; + if (!isPairableLdStInst(FirstLdSt) || !isPairableLdStInst(SecondLdSt)) + return false; + // Can we pair these instructions based on their opcodes? unsigned FirstOpc = FirstLdSt.getOpcode(); unsigned SecondOpc = SecondLdSt.getOpcode(); @@ -1799,41 +1878,82 @@ bool AArch64InstrInfo::shouldClusterMemOps(MachineInstr &FirstLdSt, return Offset1 + 1 == Offset2; } -bool AArch64InstrInfo::shouldScheduleAdjacent(MachineInstr &First, - MachineInstr &Second) const { - if (Subtarget.hasMacroOpFusion()) { +bool AArch64InstrInfo::shouldScheduleAdjacent( + const MachineInstr &First, const MachineInstr &Second) const { + if (Subtarget.hasArithmeticBccFusion()) { // Fuse CMN, CMP, TST followed by Bcc. unsigned SecondOpcode = Second.getOpcode(); if (SecondOpcode == AArch64::Bcc) { switch (First.getOpcode()) { default: return false; - case AArch64::SUBSWri: case AArch64::ADDSWri: - case AArch64::ANDSWri: - case AArch64::SUBSXri: + case AArch64::ADDSWrr: case AArch64::ADDSXri: + case AArch64::ADDSXrr: + case AArch64::ANDSWri: + case AArch64::ANDSWrr: case AArch64::ANDSXri: + case AArch64::ANDSXrr: + case AArch64::SUBSWri: + case AArch64::SUBSWrr: + case AArch64::SUBSXri: + case AArch64::SUBSXrr: + case AArch64::BICSWrr: + case AArch64::BICSXrr: return true; + case AArch64::ADDSWrs: + case AArch64::ADDSXrs: + case AArch64::ANDSWrs: + case AArch64::ANDSXrs: + case AArch64::SUBSWrs: + case AArch64::SUBSXrs: + case AArch64::BICSWrs: + case AArch64::BICSXrs: + // Shift value can be 0 making these behave like the "rr" variant... + return !hasShiftedReg(Second); } } + } + if (Subtarget.hasArithmeticCbzFusion()) { // Fuse ALU operations followed by CBZ/CBNZ. + unsigned SecondOpcode = Second.getOpcode(); if (SecondOpcode == AArch64::CBNZW || SecondOpcode == AArch64::CBNZX || SecondOpcode == AArch64::CBZW || SecondOpcode == AArch64::CBZX) { switch (First.getOpcode()) { default: return false; case AArch64::ADDWri: + case AArch64::ADDWrr: case AArch64::ADDXri: + case AArch64::ADDXrr: case AArch64::ANDWri: + case AArch64::ANDWrr: case AArch64::ANDXri: + case AArch64::ANDXrr: case AArch64::EORWri: + case AArch64::EORWrr: case AArch64::EORXri: + case AArch64::EORXrr: case AArch64::ORRWri: + case AArch64::ORRWrr: case AArch64::ORRXri: + case AArch64::ORRXrr: case AArch64::SUBWri: + case AArch64::SUBWrr: case AArch64::SUBXri: + case AArch64::SUBXrr: return true; + case AArch64::ADDWrs: + case AArch64::ADDXrs: + case AArch64::ANDWrs: + case AArch64::ANDXrs: + case AArch64::SUBWrs: + case AArch64::SUBXrs: + case AArch64::BICWrs: + case AArch64::BICXrs: + // Shift value can be 0 making these behave like the "rr" variant... + return !hasShiftedReg(Second); } } } @@ -2186,7 +2306,7 @@ void AArch64InstrInfo::storeRegToStackSlot( if (MBBI != MBB.end()) DL = MBBI->getDebugLoc(); MachineFunction &MF = *MBB.getParent(); - MachineFrameInfo &MFI = *MF.getFrameInfo(); + MachineFrameInfo &MFI = MF.getFrameInfo(); unsigned Align = MFI.getObjectAlignment(FI); MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI); @@ -2290,7 +2410,7 @@ void AArch64InstrInfo::loadRegFromStackSlot( if (MBBI != MBB.end()) DL = MBBI->getDebugLoc(); MachineFunction &MF = *MBB.getParent(); - MachineFrameInfo &MFI = *MF.getFrameInfo(); + MachineFrameInfo &MFI = MF.getFrameInfo(); unsigned Align = MFI.getObjectAlignment(FI); MachinePointerInfo PtrInfo = MachinePointerInfo::getFixedStack(MF, FI); MachineMemOperand *MMO = MF.getMachineMemOperand( @@ -2478,6 +2598,57 @@ MachineInstr *AArch64InstrInfo::foldMemoryOperandImpl( } } + // Handle the case where a copy is being spilled or refilled but the source + // and destination register class don't match. For example: + // + // %vreg0 = COPY %XZR; GPR64common:%vreg0 + // + // In this case we can still safely fold away the COPY and generate the + // following spill code: + // + // STRXui %XZR, + // + // This also eliminates spilled cross register class COPYs (e.g. between x and + // d regs) of the same size. For example: + // + // %vreg0 = COPY %vreg1; GPR64:%vreg0, FPR64:%vreg1 + // + // will be refilled as + // + // LDRDui %vreg0, fi<#0> + // + // instead of + // + // LDRXui %vregTemp, fi<#0> + // %vreg0 = FMOV %vregTemp + // + if (MI.isFullCopy() && Ops.size() == 1 && + // Make sure we're only folding the explicit COPY defs/uses. + (Ops[0] == 0 || Ops[0] == 1)) { + const TargetRegisterInfo &TRI = *MF.getSubtarget().getRegisterInfo(); + const MachineRegisterInfo &MRI = MF.getRegInfo(); + MachineBasicBlock &MBB = *MI.getParent(); + const MachineOperand &DstMO = MI.getOperand(0); + const MachineOperand &SrcMO = MI.getOperand(1); + unsigned DstReg = DstMO.getReg(); + unsigned SrcReg = SrcMO.getReg(); + auto getRegClass = [&](unsigned Reg) { + return TargetRegisterInfo::isVirtualRegister(Reg) + ? MRI.getRegClass(Reg) + : TRI.getMinimalPhysRegClass(Reg); + }; + const TargetRegisterClass &DstRC = *getRegClass(DstReg); + const TargetRegisterClass &SrcRC = *getRegClass(SrcReg); + if (DstRC.getSize() == SrcRC.getSize()) { + if (Ops[0] == 0) + storeRegToStackSlot(MBB, InsertPt, SrcReg, SrcMO.isKill(), FrameIndex, + &SrcRC, &TRI); + else + loadRegFromStackSlot(MBB, InsertPt, DstReg, FrameIndex, &DstRC, &TRI); + return &*--InsertPt; + } + } + // Cannot fold. return nullptr; } @@ -2826,6 +2997,8 @@ static bool isCombineInstrCandidate64(unsigned Opc) { // FP Opcodes that can be combined with a FMUL static bool isCombineInstrCandidateFP(const MachineInstr &Inst) { switch (Inst.getOpcode()) { + default: + break; case AArch64::FADDSrr: case AArch64::FADDDrr: case AArch64::FADDv2f32: @@ -2836,9 +3009,9 @@ static bool isCombineInstrCandidateFP(const MachineInstr &Inst) { case AArch64::FSUBv2f32: case AArch64::FSUBv2f64: case AArch64::FSUBv4f32: - return Inst.getParent()->getParent()->getTarget().Options.UnsafeFPMath; - default: - break; + TargetOptions Options = Inst.getParent()->getParent()->getTarget().Options; + return (Options.UnsafeFPMath || + Options.AllowFPOpFusion == FPOpFusion::Fast); } return false; } @@ -3462,7 +3635,7 @@ void AArch64InstrInfo::genAlternativeCodeSequence( unsigned Val = Root.getOperand(3).getImm(); Imm = Imm << Val; } - uint64_t UImm = Imm << (64 - BitSize) >> (64 - BitSize); + uint64_t UImm = SignExtend64(Imm, BitSize); uint64_t Encoding; if (AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding)) { MachineInstrBuilder MIB1 = @@ -3548,12 +3721,12 @@ void AArch64InstrInfo::genAlternativeCodeSequence( RC = &AArch64::GPR64RegClass; } unsigned NewVR = MRI.createVirtualRegister(OrrRC); - int Imm = Root.getOperand(2).getImm(); + uint64_t Imm = Root.getOperand(2).getImm(); if (Root.getOperand(3).isImm()) { unsigned Val = Root.getOperand(3).getImm(); Imm = Imm << Val; } - uint64_t UImm = -Imm << (64 - BitSize) >> (64 - BitSize); + uint64_t UImm = SignExtend64(-Imm, BitSize); uint64_t Encoding; if (AArch64_AM::processLogicalImmediate(UImm, BitSize, Encoding)) { MachineInstrBuilder MIB1 = diff --git a/lib/Target/AArch64/AArch64InstrInfo.h b/lib/Target/AArch64/AArch64InstrInfo.h index 24bc0e639747..90b2c0896872 100644 --- a/lib/Target/AArch64/AArch64InstrInfo.h +++ b/lib/Target/AArch64/AArch64InstrInfo.h @@ -27,7 +27,7 @@ namespace llvm { class AArch64Subtarget; class AArch64TargetMachine; -class AArch64InstrInfo : public AArch64GenInstrInfo { +class AArch64InstrInfo final : public AArch64GenInstrInfo { const AArch64RegisterInfo RI; const AArch64Subtarget &Subtarget; @@ -39,7 +39,7 @@ public: /// always be able to get register info as well (through this method). const AArch64RegisterInfo &getRegisterInfo() const { return RI; } - unsigned GetInstSizeInBytes(const MachineInstr &MI) const; + unsigned getInstSizeInBytes(const MachineInstr &MI) const override; bool isAsCheapAsAMove(const MachineInstr &MI) const override; @@ -87,6 +87,38 @@ public: /// Return true if this is an unscaled load/store. bool isUnscaledLdSt(MachineInstr &MI) const; + static bool isPairableLdStInst(const MachineInstr &MI) { + switch (MI.getOpcode()) { + default: + return false; + // Scaled instructions. + case AArch64::STRSui: + case AArch64::STRDui: + case AArch64::STRQui: + case AArch64::STRXui: + case AArch64::STRWui: + case AArch64::LDRSui: + case AArch64::LDRDui: + case AArch64::LDRQui: + case AArch64::LDRXui: + case AArch64::LDRWui: + case AArch64::LDRSWui: + // Unscaled instructions. + case AArch64::STURSi: + case AArch64::STURDi: + case AArch64::STURQi: + case AArch64::STURWi: + case AArch64::STURXi: + case AArch64::LDURSi: + case AArch64::LDURDi: + case AArch64::LDURQi: + case AArch64::LDURWi: + case AArch64::LDURXi: + case AArch64::LDURSWi: + return true; + } + } + /// Return true if this is a load/store that can be potentially paired/merged. bool isCandidateToMergeOrPair(MachineInstr &MI) const; @@ -101,15 +133,11 @@ public: int64_t &Offset, unsigned &Width, const TargetRegisterInfo *TRI) const; - bool enableClusterLoads() const override { return true; } - - bool enableClusterStores() const override { return true; } - bool shouldClusterMemOps(MachineInstr &FirstLdSt, MachineInstr &SecondLdSt, unsigned NumLoads) const override; - bool shouldScheduleAdjacent(MachineInstr &First, - MachineInstr &Second) const override; + bool shouldScheduleAdjacent(const MachineInstr &First, + const MachineInstr &Second) const override; MachineInstr *emitFrameIndexDebugValue(MachineFunction &MF, int FrameIx, uint64_t Offset, const MDNode *Var, @@ -141,16 +169,25 @@ public: MachineBasicBlock::iterator InsertPt, int FrameIndex, LiveIntervals *LIS = nullptr) const override; + /// \returns true if a branch from an instruction with opcode \p BranchOpc + /// bytes is capable of jumping to a position \p BrOffset bytes away. + bool isBranchOffsetInRange(unsigned BranchOpc, + int64_t BrOffset) const override; + + MachineBasicBlock *getBranchDestBlock(const MachineInstr &MI) const override; + bool analyzeBranch(MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB, SmallVectorImpl &Cond, bool AllowModify = false) const override; - unsigned RemoveBranch(MachineBasicBlock &MBB) const override; - unsigned InsertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, + unsigned removeBranch(MachineBasicBlock &MBB, + int *BytesRemoved = nullptr) const override; + unsigned insertBranch(MachineBasicBlock &MBB, MachineBasicBlock *TBB, MachineBasicBlock *FBB, ArrayRef Cond, - const DebugLoc &DL) const override; + const DebugLoc &DL, + int *BytesAdded = nullptr) const override; bool - ReverseBranchCondition(SmallVectorImpl &Cond) const override; + reverseBranchCondition(SmallVectorImpl &Cond) const override; bool canInsertSelect(const MachineBasicBlock &, ArrayRef Cond, unsigned, unsigned, int &, int &, int &) const override; void insertSelect(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, diff --git a/lib/Target/AArch64/AArch64InstrInfo.td b/lib/Target/AArch64/AArch64InstrInfo.td index af9ed812e6da..c5b95f282ea8 100644 --- a/lib/Target/AArch64/AArch64InstrInfo.td +++ b/lib/Target/AArch64/AArch64InstrInfo.td @@ -26,6 +26,8 @@ def HasCrypto : Predicate<"Subtarget->hasCrypto()">, AssemblerPredicate<"FeatureCrypto", "crypto">; def HasCRC : Predicate<"Subtarget->hasCRC()">, AssemblerPredicate<"FeatureCRC", "crc">; +def HasLSE : Predicate<"Subtarget->hasLSE()">, + AssemblerPredicate<"FeatureLSE", "lse">; def HasRAS : Predicate<"Subtarget->hasRAS()">, AssemblerPredicate<"FeatureRAS", "ras">; def HasPerfMon : Predicate<"Subtarget->hasPerfMon()">; @@ -287,7 +289,9 @@ def AArch64smull : SDNode<"AArch64ISD::SMULL", SDT_AArch64mull>; def AArch64umull : SDNode<"AArch64ISD::UMULL", SDT_AArch64mull>; def AArch64frecpe : SDNode<"AArch64ISD::FRECPE", SDTFPUnaryOp>; +def AArch64frecps : SDNode<"AArch64ISD::FRECPS", SDTFPBinOp>; def AArch64frsqrte : SDNode<"AArch64ISD::FRSQRTE", SDTFPUnaryOp>; +def AArch64frsqrts : SDNode<"AArch64ISD::FRSQRTS", SDTFPBinOp>; def AArch64saddv : SDNode<"AArch64ISD::SADDV", SDT_AArch64UnaryVec>; def AArch64uaddv : SDNode<"AArch64ISD::UADDV", SDT_AArch64UnaryVec>; @@ -1133,6 +1137,14 @@ def : Pat<(AArch64csel (i32 0), (i32 1), (i32 imm:$cc), NZCV), (CSINCWr WZR, WZR, (i32 imm:$cc))>; def : Pat<(AArch64csel (i64 0), (i64 1), (i32 imm:$cc), NZCV), (CSINCXr XZR, XZR, (i32 imm:$cc))>; +def : Pat<(AArch64csel GPR32:$tval, (i32 1), (i32 imm:$cc), NZCV), + (CSINCWr GPR32:$tval, WZR, (i32 imm:$cc))>; +def : Pat<(AArch64csel GPR64:$tval, (i64 1), (i32 imm:$cc), NZCV), + (CSINCXr GPR64:$tval, XZR, (i32 imm:$cc))>; +def : Pat<(AArch64csel (i32 1), GPR32:$fval, (i32 imm:$cc), NZCV), + (CSINCWr GPR32:$fval, WZR, (i32 (inv_cond_XFORM imm:$cc)))>; +def : Pat<(AArch64csel (i64 1), GPR64:$fval, (i32 imm:$cc), NZCV), + (CSINCXr GPR64:$fval, XZR, (i32 (inv_cond_XFORM imm:$cc)))>; def : Pat<(AArch64csel (i32 0), (i32 -1), (i32 imm:$cc), NZCV), (CSINVWr WZR, WZR, (i32 imm:$cc))>; def : Pat<(AArch64csel (i64 0), (i64 -1), (i32 imm:$cc), NZCV), @@ -2545,8 +2557,8 @@ defm : FPToIntegerPats; defm : FPToIntegerPats; defm : FPToIntegerPats; defm : FPToIntegerPats; -defm : FPToIntegerPats; -defm : FPToIntegerPats; +defm : FPToIntegerPats; +defm : FPToIntegerPats; //===----------------------------------------------------------------------===// // Scaled integer to floating point conversion instructions. @@ -2582,7 +2594,7 @@ defm FCVT : FPConversion<"fcvt">; defm FABS : SingleOperandFPData<0b0001, "fabs", fabs>; defm FMOV : SingleOperandFPData<0b0000, "fmov">; defm FNEG : SingleOperandFPData<0b0010, "fneg", fneg>; -defm FRINTA : SingleOperandFPData<0b1100, "frinta", frnd>; +defm FRINTA : SingleOperandFPData<0b1100, "frinta", fround>; defm FRINTI : SingleOperandFPData<0b1111, "frinti", fnearbyint>; defm FRINTM : SingleOperandFPData<0b1010, "frintm", ffloor>; defm FRINTN : SingleOperandFPData<0b1000, "frintn", int_aarch64_neon_frintn>; @@ -2788,13 +2800,13 @@ def : Pat<(v4f32 (int_aarch64_neon_vcvthf2fp (v4i16 V64:$Rn))), def : Pat<(v4f32 (int_aarch64_neon_vcvthf2fp (extract_subvector (v8i16 V128:$Rn), (i64 4)))), (FCVTLv8i16 V128:$Rn)>; -def : Pat<(v2f64 (fextend (v2f32 V64:$Rn))), (FCVTLv2i32 V64:$Rn)>; -def : Pat<(v2f64 (fextend (v2f32 (extract_subvector (v4f32 V128:$Rn), +def : Pat<(v2f64 (fpextend (v2f32 V64:$Rn))), (FCVTLv2i32 V64:$Rn)>; +def : Pat<(v2f64 (fpextend (v2f32 (extract_subvector (v4f32 V128:$Rn), (i64 2))))), (FCVTLv4i32 V128:$Rn)>; -def : Pat<(v4f32 (fextend (v4f16 V64:$Rn))), (FCVTLv4i16 V64:$Rn)>; -def : Pat<(v4f32 (fextend (v4f16 (extract_subvector (v8f16 V128:$Rn), +def : Pat<(v4f32 (fpextend (v4f16 V64:$Rn))), (FCVTLv4i16 V64:$Rn)>; +def : Pat<(v4f32 (fpextend (v4f16 (extract_subvector (v8f16 V128:$Rn), (i64 4))))), (FCVTLv8i16 V128:$Rn)>; @@ -2808,9 +2820,9 @@ def : Pat<(v4i16 (int_aarch64_neon_vcvtfp2hf (v4f32 V128:$Rn))), def : Pat<(concat_vectors V64:$Rd, (v4i16 (int_aarch64_neon_vcvtfp2hf (v4f32 V128:$Rn)))), (FCVTNv8i16 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rd, dsub), V128:$Rn)>; -def : Pat<(v2f32 (fround (v2f64 V128:$Rn))), (FCVTNv2i32 V128:$Rn)>; -def : Pat<(v4f16 (fround (v4f32 V128:$Rn))), (FCVTNv4i16 V128:$Rn)>; -def : Pat<(concat_vectors V64:$Rd, (v2f32 (fround (v2f64 V128:$Rn)))), +def : Pat<(v2f32 (fpround (v2f64 V128:$Rn))), (FCVTNv2i32 V128:$Rn)>; +def : Pat<(v4f16 (fpround (v4f32 V128:$Rn))), (FCVTNv4i16 V128:$Rn)>; +def : Pat<(concat_vectors V64:$Rd, (v2f32 (fpround (v2f64 V128:$Rn)))), (FCVTNv4i32 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rd, dsub), V128:$Rn)>; defm FCVTPS : SIMDTwoVectorFPToInt<0,1,0b11010, "fcvtps",int_aarch64_neon_fcvtps>; defm FCVTPU : SIMDTwoVectorFPToInt<1,1,0b11010, "fcvtpu",int_aarch64_neon_fcvtpu>; @@ -2833,7 +2845,7 @@ def : Pat<(v2i64 (int_aarch64_neon_fcvtzu v2f64:$Rn)), (FCVTZUv2f64 $Rn)>; defm FNEG : SIMDTwoVectorFP<1, 1, 0b01111, "fneg", fneg>; defm FRECPE : SIMDTwoVectorFP<0, 1, 0b11101, "frecpe", int_aarch64_neon_frecpe>; -defm FRINTA : SIMDTwoVectorFP<1, 0, 0b11000, "frinta", frnd>; +defm FRINTA : SIMDTwoVectorFP<1, 0, 0b11000, "frinta", fround>; defm FRINTI : SIMDTwoVectorFP<1, 1, 0b11001, "frinti", fnearbyint>; defm FRINTM : SIMDTwoVectorFP<0, 0, 0b11001, "frintm", ffloor>; defm FRINTN : SIMDTwoVectorFP<0, 0, 0b11000, "frintn", int_aarch64_neon_frintn>; @@ -3414,6 +3426,17 @@ def : Pat<(v1f64 (AArch64frecpe (v1f64 FPR64:$Rn))), def : Pat<(v2f64 (AArch64frecpe (v2f64 FPR128:$Rn))), (FRECPEv2f64 FPR128:$Rn)>; +def : Pat<(f32 (AArch64frecps (f32 FPR32:$Rn), (f32 FPR32:$Rm))), + (FRECPS32 FPR32:$Rn, FPR32:$Rm)>; +def : Pat<(v2f32 (AArch64frecps (v2f32 V64:$Rn), (v2f32 V64:$Rm))), + (FRECPSv2f32 V64:$Rn, V64:$Rm)>; +def : Pat<(v4f32 (AArch64frecps (v4f32 FPR128:$Rn), (v4f32 FPR128:$Rm))), + (FRECPSv4f32 FPR128:$Rn, FPR128:$Rm)>; +def : Pat<(f64 (AArch64frecps (f64 FPR64:$Rn), (f64 FPR64:$Rm))), + (FRECPS64 FPR64:$Rn, FPR64:$Rm)>; +def : Pat<(v2f64 (AArch64frecps (v2f64 FPR128:$Rn), (v2f64 FPR128:$Rm))), + (FRECPSv2f64 FPR128:$Rn, FPR128:$Rm)>; + def : Pat<(f32 (int_aarch64_neon_frecpx (f32 FPR32:$Rn))), (FRECPXv1i32 FPR32:$Rn)>; def : Pat<(f64 (int_aarch64_neon_frecpx (f64 FPR64:$Rn))), @@ -3439,6 +3462,17 @@ def : Pat<(v1f64 (AArch64frsqrte (v1f64 FPR64:$Rn))), def : Pat<(v2f64 (AArch64frsqrte (v2f64 FPR128:$Rn))), (FRSQRTEv2f64 FPR128:$Rn)>; +def : Pat<(f32 (AArch64frsqrts (f32 FPR32:$Rn), (f32 FPR32:$Rm))), + (FRSQRTS32 FPR32:$Rn, FPR32:$Rm)>; +def : Pat<(v2f32 (AArch64frsqrts (v2f32 V64:$Rn), (v2f32 V64:$Rm))), + (FRSQRTSv2f32 V64:$Rn, V64:$Rm)>; +def : Pat<(v4f32 (AArch64frsqrts (v4f32 FPR128:$Rn), (v4f32 FPR128:$Rm))), + (FRSQRTSv4f32 FPR128:$Rn, FPR128:$Rm)>; +def : Pat<(f64 (AArch64frsqrts (f64 FPR64:$Rn), (f64 FPR64:$Rm))), + (FRSQRTS64 FPR64:$Rn, FPR64:$Rm)>; +def : Pat<(v2f64 (AArch64frsqrts (v2f64 FPR128:$Rn), (v2f64 FPR128:$Rm))), + (FRSQRTSv2f64 FPR128:$Rn, FPR128:$Rm)>; + // If an integer is about to be converted to a floating point value, // just load it on the floating point unit. // Here are the patterns for 8 and 16-bits to float. @@ -5293,15 +5327,8 @@ def SHA256SU0rr : SHATiedInstVV<0b0010, "sha256su0",int_aarch64_crypto_sha256su0 //---------------------------------------------------------------------------- // FIXME: Like for X86, these should go in their own separate .td file. -// Any instruction that defines a 32-bit result leaves the high half of the -// register. Truncate can be lowered to EXTRACT_SUBREG. CopyFromReg may -// be copying from a truncate. But any other 32-bit operation will zero-extend -// up to 64 bits. -// FIXME: X86 also checks for CMOV here. Do we need something similar? def def32 : PatLeaf<(i32 GPR32:$src), [{ - return N->getOpcode() != ISD::TRUNCATE && - N->getOpcode() != TargetOpcode::EXTRACT_SUBREG && - N->getOpcode() != ISD::CopyFromReg; + return isDef32(*N); }]>; // In the case of a 32-bit def that is known to implicitly zero-extend, diff --git a/lib/Target/AArch64/AArch64InstructionSelector.cpp b/lib/Target/AArch64/AArch64InstructionSelector.cpp new file mode 100644 index 000000000000..20de07424c53 --- /dev/null +++ b/lib/Target/AArch64/AArch64InstructionSelector.cpp @@ -0,0 +1,1161 @@ +//===- AArch64InstructionSelector.cpp ----------------------------*- C++ -*-==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// This file implements the targeting of the InstructionSelector class for +/// AArch64. +/// \todo This should be generated by TableGen. +//===----------------------------------------------------------------------===// + +#include "AArch64InstructionSelector.h" +#include "AArch64InstrInfo.h" +#include "AArch64RegisterBankInfo.h" +#include "AArch64RegisterInfo.h" +#include "AArch64Subtarget.h" +#include "AArch64TargetMachine.h" +#include "MCTargetDesc/AArch64AddressingModes.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/IR/Type.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/raw_ostream.h" + +#define DEBUG_TYPE "aarch64-isel" + +using namespace llvm; + +#ifndef LLVM_BUILD_GLOBAL_ISEL +#error "You shouldn't build this" +#endif + +#include "AArch64GenGlobalISel.inc" + +AArch64InstructionSelector::AArch64InstructionSelector( + const AArch64TargetMachine &TM, const AArch64Subtarget &STI, + const AArch64RegisterBankInfo &RBI) + : InstructionSelector(), TM(TM), STI(STI), TII(*STI.getInstrInfo()), + TRI(*STI.getRegisterInfo()), RBI(RBI) {} + +// FIXME: This should be target-independent, inferred from the types declared +// for each class in the bank. +static const TargetRegisterClass * +getRegClassForTypeOnBank(LLT Ty, const RegisterBank &RB, + const RegisterBankInfo &RBI) { + if (RB.getID() == AArch64::GPRRegBankID) { + if (Ty.getSizeInBits() <= 32) + return &AArch64::GPR32RegClass; + if (Ty.getSizeInBits() == 64) + return &AArch64::GPR64RegClass; + return nullptr; + } + + if (RB.getID() == AArch64::FPRRegBankID) { + if (Ty.getSizeInBits() == 32) + return &AArch64::FPR32RegClass; + if (Ty.getSizeInBits() == 64) + return &AArch64::FPR64RegClass; + if (Ty.getSizeInBits() == 128) + return &AArch64::FPR128RegClass; + return nullptr; + } + + return nullptr; +} + +/// Check whether \p I is a currently unsupported binary operation: +/// - it has an unsized type +/// - an operand is not a vreg +/// - all operands are not in the same bank +/// These are checks that should someday live in the verifier, but right now, +/// these are mostly limitations of the aarch64 selector. +static bool unsupportedBinOp(const MachineInstr &I, + const AArch64RegisterBankInfo &RBI, + const MachineRegisterInfo &MRI, + const AArch64RegisterInfo &TRI) { + LLT Ty = MRI.getType(I.getOperand(0).getReg()); + if (!Ty.isValid()) { + DEBUG(dbgs() << "Generic binop register should be typed\n"); + return true; + } + + const RegisterBank *PrevOpBank = nullptr; + for (auto &MO : I.operands()) { + // FIXME: Support non-register operands. + if (!MO.isReg()) { + DEBUG(dbgs() << "Generic inst non-reg operands are unsupported\n"); + return true; + } + + // FIXME: Can generic operations have physical registers operands? If + // so, this will need to be taught about that, and we'll need to get the + // bank out of the minimal class for the register. + // Either way, this needs to be documented (and possibly verified). + if (!TargetRegisterInfo::isVirtualRegister(MO.getReg())) { + DEBUG(dbgs() << "Generic inst has physical register operand\n"); + return true; + } + + const RegisterBank *OpBank = RBI.getRegBank(MO.getReg(), MRI, TRI); + if (!OpBank) { + DEBUG(dbgs() << "Generic register has no bank or class\n"); + return true; + } + + if (PrevOpBank && OpBank != PrevOpBank) { + DEBUG(dbgs() << "Generic inst operands have different banks\n"); + return true; + } + PrevOpBank = OpBank; + } + return false; +} + +/// Select the AArch64 opcode for the basic binary operation \p GenericOpc +/// (such as G_OR or G_ADD), appropriate for the register bank \p RegBankID +/// and of size \p OpSize. +/// \returns \p GenericOpc if the combination is unsupported. +static unsigned selectBinaryOp(unsigned GenericOpc, unsigned RegBankID, + unsigned OpSize) { + switch (RegBankID) { + case AArch64::GPRRegBankID: + if (OpSize <= 32) { + assert((OpSize == 32 || (GenericOpc != TargetOpcode::G_SDIV && + GenericOpc != TargetOpcode::G_UDIV && + GenericOpc != TargetOpcode::G_LSHR && + GenericOpc != TargetOpcode::G_ASHR)) && + "operation should have been legalized before now"); + + switch (GenericOpc) { + case TargetOpcode::G_OR: + return AArch64::ORRWrr; + case TargetOpcode::G_XOR: + return AArch64::EORWrr; + case TargetOpcode::G_AND: + return AArch64::ANDWrr; + case TargetOpcode::G_ADD: + assert(OpSize != 32 && "s32 G_ADD should have been selected"); + return AArch64::ADDWrr; + case TargetOpcode::G_SUB: + return AArch64::SUBWrr; + case TargetOpcode::G_SHL: + return AArch64::LSLVWr; + case TargetOpcode::G_LSHR: + return AArch64::LSRVWr; + case TargetOpcode::G_ASHR: + return AArch64::ASRVWr; + case TargetOpcode::G_SDIV: + return AArch64::SDIVWr; + case TargetOpcode::G_UDIV: + return AArch64::UDIVWr; + default: + return GenericOpc; + } + } else if (OpSize == 64) { + switch (GenericOpc) { + case TargetOpcode::G_OR: + return AArch64::ORRXrr; + case TargetOpcode::G_XOR: + return AArch64::EORXrr; + case TargetOpcode::G_AND: + return AArch64::ANDXrr; + case TargetOpcode::G_GEP: + return AArch64::ADDXrr; + case TargetOpcode::G_SUB: + return AArch64::SUBXrr; + case TargetOpcode::G_SHL: + return AArch64::LSLVXr; + case TargetOpcode::G_LSHR: + return AArch64::LSRVXr; + case TargetOpcode::G_ASHR: + return AArch64::ASRVXr; + case TargetOpcode::G_SDIV: + return AArch64::SDIVXr; + case TargetOpcode::G_UDIV: + return AArch64::UDIVXr; + default: + return GenericOpc; + } + } + case AArch64::FPRRegBankID: + switch (OpSize) { + case 32: + switch (GenericOpc) { + case TargetOpcode::G_FADD: + return AArch64::FADDSrr; + case TargetOpcode::G_FSUB: + return AArch64::FSUBSrr; + case TargetOpcode::G_FMUL: + return AArch64::FMULSrr; + case TargetOpcode::G_FDIV: + return AArch64::FDIVSrr; + default: + return GenericOpc; + } + case 64: + switch (GenericOpc) { + case TargetOpcode::G_FADD: + return AArch64::FADDDrr; + case TargetOpcode::G_FSUB: + return AArch64::FSUBDrr; + case TargetOpcode::G_FMUL: + return AArch64::FMULDrr; + case TargetOpcode::G_FDIV: + return AArch64::FDIVDrr; + case TargetOpcode::G_OR: + return AArch64::ORRv8i8; + default: + return GenericOpc; + } + } + }; + return GenericOpc; +} + +/// Select the AArch64 opcode for the G_LOAD or G_STORE operation \p GenericOpc, +/// appropriate for the (value) register bank \p RegBankID and of memory access +/// size \p OpSize. This returns the variant with the base+unsigned-immediate +/// addressing mode (e.g., LDRXui). +/// \returns \p GenericOpc if the combination is unsupported. +static unsigned selectLoadStoreUIOp(unsigned GenericOpc, unsigned RegBankID, + unsigned OpSize) { + const bool isStore = GenericOpc == TargetOpcode::G_STORE; + switch (RegBankID) { + case AArch64::GPRRegBankID: + switch (OpSize) { + case 8: + return isStore ? AArch64::STRBBui : AArch64::LDRBBui; + case 16: + return isStore ? AArch64::STRHHui : AArch64::LDRHHui; + case 32: + return isStore ? AArch64::STRWui : AArch64::LDRWui; + case 64: + return isStore ? AArch64::STRXui : AArch64::LDRXui; + } + case AArch64::FPRRegBankID: + switch (OpSize) { + case 8: + return isStore ? AArch64::STRBui : AArch64::LDRBui; + case 16: + return isStore ? AArch64::STRHui : AArch64::LDRHui; + case 32: + return isStore ? AArch64::STRSui : AArch64::LDRSui; + case 64: + return isStore ? AArch64::STRDui : AArch64::LDRDui; + } + }; + return GenericOpc; +} + +static bool selectCopy(MachineInstr &I, const TargetInstrInfo &TII, + MachineRegisterInfo &MRI, const TargetRegisterInfo &TRI, + const RegisterBankInfo &RBI) { + + unsigned DstReg = I.getOperand(0).getReg(); + if (TargetRegisterInfo::isPhysicalRegister(DstReg)) { + assert(I.isCopy() && "Generic operators do not allow physical registers"); + return true; + } + + const RegisterBank &RegBank = *RBI.getRegBank(DstReg, MRI, TRI); + const unsigned DstSize = MRI.getType(DstReg).getSizeInBits(); + unsigned SrcReg = I.getOperand(1).getReg(); + const unsigned SrcSize = RBI.getSizeInBits(SrcReg, MRI, TRI); + (void)SrcSize; + assert((!TargetRegisterInfo::isPhysicalRegister(SrcReg) || I.isCopy()) && + "No phys reg on generic operators"); + assert( + (DstSize == SrcSize || + // Copies are a mean to setup initial types, the number of + // bits may not exactly match. + (TargetRegisterInfo::isPhysicalRegister(SrcReg) && + DstSize <= RBI.getSizeInBits(SrcReg, MRI, TRI)) || + // Copies are a mean to copy bits around, as long as we are + // on the same register class, that's fine. Otherwise, that + // means we need some SUBREG_TO_REG or AND & co. + (((DstSize + 31) / 32 == (SrcSize + 31) / 32) && DstSize > SrcSize)) && + "Copy with different width?!"); + assert((DstSize <= 64 || RegBank.getID() == AArch64::FPRRegBankID) && + "GPRs cannot get more than 64-bit width values"); + const TargetRegisterClass *RC = nullptr; + + if (RegBank.getID() == AArch64::FPRRegBankID) { + if (DstSize <= 32) + RC = &AArch64::FPR32RegClass; + else if (DstSize <= 64) + RC = &AArch64::FPR64RegClass; + else if (DstSize <= 128) + RC = &AArch64::FPR128RegClass; + else { + DEBUG(dbgs() << "Unexpected bitcast size " << DstSize << '\n'); + return false; + } + } else { + assert(RegBank.getID() == AArch64::GPRRegBankID && + "Bitcast for the flags?"); + RC = + DstSize <= 32 ? &AArch64::GPR32allRegClass : &AArch64::GPR64allRegClass; + } + + // No need to constrain SrcReg. It will get constrained when + // we hit another of its use or its defs. + // Copies do not have constraints. + if (!RBI.constrainGenericRegister(DstReg, *RC, MRI)) { + DEBUG(dbgs() << "Failed to constrain " << TII.getName(I.getOpcode()) + << " operand\n"); + return false; + } + I.setDesc(TII.get(AArch64::COPY)); + return true; +} + +static unsigned selectFPConvOpc(unsigned GenericOpc, LLT DstTy, LLT SrcTy) { + if (!DstTy.isScalar() || !SrcTy.isScalar()) + return GenericOpc; + + const unsigned DstSize = DstTy.getSizeInBits(); + const unsigned SrcSize = SrcTy.getSizeInBits(); + + switch (DstSize) { + case 32: + switch (SrcSize) { + case 32: + switch (GenericOpc) { + case TargetOpcode::G_SITOFP: + return AArch64::SCVTFUWSri; + case TargetOpcode::G_UITOFP: + return AArch64::UCVTFUWSri; + case TargetOpcode::G_FPTOSI: + return AArch64::FCVTZSUWSr; + case TargetOpcode::G_FPTOUI: + return AArch64::FCVTZUUWSr; + default: + return GenericOpc; + } + case 64: + switch (GenericOpc) { + case TargetOpcode::G_SITOFP: + return AArch64::SCVTFUXSri; + case TargetOpcode::G_UITOFP: + return AArch64::UCVTFUXSri; + case TargetOpcode::G_FPTOSI: + return AArch64::FCVTZSUWDr; + case TargetOpcode::G_FPTOUI: + return AArch64::FCVTZUUWDr; + default: + return GenericOpc; + } + default: + return GenericOpc; + } + case 64: + switch (SrcSize) { + case 32: + switch (GenericOpc) { + case TargetOpcode::G_SITOFP: + return AArch64::SCVTFUWDri; + case TargetOpcode::G_UITOFP: + return AArch64::UCVTFUWDri; + case TargetOpcode::G_FPTOSI: + return AArch64::FCVTZSUXSr; + case TargetOpcode::G_FPTOUI: + return AArch64::FCVTZUUXSr; + default: + return GenericOpc; + } + case 64: + switch (GenericOpc) { + case TargetOpcode::G_SITOFP: + return AArch64::SCVTFUXDri; + case TargetOpcode::G_UITOFP: + return AArch64::UCVTFUXDri; + case TargetOpcode::G_FPTOSI: + return AArch64::FCVTZSUXDr; + case TargetOpcode::G_FPTOUI: + return AArch64::FCVTZUUXDr; + default: + return GenericOpc; + } + default: + return GenericOpc; + } + default: + return GenericOpc; + }; + return GenericOpc; +} + +static AArch64CC::CondCode changeICMPPredToAArch64CC(CmpInst::Predicate P) { + switch (P) { + default: + llvm_unreachable("Unknown condition code!"); + case CmpInst::ICMP_NE: + return AArch64CC::NE; + case CmpInst::ICMP_EQ: + return AArch64CC::EQ; + case CmpInst::ICMP_SGT: + return AArch64CC::GT; + case CmpInst::ICMP_SGE: + return AArch64CC::GE; + case CmpInst::ICMP_SLT: + return AArch64CC::LT; + case CmpInst::ICMP_SLE: + return AArch64CC::LE; + case CmpInst::ICMP_UGT: + return AArch64CC::HI; + case CmpInst::ICMP_UGE: + return AArch64CC::HS; + case CmpInst::ICMP_ULT: + return AArch64CC::LO; + case CmpInst::ICMP_ULE: + return AArch64CC::LS; + } +} + +static void changeFCMPPredToAArch64CC(CmpInst::Predicate P, + AArch64CC::CondCode &CondCode, + AArch64CC::CondCode &CondCode2) { + CondCode2 = AArch64CC::AL; + switch (P) { + default: + llvm_unreachable("Unknown FP condition!"); + case CmpInst::FCMP_OEQ: + CondCode = AArch64CC::EQ; + break; + case CmpInst::FCMP_OGT: + CondCode = AArch64CC::GT; + break; + case CmpInst::FCMP_OGE: + CondCode = AArch64CC::GE; + break; + case CmpInst::FCMP_OLT: + CondCode = AArch64CC::MI; + break; + case CmpInst::FCMP_OLE: + CondCode = AArch64CC::LS; + break; + case CmpInst::FCMP_ONE: + CondCode = AArch64CC::MI; + CondCode2 = AArch64CC::GT; + break; + case CmpInst::FCMP_ORD: + CondCode = AArch64CC::VC; + break; + case CmpInst::FCMP_UNO: + CondCode = AArch64CC::VS; + break; + case CmpInst::FCMP_UEQ: + CondCode = AArch64CC::EQ; + CondCode2 = AArch64CC::VS; + break; + case CmpInst::FCMP_UGT: + CondCode = AArch64CC::HI; + break; + case CmpInst::FCMP_UGE: + CondCode = AArch64CC::PL; + break; + case CmpInst::FCMP_ULT: + CondCode = AArch64CC::LT; + break; + case CmpInst::FCMP_ULE: + CondCode = AArch64CC::LE; + break; + case CmpInst::FCMP_UNE: + CondCode = AArch64CC::NE; + break; + } +} + +bool AArch64InstructionSelector::select(MachineInstr &I) const { + assert(I.getParent() && "Instruction should be in a basic block!"); + assert(I.getParent()->getParent() && "Instruction should be in a function!"); + + MachineBasicBlock &MBB = *I.getParent(); + MachineFunction &MF = *MBB.getParent(); + MachineRegisterInfo &MRI = MF.getRegInfo(); + + unsigned Opcode = I.getOpcode(); + if (!isPreISelGenericOpcode(I.getOpcode())) { + // Certain non-generic instructions also need some special handling. + + if (Opcode == TargetOpcode::LOAD_STACK_GUARD) + return constrainSelectedInstRegOperands(I, TII, TRI, RBI); + + if (Opcode == TargetOpcode::PHI) { + const unsigned DefReg = I.getOperand(0).getReg(); + const LLT DefTy = MRI.getType(DefReg); + + const TargetRegisterClass *DefRC = nullptr; + if (TargetRegisterInfo::isPhysicalRegister(DefReg)) { + DefRC = TRI.getRegClass(DefReg); + } else { + const RegClassOrRegBank &RegClassOrBank = + MRI.getRegClassOrRegBank(DefReg); + + DefRC = RegClassOrBank.dyn_cast(); + if (!DefRC) { + if (!DefTy.isValid()) { + DEBUG(dbgs() << "PHI operand has no type, not a gvreg?\n"); + return false; + } + const RegisterBank &RB = *RegClassOrBank.get(); + DefRC = getRegClassForTypeOnBank(DefTy, RB, RBI); + if (!DefRC) { + DEBUG(dbgs() << "PHI operand has unexpected size/bank\n"); + return false; + } + } + } + + return RBI.constrainGenericRegister(DefReg, *DefRC, MRI); + } + + if (I.isCopy()) + return selectCopy(I, TII, MRI, TRI, RBI); + + return true; + } + + + if (I.getNumOperands() != I.getNumExplicitOperands()) { + DEBUG(dbgs() << "Generic instruction has unexpected implicit operands\n"); + return false; + } + + if (selectImpl(I)) + return true; + + LLT Ty = + I.getOperand(0).isReg() ? MRI.getType(I.getOperand(0).getReg()) : LLT{}; + + switch (Opcode) { + case TargetOpcode::G_BRCOND: { + if (Ty.getSizeInBits() > 32) { + // We shouldn't need this on AArch64, but it would be implemented as an + // EXTRACT_SUBREG followed by a TBNZW because TBNZX has no encoding if the + // bit being tested is < 32. + DEBUG(dbgs() << "G_BRCOND has type: " << Ty + << ", expected at most 32-bits"); + return false; + } + + const unsigned CondReg = I.getOperand(0).getReg(); + MachineBasicBlock *DestMBB = I.getOperand(1).getMBB(); + + auto MIB = BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::TBNZW)) + .addUse(CondReg) + .addImm(/*bit offset=*/0) + .addMBB(DestMBB); + + I.eraseFromParent(); + return constrainSelectedInstRegOperands(*MIB.getInstr(), TII, TRI, RBI); + } + + case TargetOpcode::G_FCONSTANT: + case TargetOpcode::G_CONSTANT: { + const bool isFP = Opcode == TargetOpcode::G_FCONSTANT; + + const LLT s32 = LLT::scalar(32); + const LLT s64 = LLT::scalar(64); + const LLT p0 = LLT::pointer(0, 64); + + const unsigned DefReg = I.getOperand(0).getReg(); + const LLT DefTy = MRI.getType(DefReg); + const unsigned DefSize = DefTy.getSizeInBits(); + const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI); + + // FIXME: Redundant check, but even less readable when factored out. + if (isFP) { + if (Ty != s32 && Ty != s64) { + DEBUG(dbgs() << "Unable to materialize FP " << Ty + << " constant, expected: " << s32 << " or " << s64 + << '\n'); + return false; + } + + if (RB.getID() != AArch64::FPRRegBankID) { + DEBUG(dbgs() << "Unable to materialize FP " << Ty + << " constant on bank: " << RB << ", expected: FPR\n"); + return false; + } + } else { + if (Ty != s32 && Ty != s64 && Ty != p0) { + DEBUG(dbgs() << "Unable to materialize integer " << Ty + << " constant, expected: " << s32 << ", " << s64 << ", or " + << p0 << '\n'); + return false; + } + + if (RB.getID() != AArch64::GPRRegBankID) { + DEBUG(dbgs() << "Unable to materialize integer " << Ty + << " constant on bank: " << RB << ", expected: GPR\n"); + return false; + } + } + + const unsigned MovOpc = + DefSize == 32 ? AArch64::MOVi32imm : AArch64::MOVi64imm; + + I.setDesc(TII.get(MovOpc)); + + if (isFP) { + const TargetRegisterClass &GPRRC = + DefSize == 32 ? AArch64::GPR32RegClass : AArch64::GPR64RegClass; + const TargetRegisterClass &FPRRC = + DefSize == 32 ? AArch64::FPR32RegClass : AArch64::FPR64RegClass; + + const unsigned DefGPRReg = MRI.createVirtualRegister(&GPRRC); + MachineOperand &RegOp = I.getOperand(0); + RegOp.setReg(DefGPRReg); + + BuildMI(MBB, std::next(I.getIterator()), I.getDebugLoc(), + TII.get(AArch64::COPY)) + .addDef(DefReg) + .addUse(DefGPRReg); + + if (!RBI.constrainGenericRegister(DefReg, FPRRC, MRI)) { + DEBUG(dbgs() << "Failed to constrain G_FCONSTANT def operand\n"); + return false; + } + + MachineOperand &ImmOp = I.getOperand(1); + // FIXME: Is going through int64_t always correct? + ImmOp.ChangeToImmediate( + ImmOp.getFPImm()->getValueAPF().bitcastToAPInt().getZExtValue()); + } else { + uint64_t Val = I.getOperand(1).getCImm()->getZExtValue(); + I.getOperand(1).ChangeToImmediate(Val); + } + + constrainSelectedInstRegOperands(I, TII, TRI, RBI); + return true; + } + + case TargetOpcode::G_FRAME_INDEX: { + // allocas and G_FRAME_INDEX are only supported in addrspace(0). + if (Ty != LLT::pointer(0, 64)) { + DEBUG(dbgs() << "G_FRAME_INDEX pointer has type: " << Ty + << ", expected: " << LLT::pointer(0, 64) << '\n'); + return false; + } + + I.setDesc(TII.get(AArch64::ADDXri)); + + // MOs for a #0 shifted immediate. + I.addOperand(MachineOperand::CreateImm(0)); + I.addOperand(MachineOperand::CreateImm(0)); + + return constrainSelectedInstRegOperands(I, TII, TRI, RBI); + } + + case TargetOpcode::G_GLOBAL_VALUE: { + auto GV = I.getOperand(1).getGlobal(); + if (GV->isThreadLocal()) { + // FIXME: we don't support TLS yet. + return false; + } + unsigned char OpFlags = STI.ClassifyGlobalReference(GV, TM); + if (OpFlags & AArch64II::MO_GOT) { + I.setDesc(TII.get(AArch64::LOADgot)); + I.getOperand(1).setTargetFlags(OpFlags); + } else { + I.setDesc(TII.get(AArch64::MOVaddr)); + I.getOperand(1).setTargetFlags(OpFlags | AArch64II::MO_PAGE); + MachineInstrBuilder MIB(MF, I); + MIB.addGlobalAddress(GV, I.getOperand(1).getOffset(), + OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC); + } + return constrainSelectedInstRegOperands(I, TII, TRI, RBI); + } + + case TargetOpcode::G_LOAD: + case TargetOpcode::G_STORE: { + LLT MemTy = Ty; + LLT PtrTy = MRI.getType(I.getOperand(1).getReg()); + + if (PtrTy != LLT::pointer(0, 64)) { + DEBUG(dbgs() << "Load/Store pointer has type: " << PtrTy + << ", expected: " << LLT::pointer(0, 64) << '\n'); + return false; + } + +#ifndef NDEBUG + // Sanity-check the pointer register. + const unsigned PtrReg = I.getOperand(1).getReg(); + const RegisterBank &PtrRB = *RBI.getRegBank(PtrReg, MRI, TRI); + assert(PtrRB.getID() == AArch64::GPRRegBankID && + "Load/Store pointer operand isn't a GPR"); + assert(MRI.getType(PtrReg).isPointer() && + "Load/Store pointer operand isn't a pointer"); +#endif + + const unsigned ValReg = I.getOperand(0).getReg(); + const RegisterBank &RB = *RBI.getRegBank(ValReg, MRI, TRI); + + const unsigned NewOpc = + selectLoadStoreUIOp(I.getOpcode(), RB.getID(), MemTy.getSizeInBits()); + if (NewOpc == I.getOpcode()) + return false; + + I.setDesc(TII.get(NewOpc)); + + I.addOperand(MachineOperand::CreateImm(0)); + return constrainSelectedInstRegOperands(I, TII, TRI, RBI); + } + + case TargetOpcode::G_MUL: { + // Reject the various things we don't support yet. + if (unsupportedBinOp(I, RBI, MRI, TRI)) + return false; + + const unsigned DefReg = I.getOperand(0).getReg(); + const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI); + + if (RB.getID() != AArch64::GPRRegBankID) { + DEBUG(dbgs() << "G_MUL on bank: " << RB << ", expected: GPR\n"); + return false; + } + + unsigned ZeroReg; + unsigned NewOpc; + if (Ty.isScalar() && Ty.getSizeInBits() <= 32) { + NewOpc = AArch64::MADDWrrr; + ZeroReg = AArch64::WZR; + } else if (Ty == LLT::scalar(64)) { + NewOpc = AArch64::MADDXrrr; + ZeroReg = AArch64::XZR; + } else { + DEBUG(dbgs() << "G_MUL has type: " << Ty << ", expected: " + << LLT::scalar(32) << " or " << LLT::scalar(64) << '\n'); + return false; + } + + I.setDesc(TII.get(NewOpc)); + + I.addOperand(MachineOperand::CreateReg(ZeroReg, /*isDef=*/false)); + + // Now that we selected an opcode, we need to constrain the register + // operands to use appropriate classes. + return constrainSelectedInstRegOperands(I, TII, TRI, RBI); + } + + case TargetOpcode::G_FADD: + case TargetOpcode::G_FSUB: + case TargetOpcode::G_FMUL: + case TargetOpcode::G_FDIV: + + case TargetOpcode::G_OR: + case TargetOpcode::G_XOR: + case TargetOpcode::G_AND: + case TargetOpcode::G_SHL: + case TargetOpcode::G_LSHR: + case TargetOpcode::G_ASHR: + case TargetOpcode::G_SDIV: + case TargetOpcode::G_UDIV: + case TargetOpcode::G_ADD: + case TargetOpcode::G_SUB: + case TargetOpcode::G_GEP: { + // Reject the various things we don't support yet. + if (unsupportedBinOp(I, RBI, MRI, TRI)) + return false; + + const unsigned OpSize = Ty.getSizeInBits(); + + const unsigned DefReg = I.getOperand(0).getReg(); + const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI); + + const unsigned NewOpc = selectBinaryOp(I.getOpcode(), RB.getID(), OpSize); + if (NewOpc == I.getOpcode()) + return false; + + I.setDesc(TII.get(NewOpc)); + // FIXME: Should the type be always reset in setDesc? + + // Now that we selected an opcode, we need to constrain the register + // operands to use appropriate classes. + return constrainSelectedInstRegOperands(I, TII, TRI, RBI); + } + + case TargetOpcode::G_PTRTOINT: + case TargetOpcode::G_TRUNC: { + const LLT DstTy = MRI.getType(I.getOperand(0).getReg()); + const LLT SrcTy = MRI.getType(I.getOperand(1).getReg()); + + const unsigned DstReg = I.getOperand(0).getReg(); + const unsigned SrcReg = I.getOperand(1).getReg(); + + const RegisterBank &DstRB = *RBI.getRegBank(DstReg, MRI, TRI); + const RegisterBank &SrcRB = *RBI.getRegBank(SrcReg, MRI, TRI); + + if (DstRB.getID() != SrcRB.getID()) { + DEBUG(dbgs() << "G_TRUNC input/output on different banks\n"); + return false; + } + + if (DstRB.getID() == AArch64::GPRRegBankID) { + const TargetRegisterClass *DstRC = + getRegClassForTypeOnBank(DstTy, DstRB, RBI); + if (!DstRC) + return false; + + const TargetRegisterClass *SrcRC = + getRegClassForTypeOnBank(SrcTy, SrcRB, RBI); + if (!SrcRC) + return false; + + if (!RBI.constrainGenericRegister(SrcReg, *SrcRC, MRI) || + !RBI.constrainGenericRegister(DstReg, *DstRC, MRI)) { + DEBUG(dbgs() << "Failed to constrain G_TRUNC\n"); + return false; + } + + if (DstRC == SrcRC) { + // Nothing to be done + } else if (DstRC == &AArch64::GPR32RegClass && + SrcRC == &AArch64::GPR64RegClass) { + I.getOperand(1).setSubReg(AArch64::sub_32); + } else { + return false; + } + + I.setDesc(TII.get(TargetOpcode::COPY)); + return true; + } else if (DstRB.getID() == AArch64::FPRRegBankID) { + if (DstTy == LLT::vector(4, 16) && SrcTy == LLT::vector(4, 32)) { + I.setDesc(TII.get(AArch64::XTNv4i16)); + constrainSelectedInstRegOperands(I, TII, TRI, RBI); + return true; + } + } + + return false; + } + + case TargetOpcode::G_ANYEXT: { + const unsigned DstReg = I.getOperand(0).getReg(); + const unsigned SrcReg = I.getOperand(1).getReg(); + + const RegisterBank &RBDst = *RBI.getRegBank(DstReg, MRI, TRI); + if (RBDst.getID() != AArch64::GPRRegBankID) { + DEBUG(dbgs() << "G_ANYEXT on bank: " << RBDst << ", expected: GPR\n"); + return false; + } + + const RegisterBank &RBSrc = *RBI.getRegBank(SrcReg, MRI, TRI); + if (RBSrc.getID() != AArch64::GPRRegBankID) { + DEBUG(dbgs() << "G_ANYEXT on bank: " << RBSrc << ", expected: GPR\n"); + return false; + } + + const unsigned DstSize = MRI.getType(DstReg).getSizeInBits(); + + if (DstSize == 0) { + DEBUG(dbgs() << "G_ANYEXT operand has no size, not a gvreg?\n"); + return false; + } + + if (DstSize != 64 && DstSize > 32) { + DEBUG(dbgs() << "G_ANYEXT to size: " << DstSize + << ", expected: 32 or 64\n"); + return false; + } + // At this point G_ANYEXT is just like a plain COPY, but we need + // to explicitly form the 64-bit value if any. + if (DstSize > 32) { + unsigned ExtSrc = MRI.createVirtualRegister(&AArch64::GPR64allRegClass); + BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG)) + .addDef(ExtSrc) + .addImm(0) + .addUse(SrcReg) + .addImm(AArch64::sub_32); + I.getOperand(1).setReg(ExtSrc); + } + return selectCopy(I, TII, MRI, TRI, RBI); + } + + case TargetOpcode::G_ZEXT: + case TargetOpcode::G_SEXT: { + unsigned Opcode = I.getOpcode(); + const LLT DstTy = MRI.getType(I.getOperand(0).getReg()), + SrcTy = MRI.getType(I.getOperand(1).getReg()); + const bool isSigned = Opcode == TargetOpcode::G_SEXT; + const unsigned DefReg = I.getOperand(0).getReg(); + const unsigned SrcReg = I.getOperand(1).getReg(); + const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI); + + if (RB.getID() != AArch64::GPRRegBankID) { + DEBUG(dbgs() << TII.getName(I.getOpcode()) << " on bank: " << RB + << ", expected: GPR\n"); + return false; + } + + MachineInstr *ExtI; + if (DstTy == LLT::scalar(64)) { + // FIXME: Can we avoid manually doing this? + if (!RBI.constrainGenericRegister(SrcReg, AArch64::GPR32RegClass, MRI)) { + DEBUG(dbgs() << "Failed to constrain " << TII.getName(Opcode) + << " operand\n"); + return false; + } + + const unsigned SrcXReg = + MRI.createVirtualRegister(&AArch64::GPR64RegClass); + BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::SUBREG_TO_REG)) + .addDef(SrcXReg) + .addImm(0) + .addUse(SrcReg) + .addImm(AArch64::sub_32); + + const unsigned NewOpc = isSigned ? AArch64::SBFMXri : AArch64::UBFMXri; + ExtI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(NewOpc)) + .addDef(DefReg) + .addUse(SrcXReg) + .addImm(0) + .addImm(SrcTy.getSizeInBits() - 1); + } else if (DstTy.isScalar() && DstTy.getSizeInBits() <= 32) { + const unsigned NewOpc = isSigned ? AArch64::SBFMWri : AArch64::UBFMWri; + ExtI = BuildMI(MBB, I, I.getDebugLoc(), TII.get(NewOpc)) + .addDef(DefReg) + .addUse(SrcReg) + .addImm(0) + .addImm(SrcTy.getSizeInBits() - 1); + } else { + return false; + } + + constrainSelectedInstRegOperands(*ExtI, TII, TRI, RBI); + + I.eraseFromParent(); + return true; + } + + case TargetOpcode::G_SITOFP: + case TargetOpcode::G_UITOFP: + case TargetOpcode::G_FPTOSI: + case TargetOpcode::G_FPTOUI: { + const LLT DstTy = MRI.getType(I.getOperand(0).getReg()), + SrcTy = MRI.getType(I.getOperand(1).getReg()); + const unsigned NewOpc = selectFPConvOpc(Opcode, DstTy, SrcTy); + if (NewOpc == Opcode) + return false; + + I.setDesc(TII.get(NewOpc)); + constrainSelectedInstRegOperands(I, TII, TRI, RBI); + + return true; + } + + + case TargetOpcode::G_INTTOPTR: + case TargetOpcode::G_BITCAST: + return selectCopy(I, TII, MRI, TRI, RBI); + + case TargetOpcode::G_FPEXT: { + if (MRI.getType(I.getOperand(0).getReg()) != LLT::scalar(64)) { + DEBUG(dbgs() << "G_FPEXT to type " << Ty + << ", expected: " << LLT::scalar(64) << '\n'); + return false; + } + + if (MRI.getType(I.getOperand(1).getReg()) != LLT::scalar(32)) { + DEBUG(dbgs() << "G_FPEXT from type " << Ty + << ", expected: " << LLT::scalar(32) << '\n'); + return false; + } + + const unsigned DefReg = I.getOperand(0).getReg(); + const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI); + + if (RB.getID() != AArch64::FPRRegBankID) { + DEBUG(dbgs() << "G_FPEXT on bank: " << RB << ", expected: FPR\n"); + return false; + } + + I.setDesc(TII.get(AArch64::FCVTDSr)); + constrainSelectedInstRegOperands(I, TII, TRI, RBI); + + return true; + } + + case TargetOpcode::G_FPTRUNC: { + if (MRI.getType(I.getOperand(0).getReg()) != LLT::scalar(32)) { + DEBUG(dbgs() << "G_FPTRUNC to type " << Ty + << ", expected: " << LLT::scalar(32) << '\n'); + return false; + } + + if (MRI.getType(I.getOperand(1).getReg()) != LLT::scalar(64)) { + DEBUG(dbgs() << "G_FPTRUNC from type " << Ty + << ", expected: " << LLT::scalar(64) << '\n'); + return false; + } + + const unsigned DefReg = I.getOperand(0).getReg(); + const RegisterBank &RB = *RBI.getRegBank(DefReg, MRI, TRI); + + if (RB.getID() != AArch64::FPRRegBankID) { + DEBUG(dbgs() << "G_FPTRUNC on bank: " << RB << ", expected: FPR\n"); + return false; + } + + I.setDesc(TII.get(AArch64::FCVTSDr)); + constrainSelectedInstRegOperands(I, TII, TRI, RBI); + + return true; + } + + case TargetOpcode::G_SELECT: { + if (MRI.getType(I.getOperand(1).getReg()) != LLT::scalar(1)) { + DEBUG(dbgs() << "G_SELECT cond has type: " << Ty + << ", expected: " << LLT::scalar(1) << '\n'); + return false; + } + + const unsigned CondReg = I.getOperand(1).getReg(); + const unsigned TReg = I.getOperand(2).getReg(); + const unsigned FReg = I.getOperand(3).getReg(); + + unsigned CSelOpc = 0; + + if (Ty == LLT::scalar(32)) { + CSelOpc = AArch64::CSELWr; + } else if (Ty == LLT::scalar(64)) { + CSelOpc = AArch64::CSELXr; + } else { + return false; + } + + MachineInstr &TstMI = + *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ANDSWri)) + .addDef(AArch64::WZR) + .addUse(CondReg) + .addImm(AArch64_AM::encodeLogicalImmediate(1, 32)); + + MachineInstr &CSelMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CSelOpc)) + .addDef(I.getOperand(0).getReg()) + .addUse(TReg) + .addUse(FReg) + .addImm(AArch64CC::NE); + + constrainSelectedInstRegOperands(TstMI, TII, TRI, RBI); + constrainSelectedInstRegOperands(CSelMI, TII, TRI, RBI); + + I.eraseFromParent(); + return true; + } + case TargetOpcode::G_ICMP: { + if (Ty != LLT::scalar(1)) { + DEBUG(dbgs() << "G_ICMP result has type: " << Ty + << ", expected: " << LLT::scalar(1) << '\n'); + return false; + } + + unsigned CmpOpc = 0; + unsigned ZReg = 0; + + LLT CmpTy = MRI.getType(I.getOperand(2).getReg()); + if (CmpTy == LLT::scalar(32)) { + CmpOpc = AArch64::SUBSWrr; + ZReg = AArch64::WZR; + } else if (CmpTy == LLT::scalar(64) || CmpTy.isPointer()) { + CmpOpc = AArch64::SUBSXrr; + ZReg = AArch64::XZR; + } else { + return false; + } + + const AArch64CC::CondCode CC = changeICMPPredToAArch64CC( + (CmpInst::Predicate)I.getOperand(1).getPredicate()); + + MachineInstr &CmpMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc)) + .addDef(ZReg) + .addUse(I.getOperand(2).getReg()) + .addUse(I.getOperand(3).getReg()); + + MachineInstr &CSetMI = + *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr)) + .addDef(I.getOperand(0).getReg()) + .addUse(AArch64::WZR) + .addUse(AArch64::WZR) + .addImm(CC); + + constrainSelectedInstRegOperands(CmpMI, TII, TRI, RBI); + constrainSelectedInstRegOperands(CSetMI, TII, TRI, RBI); + + I.eraseFromParent(); + return true; + } + + case TargetOpcode::G_FCMP: { + if (Ty != LLT::scalar(1)) { + DEBUG(dbgs() << "G_FCMP result has type: " << Ty + << ", expected: " << LLT::scalar(1) << '\n'); + return false; + } + + unsigned CmpOpc = 0; + LLT CmpTy = MRI.getType(I.getOperand(2).getReg()); + if (CmpTy == LLT::scalar(32)) { + CmpOpc = AArch64::FCMPSrr; + } else if (CmpTy == LLT::scalar(64)) { + CmpOpc = AArch64::FCMPDrr; + } else { + return false; + } + + // FIXME: regbank + + AArch64CC::CondCode CC1, CC2; + changeFCMPPredToAArch64CC( + (CmpInst::Predicate)I.getOperand(1).getPredicate(), CC1, CC2); + + MachineInstr &CmpMI = *BuildMI(MBB, I, I.getDebugLoc(), TII.get(CmpOpc)) + .addUse(I.getOperand(2).getReg()) + .addUse(I.getOperand(3).getReg()); + + const unsigned DefReg = I.getOperand(0).getReg(); + unsigned Def1Reg = DefReg; + if (CC2 != AArch64CC::AL) + Def1Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass); + + MachineInstr &CSetMI = + *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr)) + .addDef(Def1Reg) + .addUse(AArch64::WZR) + .addUse(AArch64::WZR) + .addImm(CC1); + + if (CC2 != AArch64CC::AL) { + unsigned Def2Reg = MRI.createVirtualRegister(&AArch64::GPR32RegClass); + MachineInstr &CSet2MI = + *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::CSINCWr)) + .addDef(Def2Reg) + .addUse(AArch64::WZR) + .addUse(AArch64::WZR) + .addImm(CC2); + MachineInstr &OrMI = + *BuildMI(MBB, I, I.getDebugLoc(), TII.get(AArch64::ORRWrr)) + .addDef(DefReg) + .addUse(Def1Reg) + .addUse(Def2Reg); + constrainSelectedInstRegOperands(OrMI, TII, TRI, RBI); + constrainSelectedInstRegOperands(CSet2MI, TII, TRI, RBI); + } + + constrainSelectedInstRegOperands(CmpMI, TII, TRI, RBI); + constrainSelectedInstRegOperands(CSetMI, TII, TRI, RBI); + + I.eraseFromParent(); + return true; + } + } + + return false; +} diff --git a/lib/Target/AArch64/AArch64InstructionSelector.h b/lib/Target/AArch64/AArch64InstructionSelector.h new file mode 100644 index 000000000000..0d44e696ac20 --- /dev/null +++ b/lib/Target/AArch64/AArch64InstructionSelector.h @@ -0,0 +1,47 @@ +//===- AArch64InstructionSelector --------------------------------*- C++ -*-==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// This file declares the targeting of the InstructionSelector class for +/// AArch64. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_AARCH64_AARCH64INSTRUCTIONSELECTOR_H +#define LLVM_LIB_TARGET_AARCH64_AARCH64INSTRUCTIONSELECTOR_H + +#include "llvm/CodeGen/GlobalISel/InstructionSelector.h" + +namespace llvm { +class AArch64InstrInfo; +class AArch64RegisterBankInfo; +class AArch64RegisterInfo; +class AArch64Subtarget; +class AArch64TargetMachine; + +class AArch64InstructionSelector : public InstructionSelector { +public: + AArch64InstructionSelector(const AArch64TargetMachine &TM, + const AArch64Subtarget &STI, + const AArch64RegisterBankInfo &RBI); + + virtual bool select(MachineInstr &I) const override; + +private: + /// tblgen-erated 'select' implementation, used as the initial selector for + /// the patterns that don't require complex C++. + bool selectImpl(MachineInstr &I) const; + + const AArch64TargetMachine &TM; + const AArch64Subtarget &STI; + const AArch64InstrInfo &TII; + const AArch64RegisterInfo &TRI; + const AArch64RegisterBankInfo &RBI; +}; + +} // End llvm namespace. +#endif diff --git a/lib/Target/AArch64/AArch64LegalizerInfo.cpp b/lib/Target/AArch64/AArch64LegalizerInfo.cpp new file mode 100644 index 000000000000..83f276a8161b --- /dev/null +++ b/lib/Target/AArch64/AArch64LegalizerInfo.cpp @@ -0,0 +1,204 @@ +//===- AArch64LegalizerInfo.cpp ----------------------------------*- C++ -*-==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// This file implements the targeting of the Machinelegalizer class for +/// AArch64. +/// \todo This should be generated by TableGen. +//===----------------------------------------------------------------------===// + +#include "AArch64LegalizerInfo.h" +#include "llvm/CodeGen/ValueTypes.h" +#include "llvm/IR/Type.h" +#include "llvm/IR/DerivedTypes.h" +#include "llvm/Target/TargetOpcodes.h" + +using namespace llvm; + +#ifndef LLVM_BUILD_GLOBAL_ISEL +#error "You shouldn't build this" +#endif + +AArch64LegalizerInfo::AArch64LegalizerInfo() { + using namespace TargetOpcode; + const LLT p0 = LLT::pointer(0, 64); + const LLT s1 = LLT::scalar(1); + const LLT s8 = LLT::scalar(8); + const LLT s16 = LLT::scalar(16); + const LLT s32 = LLT::scalar(32); + const LLT s64 = LLT::scalar(64); + const LLT v2s32 = LLT::vector(2, 32); + const LLT v4s32 = LLT::vector(4, 32); + const LLT v2s64 = LLT::vector(2, 64); + + for (auto BinOp : {G_ADD, G_SUB, G_MUL, G_AND, G_OR, G_XOR, G_SHL}) { + // These operations naturally get the right answer when used on + // GPR32, even if the actual type is narrower. + for (auto Ty : {s1, s8, s16, s32, s64, v2s32, v4s32, v2s64}) + setAction({BinOp, Ty}, Legal); + } + + setAction({G_GEP, p0}, Legal); + setAction({G_GEP, 1, s64}, Legal); + + for (auto Ty : {s1, s8, s16, s32}) + setAction({G_GEP, 1, Ty}, WidenScalar); + + for (auto BinOp : {G_LSHR, G_ASHR, G_SDIV, G_UDIV}) { + for (auto Ty : {s32, s64}) + setAction({BinOp, Ty}, Legal); + + for (auto Ty : {s1, s8, s16}) + setAction({BinOp, Ty}, WidenScalar); + } + + for (auto BinOp : { G_SREM, G_UREM }) + for (auto Ty : { s1, s8, s16, s32, s64 }) + setAction({BinOp, Ty}, Lower); + + for (auto Op : { G_UADDE, G_USUBE, G_SADDO, G_SSUBO, G_SMULO, G_UMULO }) { + for (auto Ty : { s32, s64 }) + setAction({Op, Ty}, Legal); + + setAction({Op, 1, s1}, Legal); + } + + for (auto BinOp : {G_FADD, G_FSUB, G_FMUL, G_FDIV}) + for (auto Ty : {s32, s64}) + setAction({BinOp, Ty}, Legal); + + setAction({G_FREM, s32}, Libcall); + setAction({G_FREM, s64}, Libcall); + + for (auto MemOp : {G_LOAD, G_STORE}) { + for (auto Ty : {s8, s16, s32, s64, p0, v2s32}) + setAction({MemOp, Ty}, Legal); + + setAction({MemOp, s1}, WidenScalar); + + // And everything's fine in addrspace 0. + setAction({MemOp, 1, p0}, Legal); + } + + // Constants + for (auto Ty : {s32, s64}) { + setAction({TargetOpcode::G_CONSTANT, Ty}, Legal); + setAction({TargetOpcode::G_FCONSTANT, Ty}, Legal); + } + + setAction({G_CONSTANT, p0}, Legal); + + for (auto Ty : {s1, s8, s16}) + setAction({TargetOpcode::G_CONSTANT, Ty}, WidenScalar); + + setAction({TargetOpcode::G_FCONSTANT, s16}, WidenScalar); + + setAction({G_ICMP, s1}, Legal); + setAction({G_ICMP, 1, s32}, Legal); + setAction({G_ICMP, 1, s64}, Legal); + setAction({G_ICMP, 1, p0}, Legal); + + for (auto Ty : {s1, s8, s16}) { + setAction({G_ICMP, 1, Ty}, WidenScalar); + } + + setAction({G_FCMP, s1}, Legal); + setAction({G_FCMP, 1, s32}, Legal); + setAction({G_FCMP, 1, s64}, Legal); + + // Extensions + for (auto Ty : { s1, s8, s16, s32, s64 }) { + setAction({G_ZEXT, Ty}, Legal); + setAction({G_SEXT, Ty}, Legal); + setAction({G_ANYEXT, Ty}, Legal); + } + + for (auto Ty : { s1, s8, s16, s32 }) { + setAction({G_ZEXT, 1, Ty}, Legal); + setAction({G_SEXT, 1, Ty}, Legal); + setAction({G_ANYEXT, 1, Ty}, Legal); + } + + setAction({G_FPEXT, s64}, Legal); + setAction({G_FPEXT, 1, s32}, Legal); + + // Truncations + for (auto Ty : { s16, s32 }) + setAction({G_FPTRUNC, Ty}, Legal); + + for (auto Ty : { s32, s64 }) + setAction({G_FPTRUNC, 1, Ty}, Legal); + + for (auto Ty : { s1, s8, s16, s32 }) + setAction({G_TRUNC, Ty}, Legal); + + for (auto Ty : { s8, s16, s32, s64 }) + setAction({G_TRUNC, 1, Ty}, Legal); + + // Conversions + for (auto Ty : { s1, s8, s16, s32, s64 }) { + setAction({G_FPTOSI, 0, Ty}, Legal); + setAction({G_FPTOUI, 0, Ty}, Legal); + setAction({G_SITOFP, 1, Ty}, Legal); + setAction({G_UITOFP, 1, Ty}, Legal); + } + + for (auto Ty : { s32, s64 }) { + setAction({G_FPTOSI, 1, Ty}, Legal); + setAction({G_FPTOUI, 1, Ty}, Legal); + setAction({G_SITOFP, 0, Ty}, Legal); + setAction({G_UITOFP, 0, Ty}, Legal); + } + + // Control-flow + for (auto Ty : {s1, s8, s16, s32}) + setAction({G_BRCOND, Ty}, Legal); + + // Select + for (auto Ty : {s1, s8, s16, s32, s64}) + setAction({G_SELECT, Ty}, Legal); + + setAction({G_SELECT, 1, s1}, Legal); + + // Pointer-handling + setAction({G_FRAME_INDEX, p0}, Legal); + setAction({G_GLOBAL_VALUE, p0}, Legal); + + for (auto Ty : {s1, s8, s16, s32, s64}) + setAction({G_PTRTOINT, 0, Ty}, Legal); + + setAction({G_PTRTOINT, 1, p0}, Legal); + + setAction({G_INTTOPTR, 0, p0}, Legal); + setAction({G_INTTOPTR, 1, s64}, Legal); + + // Casts for 32 and 64-bit width type are just copies. + for (auto Ty : {s1, s8, s16, s32, s64}) { + setAction({G_BITCAST, 0, Ty}, Legal); + setAction({G_BITCAST, 1, Ty}, Legal); + } + + // For the sake of copying bits around, the type does not really + // matter as long as it fits a register. + for (int EltSize = 8; EltSize <= 64; EltSize *= 2) { + setAction({G_BITCAST, 0, LLT::vector(128/EltSize, EltSize)}, Legal); + setAction({G_BITCAST, 1, LLT::vector(128/EltSize, EltSize)}, Legal); + if (EltSize >= 64) + continue; + + setAction({G_BITCAST, 0, LLT::vector(64/EltSize, EltSize)}, Legal); + setAction({G_BITCAST, 1, LLT::vector(64/EltSize, EltSize)}, Legal); + if (EltSize >= 32) + continue; + + setAction({G_BITCAST, 0, LLT::vector(32/EltSize, EltSize)}, Legal); + setAction({G_BITCAST, 1, LLT::vector(32/EltSize, EltSize)}, Legal); + } + + computeTables(); +} diff --git a/lib/Target/AArch64/AArch64LegalizerInfo.h b/lib/Target/AArch64/AArch64LegalizerInfo.h new file mode 100644 index 000000000000..feacbef9f147 --- /dev/null +++ b/lib/Target/AArch64/AArch64LegalizerInfo.h @@ -0,0 +1,30 @@ +//===- AArch64LegalizerInfo --------------------------------------*- C++ -*-==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +/// \file +/// This file declares the targeting of the Machinelegalizer class for +/// AArch64. +/// \todo This should be generated by TableGen. +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_AARCH64_AARCH64MACHINELEGALIZER_H +#define LLVM_LIB_TARGET_AARCH64_AARCH64MACHINELEGALIZER_H + +#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" + +namespace llvm { + +class LLVMContext; + +/// This class provides the information for the target register banks. +class AArch64LegalizerInfo : public LegalizerInfo { +public: + AArch64LegalizerInfo(); +}; +} // End llvm namespace. +#endif diff --git a/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp b/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp index dd2ea6a9dbd6..dcb05601e5f4 100644 --- a/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp +++ b/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp @@ -38,7 +38,6 @@ STATISTIC(NumPostFolded, "Number of post-index updates folded"); STATISTIC(NumPreFolded, "Number of pre-index updates folded"); STATISTIC(NumUnscaledPairCreated, "Number of load/store from unscaled generated"); -STATISTIC(NumNarrowLoadsPromoted, "Number of narrow loads promoted"); STATISTIC(NumZeroStoresPromoted, "Number of narrow zero stores promoted"); STATISTIC(NumLoadsFromStoresPromoted, "Number of loads from stores promoted"); @@ -51,14 +50,6 @@ static cl::opt LdStLimit("aarch64-load-store-scan-limit", static cl::opt UpdateLimit("aarch64-update-scan-limit", cl::init(100), cl::Hidden); -static cl::opt EnableNarrowLdMerge("enable-narrow-ld-merge", cl::Hidden, - cl::init(false), - cl::desc("Enable narrow load merge")); - -namespace llvm { -void initializeAArch64LoadStoreOptPass(PassRegistry &); -} - #define AARCH64_LOAD_STORE_OPT_NAME "AArch64 load / store optimization pass" namespace { @@ -111,11 +102,11 @@ struct AArch64LoadStoreOpt : public MachineFunctionPass { bool findMatchingStore(MachineBasicBlock::iterator I, unsigned Limit, MachineBasicBlock::iterator &StoreI); - // Merge the two instructions indicated into a wider instruction. + // Merge the two instructions indicated into a wider narrow store instruction. MachineBasicBlock::iterator - mergeNarrowInsns(MachineBasicBlock::iterator I, - MachineBasicBlock::iterator MergeMI, - const LdStPairFlags &Flags); + mergeNarrowZeroStores(MachineBasicBlock::iterator I, + MachineBasicBlock::iterator MergeMI, + const LdStPairFlags &Flags); // Merge the two instructions indicated into a single pair-wise instruction. MachineBasicBlock::iterator @@ -151,8 +142,8 @@ struct AArch64LoadStoreOpt : public MachineFunctionPass { mergeUpdateInsn(MachineBasicBlock::iterator I, MachineBasicBlock::iterator Update, bool IsPreIdx); - // Find and merge foldable ldr/str instructions. - bool tryToMergeLdStInst(MachineBasicBlock::iterator &MBBI); + // Find and merge zero store instructions. + bool tryToMergeZeroStInst(MachineBasicBlock::iterator &MBBI); // Find and pair ldr/str instructions. bool tryToPairLdStInst(MachineBasicBlock::iterator &MBBI); @@ -160,18 +151,16 @@ struct AArch64LoadStoreOpt : public MachineFunctionPass { // Find and promote load instructions which read directly from store. bool tryToPromoteLoadFromStore(MachineBasicBlock::iterator &MBBI); - bool optimizeBlock(MachineBasicBlock &MBB, bool enableNarrowLdOpt); + bool optimizeBlock(MachineBasicBlock &MBB, bool EnableNarrowZeroStOpt); bool runOnMachineFunction(MachineFunction &Fn) override; MachineFunctionProperties getRequiredProperties() const override { return MachineFunctionProperties().set( - MachineFunctionProperties::Property::AllVRegsAllocated); + MachineFunctionProperties::Property::NoVRegs); } - const char *getPassName() const override { - return AARCH64_LOAD_STORE_OPT_NAME; - } + StringRef getPassName() const override { return AARCH64_LOAD_STORE_OPT_NAME; } }; char AArch64LoadStoreOpt::ID = 0; } // namespace @@ -179,23 +168,6 @@ char AArch64LoadStoreOpt::ID = 0; INITIALIZE_PASS(AArch64LoadStoreOpt, "aarch64-ldst-opt", AARCH64_LOAD_STORE_OPT_NAME, false, false) -static unsigned getBitExtrOpcode(MachineInstr &MI) { - switch (MI.getOpcode()) { - default: - llvm_unreachable("Unexpected opcode."); - case AArch64::LDRBBui: - case AArch64::LDURBBi: - case AArch64::LDRHHui: - case AArch64::LDURHHi: - return AArch64::UBFMWri; - case AArch64::LDRSBWui: - case AArch64::LDURSBWi: - case AArch64::LDRSHWui: - case AArch64::LDURSHWi: - return AArch64::SBFMWri; - } -} - static bool isNarrowStore(unsigned Opc) { switch (Opc) { default: @@ -208,30 +180,6 @@ static bool isNarrowStore(unsigned Opc) { } } -static bool isNarrowLoad(unsigned Opc) { - switch (Opc) { - default: - return false; - case AArch64::LDRHHui: - case AArch64::LDURHHi: - case AArch64::LDRBBui: - case AArch64::LDURBBi: - case AArch64::LDRSHWui: - case AArch64::LDURSHWi: - case AArch64::LDRSBWui: - case AArch64::LDURSBWi: - return true; - } -} - -static bool isNarrowLoad(MachineInstr &MI) { - return isNarrowLoad(MI.getOpcode()); -} - -static bool isNarrowLoadOrStore(unsigned Opc) { - return isNarrowLoad(Opc) || isNarrowStore(Opc); -} - // Scaling factor for unscaled load or store. static int getMemScale(MachineInstr &MI) { switch (MI.getOpcode()) { @@ -323,23 +271,11 @@ static unsigned getMatchingNonSExtOpcode(unsigned Opc, case AArch64::STURSi: case AArch64::LDRSui: case AArch64::LDURSi: - case AArch64::LDRHHui: - case AArch64::LDURHHi: - case AArch64::LDRBBui: - case AArch64::LDURBBi: return Opc; case AArch64::LDRSWui: return AArch64::LDRWui; case AArch64::LDURSWi: return AArch64::LDURWi; - case AArch64::LDRSBWui: - return AArch64::LDRBBui; - case AArch64::LDRSHWui: - return AArch64::LDRHHui; - case AArch64::LDURSBWi: - return AArch64::LDURBBi; - case AArch64::LDURSHWi: - return AArch64::LDURHHi; } } @@ -359,18 +295,6 @@ static unsigned getMatchingWideOpcode(unsigned Opc) { return AArch64::STURXi; case AArch64::STRWui: return AArch64::STRXui; - case AArch64::LDRHHui: - case AArch64::LDRSHWui: - return AArch64::LDRWui; - case AArch64::LDURHHi: - case AArch64::LDURSHWi: - return AArch64::LDURWi; - case AArch64::LDRBBui: - case AArch64::LDRSBWui: - return AArch64::LDRHHui; - case AArch64::LDURBBi: - case AArch64::LDURSBWi: - return AArch64::LDURHHi; } } @@ -614,23 +538,20 @@ static bool isLdOffsetInRangeOfSt(MachineInstr &LoadInst, (UnscaledLdOffset + LoadSize <= (UnscaledStOffset + StoreSize)); } -static bool isPromotableZeroStoreOpcode(unsigned Opc) { - return isNarrowStore(Opc) || Opc == AArch64::STRWui || Opc == AArch64::STURWi; -} - -static bool isPromotableZeroStoreOpcode(MachineInstr &MI) { - return isPromotableZeroStoreOpcode(MI.getOpcode()); -} - static bool isPromotableZeroStoreInst(MachineInstr &MI) { - return (isPromotableZeroStoreOpcode(MI)) && + unsigned Opc = MI.getOpcode(); + return (Opc == AArch64::STRWui || Opc == AArch64::STURWi || + isNarrowStore(Opc)) && getLdStRegOp(MI).getReg() == AArch64::WZR; } MachineBasicBlock::iterator -AArch64LoadStoreOpt::mergeNarrowInsns(MachineBasicBlock::iterator I, - MachineBasicBlock::iterator MergeMI, - const LdStPairFlags &Flags) { +AArch64LoadStoreOpt::mergeNarrowZeroStores(MachineBasicBlock::iterator I, + MachineBasicBlock::iterator MergeMI, + const LdStPairFlags &Flags) { + assert(isPromotableZeroStoreInst(*I) && isPromotableZeroStoreInst(*MergeMI) && + "Expected promotable zero stores."); + MachineBasicBlock::iterator NextI = I; ++NextI; // If NextI is the second of the two instructions to be merged, we need @@ -654,15 +575,12 @@ AArch64LoadStoreOpt::mergeNarrowInsns(MachineBasicBlock::iterator I, MergeForward ? getLdStBaseOp(*MergeMI) : getLdStBaseOp(*I); // Which register is Rt and which is Rt2 depends on the offset order. - MachineInstr *RtMI, *Rt2MI; + MachineInstr *RtMI; if (getLdStOffsetOp(*I).getImm() == - getLdStOffsetOp(*MergeMI).getImm() + OffsetStride) { + getLdStOffsetOp(*MergeMI).getImm() + OffsetStride) RtMI = &*MergeMI; - Rt2MI = &*I; - } else { + else RtMI = &*I; - Rt2MI = &*MergeMI; - } int OffsetImm = getLdStOffsetOp(*RtMI).getImm(); // Change the scaled offset from small to large type. @@ -671,105 +589,9 @@ AArch64LoadStoreOpt::mergeNarrowInsns(MachineBasicBlock::iterator I, OffsetImm /= 2; } + // Construct the new instruction. DebugLoc DL = I->getDebugLoc(); MachineBasicBlock *MBB = I->getParent(); - if (isNarrowLoad(Opc)) { - MachineInstr *RtNewDest = &*(MergeForward ? I : MergeMI); - // When merging small (< 32 bit) loads for big-endian targets, the order of - // the component parts gets swapped. - if (!Subtarget->isLittleEndian()) - std::swap(RtMI, Rt2MI); - // Construct the new load instruction. - MachineInstr *NewMemMI, *BitExtMI1, *BitExtMI2; - NewMemMI = - BuildMI(*MBB, InsertionPoint, DL, TII->get(getMatchingWideOpcode(Opc))) - .addOperand(getLdStRegOp(*RtNewDest)) - .addOperand(BaseRegOp) - .addImm(OffsetImm) - .setMemRefs(I->mergeMemRefsWith(*MergeMI)); - (void)NewMemMI; - - DEBUG( - dbgs() - << "Creating the new load and extract. Replacing instructions:\n "); - DEBUG(I->print(dbgs())); - DEBUG(dbgs() << " "); - DEBUG(MergeMI->print(dbgs())); - DEBUG(dbgs() << " with instructions:\n "); - DEBUG((NewMemMI)->print(dbgs())); - - int Width = getMemScale(*I) == 1 ? 8 : 16; - int LSBLow = 0; - int LSBHigh = Width; - int ImmsLow = LSBLow + Width - 1; - int ImmsHigh = LSBHigh + Width - 1; - MachineInstr *ExtDestMI = &*(MergeForward ? MergeMI : I); - if ((ExtDestMI == Rt2MI) == Subtarget->isLittleEndian()) { - // Create the bitfield extract for high bits. - BitExtMI1 = - BuildMI(*MBB, InsertionPoint, DL, TII->get(getBitExtrOpcode(*Rt2MI))) - .addOperand(getLdStRegOp(*Rt2MI)) - .addReg(getLdStRegOp(*RtNewDest).getReg()) - .addImm(LSBHigh) - .addImm(ImmsHigh); - // Create the bitfield extract for low bits. - if (RtMI->getOpcode() == getMatchingNonSExtOpcode(RtMI->getOpcode())) { - // For unsigned, prefer to use AND for low bits. - BitExtMI2 = BuildMI(*MBB, InsertionPoint, DL, TII->get(AArch64::ANDWri)) - .addOperand(getLdStRegOp(*RtMI)) - .addReg(getLdStRegOp(*RtNewDest).getReg()) - .addImm(ImmsLow); - } else { - BitExtMI2 = - BuildMI(*MBB, InsertionPoint, DL, TII->get(getBitExtrOpcode(*RtMI))) - .addOperand(getLdStRegOp(*RtMI)) - .addReg(getLdStRegOp(*RtNewDest).getReg()) - .addImm(LSBLow) - .addImm(ImmsLow); - } - } else { - // Create the bitfield extract for low bits. - if (RtMI->getOpcode() == getMatchingNonSExtOpcode(RtMI->getOpcode())) { - // For unsigned, prefer to use AND for low bits. - BitExtMI1 = BuildMI(*MBB, InsertionPoint, DL, TII->get(AArch64::ANDWri)) - .addOperand(getLdStRegOp(*RtMI)) - .addReg(getLdStRegOp(*RtNewDest).getReg()) - .addImm(ImmsLow); - } else { - BitExtMI1 = - BuildMI(*MBB, InsertionPoint, DL, TII->get(getBitExtrOpcode(*RtMI))) - .addOperand(getLdStRegOp(*RtMI)) - .addReg(getLdStRegOp(*RtNewDest).getReg()) - .addImm(LSBLow) - .addImm(ImmsLow); - } - - // Create the bitfield extract for high bits. - BitExtMI2 = - BuildMI(*MBB, InsertionPoint, DL, TII->get(getBitExtrOpcode(*Rt2MI))) - .addOperand(getLdStRegOp(*Rt2MI)) - .addReg(getLdStRegOp(*RtNewDest).getReg()) - .addImm(LSBHigh) - .addImm(ImmsHigh); - } - (void)BitExtMI1; - (void)BitExtMI2; - - DEBUG(dbgs() << " "); - DEBUG((BitExtMI1)->print(dbgs())); - DEBUG(dbgs() << " "); - DEBUG((BitExtMI2)->print(dbgs())); - DEBUG(dbgs() << "\n"); - - // Erase the old instructions. - I->eraseFromParent(); - MergeMI->eraseFromParent(); - return NextI; - } - assert(isPromotableZeroStoreInst(*I) && isPromotableZeroStoreInst(*MergeMI) && - "Expected promotable zero store"); - - // Construct the new instruction. MachineInstrBuilder MIB; MIB = BuildMI(*MBB, InsertionPoint, DL, TII->get(getMatchingWideOpcode(Opc))) .addReg(isNarrowStore(Opc) ? AArch64::WZR : AArch64::XZR) @@ -778,7 +600,7 @@ AArch64LoadStoreOpt::mergeNarrowInsns(MachineBasicBlock::iterator I, .setMemRefs(I->mergeMemRefsWith(*MergeMI)); (void)MIB; - DEBUG(dbgs() << "Creating wider load/store. Replacing instructions:\n "); + DEBUG(dbgs() << "Creating wider store. Replacing instructions:\n "); DEBUG(I->print(dbgs())); DEBUG(dbgs() << " "); DEBUG(MergeMI->print(dbgs())); @@ -945,6 +767,7 @@ AArch64LoadStoreOpt::promoteLoadFromStore(MachineBasicBlock::iterator LoadI, // Remove the load, if the destination register of the loads is the same // register for stored value. if (StRt == LdRt && LoadSize == 8) { + StoreI->clearRegisterKills(StRt, TRI); DEBUG(dbgs() << "Remove load instruction:\n "); DEBUG(LoadI->print(dbgs())); DEBUG(dbgs() << "\n"); @@ -1009,6 +832,8 @@ AArch64LoadStoreOpt::promoteLoadFromStore(MachineBasicBlock::iterator LoadI, .addImm(Imms); } } + StoreI->clearRegisterKills(StRt, TRI); + (void)BitExtMI; DEBUG(dbgs() << "Promoting load by replacing :\n "); @@ -1041,8 +866,10 @@ static void trackRegDefsUses(const MachineInstr &MI, BitVector &ModifiedRegs, if (!Reg) continue; if (MO.isDef()) { - for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) - ModifiedRegs.set(*AI); + // WZR/XZR are not modified even when used as a destination register. + if (Reg != AArch64::WZR && Reg != AArch64::XZR) + for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) + ModifiedRegs.set(*AI); } else { assert(MO.isUse() && "Reg operand not a def and not a use?!?"); for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) @@ -1118,8 +945,9 @@ bool AArch64LoadStoreOpt::findMatchingStore( --MBBI; MachineInstr &MI = *MBBI; - // Don't count DBG_VALUE instructions towards the search limit. - if (!MI.isDebugValue()) + // Don't count transient instructions towards the search limit since there + // may be different numbers of them if e.g. debug information is present. + if (!MI.isTransient()) ++Count; // If the load instruction reads directly from the address to which the @@ -1184,13 +1012,14 @@ static bool areCandidatesToMergeOrPair(MachineInstr &FirstMI, MachineInstr &MI, return true; } - // If the second instruction isn't even a load/store, bail out. + // If the second instruction isn't even a mergable/pairable load/store, bail + // out. if (!PairIsValidLdStrOpc) return false; - // FIXME: We don't support merging narrow loads/stores with mixed - // scaled/unscaled offsets. - if (isNarrowLoadOrStore(OpcA) || isNarrowLoadOrStore(OpcB)) + // FIXME: We don't support merging narrow stores with mixed scaled/unscaled + // offsets. + if (isNarrowStore(OpcA) || isNarrowStore(OpcB)) return false; // Try to match an unscaled load/store with a scaled load/store. @@ -1229,13 +1058,11 @@ AArch64LoadStoreOpt::findMatchingInsn(MachineBasicBlock::iterator I, for (unsigned Count = 0; MBBI != E && Count < Limit; ++MBBI) { MachineInstr &MI = *MBBI; - // Skip DBG_VALUE instructions. Otherwise debug info can affect the - // optimization by changing how far we scan. - if (MI.isDebugValue()) - continue; - // Now that we know this is a real instruction, count it. - ++Count; + // Don't count transient instructions towards the search limit since there + // may be different numbers of them if e.g. debug information is present. + if (!MI.isTransient()) + ++Count; Flags.setSExtIdx(-1); if (areCandidatesToMergeOrPair(FirstMI, MI, Flags, TII) && @@ -1505,12 +1332,11 @@ MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnForward( ++MBBI; for (unsigned Count = 0; MBBI != E && Count < Limit; ++MBBI) { MachineInstr &MI = *MBBI; - // Skip DBG_VALUE instructions. - if (MI.isDebugValue()) - continue; - // Now that we know this is a real instruction, count it. - ++Count; + // Don't count transient instructions towards the search limit since there + // may be different numbers of them if e.g. debug information is present. + if (!MI.isTransient()) + ++Count; // If we found a match, return it. if (isMatchingUpdateInsn(*I, MI, BaseReg, UnscaledOffset)) @@ -1559,8 +1385,9 @@ MachineBasicBlock::iterator AArch64LoadStoreOpt::findMatchingUpdateInsnBackward( --MBBI; MachineInstr &MI = *MBBI; - // Don't count DBG_VALUE instructions towards the search limit. - if (!MI.isDebugValue()) + // Don't count transient instructions towards the search limit since there + // may be different numbers of them if e.g. debug information is present. + if (!MI.isTransient()) ++Count; // If we found a match, return it. @@ -1603,37 +1430,26 @@ bool AArch64LoadStoreOpt::tryToPromoteLoadFromStore( return false; } -// Find narrow loads that can be converted into a single wider load with -// bitfield extract instructions. Also merge adjacent zero stores into a wider -// store. -bool AArch64LoadStoreOpt::tryToMergeLdStInst( +// Merge adjacent zero stores into a wider store. +bool AArch64LoadStoreOpt::tryToMergeZeroStInst( MachineBasicBlock::iterator &MBBI) { - assert((isNarrowLoad(*MBBI) || isPromotableZeroStoreOpcode(*MBBI)) && - "Expected narrow op."); + assert(isPromotableZeroStoreInst(*MBBI) && "Expected narrow store."); MachineInstr &MI = *MBBI; MachineBasicBlock::iterator E = MI.getParent()->end(); if (!TII->isCandidateToMergeOrPair(MI)) return false; - // For promotable zero stores, the stored value should be WZR. - if (isPromotableZeroStoreOpcode(MI) && - getLdStRegOp(MI).getReg() != AArch64::WZR) - return false; - // Look ahead up to LdStLimit instructions for a mergable instruction. LdStPairFlags Flags; MachineBasicBlock::iterator MergeMI = findMatchingInsn(MBBI, Flags, LdStLimit, /* FindNarrowMerge = */ true); if (MergeMI != E) { - if (isNarrowLoad(MI)) { - ++NumNarrowLoadsPromoted; - } else if (isPromotableZeroStoreInst(MI)) { - ++NumZeroStoresPromoted; - } + ++NumZeroStoresPromoted; + // Keeping the iterator straight is a pain, so we let the merge routine tell // us what the next instruction is after it's done mucking about. - MBBI = mergeNarrowInsns(MBBI, MergeMI, Flags); + MBBI = mergeNarrowZeroStores(MBBI, MergeMI, Flags); return true; } return false; @@ -1674,7 +1490,7 @@ bool AArch64LoadStoreOpt::tryToPairLdStInst(MachineBasicBlock::iterator &MBBI) { } bool AArch64LoadStoreOpt::optimizeBlock(MachineBasicBlock &MBB, - bool enableNarrowLdOpt) { + bool EnableNarrowZeroStOpt) { bool Modified = false; // Four tranformations to do here: // 1) Find loads that directly read from stores and promote them by @@ -1713,29 +1529,21 @@ bool AArch64LoadStoreOpt::optimizeBlock(MachineBasicBlock &MBB, } } } - // 2) Find narrow loads that can be converted into a single wider load - // with bitfield extract instructions. - // e.g., - // ldrh w0, [x2] - // ldrh w1, [x2, #2] - // ; becomes - // ldr w0, [x2] - // ubfx w1, w0, #16, #16 - // and w0, w0, #ffff - // - // Also merge adjacent zero stores into a wider store. + // 2) Merge adjacent zero stores into a wider store. // e.g., // strh wzr, [x0] // strh wzr, [x0, #2] // ; becomes // str wzr, [x0] + // e.g., + // str wzr, [x0] + // str wzr, [x0, #4] + // ; becomes + // str xzr, [x0] for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); - enableNarrowLdOpt && MBBI != E;) { - MachineInstr &MI = *MBBI; - unsigned Opc = MI.getOpcode(); - if (isPromotableZeroStoreOpcode(Opc) || - (EnableNarrowLdMerge && isNarrowLoad(Opc))) { - if (tryToMergeLdStInst(MBBI)) { + EnableNarrowZeroStOpt && MBBI != E;) { + if (isPromotableZeroStoreInst(*MBBI)) { + if (tryToMergeZeroStInst(MBBI)) { Modified = true; } else ++MBBI; @@ -1752,44 +1560,10 @@ bool AArch64LoadStoreOpt::optimizeBlock(MachineBasicBlock &MBB, // ldp x0, x1, [x2] for (MachineBasicBlock::iterator MBBI = MBB.begin(), E = MBB.end(); MBBI != E;) { - MachineInstr &MI = *MBBI; - switch (MI.getOpcode()) { - default: - // Just move on to the next instruction. + if (TII->isPairableLdStInst(*MBBI) && tryToPairLdStInst(MBBI)) + Modified = true; + else ++MBBI; - break; - // Scaled instructions. - case AArch64::STRSui: - case AArch64::STRDui: - case AArch64::STRQui: - case AArch64::STRXui: - case AArch64::STRWui: - case AArch64::LDRSui: - case AArch64::LDRDui: - case AArch64::LDRQui: - case AArch64::LDRXui: - case AArch64::LDRWui: - case AArch64::LDRSWui: - // Unscaled instructions. - case AArch64::STURSi: - case AArch64::STURDi: - case AArch64::STURQi: - case AArch64::STURWi: - case AArch64::STURXi: - case AArch64::LDURSi: - case AArch64::LDURDi: - case AArch64::LDURQi: - case AArch64::LDURWi: - case AArch64::LDURXi: - case AArch64::LDURSWi: { - if (tryToPairLdStInst(MBBI)) { - Modified = true; - break; - } - ++MBBI; - break; - } - } } // 4) Find base register updates that can be merged into the load or store // as a base-reg writeback. @@ -1930,16 +1704,17 @@ bool AArch64LoadStoreOpt::runOnMachineFunction(MachineFunction &Fn) { UsedRegs.resize(TRI->getNumRegs()); bool Modified = false; - bool enableNarrowLdOpt = - Subtarget->mergeNarrowLoads() && !Subtarget->requiresStrictAlign(); + bool enableNarrowZeroStOpt = !Subtarget->requiresStrictAlign(); for (auto &MBB : Fn) - Modified |= optimizeBlock(MBB, enableNarrowLdOpt); + Modified |= optimizeBlock(MBB, enableNarrowZeroStOpt); return Modified; } -// FIXME: Do we need/want a pre-alloc pass like ARM has to try to keep -// loads and stores near one another? +// FIXME: Do we need/want a pre-alloc pass like ARM has to try to keep loads and +// stores near one another? Note: The pre-RA instruction scheduler already has +// hooks to try and schedule pairable loads/stores together to improve pairing +// opportunities. Thus, pre-RA pairing pass may not be worth the effort. // FIXME: When pairing store instructions it's very possible for this pass to // hoist a store with a KILL marker above another use (without a KILL marker). diff --git a/lib/Target/AArch64/AArch64MCInstLower.cpp b/lib/Target/AArch64/AArch64MCInstLower.cpp index 2b4cdf1083be..45083df7ab45 100644 --- a/lib/Target/AArch64/AArch64MCInstLower.cpp +++ b/lib/Target/AArch64/AArch64MCInstLower.cpp @@ -29,7 +29,7 @@ using namespace llvm; extern cl::opt EnableAArch64ELFLocalDynamicTLSGeneration; AArch64MCInstLower::AArch64MCInstLower(MCContext &ctx, AsmPrinter &printer) - : Ctx(ctx), Printer(printer), TargetTriple(printer.getTargetTriple()) {} + : Ctx(ctx), Printer(printer) {} MCSymbol * AArch64MCInstLower::GetGlobalAddressSymbol(const MachineOperand &MO) const { @@ -153,10 +153,11 @@ MCOperand AArch64MCInstLower::lowerSymbolOperandELF(const MachineOperand &MO, MCOperand AArch64MCInstLower::LowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym) const { - if (TargetTriple.isOSDarwin()) + if (Printer.TM.getTargetTriple().isOSDarwin()) return lowerSymbolOperandDarwin(MO, Sym); - assert(TargetTriple.isOSBinFormatELF() && "Expect Darwin or ELF target"); + assert(Printer.TM.getTargetTriple().isOSBinFormatELF() && + "Expect Darwin or ELF target"); return lowerSymbolOperandELF(MO, Sym); } diff --git a/lib/Target/AArch64/AArch64MachineFunctionInfo.h b/lib/Target/AArch64/AArch64MachineFunctionInfo.h index 49e7767741ea..ca2860afe13d 100644 --- a/lib/Target/AArch64/AArch64MachineFunctionInfo.h +++ b/lib/Target/AArch64/AArch64MachineFunctionInfo.h @@ -23,7 +23,7 @@ namespace llvm { /// AArch64FunctionInfo - This class is derived from MachineFunctionInfo and /// contains private AArch64-specific information for each MachineFunction. -class AArch64FunctionInfo : public MachineFunctionInfo { +class AArch64FunctionInfo final : public MachineFunctionInfo { /// Number of bytes of arguments this function has on the stack. If the callee /// is expected to restore the argument stack this should be a multiple of 16, diff --git a/lib/Target/AArch64/AArch64PromoteConstant.cpp b/lib/Target/AArch64/AArch64PromoteConstant.cpp index b1e40510b2ae..8693f76d7c32 100644 --- a/lib/Target/AArch64/AArch64PromoteConstant.cpp +++ b/lib/Target/AArch64/AArch64PromoteConstant.cpp @@ -101,9 +101,11 @@ public: }; static char ID; - AArch64PromoteConstant() : ModulePass(ID) {} + AArch64PromoteConstant() : ModulePass(ID) { + initializeAArch64PromoteConstantPass(*PassRegistry::getPassRegistry()); + } - const char *getPassName() const override { return "AArch64 Promote Constant"; } + StringRef getPassName() const override { return "AArch64 Promote Constant"; } /// Iterate over the functions and promote the interesting constants into /// global variables with module scope. @@ -214,10 +216,6 @@ private: char AArch64PromoteConstant::ID = 0; -namespace llvm { -void initializeAArch64PromoteConstantPass(PassRegistry &); -} - INITIALIZE_PASS_BEGIN(AArch64PromoteConstant, "aarch64-promote-const", "AArch64 Promote Constant Pass", false, false) INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass) diff --git a/lib/Target/AArch64/AArch64RedundantCopyElimination.cpp b/lib/Target/AArch64/AArch64RedundantCopyElimination.cpp index 60d8bbd260bb..8f45e6a80a36 100644 --- a/lib/Target/AArch64/AArch64RedundantCopyElimination.cpp +++ b/lib/Target/AArch64/AArch64RedundantCopyElimination.cpp @@ -39,10 +39,6 @@ using namespace llvm; STATISTIC(NumCopiesRemoved, "Number of copies removed."); -namespace llvm { -void initializeAArch64RedundantCopyEliminationPass(PassRegistry &); -} - namespace { class AArch64RedundantCopyElimination : public MachineFunctionPass { const MachineRegisterInfo *MRI; @@ -50,14 +46,17 @@ class AArch64RedundantCopyElimination : public MachineFunctionPass { public: static char ID; - AArch64RedundantCopyElimination() : MachineFunctionPass(ID) {} + AArch64RedundantCopyElimination() : MachineFunctionPass(ID) { + initializeAArch64RedundantCopyEliminationPass( + *PassRegistry::getPassRegistry()); + } bool optimizeCopy(MachineBasicBlock *MBB); bool runOnMachineFunction(MachineFunction &MF) override; MachineFunctionProperties getRequiredProperties() const override { return MachineFunctionProperties().set( - MachineFunctionProperties::Property::AllVRegsAllocated); + MachineFunctionProperties::Property::NoVRegs); } - const char *getPassName() const override { + StringRef getPassName() const override { return "AArch64 Redundant Copy Elimination"; } }; @@ -153,13 +152,11 @@ bool AArch64RedundantCopyElimination::optimizeCopy(MachineBasicBlock *MBB) { // CBZ/CBNZ. Conservatively mark as much as we can live. CompBr->clearRegisterKills(SmallestDef, TRI); - if (std::none_of(TargetRegs.begin(), TargetRegs.end(), - [&](unsigned Reg) { return MBB->isLiveIn(Reg); })) + if (none_of(TargetRegs, [&](unsigned Reg) { return MBB->isLiveIn(Reg); })) MBB->addLiveIn(TargetReg); // Clear any kills of TargetReg between CompBr and the last removed COPY. - for (MachineInstr &MMI : - make_range(MBB->begin()->getIterator(), LastChange->getIterator())) + for (MachineInstr &MMI : make_range(MBB->begin(), LastChange)) MMI.clearRegisterKills(SmallestDef, TRI); return true; diff --git a/lib/Target/AArch64/AArch64RegisterBankInfo.cpp b/lib/Target/AArch64/AArch64RegisterBankInfo.cpp index 0a1831bd9a8c..a5fd2fbdde19 100644 --- a/lib/Target/AArch64/AArch64RegisterBankInfo.cpp +++ b/lib/Target/AArch64/AArch64RegisterBankInfo.cpp @@ -14,11 +14,16 @@ #include "AArch64RegisterBankInfo.h" #include "AArch64InstrInfo.h" // For XXXRegClassID. +#include "llvm/CodeGen/LowLevelType.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/GlobalISel/RegisterBank.h" #include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h" #include "llvm/Target/TargetRegisterInfo.h" #include "llvm/Target/TargetSubtargetInfo.h" +// This file will be TableGen'ed at some point. +#include "AArch64GenRegisterBankInfo.def" + using namespace llvm; #ifndef LLVM_BUILD_GLOBAL_ISEL @@ -26,7 +31,16 @@ using namespace llvm; #endif AArch64RegisterBankInfo::AArch64RegisterBankInfo(const TargetRegisterInfo &TRI) - : RegisterBankInfo(AArch64::NumRegisterBanks) { + : RegisterBankInfo(AArch64::RegBanks, AArch64::NumRegisterBanks) { + static bool AlreadyInit = false; + // We have only one set of register banks, whatever the subtarget + // is. Therefore, the initialization of the RegBanks table should be + // done only once. Indeed the table of all register banks + // (AArch64::RegBanks) is unique in the compiler. At some point, it + // will get tablegen'ed and the whole constructor becomes empty. + if (AlreadyInit) + return; + AlreadyInit = true; // Initialize the GPR bank. createRegisterBank(AArch64::GPRRegBankID, "GPR"); // The GPR register bank is fully defined by all the registers in @@ -34,6 +48,8 @@ AArch64RegisterBankInfo::AArch64RegisterBankInfo(const TargetRegisterInfo &TRI) addRegBankCoverage(AArch64::GPRRegBankID, AArch64::GPR64allRegClassID, TRI); const RegisterBank &RBGPR = getRegBank(AArch64::GPRRegBankID); (void)RBGPR; + assert(&AArch64::GPRRegBank == &RBGPR && + "The order in RegBanks is messed up"); assert(RBGPR.covers(*TRI.getRegClass(AArch64::GPR32RegClassID)) && "Subclass not added?"); assert(RBGPR.getSize() == 64 && "GPRs should hold up to 64-bit"); @@ -45,6 +61,8 @@ AArch64RegisterBankInfo::AArch64RegisterBankInfo(const TargetRegisterInfo &TRI) addRegBankCoverage(AArch64::FPRRegBankID, AArch64::QQQQRegClassID, TRI); const RegisterBank &RBFPR = getRegBank(AArch64::FPRRegBankID); (void)RBFPR; + assert(&AArch64::FPRRegBank == &RBFPR && + "The order in RegBanks is messed up"); assert(RBFPR.covers(*TRI.getRegClass(AArch64::QQRegClassID)) && "Subclass not added?"); assert(RBFPR.covers(*TRI.getRegClass(AArch64::FPR64RegClassID)) && @@ -57,10 +75,133 @@ AArch64RegisterBankInfo::AArch64RegisterBankInfo(const TargetRegisterInfo &TRI) addRegBankCoverage(AArch64::CCRRegBankID, AArch64::CCRRegClassID, TRI); const RegisterBank &RBCCR = getRegBank(AArch64::CCRRegBankID); (void)RBCCR; + assert(&AArch64::CCRRegBank == &RBCCR && + "The order in RegBanks is messed up"); assert(RBCCR.covers(*TRI.getRegClass(AArch64::CCRRegClassID)) && "Class not added?"); assert(RBCCR.getSize() == 32 && "CCR should hold up to 32-bit"); + // Check that the TableGen'ed like file is in sync we our expectations. + // First, the Idx. + assert(AArch64::PartialMappingIdx::PMI_GPR32 == + AArch64::PartialMappingIdx::PMI_FirstGPR && + "GPR32 index not first in the GPR list"); + assert(AArch64::PartialMappingIdx::PMI_GPR64 == + AArch64::PartialMappingIdx::PMI_LastGPR && + "GPR64 index not last in the GPR list"); + assert(AArch64::PartialMappingIdx::PMI_FirstGPR <= + AArch64::PartialMappingIdx::PMI_LastGPR && + "GPR list is backward"); + assert(AArch64::PartialMappingIdx::PMI_FPR32 == + AArch64::PartialMappingIdx::PMI_FirstFPR && + "FPR32 index not first in the FPR list"); + assert(AArch64::PartialMappingIdx::PMI_FPR512 == + AArch64::PartialMappingIdx::PMI_LastFPR && + "FPR512 index not last in the FPR list"); + assert(AArch64::PartialMappingIdx::PMI_FirstFPR <= + AArch64::PartialMappingIdx::PMI_LastFPR && + "FPR list is backward"); + assert(AArch64::PartialMappingIdx::PMI_FPR32 + 1 == + AArch64::PartialMappingIdx::PMI_FPR64 && + AArch64::PartialMappingIdx::PMI_FPR64 + 1 == + AArch64::PartialMappingIdx::PMI_FPR128 && + AArch64::PartialMappingIdx::PMI_FPR128 + 1 == + AArch64::PartialMappingIdx::PMI_FPR256 && + AArch64::PartialMappingIdx::PMI_FPR256 + 1 == + AArch64::PartialMappingIdx::PMI_FPR512 && + "FPR indices not properly ordered"); +// Now, the content. +// Check partial mapping. +#define CHECK_PARTIALMAP(Idx, ValStartIdx, ValLength, RB) \ + do { \ + const PartialMapping &Map = \ + AArch64::PartMappings[AArch64::PartialMappingIdx::Idx - \ + AArch64::PartialMappingIdx::PMI_Min]; \ + (void)Map; \ + assert(Map.StartIdx == ValStartIdx && Map.Length == ValLength && \ + Map.RegBank == &RB && #Idx " is incorrectly initialized"); \ + } while (0) + + CHECK_PARTIALMAP(PMI_GPR32, 0, 32, RBGPR); + CHECK_PARTIALMAP(PMI_GPR64, 0, 64, RBGPR); + CHECK_PARTIALMAP(PMI_FPR32, 0, 32, RBFPR); + CHECK_PARTIALMAP(PMI_FPR64, 0, 64, RBFPR); + CHECK_PARTIALMAP(PMI_FPR128, 0, 128, RBFPR); + CHECK_PARTIALMAP(PMI_FPR256, 0, 256, RBFPR); + CHECK_PARTIALMAP(PMI_FPR512, 0, 512, RBFPR); + +// Check value mapping. +#define CHECK_VALUEMAP_IMPL(RBName, Size, Offset) \ + do { \ + unsigned PartialMapBaseIdx = \ + AArch64::PartialMappingIdx::PMI_##RBName##Size - \ + AArch64::PartialMappingIdx::PMI_Min; \ + (void)PartialMapBaseIdx; \ + const ValueMapping &Map = AArch64::getValueMapping( \ + AArch64::PartialMappingIdx::PMI_First##RBName, Size)[Offset]; \ + (void)Map; \ + assert(Map.BreakDown == &AArch64::PartMappings[PartialMapBaseIdx] && \ + Map.NumBreakDowns == 1 && #RBName #Size \ + " " #Offset " is incorrectly initialized"); \ + } while (0) + +#define CHECK_VALUEMAP(RBName, Size) CHECK_VALUEMAP_IMPL(RBName, Size, 0) + + CHECK_VALUEMAP(GPR, 32); + CHECK_VALUEMAP(GPR, 64); + CHECK_VALUEMAP(FPR, 32); + CHECK_VALUEMAP(FPR, 64); + CHECK_VALUEMAP(FPR, 128); + CHECK_VALUEMAP(FPR, 256); + CHECK_VALUEMAP(FPR, 512); + +// Check the value mapping for 3-operands instructions where all the operands +// map to the same value mapping. +#define CHECK_VALUEMAP_3OPS(RBName, Size) \ + do { \ + CHECK_VALUEMAP_IMPL(RBName, Size, 0); \ + CHECK_VALUEMAP_IMPL(RBName, Size, 1); \ + CHECK_VALUEMAP_IMPL(RBName, Size, 2); \ + } while (0) + + CHECK_VALUEMAP_3OPS(GPR, 32); + CHECK_VALUEMAP_3OPS(GPR, 64); + CHECK_VALUEMAP_3OPS(FPR, 32); + CHECK_VALUEMAP_3OPS(FPR, 64); + CHECK_VALUEMAP_3OPS(FPR, 128); + CHECK_VALUEMAP_3OPS(FPR, 256); + CHECK_VALUEMAP_3OPS(FPR, 512); + +#define CHECK_VALUEMAP_CROSSREGCPY(RBNameDst, RBNameSrc, Size) \ + do { \ + unsigned PartialMapDstIdx = \ + AArch64::PMI_##RBNameDst##Size - AArch64::PMI_Min; \ + unsigned PartialMapSrcIdx = \ + AArch64::PMI_##RBNameSrc##Size - AArch64::PMI_Min; \ + (void) PartialMapDstIdx; \ + (void) PartialMapSrcIdx; \ + const ValueMapping *Map = AArch64::getCopyMapping( \ + AArch64::PMI_First##RBNameDst == AArch64::PMI_FirstGPR, \ + AArch64::PMI_First##RBNameSrc == AArch64::PMI_FirstGPR, Size); \ + (void) Map; \ + assert(Map[0].BreakDown == &AArch64::PartMappings[PartialMapDstIdx] && \ + Map[0].NumBreakDowns == 1 && #RBNameDst #Size \ + " Dst is incorrectly initialized"); \ + assert(Map[1].BreakDown == &AArch64::PartMappings[PartialMapSrcIdx] && \ + Map[1].NumBreakDowns == 1 && #RBNameSrc #Size \ + " Src is incorrectly initialized"); \ + \ + } while (0) + + CHECK_VALUEMAP_CROSSREGCPY(GPR, GPR, 32); + CHECK_VALUEMAP_CROSSREGCPY(GPR, FPR, 32); + CHECK_VALUEMAP_CROSSREGCPY(GPR, GPR, 64); + CHECK_VALUEMAP_CROSSREGCPY(GPR, FPR, 64); + CHECK_VALUEMAP_CROSSREGCPY(FPR, FPR, 32); + CHECK_VALUEMAP_CROSSREGCPY(FPR, GPR, 32); + CHECK_VALUEMAP_CROSSREGCPY(FPR, FPR, 64); + CHECK_VALUEMAP_CROSSREGCPY(FPR, GPR, 64); + assert(verify(TRI) && "Invalid register bank information"); } @@ -72,7 +213,16 @@ unsigned AArch64RegisterBankInfo::copyCost(const RegisterBank &A, // Will introduce other hooks for different size: // * extract cost. // * build_sequence cost. - // TODO: Add more accurate cost for FPR to/from GPR. + + // Copy from (resp. to) GPR to (resp. from) FPR involves FMOV. + // FIXME: This should be deduced from the scheduling model. + if (&A == &AArch64::GPRRegBank && &B == &AArch64::FPRRegBank) + // FMOVXDr or FMOVWSr. + return 5; + if (&A == &AArch64::FPRRegBank && &B == &AArch64::GPRRegBank) + // FMOVDXr or FMOVSWr. + return 4; + return RegisterBankInfo::copyCost(A, B, Size); } @@ -116,15 +266,15 @@ const RegisterBank &AArch64RegisterBankInfo::getRegBankFromRegClass( RegisterBankInfo::InstructionMappings AArch64RegisterBankInfo::getInstrAlternativeMappings( const MachineInstr &MI) const { + const MachineFunction &MF = *MI.getParent()->getParent(); + const TargetSubtargetInfo &STI = MF.getSubtarget(); + const TargetRegisterInfo &TRI = *STI.getRegisterInfo(); + const MachineRegisterInfo &MRI = MF.getRegInfo(); + switch (MI.getOpcode()) { case TargetOpcode::G_OR: { // 32 and 64-bit or can be mapped on either FPR or // GPR for the same cost. - const MachineFunction &MF = *MI.getParent()->getParent(); - const TargetSubtargetInfo &STI = MF.getSubtarget(); - const TargetRegisterInfo &TRI = *STI.getRegisterInfo(); - const MachineRegisterInfo &MRI = MF.getRegInfo(); - unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI); if (Size != 32 && Size != 64) break; @@ -134,14 +284,81 @@ AArch64RegisterBankInfo::getInstrAlternativeMappings( if (MI.getNumOperands() != 3) break; InstructionMappings AltMappings; - InstructionMapping GPRMapping(/*ID*/ 1, /*Cost*/ 1, /*NumOperands*/ 3); - InstructionMapping FPRMapping(/*ID*/ 2, /*Cost*/ 1, /*NumOperands*/ 3); - for (unsigned Idx = 0; Idx != 3; ++Idx) { - GPRMapping.setOperandMapping(Idx, Size, - getRegBank(AArch64::GPRRegBankID)); - FPRMapping.setOperandMapping(Idx, Size, - getRegBank(AArch64::FPRRegBankID)); - } + InstructionMapping GPRMapping( + /*ID*/ 1, /*Cost*/ 1, + AArch64::getValueMapping(AArch64::PMI_FirstGPR, Size), + /*NumOperands*/ 3); + InstructionMapping FPRMapping( + /*ID*/ 2, /*Cost*/ 1, + AArch64::getValueMapping(AArch64::PMI_FirstFPR, Size), + /*NumOperands*/ 3); + + AltMappings.emplace_back(std::move(GPRMapping)); + AltMappings.emplace_back(std::move(FPRMapping)); + return AltMappings; + } + case TargetOpcode::G_BITCAST: { + unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI); + if (Size != 32 && Size != 64) + break; + + // If the instruction has any implicit-defs or uses, + // do not mess with it. + if (MI.getNumOperands() != 2) + break; + + InstructionMappings AltMappings; + InstructionMapping GPRMapping( + /*ID*/ 1, /*Cost*/ 1, + AArch64::getCopyMapping(/*DstIsGPR*/ true, /*SrcIsGPR*/ true, Size), + /*NumOperands*/ 2); + InstructionMapping FPRMapping( + /*ID*/ 2, /*Cost*/ 1, + AArch64::getCopyMapping(/*DstIsGPR*/ false, /*SrcIsGPR*/ false, Size), + /*NumOperands*/ 2); + InstructionMapping GPRToFPRMapping( + /*ID*/ 3, + /*Cost*/ copyCost(AArch64::GPRRegBank, AArch64::FPRRegBank, Size), + AArch64::getCopyMapping(/*DstIsGPR*/ false, /*SrcIsGPR*/ true, Size), + /*NumOperands*/ 2); + InstructionMapping FPRToGPRMapping( + /*ID*/ 3, + /*Cost*/ copyCost(AArch64::GPRRegBank, AArch64::FPRRegBank, Size), + AArch64::getCopyMapping(/*DstIsGPR*/ true, /*SrcIsGPR*/ false, Size), + /*NumOperands*/ 2); + + AltMappings.emplace_back(std::move(GPRMapping)); + AltMappings.emplace_back(std::move(FPRMapping)); + AltMappings.emplace_back(std::move(GPRToFPRMapping)); + AltMappings.emplace_back(std::move(FPRToGPRMapping)); + return AltMappings; + } + case TargetOpcode::G_LOAD: { + unsigned Size = getSizeInBits(MI.getOperand(0).getReg(), MRI, TRI); + if (Size != 64) + break; + + // If the instruction has any implicit-defs or uses, + // do not mess with it. + if (MI.getNumOperands() != 2) + break; + + InstructionMappings AltMappings; + InstructionMapping GPRMapping( + /*ID*/ 1, /*Cost*/ 1, + getOperandsMapping( + {AArch64::getValueMapping(AArch64::PMI_FirstGPR, Size), + // Addresses are GPR 64-bit. + AArch64::getValueMapping(AArch64::PMI_FirstGPR, 64)}), + /*NumOperands*/ 2); + InstructionMapping FPRMapping( + /*ID*/ 2, /*Cost*/ 1, + getOperandsMapping( + {AArch64::getValueMapping(AArch64::PMI_FirstFPR, Size), + // Addresses are GPR 64-bit. + AArch64::getValueMapping(AArch64::PMI_FirstGPR, 64)}), + /*NumOperands*/ 2); + AltMappings.emplace_back(std::move(GPRMapping)); AltMappings.emplace_back(std::move(FPRMapping)); return AltMappings; @@ -155,10 +372,12 @@ AArch64RegisterBankInfo::getInstrAlternativeMappings( void AArch64RegisterBankInfo::applyMappingImpl( const OperandsMapper &OpdMapper) const { switch (OpdMapper.getMI().getOpcode()) { - case TargetOpcode::G_OR: { + case TargetOpcode::G_OR: + case TargetOpcode::G_BITCAST: + case TargetOpcode::G_LOAD: { // Those ID must match getInstrAlternativeMappings. - assert((OpdMapper.getInstrMapping().getID() == 1 || - OpdMapper.getInstrMapping().getID() == 2) && + assert((OpdMapper.getInstrMapping().getID() >= 1 && + OpdMapper.getInstrMapping().getID() <= 4) && "Don't know how to handle that ID"); return applyDefaultMapping(OpdMapper); } @@ -166,3 +385,193 @@ void AArch64RegisterBankInfo::applyMappingImpl( llvm_unreachable("Don't know how to handle that operation"); } } + +/// Returns whether opcode \p Opc is a pre-isel generic floating-point opcode, +/// having only floating-point operands. +static bool isPreISelGenericFloatingPointOpcode(unsigned Opc) { + switch (Opc) { + case TargetOpcode::G_FADD: + case TargetOpcode::G_FSUB: + case TargetOpcode::G_FMUL: + case TargetOpcode::G_FDIV: + case TargetOpcode::G_FCONSTANT: + case TargetOpcode::G_FPEXT: + case TargetOpcode::G_FPTRUNC: + return true; + } + return false; +} + +RegisterBankInfo::InstructionMapping +AArch64RegisterBankInfo::getSameKindOfOperandsMapping(const MachineInstr &MI) { + const unsigned Opc = MI.getOpcode(); + const MachineFunction &MF = *MI.getParent()->getParent(); + const MachineRegisterInfo &MRI = MF.getRegInfo(); + + unsigned NumOperands = MI.getNumOperands(); + assert(NumOperands <= 3 && + "This code is for instructions with 3 or less operands"); + + LLT Ty = MRI.getType(MI.getOperand(0).getReg()); + unsigned Size = Ty.getSizeInBits(); + bool IsFPR = Ty.isVector() || isPreISelGenericFloatingPointOpcode(Opc); + +#ifndef NDEBUG + // Make sure all the operands are using similar size and type. + // Should probably be checked by the machine verifier. + // This code won't catch cases where the number of lanes is + // different between the operands. + // If we want to go to that level of details, it is probably + // best to check that the types are the same, period. + // Currently, we just check that the register banks are the same + // for each types. + for (unsigned Idx = 1; Idx != NumOperands; ++Idx) { + LLT OpTy = MRI.getType(MI.getOperand(Idx).getReg()); + assert(AArch64::getRegBankBaseIdxOffset(OpTy.getSizeInBits()) == + AArch64::getRegBankBaseIdxOffset(Size) && + "Operand has incompatible size"); + bool OpIsFPR = OpTy.isVector() || isPreISelGenericFloatingPointOpcode(Opc); + (void)OpIsFPR; + assert(IsFPR == OpIsFPR && "Operand has incompatible type"); + } +#endif // End NDEBUG. + + AArch64::PartialMappingIdx RBIdx = + IsFPR ? AArch64::PMI_FirstFPR : AArch64::PMI_FirstGPR; + + return InstructionMapping{DefaultMappingID, 1, + AArch64::getValueMapping(RBIdx, Size), NumOperands}; +} + +RegisterBankInfo::InstructionMapping +AArch64RegisterBankInfo::getInstrMapping(const MachineInstr &MI) const { + const unsigned Opc = MI.getOpcode(); + const MachineFunction &MF = *MI.getParent()->getParent(); + const MachineRegisterInfo &MRI = MF.getRegInfo(); + + // Try the default logic for non-generic instructions that are either copies + // or already have some operands assigned to banks. + if (!isPreISelGenericOpcode(Opc)) { + RegisterBankInfo::InstructionMapping Mapping = getInstrMappingImpl(MI); + if (Mapping.isValid()) + return Mapping; + } + + switch (Opc) { + // G_{F|S|U}REM are not listed because they are not legal. + // Arithmetic ops. + case TargetOpcode::G_ADD: + case TargetOpcode::G_SUB: + case TargetOpcode::G_GEP: + case TargetOpcode::G_MUL: + case TargetOpcode::G_SDIV: + case TargetOpcode::G_UDIV: + // Bitwise ops. + case TargetOpcode::G_AND: + case TargetOpcode::G_OR: + case TargetOpcode::G_XOR: + // Shifts. + case TargetOpcode::G_SHL: + case TargetOpcode::G_LSHR: + case TargetOpcode::G_ASHR: + // Floating point ops. + case TargetOpcode::G_FADD: + case TargetOpcode::G_FSUB: + case TargetOpcode::G_FMUL: + case TargetOpcode::G_FDIV: + return getSameKindOfOperandsMapping(MI); + case TargetOpcode::G_BITCAST: { + LLT DstTy = MRI.getType(MI.getOperand(0).getReg()); + LLT SrcTy = MRI.getType(MI.getOperand(1).getReg()); + unsigned Size = DstTy.getSizeInBits(); + bool DstIsGPR = !DstTy.isVector(); + bool SrcIsGPR = !SrcTy.isVector(); + const RegisterBank &DstRB = + DstIsGPR ? AArch64::GPRRegBank : AArch64::FPRRegBank; + const RegisterBank &SrcRB = + SrcIsGPR ? AArch64::GPRRegBank : AArch64::FPRRegBank; + return InstructionMapping{DefaultMappingID, copyCost(DstRB, SrcRB, Size), + AArch64::getCopyMapping(DstIsGPR, SrcIsGPR, Size), + /*NumOperands*/ 2}; + } + case TargetOpcode::G_SEQUENCE: + // FIXME: support this, but the generic code is really not going to do + // anything sane. + return InstructionMapping(); + default: + break; + } + + unsigned NumOperands = MI.getNumOperands(); + + // Track the size and bank of each register. We don't do partial mappings. + SmallVector OpSize(NumOperands); + SmallVector OpRegBankIdx(NumOperands); + for (unsigned Idx = 0; Idx < NumOperands; ++Idx) { + auto &MO = MI.getOperand(Idx); + if (!MO.isReg()) + continue; + + LLT Ty = MRI.getType(MO.getReg()); + OpSize[Idx] = Ty.getSizeInBits(); + + // As a top-level guess, vectors go in FPRs, scalars and pointers in GPRs. + // For floating-point instructions, scalars go in FPRs. + if (Ty.isVector() || isPreISelGenericFloatingPointOpcode(Opc)) + OpRegBankIdx[Idx] = AArch64::PMI_FirstFPR; + else + OpRegBankIdx[Idx] = AArch64::PMI_FirstGPR; + } + + unsigned Cost = 1; + // Some of the floating-point instructions have mixed GPR and FPR operands: + // fine-tune the computed mapping. + switch (Opc) { + case TargetOpcode::G_SITOFP: + case TargetOpcode::G_UITOFP: { + OpRegBankIdx = {AArch64::PMI_FirstFPR, AArch64::PMI_FirstGPR}; + break; + } + case TargetOpcode::G_FPTOSI: + case TargetOpcode::G_FPTOUI: { + OpRegBankIdx = {AArch64::PMI_FirstGPR, AArch64::PMI_FirstFPR}; + break; + } + case TargetOpcode::G_FCMP: { + OpRegBankIdx = {AArch64::PMI_FirstGPR, + /* Predicate */ AArch64::PMI_None, AArch64::PMI_FirstFPR, + AArch64::PMI_FirstFPR}; + break; + } + case TargetOpcode::G_BITCAST: { + // This is going to be a cross register bank copy and this is expensive. + if (OpRegBankIdx[0] != OpRegBankIdx[1]) + Cost = + copyCost(*AArch64::PartMappings[OpRegBankIdx[0]].RegBank, + *AArch64::PartMappings[OpRegBankIdx[1]].RegBank, OpSize[0]); + break; + } + case TargetOpcode::G_LOAD: { + // Loading in vector unit is slightly more expensive. + // This is actually only true for the LD1R and co instructions, + // but anyway for the fast mode this number does not matter and + // for the greedy mode the cost of the cross bank copy will + // offset this number. + // FIXME: Should be derived from the scheduling model. + if (OpRegBankIdx[0] >= AArch64::PMI_FirstFPR) + Cost = 2; + } + } + + // Finally construct the computed mapping. + RegisterBankInfo::InstructionMapping Mapping = + InstructionMapping{DefaultMappingID, Cost, nullptr, NumOperands}; + SmallVector OpdsMapping(NumOperands); + for (unsigned Idx = 0; Idx < NumOperands; ++Idx) + if (MI.getOperand(Idx).isReg()) + OpdsMapping[Idx] = + AArch64::getValueMapping(OpRegBankIdx[Idx], OpSize[Idx]); + + Mapping.setOperandsMapping(getOperandsMapping(OpdsMapping)); + return Mapping; +} diff --git a/lib/Target/AArch64/AArch64RegisterBankInfo.h b/lib/Target/AArch64/AArch64RegisterBankInfo.h index 907bcfdea161..f763235049d4 100644 --- a/lib/Target/AArch64/AArch64RegisterBankInfo.h +++ b/lib/Target/AArch64/AArch64RegisterBankInfo.h @@ -27,43 +27,40 @@ enum { CCRRegBankID = 2, /// Conditional register: NZCV. NumRegisterBanks }; + +extern RegisterBank GPRRegBank; +extern RegisterBank FPRRegBank; +extern RegisterBank CCRRegBank; } // End AArch64 namespace. /// This class provides the information for the target register banks. -class AArch64RegisterBankInfo : public RegisterBankInfo { +class AArch64RegisterBankInfo final : public RegisterBankInfo { /// See RegisterBankInfo::applyMapping. void applyMappingImpl(const OperandsMapper &OpdMapper) const override; + /// Get an instruction mapping where all the operands map to + /// the same register bank and have similar size. + /// + /// \pre MI.getNumOperands() <= 3 + /// + /// \return An InstructionMappings with a statically allocated + /// OperandsMapping. + static InstructionMapping + getSameKindOfOperandsMapping(const MachineInstr &MI); + public: AArch64RegisterBankInfo(const TargetRegisterInfo &TRI); - /// Get the cost of a copy from \p B to \p A, or put differently, - /// get the cost of A = COPY B. Since register banks may cover - /// different size, \p Size specifies what will be the size in bits - /// that will be copied around. - /// - /// \note Since this is a copy, both registers have the same size. + unsigned copyCost(const RegisterBank &A, const RegisterBank &B, unsigned Size) const override; - /// Get a register bank that covers \p RC. - /// - /// \pre \p RC is a user-defined register class (as opposed as one - /// generated by TableGen). - /// - /// \note The mapping RC -> RegBank could be built while adding the - /// coverage for the register banks. However, we do not do it, because, - /// at least for now, we only need this information for register classes - /// that are used in the description of instruction. In other words, - /// there are just a handful of them and we do not want to waste space. - /// - /// \todo This should be TableGen'ed. const RegisterBank & getRegBankFromRegClass(const TargetRegisterClass &RC) const override; - /// Get the alternative mappings for \p MI. - /// Alternative in the sense different from getInstrMapping. InstructionMappings getInstrAlternativeMappings(const MachineInstr &MI) const override; + + InstructionMapping getInstrMapping(const MachineInstr &MI) const override; }; } // End llvm namespace. #endif diff --git a/lib/Target/AArch64/AArch64RegisterInfo.cpp b/lib/Target/AArch64/AArch64RegisterInfo.cpp index af867da4823d..98fad71aa18a 100644 --- a/lib/Target/AArch64/AArch64RegisterInfo.cpp +++ b/lib/Target/AArch64/AArch64RegisterInfo.cpp @@ -118,26 +118,27 @@ AArch64RegisterInfo::getReservedRegs(const MachineFunction &MF) const { // FIXME: avoid re-calculating this every time. BitVector Reserved(getNumRegs()); - Reserved.set(AArch64::SP); - Reserved.set(AArch64::XZR); - Reserved.set(AArch64::WSP); - Reserved.set(AArch64::WZR); + markSuperRegs(Reserved, AArch64::SP); + markSuperRegs(Reserved, AArch64::XZR); + markSuperRegs(Reserved, AArch64::WSP); + markSuperRegs(Reserved, AArch64::WZR); if (TFI->hasFP(MF) || TT.isOSDarwin()) { - Reserved.set(AArch64::FP); - Reserved.set(AArch64::W29); + markSuperRegs(Reserved, AArch64::FP); + markSuperRegs(Reserved, AArch64::W29); } if (MF.getSubtarget().isX18Reserved()) { - Reserved.set(AArch64::X18); // Platform register - Reserved.set(AArch64::W18); + markSuperRegs(Reserved, AArch64::X18); // Platform register + markSuperRegs(Reserved, AArch64::W18); } if (hasBasePointer(MF)) { - Reserved.set(AArch64::X19); - Reserved.set(AArch64::W19); + markSuperRegs(Reserved, AArch64::X19); + markSuperRegs(Reserved, AArch64::W19); } + assert(checkAllSuperRegsMarked(Reserved)); return Reserved; } @@ -167,6 +168,10 @@ bool AArch64RegisterInfo::isReservedReg(const MachineFunction &MF, return false; } +bool AArch64RegisterInfo::isConstantPhysReg(unsigned PhysReg) const { + return PhysReg == AArch64::WZR || PhysReg == AArch64::XZR; +} + const TargetRegisterClass * AArch64RegisterInfo::getPointerRegClass(const MachineFunction &MF, unsigned Kind) const { @@ -183,7 +188,7 @@ AArch64RegisterInfo::getCrossCopyRegClass(const TargetRegisterClass *RC) const { unsigned AArch64RegisterInfo::getBaseRegister() const { return AArch64::X19; } bool AArch64RegisterInfo::hasBasePointer(const MachineFunction &MF) const { - const MachineFrameInfo *MFI = MF.getFrameInfo(); + const MachineFrameInfo &MFI = MF.getFrameInfo(); // In the presence of variable sized objects, if the fixed stack size is // large enough that referencing from the FP won't result in things being @@ -192,7 +197,7 @@ bool AArch64RegisterInfo::hasBasePointer(const MachineFunction &MF) const { // Furthermore, if both variable sized objects are present, and the // stack needs to be dynamically re-aligned, the base pointer is the only // reliable way to reference the locals. - if (MFI->hasVarSizedObjects()) { + if (MFI.hasVarSizedObjects()) { if (needsStackRealignment(MF)) return true; // Conservatively estimate whether the negative offset from the frame @@ -202,7 +207,7 @@ bool AArch64RegisterInfo::hasBasePointer(const MachineFunction &MF) const { // If it's wrong, we'll materialize the constant and still get to the // object; it's just suboptimal. Negative offsets use the unscaled // load/store instructions, which have a 9-bit signed immediate. - return MFI->getLocalFrameSize() >= 256; + return MFI.getLocalFrameSize() >= 256; } return false; @@ -226,11 +231,11 @@ bool AArch64RegisterInfo::requiresVirtualBaseRegisters( bool AArch64RegisterInfo::useFPForScavengingIndex(const MachineFunction &MF) const { - const MachineFrameInfo *MFI = MF.getFrameInfo(); + const MachineFrameInfo &MFI = MF.getFrameInfo(); // AArch64FrameLowering::resolveFrameIndexReference() can always fall back // to the stack pointer, so only put the emergency spill slot next to the // FP when there's no better way to access it (SP or base pointer). - return MFI->hasVarSizedObjects() && !hasBasePointer(MF); + return MFI.hasVarSizedObjects() && !hasBasePointer(MF); } bool AArch64RegisterInfo::requiresFrameIndexScavenging( @@ -240,10 +245,10 @@ bool AArch64RegisterInfo::requiresFrameIndexScavenging( bool AArch64RegisterInfo::cannotEliminateFrame(const MachineFunction &MF) const { - const MachineFrameInfo *MFI = MF.getFrameInfo(); - if (MF.getTarget().Options.DisableFramePointerElim(MF) && MFI->adjustsStack()) + const MachineFrameInfo &MFI = MF.getFrameInfo(); + if (MF.getTarget().Options.DisableFramePointerElim(MF) && MFI.adjustsStack()) return true; - return MFI->hasVarSizedObjects() || MFI->isFrameAddressTaken(); + return MFI.hasVarSizedObjects() || MFI.isFrameAddressTaken(); } /// needsFrameBaseReg - Returns true if the instruction's frame index @@ -275,7 +280,7 @@ bool AArch64RegisterInfo::needsFrameBaseReg(MachineInstr *MI, // so it'll be negative. MachineFunction &MF = *MI->getParent()->getParent(); const AArch64FrameLowering *TFI = getFrameLowering(MF); - MachineFrameInfo *MFI = MF.getFrameInfo(); + MachineFrameInfo &MFI = MF.getFrameInfo(); // Estimate an offset from the frame pointer. // Conservatively assume all GPR callee-saved registers get pushed. @@ -285,7 +290,7 @@ bool AArch64RegisterInfo::needsFrameBaseReg(MachineInstr *MI, // The incoming offset is relating to the SP at the start of the function, // but when we access the local it'll be relative to the SP after local // allocation, so adjust our SP-relative offset by that allocation size. - Offset += MFI->getLocalFrameSize(); + Offset += MFI.getLocalFrameSize(); // Assume that we'll have at least some spill slots allocated. // FIXME: This is a total SWAG number. We should run some statistics // and pick a real one. diff --git a/lib/Target/AArch64/AArch64RegisterInfo.h b/lib/Target/AArch64/AArch64RegisterInfo.h index f33f788fd437..8ce893516fe2 100644 --- a/lib/Target/AArch64/AArch64RegisterInfo.h +++ b/lib/Target/AArch64/AArch64RegisterInfo.h @@ -24,8 +24,7 @@ class RegScavenger; class TargetRegisterClass; class Triple; -struct AArch64RegisterInfo : public AArch64GenRegisterInfo { -private: +class AArch64RegisterInfo final : public AArch64GenRegisterInfo { const Triple &TT; public: @@ -36,7 +35,7 @@ public: /// Code Generation virtual methods... const MCPhysReg *getCalleeSavedRegs(const MachineFunction *MF) const override; const MCPhysReg * - getCalleeSavedRegsViaCopy(const MachineFunction *MF) const override; + getCalleeSavedRegsViaCopy(const MachineFunction *MF) const; const uint32_t *getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const override; @@ -63,6 +62,7 @@ public: CallingConv::ID) const; BitVector getReservedRegs(const MachineFunction &MF) const override; + bool isConstantPhysReg(unsigned PhysReg) const override; const TargetRegisterClass * getPointerRegClass(const MachineFunction &MF, unsigned Kind = 0) const override; @@ -95,6 +95,10 @@ public: unsigned getRegPressureLimit(const TargetRegisterClass *RC, MachineFunction &MF) const override; + + bool trackLivenessAfterRegAlloc(const MachineFunction&) const override { + return true; + } }; } // end namespace llvm diff --git a/lib/Target/AArch64/AArch64RegisterInfo.td b/lib/Target/AArch64/AArch64RegisterInfo.td index 5fbaff00a5e7..7e29ee5e9baf 100644 --- a/lib/Target/AArch64/AArch64RegisterInfo.td +++ b/lib/Target/AArch64/AArch64RegisterInfo.td @@ -422,7 +422,7 @@ def DD : RegisterClass<"AArch64", [untyped], 64, (add DSeqPairs)> { let Size = 128; } def DDD : RegisterClass<"AArch64", [untyped], 64, (add DSeqTriples)> { - let Size = 196; + let Size = 192; } def DDDD : RegisterClass<"AArch64", [untyped], 64, (add DSeqQuads)> { let Size = 256; diff --git a/lib/Target/AArch64/AArch64SchedA57.td b/lib/Target/AArch64/AArch64SchedA57.td index a266351f7ffc..99c48d0146e4 100644 --- a/lib/Target/AArch64/AArch64SchedA57.td +++ b/lib/Target/AArch64/AArch64SchedA57.td @@ -92,7 +92,7 @@ def : SchedAlias; def : SchedAlias; def : SchedAlias; def : SchedAlias; -def : SchedAlias; +def : SchedAlias; def : SchedAlias; def : SchedAlias; def : SchedAlias; @@ -444,19 +444,19 @@ def : InstRW<[A57Write_5cyc_1V], (instregex "^[FVSU]CVT([AMNPZ][SU])?(_Int)?(v2f def : InstRW<[A57Write_5cyc_2V], (instregex "^[FVSU]CVT([AMNPZ][SU])?(_Int)?(v4f32|v2f64|v4i32|v2i64)")>; // ASIMD FP divide, D-form, F32 -def : InstRW<[A57Write_18cyc_1X], (instregex "FDIVv2f32")>; +def : InstRW<[A57Write_17cyc_1W], (instregex "FDIVv2f32")>; // ASIMD FP divide, Q-form, F32 -def : InstRW<[A57Write_36cyc_2X], (instregex "FDIVv4f32")>; +def : InstRW<[A57Write_34cyc_2W], (instregex "FDIVv4f32")>; // ASIMD FP divide, Q-form, F64 -def : InstRW<[A57Write_64cyc_2X], (instregex "FDIVv2f64")>; +def : InstRW<[A57Write_64cyc_2W], (instregex "FDIVv2f64")>; // Note: These were simply duplicated from ASIMD FDIV because of missing documentation // ASIMD FP square root, D-form, F32 -def : InstRW<[A57Write_18cyc_1X], (instregex "FSQRTv2f32")>; +def : InstRW<[A57Write_17cyc_1W], (instregex "FSQRTv2f32")>; // ASIMD FP square root, Q-form, F32 -def : InstRW<[A57Write_36cyc_2X], (instregex "FSQRTv4f32")>; +def : InstRW<[A57Write_34cyc_2W], (instregex "FSQRTv4f32")>; // ASIMD FP square root, Q-form, F64 -def : InstRW<[A57Write_64cyc_2X], (instregex "FSQRTv2f64")>; +def : InstRW<[A57Write_64cyc_2W], (instregex "FSQRTv2f64")>; // ASIMD FP max/min, normal, D-form def : InstRW<[A57Write_5cyc_1V], (instregex "^(FMAX|FMIN)(NM)?(v2f32)")>; @@ -551,15 +551,15 @@ def : InstRW<[A57WriteFPMA, A57ReadFPM, A57ReadFPM, A57ReadFPMA5], (instregex "^ def : InstRW<[A57Write_10cyc_1L_1V], (instregex "^[FSU]CVT[AMNPZ][SU](_Int)?[SU]?[XW]?[DS]?[rds]i?")>; def : InstRW<[A57Write_10cyc_1L_1V], (instregex "^[SU]CVTF")>; -def : InstRW<[A57Write_32cyc_1X], (instrs FDIVDrr)>; -def : InstRW<[A57Write_18cyc_1X], (instrs FDIVSrr)>; +def : InstRW<[A57Write_32cyc_1W], (instrs FDIVDrr)>; +def : InstRW<[A57Write_17cyc_1W], (instrs FDIVSrr)>; def : InstRW<[A57Write_5cyc_1V], (instregex "^F(MAX|MIN).+rr")>; def : InstRW<[A57Write_5cyc_1V], (instregex "^FRINT.+r")>; -def : InstRW<[A57Write_32cyc_1X], (instrs FSQRTDr)>; -def : InstRW<[A57Write_18cyc_1X], (instrs FSQRTSr)>; +def : InstRW<[A57Write_32cyc_1W], (instrs FSQRTDr)>; +def : InstRW<[A57Write_17cyc_1W], (instrs FSQRTSr)>; def : InstRW<[A57Write_5cyc_1L, WriteLDHi], (instrs LDNPDi)>; def : InstRW<[A57Write_6cyc_2L, WriteLDHi], (instrs LDNPQi)>; diff --git a/lib/Target/AArch64/AArch64SchedA57WriteRes.td b/lib/Target/AArch64/AArch64SchedA57WriteRes.td index 6f30108a23e3..55005e1d9ed1 100644 --- a/lib/Target/AArch64/AArch64SchedA57WriteRes.td +++ b/lib/Target/AArch64/AArch64SchedA57WriteRes.td @@ -28,15 +28,15 @@ def A57Write_5cyc_1M : SchedWriteRes<[A57UnitM]> { let Latency = 5; } def A57Write_5cyc_1V : SchedWriteRes<[A57UnitV]> { let Latency = 5; } def A57Write_5cyc_1W : SchedWriteRes<[A57UnitW]> { let Latency = 5; } def A57Write_10cyc_1V : SchedWriteRes<[A57UnitV]> { let Latency = 10; } -def A57Write_18cyc_1X : SchedWriteRes<[A57UnitX]> { let Latency = 18; - let ResourceCycles = [18]; } +def A57Write_17cyc_1W : SchedWriteRes<[A57UnitW]> { let Latency = 17; + let ResourceCycles = [17]; } def A57Write_19cyc_1M : SchedWriteRes<[A57UnitM]> { let Latency = 19; let ResourceCycles = [19]; } def A57Write_1cyc_1B : SchedWriteRes<[A57UnitB]> { let Latency = 1; } def A57Write_1cyc_1I : SchedWriteRes<[A57UnitI]> { let Latency = 1; } def A57Write_1cyc_1S : SchedWriteRes<[A57UnitS]> { let Latency = 1; } def A57Write_2cyc_1M : SchedWriteRes<[A57UnitM]> { let Latency = 2; } -def A57Write_32cyc_1X : SchedWriteRes<[A57UnitX]> { let Latency = 32; +def A57Write_32cyc_1W : SchedWriteRes<[A57UnitW]> { let Latency = 32; let ResourceCycles = [32]; } def A57Write_35cyc_1M : SchedWriteRes<[A57UnitM]> { let Latency = 35; let ResourceCycles = [35]; } @@ -54,7 +54,7 @@ def A57Write_6cyc_1V : SchedWriteRes<[A57UnitV]> { let Latency = 6; } //===----------------------------------------------------------------------===// // Define Generic 2 micro-op types -def A57Write_64cyc_2X : SchedWriteRes<[A57UnitX, A57UnitX]> { +def A57Write_64cyc_2W : SchedWriteRes<[A57UnitW, A57UnitW]> { let Latency = 64; let NumMicroOps = 2; let ResourceCycles = [32, 32]; @@ -139,10 +139,10 @@ def A57Write_2cyc_2V : SchedWriteRes<[A57UnitV, A57UnitV]> { let Latency = 2; let NumMicroOps = 2; } -def A57Write_36cyc_2X : SchedWriteRes<[A57UnitX, A57UnitX]> { - let Latency = 36; +def A57Write_34cyc_2W : SchedWriteRes<[A57UnitW, A57UnitW]> { + let Latency = 34; let NumMicroOps = 2; - let ResourceCycles = [18, 18]; + let ResourceCycles = [17, 17]; } def A57Write_3cyc_1I_1M : SchedWriteRes<[A57UnitI, A57UnitM]> { diff --git a/lib/Target/AArch64/AArch64SchedFalkor.td b/lib/Target/AArch64/AArch64SchedFalkor.td new file mode 100644 index 000000000000..19a6d6f2a1ad --- /dev/null +++ b/lib/Target/AArch64/AArch64SchedFalkor.td @@ -0,0 +1,26 @@ +//==- AArch64SchedFalkor.td - Falkor Scheduling Definitions -*- tablegen -*-==// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file defines the machine model for Qualcomm Falkor to support +// instruction scheduling and other instruction cost heuristics. +// +//===----------------------------------------------------------------------===// + +//===----------------------------------------------------------------------===// +// Define the SchedMachineModel and provide basic properties for coarse grained +// instruction cost model. + +def FalkorModel : SchedMachineModel { + let IssueWidth = 4; // 4-wide issue for expanded uops. + let MicroOpBufferSize = 128; // Out-of-order with temporary unified issue buffer. + let LoopMicroOpBufferSize = 16; + let LoadLatency = 3; // Optimistic load latency. + let MispredictPenalty = 11; // Minimum branch misprediction penalty. + let CompleteModel = 0; +} diff --git a/lib/Target/AArch64/AArch64SchedM1.td b/lib/Target/AArch64/AArch64SchedM1.td index 2288b8dfc223..14d6891253fa 100644 --- a/lib/Target/AArch64/AArch64SchedM1.td +++ b/lib/Target/AArch64/AArch64SchedM1.td @@ -32,33 +32,31 @@ def ExynosM1Model : SchedMachineModel { def M1UnitA : ProcResource<2>; // Simple integer def M1UnitC : ProcResource<1>; // Simple and complex integer +def M1UnitD : ProcResource<1>; // Integer division (inside C, serialized) def M1UnitB : ProcResource<2>; // Branch def M1UnitL : ProcResource<1>; // Load def M1UnitS : ProcResource<1>; // Store def M1PipeF0 : ProcResource<1>; // FP #0 -def M1PipeF1 : ProcResource<1>; // FP #1 - let Super = M1PipeF0 in { def M1UnitFMAC : ProcResource<1>; // FP multiplication - def M1UnitFCVT : ProcResource<1>; // FP conversion - def M1UnitNAL0 : ProcResource<1>; // Simple vector. + def M1UnitNAL0 : ProcResource<1>; // Simple vector def M1UnitNMISC : ProcResource<1>; // Miscellanea + def M1UnitFCVT : ProcResource<1>; // FP conversion def M1UnitNCRYPT : ProcResource<1>; // Cryptographic } - +def M1PipeF1 : ProcResource<1>; // FP #1 let Super = M1PipeF1 in { def M1UnitFADD : ProcResource<1>; // Simple FP - let BufferSize = 1 in + def M1UnitNAL1 : ProcResource<1>; // Simple vector def M1UnitFVAR : ProcResource<1>; // FP division & square root (serialized) - def M1UnitNAL1 : ProcResource<1>; // Simple vector. def M1UnitFST : ProcResource<1>; // FP store } let SchedModel = ExynosM1Model in { def M1UnitALU : ProcResGroup<[M1UnitA, - M1UnitC]>; // All simple integer. + M1UnitC]>; // All integer def M1UnitNALU : ProcResGroup<[M1UnitNAL0, - M1UnitNAL1]>; // All simple vector. + M1UnitNAL1]>; // All simple vector } let SchedModel = ExynosM1Model in { @@ -66,11 +64,33 @@ let SchedModel = ExynosM1Model in { //===----------------------------------------------------------------------===// // Coarse scheduling model for the Exynos-M1. +def M1WriteA1 : SchedWriteRes<[M1UnitALU]> { let Latency = 1; } +def M1WriteA2 : SchedWriteRes<[M1UnitALU]> { let Latency = 2; } +def M1WriteC1 : SchedWriteRes<[M1UnitC]> { let Latency = 1; } +def M1WriteC2 : SchedWriteRes<[M1UnitC]> { let Latency = 2; } + +def M1WriteB1 : SchedWriteRes<[M1UnitB]> { let Latency = 1; } + +def M1WriteL5 : SchedWriteRes<[M1UnitL]> { let Latency = 5; } +def M1WriteLA : SchedWriteVariant<[SchedVar, + SchedVar]>; + +def M1WriteS1 : SchedWriteRes<[M1UnitS]> { let Latency = 1; } +def M1WriteS2 : SchedWriteRes<[M1UnitS]> { let Latency = 2; } +def M1WriteS4 : SchedWriteRes<[M1UnitS]> { let Latency = 4; } +def M1WriteSA : SchedWriteVariant<[SchedVar, + SchedVar]>; + +def M1ReadAdrBase : SchedReadVariant<[SchedVar, + SchedVar]>; +def : SchedAlias; + // Branch instructions. -// TODO: Non-conditional direct branches take zero cycles and units. +// NOTE: Unconditional direct branches actually take neither cycles nor units. def : WriteRes { let Latency = 1; } def : WriteRes { let Latency = 1; } -// TODO: Branch and link is much different. // Arithmetic and logical integer instructions. def : WriteRes { let Latency = 1; } @@ -83,9 +103,12 @@ def : WriteRes { let Latency = 1; } def : WriteRes { let Latency = 1; } // Divide and multiply instructions. -// TODO: Division blocks the divider inside C. -def : WriteRes { let Latency = 13; } -def : WriteRes { let Latency = 21; } +def : WriteRes { let Latency = 13; + let ResourceCycles = [1, 13]; } +def : WriteRes { let Latency = 21; + let ResourceCycles = [1, 21]; } // TODO: Long multiplication take 5 cycles and also the ALU. // TODO: Multiplication with accumulation can be advanced. def : WriteRes { let Latency = 3; } @@ -101,31 +124,27 @@ def : WriteRes { let Latency = 0; } // Load instructions. def : WriteRes { let Latency = 4; } -// TODO: Extended address requires also the ALU. -def : WriteRes { let Latency = 5; } def : WriteRes { let Latency = 4; } +def : SchedAlias; // Store instructions. def : WriteRes { let Latency = 1; } -// TODO: Extended address requires also the ALU. -def : WriteRes { let Latency = 1; } def : WriteRes { let Latency = 1; } def : WriteRes { let Latency = 1; } +def : SchedAlias; // FP data instructions. def : WriteRes { let Latency = 3; } // TODO: FCCMP is much different. def : WriteRes { let Latency = 4; } -// TODO: DP takes longer. -def : WriteRes { let Latency = 15; } -// TODO: MACC takes longer. +def : WriteRes { let Latency = 15; + let ResourceCycles = [15]; } def : WriteRes { let Latency = 4; } // FP miscellaneous instructions. // TODO: Conversion between register files is much different. def : WriteRes { let Latency = 3; } def : WriteRes { let Latency = 1; } -// TODO: Copy from FPR to GPR is much different. def : WriteRes { let Latency = 4; } // FP load instructions. @@ -137,7 +156,6 @@ def : WriteRes { let Latency = 5; } def : WriteRes { let Latency = 1; } // ASIMD FP instructions. -// TODO: Other operations are much different. def : WriteRes { let Latency = 3; } // Other miscellaneous instructions. @@ -191,24 +209,15 @@ def M1WriteNEONJ : SchedWriteRes<[M1UnitNMISC, M1UnitFMAC]> { let Latency = 6; } def M1WriteNEONK : SchedWriteRes<[M1UnitNMISC, M1UnitFMAC]> { let Latency = 7; } -def M1WriteALU1 : SchedWriteRes<[M1UnitALU]> { let Latency = 1; } -def M1WriteB : SchedWriteRes<[M1UnitB]> { let Latency = 1; } -// FIXME: This is the worst case, conditional branch and link. -def M1WriteBL : SchedWriteRes<[M1UnitB, - M1UnitALU]> { let Latency = 1; } -// FIXME: This is the worst case, when using LR. -def M1WriteBLR : SchedWriteRes<[M1UnitB, - M1UnitALU, - M1UnitALU]> { let Latency = 2; } -def M1WriteC1 : SchedWriteRes<[M1UnitC]> { let Latency = 1; } -def M1WriteC2 : SchedWriteRes<[M1UnitC]> { let Latency = 2; } def M1WriteFADD3 : SchedWriteRes<[M1UnitFADD]> { let Latency = 3; } def M1WriteFCVT3 : SchedWriteRes<[M1UnitFCVT]> { let Latency = 3; } def M1WriteFCVT4 : SchedWriteRes<[M1UnitFCVT]> { let Latency = 4; } def M1WriteFMAC4 : SchedWriteRes<[M1UnitFMAC]> { let Latency = 4; } def M1WriteFMAC5 : SchedWriteRes<[M1UnitFMAC]> { let Latency = 5; } -def M1WriteFVAR15 : SchedWriteRes<[M1UnitFVAR]> { let Latency = 15; } -def M1WriteFVAR23 : SchedWriteRes<[M1UnitFVAR]> { let Latency = 23; } +def M1WriteFVAR15 : SchedWriteRes<[M1UnitFVAR]> { let Latency = 15; + let ResourceCycles = [15]; } +def M1WriteFVAR23 : SchedWriteRes<[M1UnitFVAR]> { let Latency = 23; + let ResourceCycles = [23]; } def M1WriteNALU1 : SchedWriteRes<[M1UnitNALU]> { let Latency = 1; } def M1WriteNALU2 : SchedWriteRes<[M1UnitNALU]> { let Latency = 2; } def M1WriteNAL11 : SchedWriteRes<[M1UnitNAL1]> { let Latency = 1; } @@ -220,19 +229,22 @@ def M1WriteNMISC1 : SchedWriteRes<[M1UnitNMISC]> { let Latency = 1; } def M1WriteNMISC2 : SchedWriteRes<[M1UnitNMISC]> { let Latency = 2; } def M1WriteNMISC3 : SchedWriteRes<[M1UnitNMISC]> { let Latency = 3; } def M1WriteNMISC4 : SchedWriteRes<[M1UnitNMISC]> { let Latency = 4; } -def M1WriteS4 : SchedWriteRes<[M1UnitS]> { let Latency = 4; } def M1WriteTB : SchedWriteRes<[M1UnitC, M1UnitALU]> { let Latency = 2; } // Branch instructions -def : InstRW<[M1WriteB ], (instrs Bcc)>; -def : InstRW<[M1WriteBL], (instrs BL)>; -def : InstRW<[M1WriteBLR], (instrs BLR)>; -def : InstRW<[M1WriteC1], (instregex "^CBN?Z[WX]")>; -def : InstRW<[M1WriteTB], (instregex "^TBN?Z[WX]")>; +def : InstRW<[M1WriteB1], (instrs Bcc)>; +// NOTE: Conditional branch and link adds a B uop. +def : InstRW<[M1WriteA1], (instrs BL)>; +// NOTE: Indirect branch and link with LR adds an ALU uop. +def : InstRW<[M1WriteA1, + M1WriteC1], (instrs BLR)>; +def : InstRW<[M1WriteC1], (instregex "^CBN?Z[WX]")>; +def : InstRW<[M1WriteC1, + M1WriteA2], (instregex "^TBN?Z[WX]")>; // Arithmetic and logical integer instructions. -def : InstRW<[M1WriteALU1], (instrs COPY)>; +def : InstRW<[M1WriteA1], (instrs COPY)>; // Divide and multiply instructions. diff --git a/lib/Target/AArch64/AArch64SchedVulcan.td b/lib/Target/AArch64/AArch64SchedVulcan.td index 0aa2462eba83..35a40c314bf4 100644 --- a/lib/Target/AArch64/AArch64SchedVulcan.td +++ b/lib/Target/AArch64/AArch64SchedVulcan.td @@ -49,15 +49,12 @@ def VulcanP5 : ProcResource<1>; let SchedModel = VulcanModel in { -// Define groups for the functional units on each -// issue port. Each group created will be used -// by a WriteRes later on. +// Define groups for the functional units on each issue port. Each group +// created will be used by a WriteRes later on. // -// NOTE: Some groups only contain one member. This -// is a way to create names for the various functional -// units that share a single issue port. For example, -// "VulcanI1" for ALU ops on port 1 and "VulcanF1" for -// FP ops on port 1. +// NOTE: Some groups only contain one member. This is a way to create names for +// the various functional units that share a single issue port. For example, +// "VulcanI1" for ALU ops on port 1 and "VulcanF1" for FP ops on port 1. // Integer divide and multiply micro-ops only on port 1. def VulcanI1 : ProcResGroup<[VulcanP1]>; diff --git a/lib/Target/AArch64/AArch64StorePairSuppress.cpp b/lib/Target/AArch64/AArch64StorePairSuppress.cpp index f904b2379416..fe984ccbaf1d 100644 --- a/lib/Target/AArch64/AArch64StorePairSuppress.cpp +++ b/lib/Target/AArch64/AArch64StorePairSuppress.cpp @@ -25,6 +25,8 @@ using namespace llvm; #define DEBUG_TYPE "aarch64-stp-suppress" +#define STPSUPPRESS_PASS_NAME "AArch64 Store Pair Suppression" + namespace { class AArch64StorePairSuppress : public MachineFunctionPass { const AArch64InstrInfo *TII; @@ -36,12 +38,12 @@ class AArch64StorePairSuppress : public MachineFunctionPass { public: static char ID; - AArch64StorePairSuppress() : MachineFunctionPass(ID) {} - - const char *getPassName() const override { - return "AArch64 Store Pair Suppression"; + AArch64StorePairSuppress() : MachineFunctionPass(ID) { + initializeAArch64StorePairSuppressPass(*PassRegistry::getPassRegistry()); } + StringRef getPassName() const override { return STPSUPPRESS_PASS_NAME; } + bool runOnMachineFunction(MachineFunction &F) override; private: @@ -59,6 +61,9 @@ private: char AArch64StorePairSuppress::ID = 0; } // anonymous +INITIALIZE_PASS(AArch64StorePairSuppress, "aarch64-stp-suppress", + STPSUPPRESS_PASS_NAME, false, false) + FunctionPass *llvm::createAArch64StorePairSuppressPass() { return new AArch64StorePairSuppress(); } diff --git a/lib/Target/AArch64/AArch64Subtarget.cpp b/lib/Target/AArch64/AArch64Subtarget.cpp index 7dd8ccbe6c25..f58bbbd26132 100644 --- a/lib/Target/AArch64/AArch64Subtarget.cpp +++ b/lib/Target/AArch64/AArch64Subtarget.cpp @@ -36,7 +36,8 @@ UseAddressTopByteIgnored("aarch64-use-tbi", cl::desc("Assume that top byte of " "an address is ignored"), cl::init(false), cl::Hidden); AArch64Subtarget & -AArch64Subtarget::initializeSubtargetDependencies(StringRef FS) { +AArch64Subtarget::initializeSubtargetDependencies(StringRef FS, + StringRef CPUString) { // Determine default and user-specified characteristics if (CPUString.empty()) @@ -63,9 +64,14 @@ void AArch64Subtarget::initializeProperties() { MaxInterleaveFactor = 4; break; case ExynosM1: + MaxInterleaveFactor = 4; + MaxJumpTableSize = 8; PrefFunctionAlignment = 4; PrefLoopAlignment = 3; break; + case Falkor: + MaxInterleaveFactor = 4; + break; case Kryo: MaxInterleaveFactor = 4; VectorInsertExtractBaseCost = 2; @@ -89,8 +95,8 @@ AArch64Subtarget::AArch64Subtarget(const Triple &TT, const std::string &CPU, const std::string &FS, const TargetMachine &TM, bool LittleEndian) : AArch64GenSubtargetInfo(TT, CPU, FS), ReserveX18(TT.isOSDarwin()), - IsLittle(LittleEndian), CPUString(CPU), TargetTriple(TT), FrameLowering(), - InstrInfo(initializeSubtargetDependencies(FS)), TSInfo(), + IsLittle(LittleEndian), TargetTriple(TT), FrameLowering(), + InstrInfo(initializeSubtargetDependencies(FS, CPU)), TSInfo(), TLInfo(TM, *this), GISel() {} const CallLowering *AArch64Subtarget::getCallLowering() const { @@ -98,6 +104,16 @@ const CallLowering *AArch64Subtarget::getCallLowering() const { return GISel->getCallLowering(); } +const InstructionSelector *AArch64Subtarget::getInstructionSelector() const { + assert(GISel && "Access to GlobalISel APIs not set"); + return GISel->getInstructionSelector(); +} + +const LegalizerInfo *AArch64Subtarget::getLegalizerInfo() const { + assert(GISel && "Access to GlobalISel APIs not set"); + return GISel->getLegalizerInfo(); +} + const RegisterBankInfo *AArch64Subtarget::getRegBankInfo() const { assert(GISel && "Access to GlobalISel APIs not set"); return GISel->getRegBankInfo(); diff --git a/lib/Target/AArch64/AArch64Subtarget.h b/lib/Target/AArch64/AArch64Subtarget.h index 16a35405c892..73f63b8b9f67 100644 --- a/lib/Target/AArch64/AArch64Subtarget.h +++ b/lib/Target/AArch64/AArch64Subtarget.h @@ -32,7 +32,7 @@ class GlobalValue; class StringRef; class Triple; -class AArch64Subtarget : public AArch64GenSubtargetInfo { +class AArch64Subtarget final : public AArch64GenSubtargetInfo { public: enum ARMProcFamilyEnum : uint8_t { Others, @@ -43,6 +43,7 @@ public: CortexA73, Cyclone, ExynosM1, + Falkor, Kryo, Vulcan }; @@ -58,6 +59,7 @@ protected: bool HasNEON = false; bool HasCrypto = false; bool HasCRC = false; + bool HasLSE = false; bool HasRAS = false; bool HasPerfMon = false; bool HasFullFP16 = false; @@ -71,7 +73,6 @@ protected: // StrictAlign - Disallow unaligned memory accesses. bool StrictAlign = false; - bool MergeNarrowLoads = false; bool UseAA = false; bool PredictableSelectIsExpensive = false; bool BalanceFPOps = false; @@ -80,7 +81,8 @@ protected: bool Misaligned128StoreIsSlow = false; bool AvoidQuadLdStPairs = false; bool UseAlternateSExtLoadCVTF32Pattern = false; - bool HasMacroOpFusion = false; + bool HasArithmeticBccFusion = false; + bool HasArithmeticCbzFusion = false; bool DisableLatencySchedHeuristic = false; bool UseRSqrt = false; uint8_t MaxInterleaveFactor = 2; @@ -91,15 +93,13 @@ protected: unsigned MaxPrefetchIterationsAhead = UINT_MAX; unsigned PrefFunctionAlignment = 0; unsigned PrefLoopAlignment = 0; + unsigned MaxJumpTableSize = 0; // ReserveX18 - X18 is not available as a general purpose register. bool ReserveX18; bool IsLittle; - /// CPUString - String name of used CPU. - std::string CPUString; - /// TargetTriple - What processor and OS we're targeting. Triple TargetTriple; @@ -116,7 +116,8 @@ private: /// initializeSubtargetDependencies - Initializes using CPUString and the /// passed in feature string so that we can use initializer lists for /// subtarget initialization. - AArch64Subtarget &initializeSubtargetDependencies(StringRef FS); + AArch64Subtarget &initializeSubtargetDependencies(StringRef FS, + StringRef CPUString); /// Initialize properties based on the selected processor family. void initializeProperties(); @@ -147,6 +148,8 @@ public: return &getInstrInfo()->getRegisterInfo(); } const CallLowering *getCallLowering() const override; + const InstructionSelector *getInstructionSelector() const override; + const LegalizerInfo *getLegalizerInfo() const override; const RegisterBankInfo *getRegBankInfo() const override; const Triple &getTargetTriple() const { return TargetTriple; } bool enableMachineScheduler() const override { return true; } @@ -171,13 +174,15 @@ public: bool requiresStrictAlign() const { return StrictAlign; } + bool isXRaySupported() const override { return true; } + bool isX18Reserved() const { return ReserveX18; } bool hasFPARMv8() const { return HasFPARMv8; } bool hasNEON() const { return HasNEON; } bool hasCrypto() const { return HasCrypto; } bool hasCRC() const { return HasCRC; } + bool hasLSE() const { return HasLSE; } bool hasRAS() const { return HasRAS; } - bool mergeNarrowLoads() const { return MergeNarrowLoads; } bool balanceFPOps() const { return BalanceFPOps; } bool predictableSelectIsExpensive() const { return PredictableSelectIsExpensive; @@ -188,7 +193,8 @@ public: bool useAlternateSExtLoadCVTF32Pattern() const { return UseAlternateSExtLoadCVTF32Pattern; } - bool hasMacroOpFusion() const { return HasMacroOpFusion; } + bool hasArithmeticBccFusion() const { return HasArithmeticBccFusion; } + bool hasArithmeticCbzFusion() const { return HasArithmeticCbzFusion; } bool useRSqrt() const { return UseRSqrt; } unsigned getMaxInterleaveFactor() const { return MaxInterleaveFactor; } unsigned getVectorInsertExtractBaseCost() const { @@ -203,6 +209,8 @@ public: unsigned getPrefFunctionAlignment() const { return PrefFunctionAlignment; } unsigned getPrefLoopAlignment() const { return PrefLoopAlignment; } + unsigned getMaximumJumpTableSize() const { return MaxJumpTableSize; } + /// CPU has TBI (top byte of addresses is ignored during HW address /// translation) and OS enables it. bool supportsAddressTopByteIgnored() const; diff --git a/lib/Target/AArch64/AArch64TargetMachine.cpp b/lib/Target/AArch64/AArch64TargetMachine.cpp index 0b6345ff8011..e4ef0d4bb8db 100644 --- a/lib/Target/AArch64/AArch64TargetMachine.cpp +++ b/lib/Target/AArch64/AArch64TargetMachine.cpp @@ -12,12 +12,17 @@ #include "AArch64.h" #include "AArch64CallLowering.h" +#include "AArch64InstructionSelector.h" +#include "AArch64LegalizerInfo.h" #include "AArch64RegisterBankInfo.h" #include "AArch64TargetMachine.h" #include "AArch64TargetObjectFile.h" #include "AArch64TargetTransformInfo.h" #include "llvm/CodeGen/GlobalISel/IRTranslator.h" +#include "llvm/CodeGen/GlobalISel/InstructionSelect.h" +#include "llvm/CodeGen/GlobalISel/Legalizer.h" #include "llvm/CodeGen/GlobalISel/RegBankSelect.h" +#include "llvm/CodeGen/MachineScheduler.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/RegAllocRegistry.h" #include "llvm/CodeGen/TargetPassConfig.h" @@ -30,53 +35,56 @@ #include "llvm/Transforms/Scalar.h" using namespace llvm; -static cl::opt -EnableCCMP("aarch64-ccmp", cl::desc("Enable the CCMP formation pass"), - cl::init(true), cl::Hidden); +static cl::opt EnableCCMP("aarch64-enable-ccmp", + cl::desc("Enable the CCMP formation pass"), + cl::init(true), cl::Hidden); -static cl::opt EnableMCR("aarch64-mcr", +static cl::opt EnableMCR("aarch64-enable-mcr", cl::desc("Enable the machine combiner pass"), cl::init(true), cl::Hidden); -static cl::opt -EnableStPairSuppress("aarch64-stp-suppress", cl::desc("Suppress STP for AArch64"), - cl::init(true), cl::Hidden); +static cl::opt EnableStPairSuppress("aarch64-enable-stp-suppress", + cl::desc("Suppress STP for AArch64"), + cl::init(true), cl::Hidden); -static cl::opt -EnableAdvSIMDScalar("aarch64-simd-scalar", cl::desc("Enable use of AdvSIMD scalar" - " integer instructions"), cl::init(false), cl::Hidden); - -static cl::opt -EnablePromoteConstant("aarch64-promote-const", cl::desc("Enable the promote " - "constant pass"), cl::init(true), cl::Hidden); - -static cl::opt -EnableCollectLOH("aarch64-collect-loh", cl::desc("Enable the pass that emits the" - " linker optimization hints (LOH)"), cl::init(true), - cl::Hidden); - -static cl::opt -EnableDeadRegisterElimination("aarch64-dead-def-elimination", cl::Hidden, - cl::desc("Enable the pass that removes dead" - " definitons and replaces stores to" - " them with stores to the zero" - " register"), - cl::init(true)); +static cl::opt EnableAdvSIMDScalar( + "aarch64-enable-simd-scalar", + cl::desc("Enable use of AdvSIMD scalar integer instructions"), + cl::init(false), cl::Hidden); static cl::opt -EnableRedundantCopyElimination("aarch64-redundant-copy-elim", - cl::desc("Enable the redundant copy elimination pass"), - cl::init(true), cl::Hidden); + EnablePromoteConstant("aarch64-enable-promote-const", + cl::desc("Enable the promote constant pass"), + cl::init(true), cl::Hidden); -static cl::opt -EnableLoadStoreOpt("aarch64-load-store-opt", cl::desc("Enable the load/store pair" - " optimization pass"), cl::init(true), cl::Hidden); +static cl::opt EnableCollectLOH( + "aarch64-enable-collect-loh", + cl::desc("Enable the pass that emits the linker optimization hints (LOH)"), + cl::init(true), cl::Hidden); static cl::opt -EnableAtomicTidy("aarch64-atomic-cfg-tidy", cl::Hidden, - cl::desc("Run SimplifyCFG after expanding atomic operations" - " to make use of cmpxchg flow-based information"), - cl::init(true)); + EnableDeadRegisterElimination("aarch64-enable-dead-defs", cl::Hidden, + cl::desc("Enable the pass that removes dead" + " definitons and replaces stores to" + " them with stores to the zero" + " register"), + cl::init(true)); + +static cl::opt EnableRedundantCopyElimination( + "aarch64-enable-copyelim", + cl::desc("Enable the redundant copy elimination pass"), cl::init(true), + cl::Hidden); + +static cl::opt EnableLoadStoreOpt("aarch64-enable-ldst-opt", + cl::desc("Enable the load/store pair" + " optimization pass"), + cl::init(true), cl::Hidden); + +static cl::opt EnableAtomicTidy( + "aarch64-enable-atomic-cfg-tidy", cl::Hidden, + cl::desc("Run SimplifyCFG after expanding atomic operations" + " to make use of cmpxchg flow-based information"), + cl::init(true)); static cl::opt EnableEarlyIfConversion("aarch64-enable-early-ifcvt", cl::Hidden, @@ -84,9 +92,9 @@ EnableEarlyIfConversion("aarch64-enable-early-ifcvt", cl::Hidden, cl::init(true)); static cl::opt -EnableCondOpt("aarch64-condopt", - cl::desc("Enable the condition optimizer pass"), - cl::init(true), cl::Hidden); + EnableCondOpt("aarch64-enable-condopt", + cl::desc("Enable the condition optimizer pass"), + cl::init(true), cl::Hidden); static cl::opt EnableA53Fix835769("aarch64-fix-cortex-a53-835769", cl::Hidden, @@ -94,28 +102,51 @@ EnableA53Fix835769("aarch64-fix-cortex-a53-835769", cl::Hidden, cl::init(false)); static cl::opt -EnableGEPOpt("aarch64-gep-opt", cl::Hidden, - cl::desc("Enable optimizations on complex GEPs"), - cl::init(false)); + EnableAddressTypePromotion("aarch64-enable-type-promotion", cl::Hidden, + cl::desc("Enable the type promotion pass"), + cl::init(true)); + +static cl::opt + EnableGEPOpt("aarch64-enable-gep-opt", cl::Hidden, + cl::desc("Enable optimizations on complex GEPs"), + cl::init(false)); + +static cl::opt + BranchRelaxation("aarch64-enable-branch-relax", cl::Hidden, cl::init(true), + cl::desc("Relax out of range conditional branches")); // FIXME: Unify control over GlobalMerge. static cl::opt -EnableGlobalMerge("aarch64-global-merge", cl::Hidden, - cl::desc("Enable the global merge pass")); + EnableGlobalMerge("aarch64-enable-global-merge", cl::Hidden, + cl::desc("Enable the global merge pass")); static cl::opt - EnableLoopDataPrefetch("aarch64-loop-data-prefetch", cl::Hidden, + EnableLoopDataPrefetch("aarch64-enable-loop-data-prefetch", cl::Hidden, cl::desc("Enable the loop data prefetch pass"), cl::init(true)); extern "C" void LLVMInitializeAArch64Target() { // Register the target. - RegisterTargetMachine X(TheAArch64leTarget); - RegisterTargetMachine Y(TheAArch64beTarget); - RegisterTargetMachine Z(TheARM64Target); + RegisterTargetMachine X(getTheAArch64leTarget()); + RegisterTargetMachine Y(getTheAArch64beTarget()); + RegisterTargetMachine Z(getTheARM64Target()); auto PR = PassRegistry::getPassRegistry(); initializeGlobalISel(*PR); + initializeAArch64A53Fix835769Pass(*PR); + initializeAArch64A57FPLoadBalancingPass(*PR); + initializeAArch64AddressTypePromotionPass(*PR); + initializeAArch64AdvSIMDScalarPass(*PR); + initializeAArch64CollectLOHPass(*PR); + initializeAArch64ConditionalComparesPass(*PR); + initializeAArch64ConditionOptimizerPass(*PR); + initializeAArch64DeadRegisterDefinitionsPass(*PR); initializeAArch64ExpandPseudoPass(*PR); + initializeAArch64LoadStoreOptPass(*PR); + initializeAArch64VectorByElementOptPass(*PR); + initializeAArch64PromoteConstantPass(*PR); + initializeAArch64RedundantCopyEliminationPass(*PR); + initializeAArch64StorePairSuppressPass(*PR); + initializeLDTLSCleanupPass(*PR); } //===----------------------------------------------------------------------===// @@ -129,7 +160,11 @@ static std::unique_ptr createTLOF(const Triple &TT) { } // Helper function to build a DataLayout string -static std::string computeDataLayout(const Triple &TT, bool LittleEndian) { +static std::string computeDataLayout(const Triple &TT, + const MCTargetOptions &Options, + bool LittleEndian) { + if (Options.getABIName() == "ilp32") + return "e-m:e-p:32:32-i8:8-i16:16-i64:64-S128"; if (TT.isOSBinFormatMachO()) return "e-m:o-i64:64-i128:128-n32:64-S128"; if (LittleEndian) @@ -137,29 +172,6 @@ static std::string computeDataLayout(const Triple &TT, bool LittleEndian) { return "E-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"; } -// Helper function to set up the defaults for reciprocals. -static void initReciprocals(AArch64TargetMachine& TM, AArch64Subtarget& ST) -{ - // For the estimates, convergence is quadratic, so essentially the number of - // digits is doubled after each iteration. ARMv8, the minimum architected - // accuracy of the initial estimate is 2^-8. Therefore, the number of extra - // steps to refine the result for float (23 mantissa bits) and for double - // (52 mantissa bits) are 2 and 3, respectively. - unsigned ExtraStepsF = 2, - ExtraStepsD = ExtraStepsF + 1; - bool UseRsqrt = ST.useRSqrt(); - - TM.Options.Reciprocals.setDefaults("sqrtf", UseRsqrt, ExtraStepsF); - TM.Options.Reciprocals.setDefaults("sqrtd", UseRsqrt, ExtraStepsD); - TM.Options.Reciprocals.setDefaults("vec-sqrtf", UseRsqrt, ExtraStepsF); - TM.Options.Reciprocals.setDefaults("vec-sqrtd", UseRsqrt, ExtraStepsD); - - TM.Options.Reciprocals.setDefaults("divf", false, ExtraStepsF); - TM.Options.Reciprocals.setDefaults("divd", false, ExtraStepsD); - TM.Options.Reciprocals.setDefaults("vec-divf", false, ExtraStepsF); - TM.Options.Reciprocals.setDefaults("vec-divd", false, ExtraStepsD); -} - static Reloc::Model getEffectiveRelocModel(const Triple &TT, Optional RM) { // AArch64 Darwin is always PIC. @@ -181,11 +193,12 @@ AArch64TargetMachine::AArch64TargetMachine( CodeModel::Model CM, CodeGenOpt::Level OL, bool LittleEndian) // This nested ternary is horrible, but DL needs to be properly // initialized before TLInfo is constructed. - : LLVMTargetMachine(T, computeDataLayout(TT, LittleEndian), TT, CPU, FS, - Options, getEffectiveRelocModel(TT, RM), CM, OL), + : LLVMTargetMachine(T, computeDataLayout(TT, Options.MCOptions, + LittleEndian), + TT, CPU, FS, Options, + getEffectiveRelocModel(TT, RM), CM, OL), TLOF(createTLOF(getTargetTriple())), - Subtarget(TT, CPU, FS, *this, LittleEndian) { - initReciprocals(*this, Subtarget); + isLittle(LittleEndian) { initAsmInfo(); } @@ -195,10 +208,18 @@ AArch64TargetMachine::~AArch64TargetMachine() {} namespace { struct AArch64GISelActualAccessor : public GISelAccessor { std::unique_ptr CallLoweringInfo; + std::unique_ptr InstSelector; + std::unique_ptr Legalizer; std::unique_ptr RegBankInfo; const CallLowering *getCallLowering() const override { return CallLoweringInfo.get(); } + const InstructionSelector *getInstructionSelector() const override { + return InstSelector.get(); + } + const LegalizerInfo *getLegalizerInfo() const override { + return Legalizer.get(); + } const RegisterBankInfo *getRegBankInfo() const override { return RegBankInfo.get(); } @@ -225,16 +246,24 @@ AArch64TargetMachine::getSubtargetImpl(const Function &F) const { // function that reside in TargetOptions. resetTargetOptions(F); I = llvm::make_unique(TargetTriple, CPU, FS, *this, - Subtarget.isLittleEndian()); + isLittle); #ifndef LLVM_BUILD_GLOBAL_ISEL - GISelAccessor *GISel = new GISelAccessor(); + GISelAccessor *GISel = new GISelAccessor(); #else AArch64GISelActualAccessor *GISel = new AArch64GISelActualAccessor(); GISel->CallLoweringInfo.reset( new AArch64CallLowering(*I->getTargetLowering())); - GISel->RegBankInfo.reset( - new AArch64RegisterBankInfo(*I->getRegisterInfo())); + GISel->Legalizer.reset(new AArch64LegalizerInfo()); + + auto *RBI = new AArch64RegisterBankInfo(*I->getRegisterInfo()); + + // FIXME: At this point, we can't rely on Subtarget having RBI. + // It's awkward to mix passing RBI and the Subtarget; should we pass + // TII/TRI as well? + GISel->InstSelector.reset(new AArch64InstructionSelector(*this, *I, *RBI)); + + GISel->RegBankInfo.reset(RBI); #endif I->setGISelAccessor(*GISel); } @@ -271,12 +300,23 @@ public: return getTM(); } + ScheduleDAGInstrs * + createMachineScheduler(MachineSchedContext *C) const override { + ScheduleDAGMILive *DAG = createGenericSchedLive(C); + DAG->addMutation(createLoadClusterDAGMutation(DAG->TII, DAG->TRI)); + DAG->addMutation(createStoreClusterDAGMutation(DAG->TII, DAG->TRI)); + DAG->addMutation(createMacroFusionDAGMutation(DAG->TII)); + return DAG; + } + void addIRPasses() override; bool addPreISel() override; bool addInstSelector() override; #ifdef LLVM_BUILD_GLOBAL_ISEL bool addIRTranslator() override; + bool addLegalizeMachineIR() override; bool addRegBankSelect() override; + bool addGlobalInstructionSelect() override; #endif bool addILPOpts() override; void addPreRegAlloc() override; @@ -351,7 +391,7 @@ bool AArch64PassConfig::addPreISel() { addPass(createGlobalMergePass(TM, 4095, OnlyOptimizeForSize)); } - if (TM->getOptLevel() != CodeGenOpt::None) + if (TM->getOptLevel() != CodeGenOpt::None && EnableAddressTypePromotion) addPass(createAArch64AddressTypePromotionPass()); return false; @@ -374,10 +414,18 @@ bool AArch64PassConfig::addIRTranslator() { addPass(new IRTranslator()); return false; } +bool AArch64PassConfig::addLegalizeMachineIR() { + addPass(new Legalizer()); + return false; +} bool AArch64PassConfig::addRegBankSelect() { addPass(new RegBankSelect()); return false; } +bool AArch64PassConfig::addGlobalInstructionSelect() { + addPass(new InstructionSelect()); + return false; +} #endif bool AArch64PassConfig::addILPOpts() { @@ -391,10 +439,15 @@ bool AArch64PassConfig::addILPOpts() { addPass(&EarlyIfConverterID); if (EnableStPairSuppress) addPass(createAArch64StorePairSuppressPass()); + addPass(createAArch64VectorByElementOptPass()); return true; } void AArch64PassConfig::addPreRegAlloc() { + // Change dead register definitions to refer to the zero register. + if (TM->getOptLevel() != CodeGenOpt::None && EnableDeadRegisterElimination) + addPass(createAArch64DeadRegisterDefinitions()); + // Use AdvSIMD scalar instructions whenever profitable. if (TM->getOptLevel() != CodeGenOpt::None && EnableAdvSIMDScalar) { addPass(createAArch64AdvSIMDScalar()); @@ -409,9 +462,6 @@ void AArch64PassConfig::addPostRegAlloc() { if (TM->getOptLevel() != CodeGenOpt::None && EnableRedundantCopyElimination) addPass(createAArch64RedundantCopyEliminationPass()); - // Change dead register definitions to refer to the zero register. - if (TM->getOptLevel() != CodeGenOpt::None && EnableDeadRegisterElimination) - addPass(createAArch64DeadRegisterDefinitions()); if (TM->getOptLevel() != CodeGenOpt::None && usingDefaultRegAlloc()) // Improve performance for some FP/SIMD code for A57. addPass(createAArch64A57FPLoadBalancing()); @@ -430,7 +480,9 @@ void AArch64PassConfig::addPreEmitPass() { addPass(createAArch64A53Fix835769()); // Relax conditional branch instructions if they're otherwise out of // range of their destination. - addPass(createAArch64BranchRelaxation()); + if (BranchRelaxation) + addPass(&BranchRelaxationPassID); + if (TM->getOptLevel() != CodeGenOpt::None && EnableCollectLOH && TM->getTargetTriple().isOSBinFormatMachO()) addPass(createAArch64CollectLOHPass()); diff --git a/lib/Target/AArch64/AArch64TargetMachine.h b/lib/Target/AArch64/AArch64TargetMachine.h index b44107b065bd..6fa5e83957e1 100644 --- a/lib/Target/AArch64/AArch64TargetMachine.h +++ b/lib/Target/AArch64/AArch64TargetMachine.h @@ -46,7 +46,7 @@ public: } private: - AArch64Subtarget Subtarget; + bool isLittle; }; // AArch64 little endian target machine. diff --git a/lib/Target/AArch64/AArch64TargetObjectFile.cpp b/lib/Target/AArch64/AArch64TargetObjectFile.cpp index 18ee4a9c72b5..8875f9b72647 100644 --- a/lib/Target/AArch64/AArch64TargetObjectFile.cpp +++ b/lib/Target/AArch64/AArch64TargetObjectFile.cpp @@ -30,15 +30,14 @@ AArch64_MachoTargetObjectFile::AArch64_MachoTargetObjectFile() } const MCExpr *AArch64_MachoTargetObjectFile::getTTypeGlobalReference( - const GlobalValue *GV, unsigned Encoding, Mangler &Mang, - const TargetMachine &TM, MachineModuleInfo *MMI, - MCStreamer &Streamer) const { + const GlobalValue *GV, unsigned Encoding, const TargetMachine &TM, + MachineModuleInfo *MMI, MCStreamer &Streamer) const { // On Darwin, we can reference dwarf symbols with foo@GOT-., which // is an indirect pc-relative reference. The default implementation // won't reference using the GOT, so we need this target-specific // version. if (Encoding & (DW_EH_PE_indirect | DW_EH_PE_pcrel)) { - const MCSymbol *Sym = TM.getSymbol(GV, Mang); + const MCSymbol *Sym = TM.getSymbol(GV); const MCExpr *Res = MCSymbolRefExpr::create(Sym, MCSymbolRefExpr::VK_GOT, getContext()); MCSymbol *PCSym = getContext().createTempSymbol(); @@ -48,13 +47,13 @@ const MCExpr *AArch64_MachoTargetObjectFile::getTTypeGlobalReference( } return TargetLoweringObjectFileMachO::getTTypeGlobalReference( - GV, Encoding, Mang, TM, MMI, Streamer); + GV, Encoding, TM, MMI, Streamer); } MCSymbol *AArch64_MachoTargetObjectFile::getCFIPersonalitySymbol( - const GlobalValue *GV, Mangler &Mang, const TargetMachine &TM, + const GlobalValue *GV, const TargetMachine &TM, MachineModuleInfo *MMI) const { - return TM.getSymbol(GV, Mang); + return TM.getSymbol(GV); } const MCExpr *AArch64_MachoTargetObjectFile::getIndirectSymViaGOTPCRel( diff --git a/lib/Target/AArch64/AArch64TargetObjectFile.h b/lib/Target/AArch64/AArch64TargetObjectFile.h index d41f445292cf..05e1dfa9e6c9 100644 --- a/lib/Target/AArch64/AArch64TargetObjectFile.h +++ b/lib/Target/AArch64/AArch64TargetObjectFile.h @@ -27,12 +27,12 @@ public: AArch64_MachoTargetObjectFile(); const MCExpr *getTTypeGlobalReference(const GlobalValue *GV, - unsigned Encoding, Mangler &Mang, + unsigned Encoding, const TargetMachine &TM, MachineModuleInfo *MMI, MCStreamer &Streamer) const override; - MCSymbol *getCFIPersonalitySymbol(const GlobalValue *GV, Mangler &Mang, + MCSymbol *getCFIPersonalitySymbol(const GlobalValue *GV, const TargetMachine &TM, MachineModuleInfo *MMI) const override; diff --git a/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index ecf4d93068a4..88c98865bbc6 100644 --- a/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -467,7 +467,8 @@ int AArch64TTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src, unsigned Alignment, unsigned AddressSpace) { std::pair LT = TLI->getTypeLegalizationCost(DL, Src); - if (Opcode == Instruction::Store && Src->isVectorTy() && Alignment != 16 && + if (ST->isMisaligned128StoreSlow() && Opcode == Instruction::Store && + Src->isVectorTy() && Alignment != 16 && Src->getVectorElementType()->isIntegerTy(64)) { // Unaligned stores are extremely inefficient. We don't split // unaligned v2i64 stores because the negative impact that has shown in diff --git a/lib/Target/AArch64/AArch64TargetTransformInfo.h b/lib/Target/AArch64/AArch64TargetTransformInfo.h index 4f2e8310d769..24642cb1698e 100644 --- a/lib/Target/AArch64/AArch64TargetTransformInfo.h +++ b/lib/Target/AArch64/AArch64TargetTransformInfo.h @@ -52,13 +52,6 @@ public: : BaseT(TM, F.getParent()->getDataLayout()), ST(TM->getSubtargetImpl(F)), TLI(ST->getTargetLowering()) {} - // Provide value semantics. MSVC requires that we spell all of these out. - AArch64TTIImpl(const AArch64TTIImpl &Arg) - : BaseT(static_cast(Arg)), ST(Arg.ST), TLI(Arg.TLI) {} - AArch64TTIImpl(AArch64TTIImpl &&Arg) - : BaseT(std::move(static_cast(Arg))), ST(std::move(Arg.ST)), - TLI(std::move(Arg.TLI)) {} - /// \name Scalar TTI Implementations /// @{ diff --git a/lib/Target/AArch64/AArch64VectorByElementOpt.cpp b/lib/Target/AArch64/AArch64VectorByElementOpt.cpp new file mode 100644 index 000000000000..e3b1d7cea48d --- /dev/null +++ b/lib/Target/AArch64/AArch64VectorByElementOpt.cpp @@ -0,0 +1,371 @@ +//=- AArch64VectorByElementOpt.cpp - AArch64 vector by element inst opt pass =// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains a pass that performs optimization for vector by element +// SIMD instructions. +// +// Certain SIMD instructions with vector element operand are not efficient. +// Rewrite them into SIMD instructions with vector operands. This rewrite +// is driven by the latency of the instructions. +// +// Example: +// fmla v0.4s, v1.4s, v2.s[1] +// is rewritten into +// dup v3.4s, v2.s[1] +// fmla v0.4s, v1.4s, v3.4s +//===----------------------------------------------------------------------===// + +#include "AArch64InstrInfo.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/TargetSchedule.h" + +using namespace llvm; + +#define DEBUG_TYPE "aarch64-vectorbyelement-opt" + +STATISTIC(NumModifiedInstr, + "Number of vector by element instructions modified"); + +#define AARCH64_VECTOR_BY_ELEMENT_OPT_NAME \ + "AArch64 vector by element instruction optimization pass" + +namespace { + +struct AArch64VectorByElementOpt : public MachineFunctionPass { + static char ID; + AArch64VectorByElementOpt() : MachineFunctionPass(ID) { + initializeAArch64VectorByElementOptPass(*PassRegistry::getPassRegistry()); + } + + const TargetInstrInfo *TII; + MachineRegisterInfo *MRI; + TargetSchedModel SchedModel; + + /// Based only on latency of instructions, determine if it is cost efficient + /// to replace the instruction InstDesc by the two instructions InstDescRep1 + /// and InstDescRep2. + /// Return true if replacement is recommended. + bool + shouldReplaceInstruction(MachineFunction *MF, const MCInstrDesc *InstDesc, + const MCInstrDesc *InstDescRep1, + const MCInstrDesc *InstDescRep2, + std::map &VecInstElemTable) const; + + /// Determine if we need to exit the vector by element instruction + /// optimization pass early. This makes sure that Targets with no need + /// for this optimization do not spent any compile time on this pass. + /// This check is done by comparing the latency of an indexed FMLA + /// instruction to the latency of the DUP + the latency of a vector + /// FMLA instruction. We do not check on other related instructions such + /// as FMLS as we assume that if the situation shows up for one + /// instruction, then it is likely to show up for the related ones. + /// Return true if early exit of the pass is recommended. + bool earlyExitVectElement(MachineFunction *MF); + + /// Check whether an equivalent DUP instruction has already been + /// created or not. + /// Return true when the dup instruction already exists. In this case, + /// DestReg will point to the destination of the already created DUP. + bool reuseDUP(MachineInstr &MI, unsigned DupOpcode, unsigned SrcReg, + unsigned LaneNumber, unsigned *DestReg) const; + + /// Certain SIMD instructions with vector element operand are not efficient. + /// Rewrite them into SIMD instructions with vector operands. This rewrite + /// is driven by the latency of the instructions. + /// Return true if the SIMD instruction is modified. + bool optimizeVectElement(MachineInstr &MI, + std::map *VecInstElemTable) const; + + bool runOnMachineFunction(MachineFunction &Fn) override; + + StringRef getPassName() const override { + return AARCH64_VECTOR_BY_ELEMENT_OPT_NAME; + } +}; +char AArch64VectorByElementOpt::ID = 0; +} // namespace + +INITIALIZE_PASS(AArch64VectorByElementOpt, "aarch64-vectorbyelement-opt", + AARCH64_VECTOR_BY_ELEMENT_OPT_NAME, false, false) + +/// Based only on latency of instructions, determine if it is cost efficient +/// to replace the instruction InstDesc by the two instructions InstDescRep1 +/// and InstDescRep2. Note that it is assumed in this fuction that an +/// instruction of type InstDesc is always replaced by the same two +/// instructions as results are cached here. +/// Return true if replacement is recommended. +bool AArch64VectorByElementOpt::shouldReplaceInstruction( + MachineFunction *MF, const MCInstrDesc *InstDesc, + const MCInstrDesc *InstDescRep1, const MCInstrDesc *InstDescRep2, + std::map &VecInstElemTable) const { + // Check if replacment decision is alredy available in the cached table. + // if so, return it. + if (!VecInstElemTable.empty() && + VecInstElemTable.find(InstDesc->getOpcode()) != VecInstElemTable.end()) + return VecInstElemTable[InstDesc->getOpcode()]; + + unsigned SCIdx = InstDesc->getSchedClass(); + unsigned SCIdxRep1 = InstDescRep1->getSchedClass(); + unsigned SCIdxRep2 = InstDescRep2->getSchedClass(); + const MCSchedClassDesc *SCDesc = + SchedModel.getMCSchedModel()->getSchedClassDesc(SCIdx); + const MCSchedClassDesc *SCDescRep1 = + SchedModel.getMCSchedModel()->getSchedClassDesc(SCIdxRep1); + const MCSchedClassDesc *SCDescRep2 = + SchedModel.getMCSchedModel()->getSchedClassDesc(SCIdxRep2); + + // If a subtarget does not define resources for any of the instructions + // of interest, then return false for no replacement. + if (!SCDesc->isValid() || SCDesc->isVariant() || !SCDescRep1->isValid() || + SCDescRep1->isVariant() || !SCDescRep2->isValid() || + SCDescRep2->isVariant()) { + VecInstElemTable[InstDesc->getOpcode()] = false; + return false; + } + + if (SchedModel.computeInstrLatency(InstDesc->getOpcode()) > + SchedModel.computeInstrLatency(InstDescRep1->getOpcode()) + + SchedModel.computeInstrLatency(InstDescRep2->getOpcode())) { + VecInstElemTable[InstDesc->getOpcode()] = true; + return true; + } + VecInstElemTable[InstDesc->getOpcode()] = false; + return false; +} + +/// Determine if we need to exit the vector by element instruction +/// optimization pass early. This makes sure that Targets with no need +/// for this optimization do not spent any compile time on this pass. +/// This check is done by comparing the latency of an indexed FMLA +/// instruction to the latency of the DUP + the latency of a vector +/// FMLA instruction. We do not check on other related instructions such +/// as FMLS as we assume that if the situation shows up for one +/// instruction, then it is likely to show up for the related ones. +/// Return true if early exit of the pass is recommended. +bool AArch64VectorByElementOpt::earlyExitVectElement(MachineFunction *MF) { + std::map VecInstElemTable; + const MCInstrDesc *IndexMulMCID = &TII->get(AArch64::FMLAv4i32_indexed); + const MCInstrDesc *DupMCID = &TII->get(AArch64::DUPv4i32lane); + const MCInstrDesc *MulMCID = &TII->get(AArch64::FMULv4f32); + + if (!shouldReplaceInstruction(MF, IndexMulMCID, DupMCID, MulMCID, + VecInstElemTable)) + return true; + return false; +} + +/// Check whether an equivalent DUP instruction has already been +/// created or not. +/// Return true when the dup instruction already exists. In this case, +/// DestReg will point to the destination of the already created DUP. +bool AArch64VectorByElementOpt::reuseDUP(MachineInstr &MI, unsigned DupOpcode, + unsigned SrcReg, unsigned LaneNumber, + unsigned *DestReg) const { + for (MachineBasicBlock::iterator MII = MI, MIE = MI.getParent()->begin(); + MII != MIE;) { + MII--; + MachineInstr *CurrentMI = &*MII; + + if (CurrentMI->getOpcode() == DupOpcode && + CurrentMI->getNumOperands() == 3 && + CurrentMI->getOperand(1).getReg() == SrcReg && + CurrentMI->getOperand(2).getImm() == LaneNumber) { + *DestReg = CurrentMI->getOperand(0).getReg(); + return true; + } + } + + return false; +} + +/// Certain SIMD instructions with vector element operand are not efficient. +/// Rewrite them into SIMD instructions with vector operands. This rewrite +/// is driven by the latency of the instructions. +/// The instruction of concerns are for the time being fmla, fmls, fmul, +/// and fmulx and hence they are hardcoded. +/// +/// Example: +/// fmla v0.4s, v1.4s, v2.s[1] +/// is rewritten into +/// dup v3.4s, v2.s[1] // dup not necessary if redundant +/// fmla v0.4s, v1.4s, v3.4s +/// Return true if the SIMD instruction is modified. +bool AArch64VectorByElementOpt::optimizeVectElement( + MachineInstr &MI, std::map *VecInstElemTable) const { + const MCInstrDesc *MulMCID, *DupMCID; + const TargetRegisterClass *RC = &AArch64::FPR128RegClass; + + switch (MI.getOpcode()) { + default: + return false; + + // 4X32 instructions + case AArch64::FMLAv4i32_indexed: + DupMCID = &TII->get(AArch64::DUPv4i32lane); + MulMCID = &TII->get(AArch64::FMLAv4f32); + break; + case AArch64::FMLSv4i32_indexed: + DupMCID = &TII->get(AArch64::DUPv4i32lane); + MulMCID = &TII->get(AArch64::FMLSv4f32); + break; + case AArch64::FMULXv4i32_indexed: + DupMCID = &TII->get(AArch64::DUPv4i32lane); + MulMCID = &TII->get(AArch64::FMULXv4f32); + break; + case AArch64::FMULv4i32_indexed: + DupMCID = &TII->get(AArch64::DUPv4i32lane); + MulMCID = &TII->get(AArch64::FMULv4f32); + break; + + // 2X64 instructions + case AArch64::FMLAv2i64_indexed: + DupMCID = &TII->get(AArch64::DUPv2i64lane); + MulMCID = &TII->get(AArch64::FMLAv2f64); + break; + case AArch64::FMLSv2i64_indexed: + DupMCID = &TII->get(AArch64::DUPv2i64lane); + MulMCID = &TII->get(AArch64::FMLSv2f64); + break; + case AArch64::FMULXv2i64_indexed: + DupMCID = &TII->get(AArch64::DUPv2i64lane); + MulMCID = &TII->get(AArch64::FMULXv2f64); + break; + case AArch64::FMULv2i64_indexed: + DupMCID = &TII->get(AArch64::DUPv2i64lane); + MulMCID = &TII->get(AArch64::FMULv2f64); + break; + + // 2X32 instructions + case AArch64::FMLAv2i32_indexed: + RC = &AArch64::FPR64RegClass; + DupMCID = &TII->get(AArch64::DUPv2i32lane); + MulMCID = &TII->get(AArch64::FMLAv2f32); + break; + case AArch64::FMLSv2i32_indexed: + RC = &AArch64::FPR64RegClass; + DupMCID = &TII->get(AArch64::DUPv2i32lane); + MulMCID = &TII->get(AArch64::FMLSv2f32); + break; + case AArch64::FMULXv2i32_indexed: + RC = &AArch64::FPR64RegClass; + DupMCID = &TII->get(AArch64::DUPv2i32lane); + MulMCID = &TII->get(AArch64::FMULXv2f32); + break; + case AArch64::FMULv2i32_indexed: + RC = &AArch64::FPR64RegClass; + DupMCID = &TII->get(AArch64::DUPv2i32lane); + MulMCID = &TII->get(AArch64::FMULv2f32); + break; + } + + if (!shouldReplaceInstruction(MI.getParent()->getParent(), + &TII->get(MI.getOpcode()), DupMCID, MulMCID, + *VecInstElemTable)) + return false; + + const DebugLoc &DL = MI.getDebugLoc(); + MachineBasicBlock &MBB = *MI.getParent(); + MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); + + // get the operands of the current SIMD arithmetic instruction. + unsigned MulDest = MI.getOperand(0).getReg(); + unsigned SrcReg0 = MI.getOperand(1).getReg(); + unsigned Src0IsKill = getKillRegState(MI.getOperand(1).isKill()); + unsigned SrcReg1 = MI.getOperand(2).getReg(); + unsigned Src1IsKill = getKillRegState(MI.getOperand(2).isKill()); + unsigned DupDest; + + // Instructions of interest have either 4 or 5 operands. + if (MI.getNumOperands() == 5) { + unsigned SrcReg2 = MI.getOperand(3).getReg(); + unsigned Src2IsKill = getKillRegState(MI.getOperand(3).isKill()); + unsigned LaneNumber = MI.getOperand(4).getImm(); + + // Create a new DUP instruction. Note that if an equivalent DUP instruction + // has already been created before, then use that one instread of creating + // a new one. + if (!reuseDUP(MI, DupMCID->getOpcode(), SrcReg2, LaneNumber, &DupDest)) { + DupDest = MRI.createVirtualRegister(RC); + BuildMI(MBB, MI, DL, *DupMCID, DupDest) + .addReg(SrcReg2, Src2IsKill) + .addImm(LaneNumber); + } + BuildMI(MBB, MI, DL, *MulMCID, MulDest) + .addReg(SrcReg0, Src0IsKill) + .addReg(SrcReg1, Src1IsKill) + .addReg(DupDest, Src2IsKill); + } else if (MI.getNumOperands() == 4) { + unsigned LaneNumber = MI.getOperand(3).getImm(); + if (!reuseDUP(MI, DupMCID->getOpcode(), SrcReg1, LaneNumber, &DupDest)) { + DupDest = MRI.createVirtualRegister(RC); + BuildMI(MBB, MI, DL, *DupMCID, DupDest) + .addReg(SrcReg1, Src1IsKill) + .addImm(LaneNumber); + } + BuildMI(MBB, MI, DL, *MulMCID, MulDest) + .addReg(SrcReg0, Src0IsKill) + .addReg(DupDest, Src1IsKill); + } else { + return false; + } + + ++NumModifiedInstr; + return true; +} + +bool AArch64VectorByElementOpt::runOnMachineFunction(MachineFunction &MF) { + if (skipFunction(*MF.getFunction())) + return false; + + TII = MF.getSubtarget().getInstrInfo(); + MRI = &MF.getRegInfo(); + const TargetSubtargetInfo &ST = MF.getSubtarget(); + const AArch64InstrInfo *AAII = + static_cast(ST.getInstrInfo()); + if (!AAII) + return false; + SchedModel.init(ST.getSchedModel(), &ST, AAII); + if (!SchedModel.hasInstrSchedModel()) + return false; + + // A simple check to exit this pass early for targets that do not need it. + if (earlyExitVectElement(&MF)) + return false; + + bool Changed = false; + std::map VecInstElemTable; + SmallVector RemoveMIs; + + for (MachineBasicBlock &MBB : MF) { + for (MachineBasicBlock::iterator MII = MBB.begin(), MIE = MBB.end(); + MII != MIE;) { + MachineInstr &MI = *MII; + if (optimizeVectElement(MI, &VecInstElemTable)) { + // Add MI to the list of instructions to be removed given that it has + // been replaced. + RemoveMIs.push_back(&MI); + Changed = true; + } + ++MII; + } + } + + for (MachineInstr *MI : RemoveMIs) + MI->eraseFromParent(); + + return Changed; +} + +/// createAArch64VectorByElementOptPass - returns an instance of the +/// vector by element optimization pass. +FunctionPass *llvm::createAArch64VectorByElementOptPass() { + return new AArch64VectorByElementOpt(); +} diff --git a/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp b/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp index aebc370333e3..db84afacf30e 100644 --- a/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp +++ b/lib/Target/AArch64/AsmParser/AArch64AsmParser.cpp @@ -14,6 +14,7 @@ #include "llvm/ADT/APInt.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringExtras.h" #include "llvm/ADT/StringSwitch.h" #include "llvm/ADT/Twine.h" #include "llvm/MC/MCContext.h" @@ -28,6 +29,7 @@ #include "llvm/MC/MCStreamer.h" #include "llvm/MC/MCSubtargetInfo.h" #include "llvm/MC/MCSymbol.h" +#include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/SourceMgr.h" #include "llvm/Support/TargetParser.h" @@ -66,8 +68,6 @@ private: bool parseOperand(OperandVector &Operands, bool isCondCode, bool invertCondCode); - void Warning(SMLoc L, const Twine &Msg) { getParser().Warning(L, Msg); } - bool Error(SMLoc L, const Twine &Msg) { return getParser().Error(L, Msg); } bool showMatchError(SMLoc Loc, unsigned ErrCode); bool parseDirectiveArch(SMLoc L); @@ -117,9 +117,11 @@ public: #define GET_OPERAND_DIAGNOSTIC_TYPES #include "AArch64GenAsmMatcher.inc" }; + bool IsILP32; AArch64AsmParser(const MCSubtargetInfo &STI, MCAsmParser &Parser, const MCInstrInfo &MII, const MCTargetOptions &Options) : MCTargetAsmParser(Options, STI) { + IsILP32 = Options.getABIName() == "ilp32"; MCAsmParserExtension::Initialize(Parser); MCStreamer &S = getParser().getStreamer(); if (S.getTargetStreamer() == nullptr) @@ -208,9 +210,9 @@ private: }; struct BarrierOp { - unsigned Val; // Not the enum since not all values have names. const char *Data; unsigned Length; + unsigned Val; // Not the enum since not all values have names. }; struct SysRegOp { @@ -226,15 +228,15 @@ private: }; struct PrefetchOp { - unsigned Val; const char *Data; unsigned Length; + unsigned Val; }; struct PSBHintOp { - unsigned Val; const char *Data; unsigned Length; + unsigned Val; }; struct ShiftExtendOp { @@ -727,9 +729,13 @@ public: || ELFRefKind == AArch64MCExpr::VK_TLSDESC_LO12; } - // Otherwise it should be a real immediate in range: - const MCConstantExpr *CE = cast(Expr); - return CE->getValue() >= 0 && CE->getValue() <= 0xfff; + // If it's a constant, it should be a real immediate in range: + if (auto *CE = dyn_cast(Expr)) + return CE->getValue() >= 0 && CE->getValue() <= 0xfff; + + // If it's an expression, we hope for the best and let the fixup/relocation + // code deal with it. + return true; } bool isAddSubImmNeg() const { if (!isShiftedImm() && !isImm()) @@ -1964,7 +1970,8 @@ unsigned AArch64AsmParser::matchRegisterNameAlias(StringRef Name, int AArch64AsmParser::tryParseRegister() { MCAsmParser &Parser = getParser(); const AsmToken &Tok = Parser.getTok(); - assert(Tok.is(AsmToken::Identifier) && "Token is not an Identifier"); + if (Tok.isNot(AsmToken::Identifier)) + return -1; std::string lowerCase = Tok.getString().lower(); unsigned RegNum = matchRegisterNameAlias(lowerCase, false); @@ -2018,7 +2025,7 @@ int AArch64AsmParser::tryMatchVectorRegister(StringRef &Kind, bool expected) { } /// tryParseSysCROperand - Try to parse a system instruction CR operand name. -AArch64AsmParser::OperandMatchResultTy +OperandMatchResultTy AArch64AsmParser::tryParseSysCROperand(OperandVector &Operands) { MCAsmParser &Parser = getParser(); SMLoc S = getLoc(); @@ -2048,16 +2055,15 @@ AArch64AsmParser::tryParseSysCROperand(OperandVector &Operands) { } /// tryParsePrefetch - Try to parse a prefetch operand. -AArch64AsmParser::OperandMatchResultTy +OperandMatchResultTy AArch64AsmParser::tryParsePrefetch(OperandVector &Operands) { MCAsmParser &Parser = getParser(); SMLoc S = getLoc(); const AsmToken &Tok = Parser.getTok(); // Either an identifier for named values or a 5-bit immediate. - bool Hash = Tok.is(AsmToken::Hash); - if (Hash || Tok.is(AsmToken::Integer)) { - if (Hash) - Parser.Lex(); // Eat hash token. + // Eat optional hash. + if (parseOptionalToken(AsmToken::Hash) || + Tok.is(AsmToken::Integer)) { const MCExpr *ImmVal; if (getParser().parseExpression(ImmVal)) return MatchOperand_ParseFail; @@ -2097,7 +2103,7 @@ AArch64AsmParser::tryParsePrefetch(OperandVector &Operands) { } /// tryParsePSBHint - Try to parse a PSB operand, mapped to Hint command -AArch64AsmParser::OperandMatchResultTy +OperandMatchResultTy AArch64AsmParser::tryParsePSBHint(OperandVector &Operands) { MCAsmParser &Parser = getParser(); SMLoc S = getLoc(); @@ -2121,7 +2127,7 @@ AArch64AsmParser::tryParsePSBHint(OperandVector &Operands) { /// tryParseAdrpLabel - Parse and validate a source label for the ADRP /// instruction. -AArch64AsmParser::OperandMatchResultTy +OperandMatchResultTy AArch64AsmParser::tryParseAdrpLabel(OperandVector &Operands) { MCAsmParser &Parser = getParser(); SMLoc S = getLoc(); @@ -2172,16 +2178,12 @@ AArch64AsmParser::tryParseAdrpLabel(OperandVector &Operands) { /// tryParseAdrLabel - Parse and validate a source label for the ADR /// instruction. -AArch64AsmParser::OperandMatchResultTy +OperandMatchResultTy AArch64AsmParser::tryParseAdrLabel(OperandVector &Operands) { - MCAsmParser &Parser = getParser(); SMLoc S = getLoc(); const MCExpr *Expr; - if (Parser.getTok().is(AsmToken::Hash)) { - Parser.Lex(); // Eat hash token. - } - + parseOptionalToken(AsmToken::Hash); if (getParser().parseExpression(Expr)) return MatchOperand_ParseFail; @@ -2192,26 +2194,19 @@ AArch64AsmParser::tryParseAdrLabel(OperandVector &Operands) { } /// tryParseFPImm - A floating point immediate expression operand. -AArch64AsmParser::OperandMatchResultTy +OperandMatchResultTy AArch64AsmParser::tryParseFPImm(OperandVector &Operands) { MCAsmParser &Parser = getParser(); SMLoc S = getLoc(); - bool Hash = false; - if (Parser.getTok().is(AsmToken::Hash)) { - Parser.Lex(); // Eat '#' - Hash = true; - } + bool Hash = parseOptionalToken(AsmToken::Hash); // Handle negation, as that still comes through as a separate token. - bool isNegative = false; - if (Parser.getTok().is(AsmToken::Minus)) { - isNegative = true; - Parser.Lex(); - } + bool isNegative = parseOptionalToken(AsmToken::Minus); + const AsmToken &Tok = Parser.getTok(); if (Tok.is(AsmToken::Real)) { - APFloat RealVal(APFloat::IEEEdouble, Tok.getString()); + APFloat RealVal(APFloat::IEEEdouble(), Tok.getString()); if (isNegative) RealVal.changeSign(); @@ -2237,7 +2232,7 @@ AArch64AsmParser::tryParseFPImm(OperandVector &Operands) { return MatchOperand_ParseFail; } } else { - APFloat RealVal(APFloat::IEEEdouble, Tok.getString()); + APFloat RealVal(APFloat::IEEEdouble(), Tok.getString()); uint64_t IntVal = RealVal.bitcastToAPInt().getZExtValue(); // If we had a '-' in front, toggle the sign bit. IntVal ^= (uint64_t)isNegative << 63; @@ -2256,7 +2251,7 @@ AArch64AsmParser::tryParseFPImm(OperandVector &Operands) { } /// tryParseAddSubImm - Parse ADD/SUB shifted immediate operand -AArch64AsmParser::OperandMatchResultTy +OperandMatchResultTy AArch64AsmParser::tryParseAddSubImm(OperandVector &Operands) { MCAsmParser &Parser = getParser(); SMLoc S = getLoc(); @@ -2299,9 +2294,7 @@ AArch64AsmParser::tryParseAddSubImm(OperandVector &Operands) { // Eat 'lsl' Parser.Lex(); - if (Parser.getTok().is(AsmToken::Hash)) { - Parser.Lex(); - } + parseOptionalToken(AsmToken::Hash); if (Parser.getTok().isNot(AsmToken::Integer)) { Error(Parser.getTok().getLoc(), "only 'lsl #+N' valid after immediate"); @@ -2374,7 +2367,7 @@ bool AArch64AsmParser::parseCondCode(OperandVector &Operands, /// tryParseOptionalShift - Some operands take an optional shift argument. Parse /// them if present. -AArch64AsmParser::OperandMatchResultTy +OperandMatchResultTy AArch64AsmParser::tryParseOptionalShiftExtend(OperandVector &Operands) { MCAsmParser &Parser = getParser(); const AsmToken &Tok = Parser.getTok(); @@ -2402,7 +2395,8 @@ AArch64AsmParser::tryParseOptionalShiftExtend(OperandVector &Operands) { SMLoc S = Tok.getLoc(); Parser.Lex(); - bool Hash = getLexer().is(AsmToken::Hash); + bool Hash = parseOptionalToken(AsmToken::Hash); + if (!Hash && getLexer().isNot(AsmToken::Integer)) { if (ShOp == AArch64_AM::LSL || ShOp == AArch64_AM::LSR || ShOp == AArch64_AM::ASR || ShOp == AArch64_AM::ROR || @@ -2412,20 +2406,19 @@ AArch64AsmParser::tryParseOptionalShiftExtend(OperandVector &Operands) { return MatchOperand_ParseFail; } - // "extend" type operatoins don't need an immediate, #0 is implicit. + // "extend" type operations don't need an immediate, #0 is implicit. SMLoc E = SMLoc::getFromPointer(getLoc().getPointer() - 1); Operands.push_back( AArch64Operand::CreateShiftExtend(ShOp, 0, false, S, E, getContext())); return MatchOperand_Success; } - if (Hash) - Parser.Lex(); // Eat the '#'. - - // Make sure we do actually have a number or a parenthesized expression. + // Make sure we do actually have a number, identifier or a parenthesized + // expression. SMLoc E = Parser.getTok().getLoc(); if (!Parser.getTok().is(AsmToken::Integer) && - !Parser.getTok().is(AsmToken::LParen)) { + !Parser.getTok().is(AsmToken::LParen) && + !Parser.getTok().is(AsmToken::Identifier)) { Error(E, "expected integer shift amount"); return MatchOperand_ParseFail; } @@ -2690,20 +2683,12 @@ bool AArch64AsmParser::parseSysAlias(StringRef Name, SMLoc NameLoc, bool HasRegister = false; // Check for the optional register operand. - if (getLexer().is(AsmToken::Comma)) { - Parser.Lex(); // Eat comma. - + if (parseOptionalToken(AsmToken::Comma)) { if (Tok.isNot(AsmToken::Identifier) || parseRegister(Operands)) return TokError("expected register operand"); - HasRegister = true; } - if (getLexer().isNot(AsmToken::EndOfStatement)) { - Parser.eatToEndOfStatement(); - return TokError("unexpected token in argument list"); - } - if (ExpectRegister && !HasRegister) { return TokError("specified " + Mnemonic + " op requires a register"); } @@ -2711,21 +2696,21 @@ bool AArch64AsmParser::parseSysAlias(StringRef Name, SMLoc NameLoc, return TokError("specified " + Mnemonic + " op does not use a register"); } - Parser.Lex(); // Consume the EndOfStatement + if (parseToken(AsmToken::EndOfStatement, "unexpected token in argument list")) + return true; + return false; } -AArch64AsmParser::OperandMatchResultTy +OperandMatchResultTy AArch64AsmParser::tryParseBarrierOperand(OperandVector &Operands) { MCAsmParser &Parser = getParser(); const AsmToken &Tok = Parser.getTok(); // Can be either a #imm style literal or an option name - bool Hash = Tok.is(AsmToken::Hash); - if (Hash || Tok.is(AsmToken::Integer)) { + if (parseOptionalToken(AsmToken::Hash) || + Tok.is(AsmToken::Integer)) { // Immediate operand. - if (Hash) - Parser.Lex(); // Eat the '#' const MCExpr *ImmVal; SMLoc ExprLoc = getLoc(); if (getParser().parseExpression(ImmVal)) @@ -2769,7 +2754,7 @@ AArch64AsmParser::tryParseBarrierOperand(OperandVector &Operands) { return MatchOperand_Success; } -AArch64AsmParser::OperandMatchResultTy +OperandMatchResultTy AArch64AsmParser::tryParseSysReg(OperandVector &Operands) { MCAsmParser &Parser = getParser(); const AsmToken &Tok = Parser.getTok(); @@ -2819,10 +2804,8 @@ bool AArch64AsmParser::tryParseVectorRegister(OperandVector &Operands) { AArch64Operand::CreateToken(Kind, false, S, getContext())); // If there is an index specifier following the register, parse that too. - if (Parser.getTok().is(AsmToken::LBrac)) { - SMLoc SIdx = getLoc(); - Parser.Lex(); // Eat left bracket token. - + SMLoc SIdx = getLoc(); + if (parseOptionalToken(AsmToken::LBrac)) { const MCExpr *ImmVal; if (getParser().parseExpression(ImmVal)) return false; @@ -2833,12 +2816,9 @@ bool AArch64AsmParser::tryParseVectorRegister(OperandVector &Operands) { } SMLoc E = getLoc(); - if (Parser.getTok().isNot(AsmToken::RBrac)) { - Error(E, "']' expected"); - return false; - } - Parser.Lex(); // Eat right bracket token. + if (parseToken(AsmToken::RBrac, "']' expected")) + return false; Operands.push_back(AArch64Operand::CreateVectorIndex(MCE->getValue(), SIdx, E, getContext())); @@ -2864,18 +2844,16 @@ bool AArch64AsmParser::parseRegister(OperandVector &Operands) { // A small number of instructions (FMOVXDhighr, for example) have "[1]" // as a string token in the instruction itself. - if (getLexer().getKind() == AsmToken::LBrac) { - SMLoc LBracS = getLoc(); - Parser.Lex(); - const AsmToken &Tok = Parser.getTok(); + SMLoc LBracS = getLoc(); + const AsmToken &Tok = Parser.getTok(); + if (parseOptionalToken(AsmToken::LBrac)) { if (Tok.is(AsmToken::Integer)) { SMLoc IntS = getLoc(); int64_t Val = Tok.getIntVal(); if (Val == 1) { Parser.Lex(); - if (getLexer().getKind() == AsmToken::RBrac) { - SMLoc RBracS = getLoc(); - Parser.Lex(); + SMLoc RBracS = getLoc(); + if (parseOptionalToken(AsmToken::RBrac)) { Operands.push_back( AArch64Operand::CreateToken("[", false, LBracS, getContext())); Operands.push_back( @@ -2896,15 +2874,11 @@ bool AArch64AsmParser::parseSymbolicImmVal(const MCExpr *&ImmVal) { bool HasELFModifier = false; AArch64MCExpr::VariantKind RefKind; - if (Parser.getTok().is(AsmToken::Colon)) { - Parser.Lex(); // Eat ':" + if (parseOptionalToken(AsmToken::Colon)) { HasELFModifier = true; - if (Parser.getTok().isNot(AsmToken::Identifier)) { - Error(Parser.getTok().getLoc(), - "expect relocation specifier in operand after ':'"); - return true; - } + if (Parser.getTok().isNot(AsmToken::Identifier)) + return TokError("expect relocation specifier in operand after ':'"); std::string LowerCase = Parser.getTok().getIdentifier().lower(); RefKind = StringSwitch(LowerCase) @@ -2945,19 +2919,13 @@ bool AArch64AsmParser::parseSymbolicImmVal(const MCExpr *&ImmVal) { .Case("tlsdesc", AArch64MCExpr::VK_TLSDESC_PAGE) .Default(AArch64MCExpr::VK_INVALID); - if (RefKind == AArch64MCExpr::VK_INVALID) { - Error(Parser.getTok().getLoc(), - "expect relocation specifier in operand after ':'"); - return true; - } + if (RefKind == AArch64MCExpr::VK_INVALID) + return TokError("expect relocation specifier in operand after ':'"); Parser.Lex(); // Eat identifier - if (Parser.getTok().isNot(AsmToken::Colon)) { - Error(Parser.getTok().getLoc(), "expect ':' after relocation specifier"); + if (parseToken(AsmToken::Colon, "expect ':' after relocation specifier")) return true; - } - Parser.Lex(); // Eat ':' } if (getParser().parseExpression(ImmVal)) @@ -2982,9 +2950,7 @@ bool AArch64AsmParser::parseVectorList(OperandVector &Operands) { int64_t PrevReg = FirstReg; unsigned Count = 1; - if (Parser.getTok().is(AsmToken::Minus)) { - Parser.Lex(); // Eat the minus. - + if (parseOptionalToken(AsmToken::Minus)) { SMLoc Loc = getLoc(); StringRef NextKind; int64_t Reg = tryMatchVectorRegister(NextKind, true); @@ -3003,9 +2969,7 @@ bool AArch64AsmParser::parseVectorList(OperandVector &Operands) { Count += Space; } else { - while (Parser.getTok().is(AsmToken::Comma)) { - Parser.Lex(); // Eat the comma token. - + while (parseOptionalToken(AsmToken::Comma)) { SMLoc Loc = getLoc(); StringRef NextKind; int64_t Reg = tryMatchVectorRegister(NextKind, true); @@ -3025,9 +2989,8 @@ bool AArch64AsmParser::parseVectorList(OperandVector &Operands) { } } - if (Parser.getTok().isNot(AsmToken::RCurly)) - return Error(getLoc(), "'}' expected"); - Parser.Lex(); // Eat the '}' token. + if (parseToken(AsmToken::RCurly, "'}' expected")) + return true; if (Count > 4) return Error(S, "invalid number of vectors"); @@ -3041,10 +3004,8 @@ bool AArch64AsmParser::parseVectorList(OperandVector &Operands) { FirstReg, Count, NumElements, ElementKind, S, getLoc(), getContext())); // If there is an index specifier following the list, parse that too. - if (Parser.getTok().is(AsmToken::LBrac)) { - SMLoc SIdx = getLoc(); - Parser.Lex(); // Eat left bracket token. - + SMLoc SIdx = getLoc(); + if (parseOptionalToken(AsmToken::LBrac)) { // Eat left bracket token. const MCExpr *ImmVal; if (getParser().parseExpression(ImmVal)) return false; @@ -3055,12 +3016,8 @@ bool AArch64AsmParser::parseVectorList(OperandVector &Operands) { } SMLoc E = getLoc(); - if (Parser.getTok().isNot(AsmToken::RBrac)) { - Error(E, "']' expected"); + if (parseToken(AsmToken::RBrac, "']' expected")) return false; - } - - Parser.Lex(); // Eat right bracket token. Operands.push_back(AArch64Operand::CreateVectorIndex(MCE->getValue(), SIdx, E, getContext())); @@ -3068,7 +3025,7 @@ bool AArch64AsmParser::parseVectorList(OperandVector &Operands) { return false; } -AArch64AsmParser::OperandMatchResultTy +OperandMatchResultTy AArch64AsmParser::tryParseGPR64sp0Operand(OperandVector &Operands) { MCAsmParser &Parser = getParser(); const AsmToken &Tok = Parser.getTok(); @@ -3085,15 +3042,13 @@ AArch64AsmParser::tryParseGPR64sp0Operand(OperandVector &Operands) { SMLoc S = getLoc(); Parser.Lex(); // Eat register - if (Parser.getTok().isNot(AsmToken::Comma)) { + if (!parseOptionalToken(AsmToken::Comma)) { Operands.push_back( AArch64Operand::CreateReg(RegNum, false, S, getLoc(), Ctx)); return MatchOperand_Success; } - Parser.Lex(); // Eat comma. - if (Parser.getTok().is(AsmToken::Hash)) - Parser.Lex(); // Eat hash + parseOptionalToken(AsmToken::Hash); if (Parser.getTok().isNot(AsmToken::Integer)) { Error(getLoc(), "index must be absent or #0"); @@ -3174,7 +3129,6 @@ bool AArch64AsmParser::parseOperand(OperandVector &Operands, bool isCondCode, S = getLoc(); if (getParser().parseExpression(IdVal)) return true; - E = SMLoc::getFromPointer(getLoc().getPointer() - 1); Operands.push_back(AArch64Operand::CreateImm(IdVal, S, E, getContext())); return false; @@ -3184,8 +3138,8 @@ bool AArch64AsmParser::parseOperand(OperandVector &Operands, bool isCondCode, case AsmToken::Hash: { // #42 -> immediate. S = getLoc(); - if (getLexer().is(AsmToken::Hash)) - Parser.Lex(); + + parseOptionalToken(AsmToken::Hash); // Parse a negative sign bool isNegative = false; @@ -3202,7 +3156,7 @@ bool AArch64AsmParser::parseOperand(OperandVector &Operands, bool isCondCode, // so convert the value. const AsmToken &Tok = Parser.getTok(); if (Tok.is(AsmToken::Real)) { - APFloat RealVal(APFloat::IEEEdouble, Tok.getString()); + APFloat RealVal(APFloat::IEEEdouble(), Tok.getString()); uint64_t IntVal = RealVal.bitcastToAPInt().getZExtValue(); if (Mnemonic != "fcmp" && Mnemonic != "fcmpe" && Mnemonic != "fcmeq" && Mnemonic != "fcmge" && Mnemonic != "fcmgt" && Mnemonic != "fcmle" && @@ -3228,9 +3182,9 @@ bool AArch64AsmParser::parseOperand(OperandVector &Operands, bool isCondCode, return false; } case AsmToken::Equal: { - SMLoc Loc = Parser.getTok().getLoc(); + SMLoc Loc = getLoc(); if (Mnemonic != "ldr") // only parse for ldr pseudo (e.g. ldr r0, =val) - return Error(Loc, "unexpected token in operand"); + return TokError("unexpected token in operand"); Parser.Lex(); // Eat '=' const MCExpr *SubExprVal; if (getParser().parseExpression(SubExprVal)) @@ -3318,12 +3272,8 @@ bool AArch64AsmParser::ParseInstruction(ParseInstructionInfo &Info, StringRef Head = Name.slice(Start, Next); // IC, DC, AT, and TLBI instructions are aliases for the SYS instruction. - if (Head == "ic" || Head == "dc" || Head == "at" || Head == "tlbi") { - bool IsError = parseSysAlias(Head, NameLoc, Operands); - if (IsError && getLexer().isNot(AsmToken::EndOfStatement)) - Parser.eatToEndOfStatement(); - return IsError; - } + if (Head == "ic" || Head == "dc" || Head == "at" || Head == "tlbi") + return parseSysAlias(Head, NameLoc, Operands); Operands.push_back( AArch64Operand::CreateToken(Head, false, NameLoc, getContext())); @@ -3378,20 +3328,16 @@ bool AArch64AsmParser::ParseInstruction(ParseInstructionInfo &Info, if (getLexer().isNot(AsmToken::EndOfStatement)) { // Read the first operand. if (parseOperand(Operands, false, false)) { - Parser.eatToEndOfStatement(); return true; } unsigned N = 2; - while (getLexer().is(AsmToken::Comma)) { - Parser.Lex(); // Eat the comma. - + while (parseOptionalToken(AsmToken::Comma)) { // Parse and remember the operand. if (parseOperand(Operands, (N == 4 && condCodeFourthOperand) || (N == 3 && condCodeThirdOperand) || (N == 2 && condCodeSecondOperand), condCodeSecondOperand || condCodeThirdOperand)) { - Parser.eatToEndOfStatement(); return true; } @@ -3403,31 +3349,23 @@ bool AArch64AsmParser::ParseInstruction(ParseInstructionInfo &Info, // // It's someone else's responsibility to make sure these tokens are sane // in the given context! - if (Parser.getTok().is(AsmToken::RBrac)) { - SMLoc Loc = Parser.getTok().getLoc(); - Operands.push_back(AArch64Operand::CreateToken("]", false, Loc, - getContext())); - Parser.Lex(); - } - if (Parser.getTok().is(AsmToken::Exclaim)) { - SMLoc Loc = Parser.getTok().getLoc(); - Operands.push_back(AArch64Operand::CreateToken("!", false, Loc, - getContext())); - Parser.Lex(); - } + SMLoc RLoc = Parser.getTok().getLoc(); + if (parseOptionalToken(AsmToken::RBrac)) + Operands.push_back( + AArch64Operand::CreateToken("]", false, RLoc, getContext())); + SMLoc ELoc = Parser.getTok().getLoc(); + if (parseOptionalToken(AsmToken::Exclaim)) + Operands.push_back( + AArch64Operand::CreateToken("!", false, ELoc, getContext())); ++N; } } - if (getLexer().isNot(AsmToken::EndOfStatement)) { - SMLoc Loc = Parser.getTok().getLoc(); - Parser.eatToEndOfStatement(); - return Error(Loc, "unexpected token in argument list"); - } + if (parseToken(AsmToken::EndOfStatement, "unexpected token in argument list")) + return true; - Parser.Lex(); // Consume the EndOfStatement return false; } @@ -3455,7 +3393,7 @@ bool AArch64AsmParser::validateInstruction(MCInst &Inst, if (RI->isSubRegisterEq(Rn, Rt2)) return Error(Loc[1], "unpredictable LDP instruction, writeback base " "is also a destination"); - // FALLTHROUGH + LLVM_FALLTHROUGH; } case AArch64::LDPDi: case AArch64::LDPQi: @@ -3572,31 +3510,34 @@ bool AArch64AsmParser::validateInstruction(MCInst &Inst, AArch64MCExpr::VariantKind ELFRefKind; MCSymbolRefExpr::VariantKind DarwinRefKind; int64_t Addend; - if (!classifySymbolRef(Expr, ELFRefKind, DarwinRefKind, Addend)) { - return Error(Loc[2], "invalid immediate expression"); - } + if (classifySymbolRef(Expr, ELFRefKind, DarwinRefKind, Addend)) { - // Only allow these with ADDXri. - if ((DarwinRefKind == MCSymbolRefExpr::VK_PAGEOFF || - DarwinRefKind == MCSymbolRefExpr::VK_TLVPPAGEOFF) && - Inst.getOpcode() == AArch64::ADDXri) - return false; + // Only allow these with ADDXri. + if ((DarwinRefKind == MCSymbolRefExpr::VK_PAGEOFF || + DarwinRefKind == MCSymbolRefExpr::VK_TLVPPAGEOFF) && + Inst.getOpcode() == AArch64::ADDXri) + return false; - // Only allow these with ADDXri/ADDWri - if ((ELFRefKind == AArch64MCExpr::VK_LO12 || - ELFRefKind == AArch64MCExpr::VK_DTPREL_HI12 || - ELFRefKind == AArch64MCExpr::VK_DTPREL_LO12 || - ELFRefKind == AArch64MCExpr::VK_DTPREL_LO12_NC || - ELFRefKind == AArch64MCExpr::VK_TPREL_HI12 || - ELFRefKind == AArch64MCExpr::VK_TPREL_LO12 || - ELFRefKind == AArch64MCExpr::VK_TPREL_LO12_NC || - ELFRefKind == AArch64MCExpr::VK_TLSDESC_LO12) && - (Inst.getOpcode() == AArch64::ADDXri || - Inst.getOpcode() == AArch64::ADDWri)) - return false; + // Only allow these with ADDXri/ADDWri + if ((ELFRefKind == AArch64MCExpr::VK_LO12 || + ELFRefKind == AArch64MCExpr::VK_DTPREL_HI12 || + ELFRefKind == AArch64MCExpr::VK_DTPREL_LO12 || + ELFRefKind == AArch64MCExpr::VK_DTPREL_LO12_NC || + ELFRefKind == AArch64MCExpr::VK_TPREL_HI12 || + ELFRefKind == AArch64MCExpr::VK_TPREL_LO12 || + ELFRefKind == AArch64MCExpr::VK_TPREL_LO12_NC || + ELFRefKind == AArch64MCExpr::VK_TLSDESC_LO12) && + (Inst.getOpcode() == AArch64::ADDXri || + Inst.getOpcode() == AArch64::ADDWri)) + return false; - // Don't allow expressions in the immediate field otherwise - return Error(Loc[2], "invalid immediate expression"); + // Don't allow symbol refs in the immediate field otherwise + // Note: Loc.back() may be Loc[1] or Loc[2] depending on the number of + // operands of the original instruction (i.e. 'add w0, w1, borked' vs + // 'cmp w0, 'borked') + return Error(Loc.back(), "invalid immediate expression"); + } + // We don't validate more complex expressions here } return false; } @@ -4075,7 +4016,8 @@ bool AArch64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode, if (ErrorInfo != ~0ULL) { if (ErrorInfo >= Operands.size()) - return Error(IDLoc, "too few operands for instruction"); + return Error(IDLoc, "too few operands for instruction", + SMRange(IDLoc, getTok().getLoc())); ErrorLoc = ((AArch64Operand &)*Operands[ErrorInfo]).getStartLoc(); if (ErrorLoc == SMLoc()) @@ -4138,7 +4080,7 @@ bool AArch64AsmParser::MatchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode, case Match_MSR: case Match_MRS: { if (ErrorInfo >= Operands.size()) - return Error(IDLoc, "too few operands for instruction"); + return Error(IDLoc, "too few operands for instruction", SMRange(IDLoc, (*Operands.back()).getEndLoc())); // Any time we get here, there's nothing fancy to do. Just get the // operand SMLoc and display the diagnostic. SMLoc ErrorLoc = ((AArch64Operand &)*Operands[ErrorInfo]).getStartLoc(); @@ -4161,28 +4103,31 @@ bool AArch64AsmParser::ParseDirective(AsmToken DirectiveID) { StringRef IDVal = DirectiveID.getIdentifier(); SMLoc Loc = DirectiveID.getLoc(); if (IDVal == ".arch") - return parseDirectiveArch(Loc); - if (IDVal == ".cpu") - return parseDirectiveCPU(Loc); - if (IDVal == ".hword") - return parseDirectiveWord(2, Loc); - if (IDVal == ".word") - return parseDirectiveWord(4, Loc); - if (IDVal == ".xword") - return parseDirectiveWord(8, Loc); - if (IDVal == ".tlsdesccall") - return parseDirectiveTLSDescCall(Loc); - if (IDVal == ".ltorg" || IDVal == ".pool") - return parseDirectiveLtorg(Loc); - if (IDVal == ".unreq") - return parseDirectiveUnreq(Loc); - - if (!IsMachO && !IsCOFF) { + parseDirectiveArch(Loc); + else if (IDVal == ".cpu") + parseDirectiveCPU(Loc); + else if (IDVal == ".hword") + parseDirectiveWord(2, Loc); + else if (IDVal == ".word") + parseDirectiveWord(4, Loc); + else if (IDVal == ".xword") + parseDirectiveWord(8, Loc); + else if (IDVal == ".tlsdesccall") + parseDirectiveTLSDescCall(Loc); + else if (IDVal == ".ltorg" || IDVal == ".pool") + parseDirectiveLtorg(Loc); + else if (IDVal == ".unreq") + parseDirectiveUnreq(Loc); + else if (!IsMachO && !IsCOFF) { if (IDVal == ".inst") - return parseDirectiveInst(Loc); - } - - return parseDirectiveLOH(IDVal, Loc); + parseDirectiveInst(Loc); + else + return true; + } else if (IDVal == MCLOHDirectiveName()) + parseDirectiveLOH(IDVal, Loc); + else + return true; + return false; } static const struct { @@ -4193,9 +4138,10 @@ static const struct { { "crypto", {AArch64::FeatureCrypto} }, { "fp", {AArch64::FeatureFPARMv8} }, { "simd", {AArch64::FeatureNEON} }, + { "ras", {AArch64::FeatureRAS} }, + { "lse", {AArch64::FeatureLSE} }, // FIXME: Unsupported extensions - { "lse", {} }, { "pan", {} }, { "lor", {} }, { "rdma", {} }, @@ -4212,17 +4158,51 @@ bool AArch64AsmParser::parseDirectiveArch(SMLoc L) { getParser().parseStringToEndOfStatement().trim().split('+'); unsigned ID = AArch64::parseArch(Arch); - if (ID == ARM::AK_INVALID) { - Error(ArchLoc, "unknown arch name"); - return false; - } + if (ID == static_cast(AArch64::ArchKind::AK_INVALID)) + return Error(ArchLoc, "unknown arch name"); + + if (parseToken(AsmToken::EndOfStatement)) + return true; + + // Get the architecture and extension features. + std::vector AArch64Features; + AArch64::getArchFeatures(ID, AArch64Features); + AArch64::getExtensionFeatures(AArch64::getDefaultExtensions("generic", ID), + AArch64Features); MCSubtargetInfo &STI = copySTI(); - STI.setDefaultFeatures("", ""); + std::vector ArchFeatures(AArch64Features.begin(), AArch64Features.end()); + STI.setDefaultFeatures("generic", join(ArchFeatures.begin(), ArchFeatures.end(), ",")); + + SmallVector RequestedExtensions; if (!ExtensionString.empty()) - STI.setDefaultFeatures("", ("+" + ExtensionString).str()); - setAvailableFeatures(ComputeAvailableFeatures(STI.getFeatureBits())); + ExtensionString.split(RequestedExtensions, '+'); + + FeatureBitset Features = STI.getFeatureBits(); + for (auto Name : RequestedExtensions) { + bool EnableFeature = true; + if (Name.startswith_lower("no")) { + EnableFeature = false; + Name = Name.substr(2); + } + + for (const auto &Extension : ExtensionMap) { + if (Extension.Name != Name) + continue; + + if (Extension.Features.none()) + report_fatal_error("unsupported architectural extension: " + Name); + + FeatureBitset ToggleFeatures = EnableFeature + ? (~Features & Extension.Features) + : ( Features & Extension.Features); + uint64_t Features = + ComputeAvailableFeatures(STI.ToggleFeature(ToggleFeatures)); + setAvailableFeatures(Features); + break; + } + } return false; } @@ -4235,6 +4215,9 @@ bool AArch64AsmParser::parseDirectiveCPU(SMLoc L) { std::tie(CPU, ExtensionString) = getParser().parseStringToEndOfStatement().trim().split('+'); + if (parseToken(AsmToken::EndOfStatement)) + return true; + SmallVector RequestedExtensions; if (!ExtensionString.empty()) ExtensionString.split(RequestedExtensions, '+'); @@ -4281,67 +4264,39 @@ bool AArch64AsmParser::parseDirectiveCPU(SMLoc L) { /// parseDirectiveWord /// ::= .word [ expression (, expression)* ] bool AArch64AsmParser::parseDirectiveWord(unsigned Size, SMLoc L) { - MCAsmParser &Parser = getParser(); - if (getLexer().isNot(AsmToken::EndOfStatement)) { - for (;;) { - const MCExpr *Value; - if (getParser().parseExpression(Value)) - return true; - - getParser().getStreamer().EmitValue(Value, Size, L); - - if (getLexer().is(AsmToken::EndOfStatement)) - break; - - // FIXME: Improve diagnostic. - if (getLexer().isNot(AsmToken::Comma)) - return Error(L, "unexpected token in directive"); - Parser.Lex(); - } - } + auto parseOp = [&]() -> bool { + const MCExpr *Value; + if (getParser().parseExpression(Value)) + return true; + getParser().getStreamer().EmitValue(Value, Size, L); + return false; + }; - Parser.Lex(); + if (parseMany(parseOp)) + return true; return false; } /// parseDirectiveInst /// ::= .inst opcode [, ...] bool AArch64AsmParser::parseDirectiveInst(SMLoc Loc) { - MCAsmParser &Parser = getParser(); - if (getLexer().is(AsmToken::EndOfStatement)) { - Parser.eatToEndOfStatement(); - Error(Loc, "expected expression following directive"); - return false; - } + if (getLexer().is(AsmToken::EndOfStatement)) + return Error(Loc, "expected expression following '.inst' directive"); - for (;;) { + auto parseOp = [&]() -> bool { + SMLoc L = getLoc(); const MCExpr *Expr; - - if (getParser().parseExpression(Expr)) { - Error(Loc, "expected expression"); - return false; - } - + if (check(getParser().parseExpression(Expr), L, "expected expression")) + return true; const MCConstantExpr *Value = dyn_cast_or_null(Expr); - if (!Value) { - Error(Loc, "expected constant expression"); - return false; - } - + if (check(!Value, L, "expected constant expression")) + return true; getTargetStreamer().emitInst(Value->getValue()); + return false; + }; - if (getLexer().is(AsmToken::EndOfStatement)) - break; - - if (getLexer().isNot(AsmToken::Comma)) { - Error(Loc, "unexpected token in directive"); - return false; - } - - Parser.Lex(); // Eat comma. - } - - Parser.Lex(); + if (parseMany(parseOp)) + return addErrorSuffix(" in '.inst' directive"); return false; } @@ -4349,8 +4304,10 @@ bool AArch64AsmParser::parseDirectiveInst(SMLoc Loc) { // ::= .tlsdesccall symbol bool AArch64AsmParser::parseDirectiveTLSDescCall(SMLoc L) { StringRef Name; - if (getParser().parseIdentifier(Name)) - return Error(L, "expected symbol after directive"); + if (check(getParser().parseIdentifier(Name), L, + "expected symbol after directive") || + parseToken(AsmToken::EndOfStatement)) + return true; MCSymbol *Sym = getContext().getOrCreateSymbol(Name); const MCExpr *Expr = MCSymbolRefExpr::create(Sym, getContext()); @@ -4367,8 +4324,6 @@ bool AArch64AsmParser::parseDirectiveTLSDescCall(SMLoc L) { /// ::= .loh label1, ..., labelN /// The number of arguments depends on the loh identifier. bool AArch64AsmParser::parseDirectiveLOH(StringRef IDVal, SMLoc Loc) { - if (IDVal != MCLOHDirectiveName()) - return true; MCLOHType Kind; if (getParser().getTok().isNot(AsmToken::Identifier)) { if (getParser().getTok().isNot(AsmToken::Integer)) @@ -4405,12 +4360,13 @@ bool AArch64AsmParser::parseDirectiveLOH(StringRef IDVal, SMLoc Loc) { if (Idx + 1 == NbArgs) break; - if (getLexer().isNot(AsmToken::Comma)) - return TokError("unexpected token in '" + Twine(IDVal) + "' directive"); - Lex(); + if (parseToken(AsmToken::Comma, + "unexpected token in '" + Twine(IDVal) + "' directive")) + return true; } - if (getLexer().isNot(AsmToken::EndOfStatement)) - return TokError("unexpected token in '" + Twine(IDVal) + "' directive"); + if (parseToken(AsmToken::EndOfStatement, + "unexpected token in '" + Twine(IDVal) + "' directive")) + return true; getStreamer().EmitLOHDirective((MCLOHType)Kind, Args); return false; @@ -4419,6 +4375,8 @@ bool AArch64AsmParser::parseDirectiveLOH(StringRef IDVal, SMLoc Loc) { /// parseDirectiveLtorg /// ::= .ltorg | .pool bool AArch64AsmParser::parseDirectiveLtorg(SMLoc L) { + if (parseToken(AsmToken::EndOfStatement, "unexpected token in directive")) + return true; getTargetStreamer().emitCurrentConstantPool(); return false; } @@ -4435,46 +4393,36 @@ bool AArch64AsmParser::parseDirectiveReq(StringRef Name, SMLoc L) { if (RegNum == static_cast(-1)) { StringRef Kind; RegNum = tryMatchVectorRegister(Kind, false); - if (!Kind.empty()) { - Error(SRegLoc, "vector register without type specifier expected"); - return false; - } + if (!Kind.empty()) + return Error(SRegLoc, "vector register without type specifier expected"); IsVector = true; } - if (RegNum == static_cast(-1)) { - Parser.eatToEndOfStatement(); - Error(SRegLoc, "register name or alias expected"); - return false; - } + if (RegNum == static_cast(-1)) + return Error(SRegLoc, "register name or alias expected"); // Shouldn't be anything else. - if (Parser.getTok().isNot(AsmToken::EndOfStatement)) { - Error(Parser.getTok().getLoc(), "unexpected input in .req directive"); - Parser.eatToEndOfStatement(); - return false; - } - - Parser.Lex(); // Consume the EndOfStatement + if (parseToken(AsmToken::EndOfStatement, + "unexpected input in .req directive")) + return true; auto pair = std::make_pair(IsVector, RegNum); if (RegisterReqs.insert(std::make_pair(Name, pair)).first->second != pair) Warning(L, "ignoring redefinition of register alias '" + Name + "'"); - return true; + return false; } /// parseDirectiveUneq /// ::= .unreq registername bool AArch64AsmParser::parseDirectiveUnreq(SMLoc L) { MCAsmParser &Parser = getParser(); - if (Parser.getTok().isNot(AsmToken::Identifier)) { - Error(Parser.getTok().getLoc(), "unexpected input in .unreq directive."); - Parser.eatToEndOfStatement(); - return false; - } + if (getTok().isNot(AsmToken::Identifier)) + return TokError("unexpected input in .unreq directive."); RegisterReqs.erase(Parser.getTok().getIdentifier().lower()); Parser.Lex(); // Eat the identifier. + if (parseToken(AsmToken::EndOfStatement)) + return addErrorSuffix("in '.unreq' directive"); return false; } @@ -4530,9 +4478,9 @@ AArch64AsmParser::classifySymbolRef(const MCExpr *Expr, /// Force static initialization. extern "C" void LLVMInitializeAArch64AsmParser() { - RegisterMCAsmParser X(TheAArch64leTarget); - RegisterMCAsmParser Y(TheAArch64beTarget); - RegisterMCAsmParser Z(TheARM64Target); + RegisterMCAsmParser X(getTheAArch64leTarget()); + RegisterMCAsmParser Y(getTheAArch64beTarget()); + RegisterMCAsmParser Z(getTheARM64Target()); } #define GET_REGISTER_MATCHER @@ -4603,7 +4551,7 @@ unsigned AArch64AsmParser::validateTargetOperandClass(MCParsedAsmOperand &AsmOp, } -AArch64AsmParser::OperandMatchResultTy +OperandMatchResultTy AArch64AsmParser::tryParseGPRSeqPair(OperandVector &Operands) { SMLoc S = getLoc(); @@ -4660,7 +4608,7 @@ AArch64AsmParser::tryParseGPRSeqPair(OperandVector &Operands) { "consecutive same-size even/odd register pair"); return MatchOperand_ParseFail; } - + unsigned Pair = 0; if(isXReg) { Pair = RI->getMatchingSuperReg(FirstReg, AArch64::sube64, diff --git a/lib/Target/AArch64/CMakeLists.txt b/lib/Target/AArch64/CMakeLists.txt index a79960ea9605..6bcf67fb3fef 100644 --- a/lib/Target/AArch64/CMakeLists.txt +++ b/lib/Target/AArch64/CMakeLists.txt @@ -13,12 +13,17 @@ tablegen(LLVM AArch64GenCallingConv.inc -gen-callingconv) tablegen(LLVM AArch64GenSubtargetInfo.inc -gen-subtarget) tablegen(LLVM AArch64GenDisassemblerTables.inc -gen-disassembler) tablegen(LLVM AArch64GenSystemOperands.inc -gen-searchable-tables) +if(LLVM_BUILD_GLOBAL_ISEL) + tablegen(LLVM AArch64GenGlobalISel.inc -gen-global-isel) +endif() add_public_tablegen_target(AArch64CommonTableGen) # List of all GlobalISel files. set(GLOBAL_ISEL_FILES AArch64CallLowering.cpp + AArch64InstructionSelector.cpp + AArch64LegalizerInfo.cpp AArch64RegisterBankInfo.cpp ) @@ -36,7 +41,6 @@ add_llvm_target(AArch64CodeGen AArch64AddressTypePromotion.cpp AArch64AdvSIMDScalarPass.cpp AArch64AsmPrinter.cpp - AArch64BranchRelaxation.cpp AArch64CleanupLocalDynamicTLSPass.cpp AArch64CollectLOH.cpp AArch64ConditionalCompares.cpp @@ -61,10 +65,12 @@ add_llvm_target(AArch64CodeGen AArch64TargetMachine.cpp AArch64TargetObjectFile.cpp AArch64TargetTransformInfo.cpp + AArch64VectorByElementOpt.cpp ${GLOBAL_ISEL_BUILD_FILES} -) -add_dependencies(LLVMAArch64CodeGen intrinsics_gen) + DEPENDS + intrinsics_gen +) add_subdirectory(TargetInfo) add_subdirectory(AsmParser) diff --git a/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp b/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp index fe6ea31b9061..0d860a7eef79 100644 --- a/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp +++ b/lib/Target/AArch64/Disassembler/AArch64Disassembler.cpp @@ -237,18 +237,18 @@ createAArch64ExternalSymbolizer(const Triple &TT, LLVMOpInfoCallback GetOpInfo, } extern "C" void LLVMInitializeAArch64Disassembler() { - TargetRegistry::RegisterMCDisassembler(TheAArch64leTarget, + TargetRegistry::RegisterMCDisassembler(getTheAArch64leTarget(), createAArch64Disassembler); - TargetRegistry::RegisterMCDisassembler(TheAArch64beTarget, + TargetRegistry::RegisterMCDisassembler(getTheAArch64beTarget(), createAArch64Disassembler); - TargetRegistry::RegisterMCSymbolizer(TheAArch64leTarget, + TargetRegistry::RegisterMCSymbolizer(getTheAArch64leTarget(), createAArch64ExternalSymbolizer); - TargetRegistry::RegisterMCSymbolizer(TheAArch64beTarget, + TargetRegistry::RegisterMCSymbolizer(getTheAArch64beTarget(), createAArch64ExternalSymbolizer); - TargetRegistry::RegisterMCDisassembler(TheARM64Target, + TargetRegistry::RegisterMCDisassembler(getTheARM64Target(), createAArch64Disassembler); - TargetRegistry::RegisterMCSymbolizer(TheARM64Target, + TargetRegistry::RegisterMCSymbolizer(getTheARM64Target(), createAArch64ExternalSymbolizer); } @@ -1097,7 +1097,7 @@ static DecodeStatus DecodeExclusiveLdStInstruction(llvm::MCInst &Inst, case AArch64::STXRB: case AArch64::STXRH: DecodeGPR32RegisterClass(Inst, Rs, Addr, Decoder); - // FALLTHROUGH + LLVM_FALLTHROUGH; case AArch64::LDARW: case AArch64::LDARB: case AArch64::LDARH: @@ -1121,7 +1121,7 @@ static DecodeStatus DecodeExclusiveLdStInstruction(llvm::MCInst &Inst, case AArch64::STLXRX: case AArch64::STXRX: DecodeGPR32RegisterClass(Inst, Rs, Addr, Decoder); - // FALLTHROUGH + LLVM_FALLTHROUGH; case AArch64::LDARX: case AArch64::LDAXRX: case AArch64::LDXRX: @@ -1133,7 +1133,7 @@ static DecodeStatus DecodeExclusiveLdStInstruction(llvm::MCInst &Inst, case AArch64::STLXPW: case AArch64::STXPW: DecodeGPR32RegisterClass(Inst, Rs, Addr, Decoder); - // FALLTHROUGH + LLVM_FALLTHROUGH; case AArch64::LDAXPW: case AArch64::LDXPW: DecodeGPR32RegisterClass(Inst, Rt, Addr, Decoder); @@ -1142,7 +1142,7 @@ static DecodeStatus DecodeExclusiveLdStInstruction(llvm::MCInst &Inst, case AArch64::STLXPX: case AArch64::STXPX: DecodeGPR32RegisterClass(Inst, Rs, Addr, Decoder); - // FALLTHROUGH + LLVM_FALLTHROUGH; case AArch64::LDAXPX: case AArch64::LDXPX: DecodeGPR64RegisterClass(Inst, Rt, Addr, Decoder); @@ -1218,7 +1218,7 @@ static DecodeStatus DecodePairLdStInstruction(llvm::MCInst &Inst, uint32_t insn, case AArch64::STPXpre: case AArch64::LDPSWpre: NeedsDisjointWritebackTransfer = true; - // Fallthrough + LLVM_FALLTHROUGH; case AArch64::LDNPXi: case AArch64::STNPXi: case AArch64::LDPXi: @@ -1232,7 +1232,7 @@ static DecodeStatus DecodePairLdStInstruction(llvm::MCInst &Inst, uint32_t insn, case AArch64::LDPWpre: case AArch64::STPWpre: NeedsDisjointWritebackTransfer = true; - // Fallthrough + LLVM_FALLTHROUGH; case AArch64::LDNPWi: case AArch64::STNPWi: case AArch64::LDPWi: diff --git a/lib/Target/AArch64/Disassembler/AArch64Disassembler.h b/lib/Target/AArch64/Disassembler/AArch64Disassembler.h index e475e505e7d1..24e353cf4b96 100644 --- a/lib/Target/AArch64/Disassembler/AArch64Disassembler.h +++ b/lib/Target/AArch64/Disassembler/AArch64Disassembler.h @@ -18,7 +18,6 @@ namespace llvm { class MCInst; -class MemoryObject; class raw_ostream; class AArch64Disassembler : public MCDisassembler { diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp index 27993246eb07..14c0327f5fa8 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp @@ -11,6 +11,7 @@ #include "AArch64RegisterInfo.h" #include "MCTargetDesc/AArch64FixupKinds.h" #include "llvm/ADT/Triple.h" +#include "llvm/MC/MCAssembler.h" #include "llvm/MC/MCAsmBackend.h" #include "llvm/MC/MCContext.h" #include "llvm/MC/MCDirectives.h" @@ -520,6 +521,17 @@ public: return CompactUnwindEncoding; } + + void processFixupValue(const MCAssembler &Asm, const MCAsmLayout &Layout, + const MCFixup &Fixup, const MCFragment *DF, + const MCValue &Target, uint64_t &Value, + bool &IsResolved) override { + // Try to get the encoded value for the fixup as-if we're mapping it into + // the instruction. This allows adjustFixupValue() to issue a diagnostic + // if the value is invalid. + if (IsResolved) + (void)adjustFixupValue(Fixup, Value, &Asm.getContext()); + } }; } // end anonymous namespace @@ -529,12 +541,14 @@ namespace { class ELFAArch64AsmBackend : public AArch64AsmBackend { public: uint8_t OSABI; + bool IsILP32; - ELFAArch64AsmBackend(const Target &T, uint8_t OSABI, bool IsLittleEndian) - : AArch64AsmBackend(T, IsLittleEndian), OSABI(OSABI) {} + ELFAArch64AsmBackend(const Target &T, uint8_t OSABI, bool IsLittleEndian, + bool IsILP32) + : AArch64AsmBackend(T, IsLittleEndian), OSABI(OSABI), IsILP32(IsILP32) {} MCObjectWriter *createObjectWriter(raw_pwrite_stream &OS) const override { - return createAArch64ELFObjectWriter(OS, OSABI, IsLittleEndian); + return createAArch64ELFObjectWriter(OS, OSABI, IsLittleEndian, IsILP32); } void processFixupValue(const MCAssembler &Asm, const MCAsmLayout &Layout, @@ -574,22 +588,25 @@ void ELFAArch64AsmBackend::processFixupValue( MCAsmBackend *llvm::createAArch64leAsmBackend(const Target &T, const MCRegisterInfo &MRI, const Triple &TheTriple, - StringRef CPU) { + StringRef CPU, + const MCTargetOptions &Options) { if (TheTriple.isOSBinFormatMachO()) return new DarwinAArch64AsmBackend(T, MRI); assert(TheTriple.isOSBinFormatELF() && "Expect either MachO or ELF target"); uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(TheTriple.getOS()); - return new ELFAArch64AsmBackend(T, OSABI, /*IsLittleEndian=*/true); + bool IsILP32 = Options.getABIName() == "ilp32"; + return new ELFAArch64AsmBackend(T, OSABI, /*IsLittleEndian=*/true, IsILP32); } MCAsmBackend *llvm::createAArch64beAsmBackend(const Target &T, const MCRegisterInfo &MRI, const Triple &TheTriple, - StringRef CPU) { + StringRef CPU, + const MCTargetOptions &Options) { assert(TheTriple.isOSBinFormatELF() && "Big endian is only supported for ELF targets!"); uint8_t OSABI = MCELFObjectTargetWriter::getOSABI(TheTriple.getOS()); - return new ELFAArch64AsmBackend(T, OSABI, - /*IsLittleEndian=*/false); + bool IsILP32 = Options.getABIName() == "ilp32"; + return new ELFAArch64AsmBackend(T, OSABI, /*IsLittleEndian=*/false, IsILP32); } diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp index 4b4c4097b97b..a1edb3cef46a 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64ELFObjectWriter.cpp @@ -25,25 +25,80 @@ using namespace llvm; namespace { class AArch64ELFObjectWriter : public MCELFObjectTargetWriter { public: - AArch64ELFObjectWriter(uint8_t OSABI, bool IsLittleEndian); + AArch64ELFObjectWriter(uint8_t OSABI, bool IsLittleEndian, bool IsILP32); ~AArch64ELFObjectWriter() override; protected: unsigned getRelocType(MCContext &Ctx, const MCValue &Target, const MCFixup &Fixup, bool IsPCRel) const override; - + bool IsILP32; private: }; } AArch64ELFObjectWriter::AArch64ELFObjectWriter(uint8_t OSABI, - bool IsLittleEndian) + bool IsLittleEndian, + bool IsILP32) : MCELFObjectTargetWriter(/*Is64Bit*/ true, OSABI, ELF::EM_AARCH64, - /*HasRelocationAddend*/ true) {} + /*HasRelocationAddend*/ true), + IsILP32(IsILP32) {} AArch64ELFObjectWriter::~AArch64ELFObjectWriter() {} +#define R_CLS(rtype) \ + IsILP32 ? ELF::R_AARCH64_P32_##rtype : ELF::R_AARCH64_##rtype +#define BAD_ILP32_MOV(lp64rtype) "ILP32 absolute MOV relocation not "\ + "supported (LP64 eqv: " #lp64rtype ")" + +// assumes IsILP32 is true +static bool isNonILP32reloc(const MCFixup &Fixup, + AArch64MCExpr::VariantKind RefKind, + MCContext &Ctx) { + if ((unsigned)Fixup.getKind() != AArch64::fixup_aarch64_movw) + return false; + switch(RefKind) { + case AArch64MCExpr::VK_ABS_G3: + Ctx.reportError(Fixup.getLoc(), BAD_ILP32_MOV(MOVW_UABS_G3)); + return true; + case AArch64MCExpr::VK_ABS_G2: + Ctx.reportError(Fixup.getLoc(), BAD_ILP32_MOV(MOVW_UABS_G2)); + return true; + case AArch64MCExpr::VK_ABS_G2_S: + Ctx.reportError(Fixup.getLoc(), BAD_ILP32_MOV(MOVW_SABS_G2)); + return ELF::R_AARCH64_NONE; + case AArch64MCExpr::VK_ABS_G2_NC: + Ctx.reportError(Fixup.getLoc(), BAD_ILP32_MOV(MOVW_UABS_G2_NC)); + return ELF::R_AARCH64_NONE; + case AArch64MCExpr::VK_ABS_G1_S: + Ctx.reportError(Fixup.getLoc(), BAD_ILP32_MOV(MOVW_SABS_G1)); + return ELF::R_AARCH64_NONE; + case AArch64MCExpr::VK_ABS_G1_NC: + Ctx.reportError(Fixup.getLoc(), BAD_ILP32_MOV(MOVW_UABS_G1_NC)); + return ELF::R_AARCH64_NONE; + case AArch64MCExpr::VK_DTPREL_G2: + Ctx.reportError(Fixup.getLoc(), BAD_ILP32_MOV(TLSLD_MOVW_DTPREL_G2)); + return ELF::R_AARCH64_NONE; + case AArch64MCExpr::VK_DTPREL_G1_NC: + Ctx.reportError(Fixup.getLoc(), BAD_ILP32_MOV(TLSLD_MOVW_DTPREL_G1_NC)); + return ELF::R_AARCH64_NONE; + case AArch64MCExpr::VK_TPREL_G2: + Ctx.reportError(Fixup.getLoc(), BAD_ILP32_MOV(TLSLE_MOVW_TPREL_G2)); + return ELF::R_AARCH64_NONE; + case AArch64MCExpr::VK_TPREL_G1_NC: + Ctx.reportError(Fixup.getLoc(), BAD_ILP32_MOV(TLSLE_MOVW_TPREL_G1_NC)); + return ELF::R_AARCH64_NONE; + case AArch64MCExpr::VK_GOTTPREL_G1: + Ctx.reportError(Fixup.getLoc(), BAD_ILP32_MOV(TLSIE_MOVW_GOTTPREL_G1)); + return ELF::R_AARCH64_NONE; + case AArch64MCExpr::VK_GOTTPREL_G0_NC: + Ctx.reportError(Fixup.getLoc(), BAD_ILP32_MOV(TLSIE_MOVW_GOTTPREL_G0_NC)); + return ELF::R_AARCH64_NONE; + default: return false; + } + return false; +} + unsigned AArch64ELFObjectWriter::getRelocType(MCContext &Ctx, const MCValue &Target, const MCFixup &Fixup, @@ -67,147 +122,161 @@ unsigned AArch64ELFObjectWriter::getRelocType(MCContext &Ctx, Ctx.reportError(Fixup.getLoc(), "1-byte data relocations not supported"); return ELF::R_AARCH64_NONE; case FK_Data_2: - return ELF::R_AARCH64_PREL16; + return R_CLS(PREL16); case FK_Data_4: - return ELF::R_AARCH64_PREL32; + return R_CLS(PREL32); case FK_Data_8: - return ELF::R_AARCH64_PREL64; + if (IsILP32) { + Ctx.reportError(Fixup.getLoc(), "ILP32 8 byte PC relative data " + "relocation not supported (LP64 eqv: PREL64)"); + return ELF::R_AARCH64_NONE; + } else + return ELF::R_AARCH64_PREL64; case AArch64::fixup_aarch64_pcrel_adr_imm21: assert(SymLoc == AArch64MCExpr::VK_NONE && "unexpected ADR relocation"); - return ELF::R_AARCH64_ADR_PREL_LO21; + return R_CLS(ADR_PREL_LO21); case AArch64::fixup_aarch64_pcrel_adrp_imm21: if (SymLoc == AArch64MCExpr::VK_ABS && !IsNC) - return ELF::R_AARCH64_ADR_PREL_PG_HI21; + return R_CLS(ADR_PREL_PG_HI21); if (SymLoc == AArch64MCExpr::VK_GOT && !IsNC) - return ELF::R_AARCH64_ADR_GOT_PAGE; + return R_CLS(ADR_GOT_PAGE); if (SymLoc == AArch64MCExpr::VK_GOTTPREL && !IsNC) - return ELF::R_AARCH64_TLSIE_ADR_GOTTPREL_PAGE21; + return R_CLS(TLSIE_ADR_GOTTPREL_PAGE21); if (SymLoc == AArch64MCExpr::VK_TLSDESC && !IsNC) - return ELF::R_AARCH64_TLSDESC_ADR_PAGE21; + return R_CLS(TLSDESC_ADR_PAGE21); Ctx.reportError(Fixup.getLoc(), "invalid symbol kind for ADRP relocation"); return ELF::R_AARCH64_NONE; case AArch64::fixup_aarch64_pcrel_branch26: - return ELF::R_AARCH64_JUMP26; + return R_CLS(JUMP26); case AArch64::fixup_aarch64_pcrel_call26: - return ELF::R_AARCH64_CALL26; + return R_CLS(CALL26); case AArch64::fixup_aarch64_ldr_pcrel_imm19: if (SymLoc == AArch64MCExpr::VK_GOTTPREL) - return ELF::R_AARCH64_TLSIE_LD_GOTTPREL_PREL19; - return ELF::R_AARCH64_LD_PREL_LO19; + return R_CLS(TLSIE_LD_GOTTPREL_PREL19); + return R_CLS(LD_PREL_LO19); case AArch64::fixup_aarch64_pcrel_branch14: - return ELF::R_AARCH64_TSTBR14; + return R_CLS(TSTBR14); case AArch64::fixup_aarch64_pcrel_branch19: - return ELF::R_AARCH64_CONDBR19; + return R_CLS(CONDBR19); default: Ctx.reportError(Fixup.getLoc(), "Unsupported pc-relative fixup kind"); return ELF::R_AARCH64_NONE; } } else { + if (IsILP32 && isNonILP32reloc(Fixup, RefKind, Ctx)) + return ELF::R_AARCH64_NONE; switch ((unsigned)Fixup.getKind()) { case FK_Data_1: Ctx.reportError(Fixup.getLoc(), "1-byte data relocations not supported"); return ELF::R_AARCH64_NONE; case FK_Data_2: - return ELF::R_AARCH64_ABS16; + return R_CLS(ABS16); case FK_Data_4: - return ELF::R_AARCH64_ABS32; + return R_CLS(ABS32); case FK_Data_8: - return ELF::R_AARCH64_ABS64; + if (IsILP32) { + Ctx.reportError(Fixup.getLoc(), BAD_ILP32_MOV(ABS64)); + return ELF::R_AARCH64_NONE; + } else + return ELF::R_AARCH64_ABS64; case AArch64::fixup_aarch64_add_imm12: if (RefKind == AArch64MCExpr::VK_DTPREL_HI12) - return ELF::R_AARCH64_TLSLD_ADD_DTPREL_HI12; + return R_CLS(TLSLD_ADD_DTPREL_HI12); if (RefKind == AArch64MCExpr::VK_TPREL_HI12) - return ELF::R_AARCH64_TLSLE_ADD_TPREL_HI12; + return R_CLS(TLSLE_ADD_TPREL_HI12); if (RefKind == AArch64MCExpr::VK_DTPREL_LO12_NC) - return ELF::R_AARCH64_TLSLD_ADD_DTPREL_LO12_NC; + return R_CLS(TLSLD_ADD_DTPREL_LO12_NC); if (RefKind == AArch64MCExpr::VK_DTPREL_LO12) - return ELF::R_AARCH64_TLSLD_ADD_DTPREL_LO12; + return R_CLS(TLSLD_ADD_DTPREL_LO12); if (RefKind == AArch64MCExpr::VK_TPREL_LO12_NC) - return ELF::R_AARCH64_TLSLE_ADD_TPREL_LO12_NC; + return R_CLS(TLSLE_ADD_TPREL_LO12_NC); if (RefKind == AArch64MCExpr::VK_TPREL_LO12) - return ELF::R_AARCH64_TLSLE_ADD_TPREL_LO12; + return R_CLS(TLSLE_ADD_TPREL_LO12); if (RefKind == AArch64MCExpr::VK_TLSDESC_LO12) - return ELF::R_AARCH64_TLSDESC_ADD_LO12_NC; + return R_CLS(TLSDESC_ADD_LO12_NC); if (SymLoc == AArch64MCExpr::VK_ABS && IsNC) - return ELF::R_AARCH64_ADD_ABS_LO12_NC; + return R_CLS(ADD_ABS_LO12_NC); Ctx.reportError(Fixup.getLoc(), "invalid fixup for add (uimm12) instruction"); return ELF::R_AARCH64_NONE; case AArch64::fixup_aarch64_ldst_imm12_scale1: if (SymLoc == AArch64MCExpr::VK_ABS && IsNC) - return ELF::R_AARCH64_LDST8_ABS_LO12_NC; + return R_CLS(LDST8_ABS_LO12_NC); if (SymLoc == AArch64MCExpr::VK_DTPREL && !IsNC) - return ELF::R_AARCH64_TLSLD_LDST8_DTPREL_LO12; + return R_CLS(TLSLD_LDST8_DTPREL_LO12); if (SymLoc == AArch64MCExpr::VK_DTPREL && IsNC) - return ELF::R_AARCH64_TLSLD_LDST8_DTPREL_LO12_NC; + return R_CLS(TLSLD_LDST8_DTPREL_LO12_NC); if (SymLoc == AArch64MCExpr::VK_TPREL && !IsNC) - return ELF::R_AARCH64_TLSLE_LDST8_TPREL_LO12; + return R_CLS(TLSLE_LDST8_TPREL_LO12); if (SymLoc == AArch64MCExpr::VK_TPREL && IsNC) - return ELF::R_AARCH64_TLSLE_LDST8_TPREL_LO12_NC; + return R_CLS(TLSLE_LDST8_TPREL_LO12_NC); Ctx.reportError(Fixup.getLoc(), "invalid fixup for 8-bit load/store instruction"); return ELF::R_AARCH64_NONE; case AArch64::fixup_aarch64_ldst_imm12_scale2: if (SymLoc == AArch64MCExpr::VK_ABS && IsNC) - return ELF::R_AARCH64_LDST16_ABS_LO12_NC; + return R_CLS(LDST16_ABS_LO12_NC); if (SymLoc == AArch64MCExpr::VK_DTPREL && !IsNC) - return ELF::R_AARCH64_TLSLD_LDST16_DTPREL_LO12; + return R_CLS(TLSLD_LDST16_DTPREL_LO12); if (SymLoc == AArch64MCExpr::VK_DTPREL && IsNC) - return ELF::R_AARCH64_TLSLD_LDST16_DTPREL_LO12_NC; + return R_CLS(TLSLD_LDST16_DTPREL_LO12_NC); if (SymLoc == AArch64MCExpr::VK_TPREL && !IsNC) - return ELF::R_AARCH64_TLSLE_LDST16_TPREL_LO12; + return R_CLS(TLSLE_LDST16_TPREL_LO12); if (SymLoc == AArch64MCExpr::VK_TPREL && IsNC) - return ELF::R_AARCH64_TLSLE_LDST16_TPREL_LO12_NC; + return R_CLS(TLSLE_LDST16_TPREL_LO12_NC); Ctx.reportError(Fixup.getLoc(), "invalid fixup for 16-bit load/store instruction"); return ELF::R_AARCH64_NONE; case AArch64::fixup_aarch64_ldst_imm12_scale4: if (SymLoc == AArch64MCExpr::VK_ABS && IsNC) - return ELF::R_AARCH64_LDST32_ABS_LO12_NC; + return R_CLS(LDST32_ABS_LO12_NC); if (SymLoc == AArch64MCExpr::VK_DTPREL && !IsNC) - return ELF::R_AARCH64_TLSLD_LDST32_DTPREL_LO12; + return R_CLS(TLSLD_LDST32_DTPREL_LO12); if (SymLoc == AArch64MCExpr::VK_DTPREL && IsNC) - return ELF::R_AARCH64_TLSLD_LDST32_DTPREL_LO12_NC; + return R_CLS(TLSLD_LDST32_DTPREL_LO12_NC); if (SymLoc == AArch64MCExpr::VK_TPREL && !IsNC) - return ELF::R_AARCH64_TLSLE_LDST32_TPREL_LO12; + return R_CLS(TLSLE_LDST32_TPREL_LO12); if (SymLoc == AArch64MCExpr::VK_TPREL && IsNC) - return ELF::R_AARCH64_TLSLE_LDST32_TPREL_LO12_NC; + return R_CLS(TLSLE_LDST32_TPREL_LO12_NC); Ctx.reportError(Fixup.getLoc(), "invalid fixup for 32-bit load/store instruction"); return ELF::R_AARCH64_NONE; case AArch64::fixup_aarch64_ldst_imm12_scale8: if (SymLoc == AArch64MCExpr::VK_ABS && IsNC) - return ELF::R_AARCH64_LDST64_ABS_LO12_NC; + return R_CLS(LDST64_ABS_LO12_NC); if (SymLoc == AArch64MCExpr::VK_GOT && IsNC) - return ELF::R_AARCH64_LD64_GOT_LO12_NC; + return R_CLS(LD64_GOT_LO12_NC); if (SymLoc == AArch64MCExpr::VK_DTPREL && !IsNC) - return ELF::R_AARCH64_TLSLD_LDST64_DTPREL_LO12; + return R_CLS(TLSLD_LDST64_DTPREL_LO12); if (SymLoc == AArch64MCExpr::VK_DTPREL && IsNC) - return ELF::R_AARCH64_TLSLD_LDST64_DTPREL_LO12_NC; + return R_CLS(TLSLD_LDST64_DTPREL_LO12_NC); if (SymLoc == AArch64MCExpr::VK_TPREL && !IsNC) - return ELF::R_AARCH64_TLSLE_LDST64_TPREL_LO12; + return R_CLS(TLSLE_LDST64_TPREL_LO12); if (SymLoc == AArch64MCExpr::VK_TPREL && IsNC) - return ELF::R_AARCH64_TLSLE_LDST64_TPREL_LO12_NC; + return R_CLS(TLSLE_LDST64_TPREL_LO12_NC); if (SymLoc == AArch64MCExpr::VK_GOTTPREL && IsNC) - return ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC; + return IsILP32 ? ELF::R_AARCH64_P32_TLSIE_LD32_GOTTPREL_LO12_NC + : ELF::R_AARCH64_TLSIE_LD64_GOTTPREL_LO12_NC; if (SymLoc == AArch64MCExpr::VK_TLSDESC && IsNC) - return ELF::R_AARCH64_TLSDESC_LD64_LO12_NC; + return IsILP32 ? ELF::R_AARCH64_P32_TLSDESC_LD32_LO12_NC + : ELF::R_AARCH64_TLSDESC_LD64_LO12_NC; Ctx.reportError(Fixup.getLoc(), "invalid fixup for 64-bit load/store instruction"); return ELF::R_AARCH64_NONE; case AArch64::fixup_aarch64_ldst_imm12_scale16: if (SymLoc == AArch64MCExpr::VK_ABS && IsNC) - return ELF::R_AARCH64_LDST128_ABS_LO12_NC; + return R_CLS(LDST128_ABS_LO12_NC); Ctx.reportError(Fixup.getLoc(), "invalid fixup for 128-bit load/store instruction"); return ELF::R_AARCH64_NONE; + // ILP32 case not reached here, tested with isNonILP32reloc case AArch64::fixup_aarch64_movw: if (RefKind == AArch64MCExpr::VK_ABS_G3) return ELF::R_AARCH64_MOVW_UABS_G3; @@ -218,37 +287,37 @@ unsigned AArch64ELFObjectWriter::getRelocType(MCContext &Ctx, if (RefKind == AArch64MCExpr::VK_ABS_G2_NC) return ELF::R_AARCH64_MOVW_UABS_G2_NC; if (RefKind == AArch64MCExpr::VK_ABS_G1) - return ELF::R_AARCH64_MOVW_UABS_G1; + return R_CLS(MOVW_UABS_G1); if (RefKind == AArch64MCExpr::VK_ABS_G1_S) return ELF::R_AARCH64_MOVW_SABS_G1; if (RefKind == AArch64MCExpr::VK_ABS_G1_NC) return ELF::R_AARCH64_MOVW_UABS_G1_NC; if (RefKind == AArch64MCExpr::VK_ABS_G0) - return ELF::R_AARCH64_MOVW_UABS_G0; + return R_CLS(MOVW_UABS_G0); if (RefKind == AArch64MCExpr::VK_ABS_G0_S) - return ELF::R_AARCH64_MOVW_SABS_G0; + return R_CLS(MOVW_SABS_G0); if (RefKind == AArch64MCExpr::VK_ABS_G0_NC) - return ELF::R_AARCH64_MOVW_UABS_G0_NC; + return R_CLS(MOVW_UABS_G0_NC); if (RefKind == AArch64MCExpr::VK_DTPREL_G2) return ELF::R_AARCH64_TLSLD_MOVW_DTPREL_G2; if (RefKind == AArch64MCExpr::VK_DTPREL_G1) - return ELF::R_AARCH64_TLSLD_MOVW_DTPREL_G1; + return R_CLS(TLSLD_MOVW_DTPREL_G1); if (RefKind == AArch64MCExpr::VK_DTPREL_G1_NC) return ELF::R_AARCH64_TLSLD_MOVW_DTPREL_G1_NC; if (RefKind == AArch64MCExpr::VK_DTPREL_G0) - return ELF::R_AARCH64_TLSLD_MOVW_DTPREL_G0; + return R_CLS(TLSLD_MOVW_DTPREL_G0); if (RefKind == AArch64MCExpr::VK_DTPREL_G0_NC) - return ELF::R_AARCH64_TLSLD_MOVW_DTPREL_G0_NC; + return R_CLS(TLSLD_MOVW_DTPREL_G0_NC); if (RefKind == AArch64MCExpr::VK_TPREL_G2) return ELF::R_AARCH64_TLSLE_MOVW_TPREL_G2; if (RefKind == AArch64MCExpr::VK_TPREL_G1) - return ELF::R_AARCH64_TLSLE_MOVW_TPREL_G1; + return R_CLS(TLSLE_MOVW_TPREL_G1); if (RefKind == AArch64MCExpr::VK_TPREL_G1_NC) return ELF::R_AARCH64_TLSLE_MOVW_TPREL_G1_NC; if (RefKind == AArch64MCExpr::VK_TPREL_G0) - return ELF::R_AARCH64_TLSLE_MOVW_TPREL_G0; + return R_CLS(TLSLE_MOVW_TPREL_G0); if (RefKind == AArch64MCExpr::VK_TPREL_G0_NC) - return ELF::R_AARCH64_TLSLE_MOVW_TPREL_G0_NC; + return R_CLS(TLSLE_MOVW_TPREL_G0_NC); if (RefKind == AArch64MCExpr::VK_GOTTPREL_G1) return ELF::R_AARCH64_TLSIE_MOVW_GOTTPREL_G1; if (RefKind == AArch64MCExpr::VK_GOTTPREL_G0_NC) @@ -257,7 +326,7 @@ unsigned AArch64ELFObjectWriter::getRelocType(MCContext &Ctx, "invalid fixup for movz/movk instruction"); return ELF::R_AARCH64_NONE; case AArch64::fixup_aarch64_tlsdesc_call: - return ELF::R_AARCH64_TLSDESC_CALL; + return R_CLS(TLSDESC_CALL); default: Ctx.reportError(Fixup.getLoc(), "Unknown ELF relocation type"); return ELF::R_AARCH64_NONE; @@ -269,8 +338,9 @@ unsigned AArch64ELFObjectWriter::getRelocType(MCContext &Ctx, MCObjectWriter *llvm::createAArch64ELFObjectWriter(raw_pwrite_stream &OS, uint8_t OSABI, - bool IsLittleEndian) { + bool IsLittleEndian, + bool IsILP32) { MCELFObjectTargetWriter *MOTW = - new AArch64ELFObjectWriter(OSABI, IsLittleEndian); + new AArch64ELFObjectWriter(OSABI, IsLittleEndian, IsILP32); return createELFObjectWriter(MOTW, OS, IsLittleEndian); } diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp index fbce26e1d9a1..8fc822329595 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCAsmInfo.cpp @@ -29,8 +29,7 @@ static cl::opt AsmWriterVariant( "aarch64-neon-syntax", cl::init(Default), cl::desc("Choose style of NEON code to emit from AArch64 backend:"), cl::values(clEnumValN(Generic, "generic", "Emit generic NEON assembly"), - clEnumValN(Apple, "apple", "Emit Apple-style NEON assembly"), - clEnumValEnd)); + clEnumValN(Apple, "apple", "Emit Apple-style NEON assembly"))); AArch64MCAsmInfoDarwin::AArch64MCAsmInfoDarwin() { // We prefer NEON instructions to be printed in the short form. diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp index 7b9ff8fa0503..f7058cdf2373 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCCodeEmitter.cpp @@ -35,11 +35,13 @@ namespace { class AArch64MCCodeEmitter : public MCCodeEmitter { MCContext &Ctx; + const MCInstrInfo &MCII; AArch64MCCodeEmitter(const AArch64MCCodeEmitter &); // DO NOT IMPLEMENT void operator=(const AArch64MCCodeEmitter &); // DO NOT IMPLEMENT public: - AArch64MCCodeEmitter(const MCInstrInfo &mcii, MCContext &ctx) : Ctx(ctx) {} + AArch64MCCodeEmitter(const MCInstrInfo &mcii, MCContext &ctx) + : Ctx(ctx), MCII(mcii) {} ~AArch64MCCodeEmitter() override {} @@ -170,6 +172,11 @@ public: unsigned fixOneOperandFPComparison(const MCInst &MI, unsigned EncodedValue, const MCSubtargetInfo &STI) const; + +private: + uint64_t computeAvailableFeatures(const FeatureBitset &FB) const; + void verifyInstructionPredicates(const MCInst &MI, + uint64_t AvailableFeatures) const; }; } // end anonymous namespace @@ -253,7 +260,7 @@ AArch64MCCodeEmitter::getAddSubImmOpValue(const MCInst &MI, unsigned OpIdx, assert((ShiftVal == 0 || ShiftVal == 12) && "unexpected shift value for add/sub immediate"); if (MO.isImm()) - return MO.getImm() | (ShiftVal == 0 ? 0 : (1 << 12)); + return MO.getImm() | (ShiftVal == 0 ? 0 : (1 << ShiftVal)); assert(MO.isExpr() && "Unable to encode MCOperand!"); const MCExpr *Expr = MO.getExpr(); @@ -263,7 +270,15 @@ AArch64MCCodeEmitter::getAddSubImmOpValue(const MCInst &MI, unsigned OpIdx, ++MCNumFixups; - return 0; + // Set the shift bit of the add instruction for relocation types + // R_AARCH64_TLSLE_ADD_TPREL_HI12 and R_AARCH64_TLSLD_ADD_DTPREL_HI12. + if (const AArch64MCExpr *A64E = dyn_cast(Expr)) { + AArch64MCExpr::VariantKind RefKind = A64E->getKind(); + if (RefKind == AArch64MCExpr::VK_TPREL_HI12 || + RefKind == AArch64MCExpr::VK_DTPREL_HI12) + ShiftVal = 12; + } + return ShiftVal == 0 ? 0 : (1 << ShiftVal); } /// getCondBranchTargetOpValue - Return the encoded value for a conditional @@ -539,6 +554,9 @@ unsigned AArch64MCCodeEmitter::fixMOVZ(const MCInst &MI, unsigned EncodedValue, void AArch64MCCodeEmitter::encodeInstruction(const MCInst &MI, raw_ostream &OS, SmallVectorImpl &Fixups, const MCSubtargetInfo &STI) const { + verifyInstructionPredicates(MI, + computeAvailableFeatures(STI.getFeatureBits())); + if (MI.getOpcode() == AArch64::TLSDESCCALL) { // This is a directive which applies an R_AARCH64_TLSDESC_CALL to the // following (BLR) instruction. It doesn't emit any code itself so it @@ -581,4 +599,5 @@ unsigned AArch64MCCodeEmitter::fixOneOperandFPComparison( return EncodedValue; } +#define ENABLE_INSTR_PREDICATE_VERIFIER #include "AArch64GenMCCodeEmitter.inc" diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp index 702780621208..e9d38d3dcf10 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.cpp @@ -15,6 +15,7 @@ #include "AArch64ELFStreamer.h" #include "AArch64MCAsmInfo.h" #include "InstPrinter/AArch64InstPrinter.h" +#include "llvm/MC/MCInstrAnalysis.h" #include "llvm/MC/MCInstrInfo.h" #include "llvm/MC/MCRegisterInfo.h" #include "llvm/MC/MCStreamer.h" @@ -116,10 +117,14 @@ static MCStreamer *createMachOStreamer(MCContext &Ctx, MCAsmBackend &TAB, /*LabelSections*/ true); } +static MCInstrAnalysis *createAArch64InstrAnalysis(const MCInstrInfo *Info) { + return new MCInstrAnalysis(Info); +} + // Force static initialization. extern "C" void LLVMInitializeAArch64TargetMC() { - for (Target *T : - {&TheAArch64leTarget, &TheAArch64beTarget, &TheARM64Target}) { + for (Target *T : {&getTheAArch64leTarget(), &getTheAArch64beTarget(), + &getTheARM64Target()}) { // Register the MC asm info. RegisterMCAsmInfoFn X(*T, createAArch64MCAsmInfo); @@ -135,6 +140,9 @@ extern "C" void LLVMInitializeAArch64TargetMC() { // Register the MC subtarget info. TargetRegistry::RegisterMCSubtargetInfo(*T, createAArch64MCSubtargetInfo); + // Register the MC instruction analyzer. + TargetRegistry::RegisterMCInstrAnalysis(*T, createAArch64InstrAnalysis); + // Register the MC Code Emitter TargetRegistry::RegisterMCCodeEmitter(*T, createAArch64MCCodeEmitter); @@ -154,8 +162,8 @@ extern "C" void LLVMInitializeAArch64TargetMC() { } // Register the asm backend. - for (Target *T : {&TheAArch64leTarget, &TheARM64Target}) + for (Target *T : {&getTheAArch64leTarget(), &getTheARM64Target()}) TargetRegistry::RegisterMCAsmBackend(*T, createAArch64leAsmBackend); - TargetRegistry::RegisterMCAsmBackend(TheAArch64beTarget, + TargetRegistry::RegisterMCAsmBackend(getTheAArch64beTarget(), createAArch64beAsmBackend); } diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h b/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h index 39414cc0c6a5..615d7dab2c51 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h +++ b/lib/Target/AArch64/MCTargetDesc/AArch64MCTargetDesc.h @@ -27,6 +27,7 @@ class MCRegisterInfo; class MCObjectWriter; class MCStreamer; class MCSubtargetInfo; +class MCTargetOptions; class MCTargetStreamer; class StringRef; class Target; @@ -34,23 +35,26 @@ class Triple; class raw_ostream; class raw_pwrite_stream; -extern Target TheAArch64leTarget; -extern Target TheAArch64beTarget; -extern Target TheARM64Target; +Target &getTheAArch64leTarget(); +Target &getTheAArch64beTarget(); +Target &getTheARM64Target(); MCCodeEmitter *createAArch64MCCodeEmitter(const MCInstrInfo &MCII, const MCRegisterInfo &MRI, MCContext &Ctx); MCAsmBackend *createAArch64leAsmBackend(const Target &T, const MCRegisterInfo &MRI, - const Triple &TT, StringRef CPU); + const Triple &TT, StringRef CPU, + const MCTargetOptions &Options); MCAsmBackend *createAArch64beAsmBackend(const Target &T, const MCRegisterInfo &MRI, - const Triple &TT, StringRef CPU); + const Triple &TT, StringRef CPU, + const MCTargetOptions &Options); MCObjectWriter *createAArch64ELFObjectWriter(raw_pwrite_stream &OS, uint8_t OSABI, - bool IsLittleEndian); + bool IsLittleEndian, + bool IsILP32); MCObjectWriter *createAArch64MachObjectWriter(raw_pwrite_stream &OS, uint32_t CPUType, diff --git a/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp b/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp index 61c96f1d93c1..53a68527ee8e 100644 --- a/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp +++ b/lib/Target/AArch64/MCTargetDesc/AArch64MachObjectWriter.cpp @@ -75,7 +75,7 @@ bool AArch64MachObjectWriter::getAArch64FixupKindMachOInfo( Log2Size = llvm::Log2_32(4); switch (Sym->getKind()) { default: - llvm_unreachable("Unexpected symbol reference variant kind!"); + return false; case MCSymbolRefExpr::VK_PAGEOFF: RelocType = unsigned(MachO::ARM64_RELOC_PAGEOFF12); return true; diff --git a/lib/Target/AArch64/TargetInfo/AArch64TargetInfo.cpp b/lib/Target/AArch64/TargetInfo/AArch64TargetInfo.cpp index f42ecb1677de..7ac9a5a08484 100644 --- a/lib/Target/AArch64/TargetInfo/AArch64TargetInfo.cpp +++ b/lib/Target/AArch64/TargetInfo/AArch64TargetInfo.cpp @@ -10,23 +10,30 @@ #include "llvm/ADT/Triple.h" #include "llvm/Support/TargetRegistry.h" using namespace llvm; - namespace llvm { -Target TheAArch64leTarget; -Target TheAArch64beTarget; -Target TheARM64Target; -} // end namespace llvm +Target &getTheAArch64leTarget() { + static Target TheAArch64leTarget; + return TheAArch64leTarget; +} +Target &getTheAArch64beTarget() { + static Target TheAArch64beTarget; + return TheAArch64beTarget; +} +Target &getTheARM64Target() { + static Target TheARM64Target; + return TheARM64Target; +} +} // namespace llvm extern "C" void LLVMInitializeAArch64TargetInfo() { // Now register the "arm64" name for use with "-march". We don't want it to // take possession of the Triple::aarch64 tag though. - TargetRegistry::RegisterTarget(TheARM64Target, "arm64", + TargetRegistry::RegisterTarget(getTheARM64Target(), "arm64", "ARM64 (little endian)", [](Triple::ArchType) { return false; }, true); RegisterTarget Z( - TheAArch64leTarget, "aarch64", "AArch64 (little endian)"); + getTheAArch64leTarget(), "aarch64", "AArch64 (little endian)"); RegisterTarget W( - TheAArch64beTarget, "aarch64_be", "AArch64 (big endian)"); - + getTheAArch64beTarget(), "aarch64_be", "AArch64 (big endian)"); } diff --git a/lib/Target/AMDGPU/AMDGPU.h b/lib/Target/AMDGPU/AMDGPU.h index d4784b5463d7..7b0a7f4b6058 100644 --- a/lib/Target/AMDGPU/AMDGPU.h +++ b/lib/Target/AMDGPU/AMDGPU.h @@ -11,22 +11,18 @@ #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPU_H #define LLVM_LIB_TARGET_AMDGPU_AMDGPU_H -#include "llvm/Support/TargetRegistry.h" #include "llvm/Target/TargetMachine.h" namespace llvm { -class AMDGPUInstrPrinter; -class AMDGPUSubtarget; class AMDGPUTargetMachine; class FunctionPass; class GCNTargetMachine; -struct MachineSchedContext; -class MCAsmInfo; -class raw_ostream; -class ScheduleDAGInstrs; +class ModulePass; +class Pass; class Target; class TargetMachine; +class PassRegistry; // R600 Passes FunctionPass *createR600VectorRegMerger(TargetMachine &tm); @@ -45,16 +41,12 @@ FunctionPass *createSILowerI1CopiesPass(); FunctionPass *createSIShrinkInstructionsPass(); FunctionPass *createSILoadStoreOptimizerPass(TargetMachine &tm); FunctionPass *createSIWholeQuadModePass(); -FunctionPass *createSILowerControlFlowPass(); FunctionPass *createSIFixControlFlowLiveIntervalsPass(); FunctionPass *createSIFixSGPRCopiesPass(); -FunctionPass *createSICodeEmitterPass(formatted_raw_ostream &OS); FunctionPass *createSIDebuggerInsertNopsPass(); FunctionPass *createSIInsertWaitsPass(); FunctionPass *createAMDGPUCodeGenPreparePass(const GCNTargetMachine *TM = nullptr); -ScheduleDAGInstrs *createSIMachineScheduler(MachineSchedContext *C); - ModulePass *createAMDGPUAnnotateKernelFeaturesPass(); void initializeAMDGPUAnnotateKernelFeaturesPass(PassRegistry &); extern char &AMDGPUAnnotateKernelFeaturesID; @@ -78,21 +70,30 @@ void initializeSIWholeQuadModePass(PassRegistry &); extern char &SIWholeQuadModeID; void initializeSILowerControlFlowPass(PassRegistry &); -extern char &SILowerControlFlowPassID; +extern char &SILowerControlFlowID; + +void initializeSIInsertSkipsPass(PassRegistry &); +extern char &SIInsertSkipsPassID; +void initializeSIOptimizeExecMaskingPass(PassRegistry &); +extern char &SIOptimizeExecMaskingID; // Passes common to R600 and SI FunctionPass *createAMDGPUPromoteAlloca(const TargetMachine *TM = nullptr); void initializeAMDGPUPromoteAllocaPass(PassRegistry&); extern char &AMDGPUPromoteAllocaID; -FunctionPass *createAMDGPUAddDivergenceMetadata(const AMDGPUSubtarget &ST); Pass *createAMDGPUStructurizeCFGPass(); -FunctionPass *createAMDGPUISelDag(TargetMachine &tm); +FunctionPass *createAMDGPUISelDag(TargetMachine &TM, + CodeGenOpt::Level OptLevel); ModulePass *createAMDGPUAlwaysInlinePass(); ModulePass *createAMDGPUOpenCLImageTypeLoweringPass(); FunctionPass *createAMDGPUAnnotateUniformValues(); +FunctionPass* createAMDGPUUnifyMetadataPass(); +void initializeAMDGPUUnifyMetadataPass(PassRegistry&); +extern char &AMDGPUUnifyMetadataID; + void initializeSIFixControlFlowLiveIntervalsPass(PassRegistry&); extern char &SIFixControlFlowLiveIntervalsID; @@ -111,8 +112,8 @@ extern char &SIDebuggerInsertNopsID; void initializeSIInsertWaitsPass(PassRegistry&); extern char &SIInsertWaitsID; -extern Target TheAMDGPUTarget; -extern Target TheGCNTarget; +Target &getTheAMDGPUTarget(); +Target &getTheGCNTarget(); namespace AMDGPU { enum TargetIndex { diff --git a/lib/Target/AMDGPU/AMDGPU.td b/lib/Target/AMDGPU/AMDGPU.td index 72c455354411..0b2badff7ccf 100644 --- a/lib/Target/AMDGPU/AMDGPU.td +++ b/lib/Target/AMDGPU/AMDGPU.td @@ -67,6 +67,19 @@ def FeatureUnalignedBufferAccess : SubtargetFeature<"unaligned-buffer-access", "Support unaligned global loads and stores" >; +def FeatureUnalignedScratchAccess : SubtargetFeature<"unaligned-scratch-access", + "UnalignedScratchAccess", + "true", + "Support unaligned scratch loads and stores" +>; + +// XNACK is disabled if SH_MEM_CONFIG.ADDRESS_MODE = GPUVM on chips that support +// XNACK. The current default kernel driver setting is: +// - graphics ring: XNACK disabled +// - compute ring: XNACK enabled +// +// If XNACK is enabled, the VMEM latency can be worse. +// If XNACK is disabled, the 2 SGPRs can be used for general purposes. def FeatureXNACK : SubtargetFeature<"xnack", "EnableXNACK", "true", @@ -110,20 +123,6 @@ class SubtargetFeatureLDSBankCount : SubtargetFeature < def FeatureLDSBankCount16 : SubtargetFeatureLDSBankCount<16>; def FeatureLDSBankCount32 : SubtargetFeatureLDSBankCount<32>; -class SubtargetFeatureISAVersion - : SubtargetFeature < - "isaver"#Major#"."#Minor#"."#Stepping, - "IsaVersion", - "ISAVersion"#Major#"_"#Minor#"_"#Stepping, - "Instruction set version number" ->; - -def FeatureISAVersion7_0_0 : SubtargetFeatureISAVersion <7,0,0>; -def FeatureISAVersion7_0_1 : SubtargetFeatureISAVersion <7,0,1>; -def FeatureISAVersion8_0_0 : SubtargetFeatureISAVersion <8,0,0>; -def FeatureISAVersion8_0_1 : SubtargetFeatureISAVersion <8,0,1>; -def FeatureISAVersion8_0_3 : SubtargetFeatureISAVersion <8,0,3>; - class SubtargetFeatureLocalMemorySize : SubtargetFeature< "localmemorysize"#Value, "LocalMemorySize", @@ -161,16 +160,46 @@ def FeatureSMemRealTime : SubtargetFeature<"s-memrealtime", "Has s_memrealtime instruction" >; +def FeatureInv2PiInlineImm : SubtargetFeature<"inv-2pi-inline-imm", + "HasInv2PiInlineImm", + "true", + "Has 1 / (2 * pi) as inline immediate" +>; + def Feature16BitInsts : SubtargetFeature<"16-bit-insts", "Has16BitInsts", "true", "Has i16/f16 instructions" >; +def FeatureMovrel : SubtargetFeature<"movrel", + "HasMovrel", + "true", + "Has v_movrel*_b32 instructions" +>; + +def FeatureVGPRIndexMode : SubtargetFeature<"vgpr-index-mode", + "HasVGPRIndexMode", + "true", + "Has VGPR mode register indexing" +>; + +def FeatureScalarStores : SubtargetFeature<"scalar-stores", + "HasScalarStores", + "true", + "Has store scalar memory instructions" +>; + //===------------------------------------------------------------===// // Subtarget Features (options and debugging) //===------------------------------------------------------------===// +def FeatureFP16Denormals : SubtargetFeature<"fp16-denormals", + "FP16Denormals", + "true", + "Enable half precision denormal handling" +>; + // Some instructions do not support denormals despite this flag. Using // fp32 denormals also causes instructions to run at the double // precision rate for the device. @@ -294,23 +323,76 @@ def FeatureNorthernIslands : SubtargetFeatureGeneration<"NORTHERN_ISLANDS", def FeatureSouthernIslands : SubtargetFeatureGeneration<"SOUTHERN_ISLANDS", [FeatureFP64, FeatureLocalMemorySize32768, FeatureWavefrontSize64, FeatureGCN, FeatureGCN1Encoding, - FeatureLDSBankCount32] + FeatureLDSBankCount32, FeatureMovrel] >; def FeatureSeaIslands : SubtargetFeatureGeneration<"SEA_ISLANDS", [FeatureFP64, FeatureLocalMemorySize65536, FeatureWavefrontSize64, FeatureGCN, FeatureFlatAddressSpace, - FeatureGCN1Encoding, FeatureCIInsts] + FeatureGCN1Encoding, FeatureCIInsts, FeatureMovrel] >; def FeatureVolcanicIslands : SubtargetFeatureGeneration<"VOLCANIC_ISLANDS", [FeatureFP64, FeatureLocalMemorySize65536, FeatureWavefrontSize64, FeatureFlatAddressSpace, FeatureGCN, FeatureGCN3Encoding, FeatureCIInsts, Feature16BitInsts, - FeatureSMemRealTime + FeatureSMemRealTime, FeatureVGPRIndexMode, FeatureMovrel, + FeatureScalarStores, FeatureInv2PiInlineImm ] >; +class SubtargetFeatureISAVersion Implies> + : SubtargetFeature < + "isaver"#Major#"."#Minor#"."#Stepping, + "IsaVersion", + "ISAVersion"#Major#"_"#Minor#"_"#Stepping, + "Instruction set version number", + Implies +>; + +def FeatureISAVersion7_0_0 : SubtargetFeatureISAVersion <7,0,0, + [FeatureSeaIslands, + FeatureLDSBankCount32]>; + +def FeatureISAVersion7_0_1 : SubtargetFeatureISAVersion <7,0,1, + [FeatureSeaIslands, + HalfRate64Ops, + FeatureLDSBankCount32, + FeatureFastFMAF32]>; + +def FeatureISAVersion7_0_2 : SubtargetFeatureISAVersion <7,0,2, + [FeatureSeaIslands, + FeatureLDSBankCount16]>; + +def FeatureISAVersion8_0_0 : SubtargetFeatureISAVersion <8,0,0, + [FeatureVolcanicIslands, + FeatureLDSBankCount32, + FeatureSGPRInitBug]>; + +def FeatureISAVersion8_0_1 : SubtargetFeatureISAVersion <8,0,1, + [FeatureVolcanicIslands, + FeatureLDSBankCount32, + FeatureXNACK]>; + +def FeatureISAVersion8_0_2 : SubtargetFeatureISAVersion <8,0,2, + [FeatureVolcanicIslands, + FeatureLDSBankCount32, + FeatureSGPRInitBug]>; + +def FeatureISAVersion8_0_3 : SubtargetFeatureISAVersion <8,0,3, + [FeatureVolcanicIslands, + FeatureLDSBankCount32]>; + +def FeatureISAVersion8_0_4 : SubtargetFeatureISAVersion <8,0,4, + [FeatureVolcanicIslands, + FeatureLDSBankCount32]>; + +def FeatureISAVersion8_1_0 : SubtargetFeatureISAVersion <8,1,0, + [FeatureVolcanicIslands, + FeatureLDSBankCount16, + FeatureXNACK]>; + //===----------------------------------------------------------------------===// // Debugger related subtarget features. //===----------------------------------------------------------------------===// @@ -349,10 +431,52 @@ def AMDGPUAsmParser : AsmParser { let ShouldEmitMatchRegisterName = 0; } +def AMDGPUAsmWriter : AsmWriter { + int PassSubtarget = 1; +} + +def AMDGPUAsmVariants { + string Default = "Default"; + int Default_ID = 0; + string VOP3 = "VOP3"; + int VOP3_ID = 1; + string SDWA = "SDWA"; + int SDWA_ID = 2; + string DPP = "DPP"; + int DPP_ID = 3; + string Disable = "Disable"; + int Disable_ID = 4; +} + +def DefaultAMDGPUAsmParserVariant : AsmParserVariant { + let Variant = AMDGPUAsmVariants.Default_ID; + let Name = AMDGPUAsmVariants.Default; +} + +def VOP3AsmParserVariant : AsmParserVariant { + let Variant = AMDGPUAsmVariants.VOP3_ID; + let Name = AMDGPUAsmVariants.VOP3; +} + +def SDWAAsmParserVariant : AsmParserVariant { + let Variant = AMDGPUAsmVariants.SDWA_ID; + let Name = AMDGPUAsmVariants.SDWA; +} + +def DPPAsmParserVariant : AsmParserVariant { + let Variant = AMDGPUAsmVariants.DPP_ID; + let Name = AMDGPUAsmVariants.DPP; +} + def AMDGPU : Target { // Pull in Instruction Info: let InstructionSet = AMDGPUInstrInfo; let AssemblyParsers = [AMDGPUAsmParser]; + let AssemblyParserVariants = [DefaultAMDGPUAsmParserVariant, + VOP3AsmParserVariant, + SDWAAsmParserVariant, + DPPAsmParserVariant]; + let AssemblyWriters = [AMDGPUAsmWriter]; } // Dummy Instruction itineraries for pseudo instructions @@ -381,6 +505,8 @@ def isCIVI : Predicate < def HasFlatAddressSpace : Predicate<"Subtarget->hasFlatAddressSpace()">; +def Has16BitInsts : Predicate<"Subtarget->has16BitInsts()">; + class PredicateControl { Predicate SubtargetPredicate; Predicate SIAssemblerPredicate = isSICI; diff --git a/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp b/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp index 63f5fb3cdf00..067a16a2af7f 100644 --- a/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp +++ b/lib/Target/AMDGPU/AMDGPUAlwaysInlinePass.cpp @@ -27,7 +27,7 @@ class AMDGPUAlwaysInline : public ModulePass { public: AMDGPUAlwaysInline() : ModulePass(ID) { } bool runOnModule(Module &M) override; - const char *getPassName() const override { return "AMDGPU Always Inline Pass"; } + StringRef getPassName() const override { return "AMDGPU Always Inline Pass"; } }; } // End anonymous namespace @@ -35,8 +35,20 @@ public: char AMDGPUAlwaysInline::ID = 0; bool AMDGPUAlwaysInline::runOnModule(Module &M) { + std::vector AliasesToRemove; std::vector FuncsToClone; + for (GlobalAlias &A : M.aliases()) { + if (Function* F = dyn_cast(A.getAliasee())) { + A.replaceAllUsesWith(F); + AliasesToRemove.push_back(&A); + } + } + + for (GlobalAlias* A : AliasesToRemove) { + A->eraseFromParent(); + } + for (Function &F : M) { if (!F.hasLocalLinkage() && !F.isDeclaration() && !F.use_empty() && !F.hasFnAttribute(Attribute::NoInline)) diff --git a/lib/Target/AMDGPU/AMDGPUAnnotateKernelFeatures.cpp b/lib/Target/AMDGPU/AMDGPUAnnotateKernelFeatures.cpp index 0910b2877b09..c98d25e20185 100644 --- a/lib/Target/AMDGPU/AMDGPUAnnotateKernelFeatures.cpp +++ b/lib/Target/AMDGPU/AMDGPUAnnotateKernelFeatures.cpp @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// #include "AMDGPU.h" +#include "llvm/ADT/Triple.h" #include "llvm/IR/Constants.h" #include "llvm/IR/Instructions.h" #include "llvm/IR/Module.h" @@ -35,7 +36,7 @@ public: AMDGPUAnnotateKernelFeatures() : ModulePass(ID) { } bool runOnModule(Module &M) override; - const char *getPassName() const override { + StringRef getPassName() const override { return "AMDGPU Annotate Kernel Features"; } @@ -188,7 +189,8 @@ bool AMDGPUAnnotateKernelFeatures::runOnModule(Module &M) { static const StringRef HSAIntrinsicToAttr[][2] = { { "llvm.amdgcn.dispatch.ptr", "amdgpu-dispatch-ptr" }, - { "llvm.amdgcn.queue.ptr", "amdgpu-queue-ptr" } + { "llvm.amdgcn.queue.ptr", "amdgpu-queue-ptr" }, + { "llvm.amdgcn.dispatch.id", "amdgpu-dispatch-id" } }; // TODO: We should not add the attributes if the known compile time workgroup @@ -200,7 +202,7 @@ bool AMDGPUAnnotateKernelFeatures::runOnModule(Module &M) { // always initialized. bool Changed = addAttrsForIntrinsics(M, IntrinsicToAttr); - if (TT.getOS() == Triple::AMDHSA) { + if (TT.getOS() == Triple::AMDHSA || TT.getOS() == Triple::Mesa3D) { Changed |= addAttrsForIntrinsics(M, HSAIntrinsicToAttr); for (Function &F : M) { diff --git a/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp b/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp index 2010cc952265..c011be6fa169 100644 --- a/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp +++ b/lib/Target/AMDGPU/AMDGPUAnnotateUniformValues.cpp @@ -15,7 +15,10 @@ #include "AMDGPU.h" #include "AMDGPUIntrinsicInfo.h" +#include "llvm/ADT/SetVector.h" #include "llvm/Analysis/DivergenceAnalysis.h" +#include "llvm/Analysis/LoopInfo.h" +#include "llvm/Analysis/MemoryDependenceAnalysis.h" #include "llvm/IR/InstVisitor.h" #include "llvm/IR/IRBuilder.h" #include "llvm/Support/Debug.h" @@ -30,6 +33,10 @@ namespace { class AMDGPUAnnotateUniformValues : public FunctionPass, public InstVisitor { DivergenceAnalysis *DA; + MemoryDependenceResults *MDR; + LoopInfo *LI; + DenseMap noClobberClones; + bool isKernelFunc; public: static char ID; @@ -37,15 +44,19 @@ public: FunctionPass(ID) { } bool doInitialization(Module &M) override; bool runOnFunction(Function &F) override; - const char *getPassName() const override { return "AMDGPU Annotate Uniform Values"; } + StringRef getPassName() const override { + return "AMDGPU Annotate Uniform Values"; + } void getAnalysisUsage(AnalysisUsage &AU) const override { AU.addRequired(); + AU.addRequired(); + AU.addRequired(); AU.setPreservesAll(); } void visitBranchInst(BranchInst &I); void visitLoadInst(LoadInst &I); - + bool isClobberedInFunction(LoadInst * Load); }; } // End anonymous namespace @@ -53,6 +64,8 @@ public: INITIALIZE_PASS_BEGIN(AMDGPUAnnotateUniformValues, DEBUG_TYPE, "Add AMDGPU uniform metadata", false, false) INITIALIZE_PASS_DEPENDENCY(DivergenceAnalysis) +INITIALIZE_PASS_DEPENDENCY(MemoryDependenceWrapperPass) +INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) INITIALIZE_PASS_END(AMDGPUAnnotateUniformValues, DEBUG_TYPE, "Add AMDGPU uniform metadata", false, false) @@ -61,6 +74,46 @@ char AMDGPUAnnotateUniformValues::ID = 0; static void setUniformMetadata(Instruction *I) { I->setMetadata("amdgpu.uniform", MDNode::get(I->getContext(), {})); } +static void setNoClobberMetadata(Instruction *I) { + I->setMetadata("amdgpu.noclobber", MDNode::get(I->getContext(), {})); +} + +static void DFS(BasicBlock *Root, SetVector & Set) { + for (auto I : predecessors(Root)) + if (Set.insert(I)) + DFS(I, Set); +} + +bool AMDGPUAnnotateUniformValues::isClobberedInFunction(LoadInst * Load) { + // 1. get Loop for the Load->getparent(); + // 2. if it exists, collect all the BBs from the most outer + // loop and check for the writes. If NOT - start DFS over all preds. + // 3. Start DFS over all preds from the most outer loop header. + SetVector Checklist; + BasicBlock *Start = Load->getParent(); + Checklist.insert(Start); + const Value *Ptr = Load->getPointerOperand(); + const Loop *L = LI->getLoopFor(Start); + if (L) { + const Loop *P = L; + do { + L = P; + P = P->getParentLoop(); + } while (P); + Checklist.insert(L->block_begin(), L->block_end()); + Start = L->getHeader(); + } + + DFS(Start, Checklist); + for (auto &BB : Checklist) { + BasicBlock::iterator StartIt = (BB == Load->getParent()) ? + BasicBlock::iterator(Load) : BB->end(); + if (MDR->getPointerDependencyFrom(MemoryLocation(Ptr), + true, StartIt, BB, Load).isClobber()) + return true; + } + return false; +} void AMDGPUAnnotateUniformValues::visitBranchInst(BranchInst &I) { if (I.isUnconditional()) @@ -77,10 +130,39 @@ void AMDGPUAnnotateUniformValues::visitLoadInst(LoadInst &I) { Value *Ptr = I.getPointerOperand(); if (!DA->isUniform(Ptr)) return; - - if (Instruction *PtrI = dyn_cast(Ptr)) + auto isGlobalLoad = [](LoadInst &Load)->bool { + return Load.getPointerAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS; + }; + // We're tracking up to the Function boundaries + // We cannot go beyond because of FunctionPass restrictions + // Thus we can ensure that memory not clobbered for memory + // operations that live in kernel only. + bool NotClobbered = isKernelFunc && !isClobberedInFunction(&I); + Instruction *PtrI = dyn_cast(Ptr); + if (!PtrI && NotClobbered && isGlobalLoad(I)) { + if (isa(Ptr) || isa(Ptr)) { + // Lookup for the existing GEP + if (noClobberClones.count(Ptr)) { + PtrI = noClobberClones[Ptr]; + } else { + // Create GEP of the Value + Function *F = I.getParent()->getParent(); + Value *Idx = Constant::getIntegerValue( + Type::getInt32Ty(Ptr->getContext()), APInt(64, 0)); + // Insert GEP at the entry to make it dominate all uses + PtrI = GetElementPtrInst::Create( + Ptr->getType()->getPointerElementType(), Ptr, + ArrayRef(Idx), Twine(""), F->getEntryBlock().getFirstNonPHI()); + } + I.replaceUsesOfWith(Ptr, PtrI); + } + } + + if (PtrI) { setUniformMetadata(PtrI); - + if (NotClobbered) + setNoClobberMetadata(PtrI); + } } bool AMDGPUAnnotateUniformValues::doInitialization(Module &M) { @@ -91,9 +173,13 @@ bool AMDGPUAnnotateUniformValues::runOnFunction(Function &F) { if (skipFunction(F)) return false; - DA = &getAnalysis(); - visit(F); + DA = &getAnalysis(); + MDR = &getAnalysis().getMemDep(); + LI = &getAnalysis().getLoopInfo(); + isKernelFunc = F.getCallingConv() == CallingConv::AMDGPU_KERNEL; + visit(F); + noClobberClones.clear(); return true; } diff --git a/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp index c9c95c796a69..a8e6902c252b 100644 --- a/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp +++ b/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp @@ -39,9 +39,7 @@ #include "llvm/Support/MathExtras.h" #include "llvm/Support/TargetRegistry.h" #include "llvm/Target/TargetLoweringObjectFile.h" -#include "AMDGPURuntimeMetadata.h" -using namespace ::AMDGPU; using namespace llvm; // TODO: This should get the default rounding mode from the kernel. We just set @@ -87,13 +85,19 @@ createAMDGPUAsmPrinterPass(TargetMachine &tm, } extern "C" void LLVMInitializeAMDGPUAsmPrinter() { - TargetRegistry::RegisterAsmPrinter(TheAMDGPUTarget, createAMDGPUAsmPrinterPass); - TargetRegistry::RegisterAsmPrinter(TheGCNTarget, createAMDGPUAsmPrinterPass); + TargetRegistry::RegisterAsmPrinter(getTheAMDGPUTarget(), + createAMDGPUAsmPrinterPass); + TargetRegistry::RegisterAsmPrinter(getTheGCNTarget(), + createAMDGPUAsmPrinterPass); } AMDGPUAsmPrinter::AMDGPUAsmPrinter(TargetMachine &TM, std::unique_ptr Streamer) - : AsmPrinter(TM, std::move(Streamer)) {} + : AsmPrinter(TM, std::move(Streamer)) {} + +StringRef AMDGPUAsmPrinter::getPassName() const { + return "AMDGPU Assembly Printer"; +} void AMDGPUAsmPrinter::EmitStartOfAsmFile(Module &M) { if (TM.getTargetTriple().getOS() != Triple::AMDHSA) @@ -113,13 +117,30 @@ void AMDGPUAsmPrinter::EmitStartOfAsmFile(Module &M) { AMDGPU::IsaVersion ISA = AMDGPU::getIsaVersion(STI->getFeatureBits()); TS->EmitDirectiveHSACodeObjectISA(ISA.Major, ISA.Minor, ISA.Stepping, "AMD", "AMDGPU"); - emitStartOfRuntimeMetadata(M); + + // Emit runtime metadata. + TS->EmitRuntimeMetadata(M); } +bool AMDGPUAsmPrinter::isBlockOnlyReachableByFallthrough( + const MachineBasicBlock *MBB) const { + if (!AsmPrinter::isBlockOnlyReachableByFallthrough(MBB)) + return false; + + if (MBB->empty()) + return true; + + // If this is a block implementing a long branch, an expression relative to + // the start of the block is needed. to the start of the block. + // XXX - Is there a smarter way to check this? + return (MBB->back().getOpcode() != AMDGPU::S_SETPC_B64); +} + + void AMDGPUAsmPrinter::EmitFunctionBodyStart() { const AMDGPUSubtarget &STM = MF->getSubtarget(); SIProgramInfo KernelInfo; - if (STM.isAmdHsaOS()) { + if (STM.isAmdCodeObjectV2()) { getSIProgramInfo(KernelInfo, *MF); EmitAmdKernelCodeT(*MF, KernelInfo); } @@ -128,11 +149,12 @@ void AMDGPUAsmPrinter::EmitFunctionBodyStart() { void AMDGPUAsmPrinter::EmitFunctionEntryLabel() { const SIMachineFunctionInfo *MFI = MF->getInfo(); const AMDGPUSubtarget &STM = MF->getSubtarget(); - if (MFI->isKernel() && STM.isAmdHsaOS()) { + if (MFI->isKernel() && STM.isAmdCodeObjectV2()) { AMDGPUTargetStreamer *TS = static_cast(OutStreamer->getTargetStreamer()); - TS->EmitAMDGPUSymbolType(CurrentFnSym->getName(), - ELF::STT_AMDGPU_HSA_KERNEL); + SmallString<128> SymbolName; + getNameWithPrefix(SymbolName, MF->getFunction()), + TS->EmitAMDGPUSymbolType(SymbolName, ELF::STT_AMDGPU_HSA_KERNEL); } AsmPrinter::EmitFunctionEntryLabel(); @@ -198,6 +220,16 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) { OutStreamer->emitRawComment(" LDSByteSize: " + Twine(KernelInfo.LDSSize) + " bytes/workgroup (compile time only)", false); + OutStreamer->emitRawComment(" SGPRBlocks: " + + Twine(KernelInfo.SGPRBlocks), false); + OutStreamer->emitRawComment(" VGPRBlocks: " + + Twine(KernelInfo.VGPRBlocks), false); + + OutStreamer->emitRawComment(" NumSGPRsForWavesPerEU: " + + Twine(KernelInfo.NumSGPRsForWavesPerEU), false); + OutStreamer->emitRawComment(" NumVGPRsForWavesPerEU: " + + Twine(KernelInfo.NumVGPRsForWavesPerEU), false); + OutStreamer->emitRawComment(" ReservedVGPRFirst: " + Twine(KernelInfo.ReservedVGPRFirst), false); OutStreamer->emitRawComment(" ReservedVGPRCount: " + Twine(KernelInfo.ReservedVGPRCount), @@ -229,7 +261,7 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) { } else { R600MachineFunctionInfo *MFI = MF.getInfo(); OutStreamer->emitRawComment( - Twine("SQ_PGM_RESOURCES:STACK_SIZE = " + Twine(MFI->StackSize))); + Twine("SQ_PGM_RESOURCES:STACK_SIZE = " + Twine(MFI->CFStackSize))); } } @@ -247,8 +279,6 @@ bool AMDGPUAsmPrinter::runOnMachineFunction(MachineFunction &MF) { } } - emitRuntimeMetadata(*MF.getFunction()); - return false; } @@ -282,7 +312,7 @@ void AMDGPUAsmPrinter::EmitProgramInfoR600(const MachineFunction &MF) { if (STM.getGeneration() >= R600Subtarget::EVERGREEN) { // Evergreen / Northern Islands switch (MF.getFunction()->getCallingConv()) { - default: // Fall through + default: LLVM_FALLTHROUGH; case CallingConv::AMDGPU_CS: RsrcReg = R_0288D4_SQ_PGM_RESOURCES_LS; break; case CallingConv::AMDGPU_GS: RsrcReg = R_028878_SQ_PGM_RESOURCES_GS; break; case CallingConv::AMDGPU_PS: RsrcReg = R_028844_SQ_PGM_RESOURCES_PS; break; @@ -291,9 +321,9 @@ void AMDGPUAsmPrinter::EmitProgramInfoR600(const MachineFunction &MF) { } else { // R600 / R700 switch (MF.getFunction()->getCallingConv()) { - default: // Fall through - case CallingConv::AMDGPU_GS: // Fall through - case CallingConv::AMDGPU_CS: // Fall through + default: LLVM_FALLTHROUGH; + case CallingConv::AMDGPU_GS: LLVM_FALLTHROUGH; + case CallingConv::AMDGPU_CS: LLVM_FALLTHROUGH; case CallingConv::AMDGPU_VS: RsrcReg = R_028868_SQ_PGM_RESOURCES_VS; break; case CallingConv::AMDGPU_PS: RsrcReg = R_028850_SQ_PGM_RESOURCES_PS; break; } @@ -301,13 +331,13 @@ void AMDGPUAsmPrinter::EmitProgramInfoR600(const MachineFunction &MF) { OutStreamer->EmitIntValue(RsrcReg, 4); OutStreamer->EmitIntValue(S_NUM_GPRS(MaxGPR + 1) | - S_STACK_SIZE(MFI->StackSize), 4); + S_STACK_SIZE(MFI->CFStackSize), 4); OutStreamer->EmitIntValue(R_02880C_DB_SHADER_CONTROL, 4); OutStreamer->EmitIntValue(S_02880C_KILL_ENABLE(killPixel), 4); if (AMDGPU::isCompute(MF.getFunction()->getCallingConv())) { OutStreamer->EmitIntValue(R_0288E8_SQ_LDS_ALLOC, 4); - OutStreamer->EmitIntValue(alignTo(MFI->LDSSize, 4) >> 2, 4); + OutStreamer->EmitIntValue(alignTo(MFI->getLDSSize(), 4) >> 2, 4); } } @@ -331,7 +361,8 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo, if (MI.isDebugValue()) continue; - CodeSize += TII->getInstSizeInBytes(MI); + if (isVerbose()) + CodeSize += TII->getInstSizeInBytes(MI); unsigned numOperands = MI.getNumOperands(); for (unsigned op_idx = 0; op_idx < numOperands; op_idx++) { @@ -360,7 +391,10 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo, case AMDGPU::FLAT_SCR: case AMDGPU::FLAT_SCR_LO: case AMDGPU::FLAT_SCR_HI: - FlatUsed = true; + // Even if FLAT_SCRATCH is implicitly used, it has no effect if flat + // instructions aren't used to access the scratch buffer. + if (MFI->hasFlatScratchInit()) + FlatUsed = true; continue; case AMDGPU::TBA: @@ -369,26 +403,23 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo, case AMDGPU::TMA: case AMDGPU::TMA_LO: case AMDGPU::TMA_HI: - llvm_unreachable("Trap Handler registers should not be used"); - continue; + llvm_unreachable("trap handler registers should not be used"); default: break; } if (AMDGPU::SReg_32RegClass.contains(reg)) { - if (AMDGPU::TTMP_32RegClass.contains(reg)) { - llvm_unreachable("Trap Handler registers should not be used"); - } + assert(!AMDGPU::TTMP_32RegClass.contains(reg) && + "trap handler registers should not be used"); isSGPR = true; width = 1; } else if (AMDGPU::VGPR_32RegClass.contains(reg)) { isSGPR = false; width = 1; } else if (AMDGPU::SReg_64RegClass.contains(reg)) { - if (AMDGPU::TTMP_64RegClass.contains(reg)) { - llvm_unreachable("Trap Handler registers should not be used"); - } + assert(!AMDGPU::TTMP_64RegClass.contains(reg) && + "trap handler registers should not be used"); isSGPR = true; width = 2; } else if (AMDGPU::VReg_64RegClass.contains(reg)) { @@ -445,20 +476,15 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo, ExtraSGPRs = 6; } - MaxSGPR += ExtraSGPRs; - // Record first reserved register and reserved register count fields, and // update max register counts if "amdgpu-debugger-reserve-regs" attribute was - // specified. - if (STM.debuggerReserveRegs()) { - ProgInfo.ReservedVGPRFirst = MaxVGPR + 1; - ProgInfo.ReservedVGPRCount = MFI->getDebuggerReservedVGPRCount(); - MaxVGPR += MFI->getDebuggerReservedVGPRCount(); - } + // requested. + ProgInfo.ReservedVGPRFirst = STM.debuggerReserveRegs() ? MaxVGPR + 1 : 0; + ProgInfo.ReservedVGPRCount = RI->getNumDebuggerReservedVGPRs(STM); // Update DebuggerWavefrontPrivateSegmentOffsetSGPR and // DebuggerPrivateSegmentBufferSGPR fields if "amdgpu-debugger-emit-prologue" - // attribute was specified. + // attribute was requested. if (STM.debuggerEmitPrologue()) { ProgInfo.DebuggerWavefrontPrivateSegmentOffsetSGPR = RI->getHWRegIndex(MFI->getScratchWaveOffsetReg()); @@ -466,21 +492,59 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo, RI->getHWRegIndex(MFI->getScratchRSrcReg()); } + // Check the addressable register limit before we add ExtraSGPRs. + if (STM.getGeneration() >= AMDGPUSubtarget::VOLCANIC_ISLANDS && + !STM.hasSGPRInitBug()) { + unsigned MaxAddressableNumSGPRs = STM.getMaxNumSGPRs(); + if (MaxSGPR + 1 > MaxAddressableNumSGPRs) { + // This can happen due to a compiler bug or when using inline asm. + LLVMContext &Ctx = MF.getFunction()->getContext(); + DiagnosticInfoResourceLimit Diag(*MF.getFunction(), + "addressable scalar registers", + MaxSGPR + 1, DS_Error, + DK_ResourceLimit, MaxAddressableNumSGPRs); + Ctx.diagnose(Diag); + MaxSGPR = MaxAddressableNumSGPRs - 1; + } + } + + // Account for extra SGPRs and VGPRs reserved for debugger use. + MaxSGPR += ExtraSGPRs; + MaxVGPR += RI->getNumDebuggerReservedVGPRs(STM); + // We found the maximum register index. They start at 0, so add one to get the // number of registers. ProgInfo.NumVGPR = MaxVGPR + 1; ProgInfo.NumSGPR = MaxSGPR + 1; - if (STM.hasSGPRInitBug()) { - if (ProgInfo.NumSGPR > SISubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG) { + // Adjust number of registers used to meet default/requested minimum/maximum + // number of waves per execution unit request. + ProgInfo.NumSGPRsForWavesPerEU = std::max( + ProgInfo.NumSGPR, RI->getMinNumSGPRs(STM, MFI->getMaxWavesPerEU())); + ProgInfo.NumVGPRsForWavesPerEU = std::max( + ProgInfo.NumVGPR, RI->getMinNumVGPRs(MFI->getMaxWavesPerEU())); + + if (STM.getGeneration() <= AMDGPUSubtarget::SEA_ISLANDS || + STM.hasSGPRInitBug()) { + unsigned MaxNumSGPRs = STM.getMaxNumSGPRs(); + if (ProgInfo.NumSGPR > MaxNumSGPRs) { + // This can happen due to a compiler bug or when using inline asm to use the + // registers which are usually reserved for vcc etc. + LLVMContext &Ctx = MF.getFunction()->getContext(); DiagnosticInfoResourceLimit Diag(*MF.getFunction(), - "SGPRs with SGPR init bug", - ProgInfo.NumSGPR, DS_Error); + "scalar registers", + ProgInfo.NumSGPR, DS_Error, + DK_ResourceLimit, MaxNumSGPRs); Ctx.diagnose(Diag); + ProgInfo.NumSGPR = MaxNumSGPRs; + ProgInfo.NumSGPRsForWavesPerEU = MaxNumSGPRs; } + } + if (STM.hasSGPRInitBug()) { ProgInfo.NumSGPR = SISubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG; + ProgInfo.NumSGPRsForWavesPerEU = SISubtarget::FIXED_SGPR_COUNT_FOR_INIT_BUG; } if (MFI->NumUserSGPRs > STM.getMaxNumUserSGPRs()) { @@ -490,26 +554,34 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo, Ctx.diagnose(Diag); } - if (MFI->LDSSize > static_cast(STM.getLocalMemorySize())) { + if (MFI->getLDSSize() > static_cast(STM.getLocalMemorySize())) { LLVMContext &Ctx = MF.getFunction()->getContext(); DiagnosticInfoResourceLimit Diag(*MF.getFunction(), "local memory", - MFI->LDSSize, DS_Error); + MFI->getLDSSize(), DS_Error); Ctx.diagnose(Diag); } - ProgInfo.VGPRBlocks = (ProgInfo.NumVGPR - 1) / 4; - ProgInfo.SGPRBlocks = (ProgInfo.NumSGPR - 1) / 8; + // SGPRBlocks is actual number of SGPR blocks minus 1. + ProgInfo.SGPRBlocks = alignTo(ProgInfo.NumSGPRsForWavesPerEU, + RI->getSGPRAllocGranule()); + ProgInfo.SGPRBlocks = ProgInfo.SGPRBlocks / RI->getSGPRAllocGranule() - 1; + + // VGPRBlocks is actual number of VGPR blocks minus 1. + ProgInfo.VGPRBlocks = alignTo(ProgInfo.NumVGPRsForWavesPerEU, + RI->getVGPRAllocGranule()); + ProgInfo.VGPRBlocks = ProgInfo.VGPRBlocks / RI->getVGPRAllocGranule() - 1; + // Set the value to initialize FP_ROUND and FP_DENORM parts of the mode // register. ProgInfo.FloatMode = getFPMode(MF); - ProgInfo.IEEEMode = 0; + ProgInfo.IEEEMode = STM.enableIEEEBit(MF); // Make clamp modifier on NaN input returns 0. ProgInfo.DX10Clamp = 1; - const MachineFrameInfo *FrameInfo = MF.getFrameInfo(); - ProgInfo.ScratchSize = FrameInfo->getStackSize(); + const MachineFrameInfo &FrameInfo = MF.getFrameInfo(); + ProgInfo.ScratchSize = FrameInfo.getStackSize(); ProgInfo.FlatUsed = FlatUsed; ProgInfo.VCCUsed = VCCUsed; @@ -524,10 +596,10 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo, LDSAlignShift = 9; } - unsigned LDSSpillSize = MFI->LDSWaveSpillSize * - MFI->getMaximumWorkGroupSize(MF); + unsigned LDSSpillSize = + MFI->LDSWaveSpillSize * MFI->getMaxFlatWorkGroupSize(); - ProgInfo.LDSSize = MFI->LDSSize + LDSSpillSize; + ProgInfo.LDSSize = MFI->getLDSSize() + LDSSpillSize; ProgInfo.LDSBlocks = alignTo(ProgInfo.LDSSize, 1ULL << LDSAlignShift) >> LDSAlignShift; @@ -573,7 +645,7 @@ void AMDGPUAsmPrinter::getSIProgramInfo(SIProgramInfo &ProgInfo, static unsigned getRsrcReg(CallingConv::ID CallConv) { switch (CallConv) { - default: // Fall through + default: LLVM_FALLTHROUGH; case CallingConv::AMDGPU_CS: return R_00B848_COMPUTE_PGM_RSRC1; case CallingConv::AMDGPU_GS: return R_00B228_SPI_SHADER_PGM_RSRC1_GS; case CallingConv::AMDGPU_PS: return R_00B028_SPI_SHADER_PGM_RSRC1_PS; @@ -703,7 +775,9 @@ void AMDGPUAsmPrinter::EmitAmdKernelCodeT(const MachineFunction &MF, if (STM.isXNACKEnabled()) header.code_properties |= AMD_CODE_PROPERTY_IS_XNACK_SUPPORTED; - header.kernarg_segment_byte_size = MFI->ABIArgOffset; + // FIXME: Should use getKernArgSize + header.kernarg_segment_byte_size = + STM.getKernArgSegmentSize(MFI->getABIArgOffset()); header.wavefront_sgpr_count = KernelInfo.NumSGPR; header.workitem_vgpr_count = KernelInfo.NumVGPR; header.workitem_private_segment_byte_size = KernelInfo.ScratchSize; @@ -711,6 +785,11 @@ void AMDGPUAsmPrinter::EmitAmdKernelCodeT(const MachineFunction &MF, header.reserved_vgpr_first = KernelInfo.ReservedVGPRFirst; header.reserved_vgpr_count = KernelInfo.ReservedVGPRCount; + // These alignment values are specified in powers of two, so alignment = + // 2^n. The minimum alignment is 2^4 = 16. + header.kernarg_segment_alignment = std::max((size_t)4, + countTrailingZeros(MFI->getMaxKernArgAlign())); + if (STM.debuggerEmitPrologue()) { header.debug_wavefront_private_segment_offset_sgpr = KernelInfo.DebuggerWavefrontPrivateSegmentOffsetSGPR; @@ -745,231 +824,3 @@ bool AMDGPUAsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, *TM.getSubtargetImpl(*MF->getFunction())->getRegisterInfo()); return false; } - -// Emit a key and an integer value for runtime metadata. -static void emitRuntimeMDIntValue(std::unique_ptr &Streamer, - RuntimeMD::Key K, uint64_t V, - unsigned Size) { - Streamer->EmitIntValue(K, 1); - Streamer->EmitIntValue(V, Size); -} - -// Emit a key and a string value for runtime metadata. -static void emitRuntimeMDStringValue(std::unique_ptr &Streamer, - RuntimeMD::Key K, StringRef S) { - Streamer->EmitIntValue(K, 1); - Streamer->EmitIntValue(S.size(), 4); - Streamer->EmitBytes(S); -} - -// Emit a key and three integer values for runtime metadata. -// The three integer values are obtained from MDNode \p Node; -static void emitRuntimeMDThreeIntValues(std::unique_ptr &Streamer, - RuntimeMD::Key K, MDNode *Node, - unsigned Size) { - Streamer->EmitIntValue(K, 1); - Streamer->EmitIntValue(mdconst::extract( - Node->getOperand(0))->getZExtValue(), Size); - Streamer->EmitIntValue(mdconst::extract( - Node->getOperand(1))->getZExtValue(), Size); - Streamer->EmitIntValue(mdconst::extract( - Node->getOperand(2))->getZExtValue(), Size); -} - -void AMDGPUAsmPrinter::emitStartOfRuntimeMetadata(const Module &M) { - OutStreamer->SwitchSection(getObjFileLowering().getContext() - .getELFSection(RuntimeMD::SectionName, ELF::SHT_PROGBITS, 0)); - - emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyMDVersion, - RuntimeMD::MDVersion << 8 | RuntimeMD::MDRevision, 2); - if (auto MD = M.getNamedMetadata("opencl.ocl.version")) { - if (MD->getNumOperands()) { - auto Node = MD->getOperand(0); - if (Node->getNumOperands() > 1) { - emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyLanguage, - RuntimeMD::OpenCL_C, 1); - uint16_t Major = mdconst::extract(Node->getOperand(0)) - ->getZExtValue(); - uint16_t Minor = mdconst::extract(Node->getOperand(1)) - ->getZExtValue(); - emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyLanguageVersion, - Major * 100 + Minor * 10, 2); - } - } - } -} - -static std::string getOCLTypeName(Type *Ty, bool isSigned) { - if (VectorType* VecTy = dyn_cast(Ty)) { - Type* EleTy = VecTy->getElementType(); - unsigned Size = VecTy->getVectorNumElements(); - return (Twine(getOCLTypeName(EleTy, isSigned)) + Twine(Size)).str(); - } - switch (Ty->getTypeID()) { - case Type::HalfTyID: return "half"; - case Type::FloatTyID: return "float"; - case Type::DoubleTyID: return "double"; - case Type::IntegerTyID: { - if (!isSigned) - return (Twine('u') + Twine(getOCLTypeName(Ty, true))).str(); - auto IntTy = cast(Ty); - auto BW = IntTy->getIntegerBitWidth(); - switch (BW) { - case 8: - return "char"; - case 16: - return "short"; - case 32: - return "int"; - case 64: - return "long"; - default: - return (Twine('i') + Twine(BW)).str(); - } - } - default: - llvm_unreachable("invalid type"); - } -} - -static RuntimeMD::KernelArg::ValueType getRuntimeMDValueType( - Type *Ty, StringRef TypeName) { - if (auto VT = dyn_cast(Ty)) - return getRuntimeMDValueType(VT->getElementType(), TypeName); - else if (auto PT = dyn_cast(Ty)) - return getRuntimeMDValueType(PT->getElementType(), TypeName); - else if (Ty->isHalfTy()) - return RuntimeMD::KernelArg::F16; - else if (Ty->isFloatTy()) - return RuntimeMD::KernelArg::F32; - else if (Ty->isDoubleTy()) - return RuntimeMD::KernelArg::F64; - else if (IntegerType* intTy = dyn_cast(Ty)) { - bool Signed = !TypeName.startswith("u"); - switch (intTy->getIntegerBitWidth()) { - case 8: - return Signed ? RuntimeMD::KernelArg::I8 : RuntimeMD::KernelArg::U8; - case 16: - return Signed ? RuntimeMD::KernelArg::I16 : RuntimeMD::KernelArg::U16; - case 32: - return Signed ? RuntimeMD::KernelArg::I32 : RuntimeMD::KernelArg::U32; - case 64: - return Signed ? RuntimeMD::KernelArg::I64 : RuntimeMD::KernelArg::U64; - default: - // Runtime does not recognize other integer types. Report as - // struct type. - return RuntimeMD::KernelArg::Struct; - } - } else - return RuntimeMD::KernelArg::Struct; -} - -void AMDGPUAsmPrinter::emitRuntimeMetadata(const Function &F) { - if (!F.getMetadata("kernel_arg_type")) - return; - - MCContext &Context = getObjFileLowering().getContext(); - OutStreamer->SwitchSection( - Context.getELFSection(RuntimeMD::SectionName, ELF::SHT_PROGBITS, 0)); - OutStreamer->EmitIntValue(RuntimeMD::KeyKernelBegin, 1); - emitRuntimeMDStringValue(OutStreamer, RuntimeMD::KeyKernelName, F.getName()); - - for (auto &Arg:F.args()) { - // Emit KeyArgBegin. - unsigned I = Arg.getArgNo(); - OutStreamer->EmitIntValue(RuntimeMD::KeyArgBegin, 1); - - // Emit KeyArgSize and KeyArgAlign. - auto T = Arg.getType(); - auto DL = F.getParent()->getDataLayout(); - emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyArgSize, - DL.getTypeAllocSize(T), 4); - emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyArgAlign, - DL.getABITypeAlignment(T), 4); - - // Emit KeyArgTypeName. - auto TypeName = dyn_cast(F.getMetadata( - "kernel_arg_type")->getOperand(I))->getString(); - emitRuntimeMDStringValue(OutStreamer, RuntimeMD::KeyArgTypeName, TypeName); - - // Emit KeyArgName. - if (auto ArgNameMD = F.getMetadata("kernel_arg_name")) { - auto ArgName = cast(ArgNameMD->getOperand( - I))->getString(); - emitRuntimeMDStringValue(OutStreamer, RuntimeMD::KeyArgName, ArgName); - } - - // Emit KeyArgIsVolatile, KeyArgIsRestrict, KeyArgIsConst and KeyArgIsPipe. - auto TypeQual = cast(F.getMetadata( - "kernel_arg_type_qual")->getOperand(I))->getString(); - SmallVector SplitQ; - TypeQual.split(SplitQ, " ", -1, false/* drop empty entry*/); - for (auto &I:SplitQ) { - auto Key = StringSwitch(I) - .Case("volatile", RuntimeMD::KeyArgIsVolatile) - .Case("restrict", RuntimeMD::KeyArgIsRestrict) - .Case("const", RuntimeMD::KeyArgIsConst) - .Case("pipe", RuntimeMD::KeyArgIsPipe) - .Default(RuntimeMD::KeyNull); - OutStreamer->EmitIntValue(Key, 1); - } - - // Emit KeyArgTypeKind. - auto BaseTypeName = cast( - F.getMetadata("kernel_arg_base_type")->getOperand(I))->getString(); - auto TypeKind = StringSwitch(BaseTypeName) - .Case("sampler_t", RuntimeMD::KernelArg::Sampler) - .Case("queue_t", RuntimeMD::KernelArg::Queue) - .Cases("image1d_t", "image1d_array_t", "image1d_buffer_t", - "image2d_t" , "image2d_array_t", RuntimeMD::KernelArg::Image) - .Cases("image2d_depth_t", "image2d_array_depth_t", - "image2d_msaa_t", "image2d_array_msaa_t", - "image2d_msaa_depth_t", RuntimeMD::KernelArg::Image) - .Cases("image2d_array_msaa_depth_t", "image3d_t", - RuntimeMD::KernelArg::Image) - .Default(isa(T) ? RuntimeMD::KernelArg::Pointer : - RuntimeMD::KernelArg::Value); - emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyArgTypeKind, TypeKind, 1); - - // Emit KeyArgValueType. - emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyArgValueType, - getRuntimeMDValueType(T, BaseTypeName), 2); - - // Emit KeyArgAccQual. - auto AccQual = cast(F.getMetadata( - "kernel_arg_access_qual")->getOperand(I))->getString(); - auto AQ = StringSwitch(AccQual) - .Case("read_only", RuntimeMD::KernelArg::ReadOnly) - .Case("write_only", RuntimeMD::KernelArg::WriteOnly) - .Case("read_write", RuntimeMD::KernelArg::ReadWrite) - .Default(RuntimeMD::KernelArg::None); - emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyArgAccQual, - AQ, 1); - - // Emit KeyArgAddrQual. - if (isa(T)) - emitRuntimeMDIntValue(OutStreamer, RuntimeMD::KeyArgAddrQual, - T->getPointerAddressSpace(), 1); - - // Emit KeyArgEnd - OutStreamer->EmitIntValue(RuntimeMD::KeyArgEnd, 1); - } - - // Emit KeyReqdWorkGroupSize, KeyWorkGroupSizeHint, and KeyVecTypeHint. - if (auto RWGS = F.getMetadata("reqd_work_group_size")) - emitRuntimeMDThreeIntValues(OutStreamer, RuntimeMD::KeyReqdWorkGroupSize, - RWGS, 4); - if (auto WGSH = F.getMetadata("work_group_size_hint")) - emitRuntimeMDThreeIntValues(OutStreamer, RuntimeMD::KeyWorkGroupSizeHint, - WGSH, 4); - if (auto VTH = F.getMetadata("vec_type_hint")) { - auto TypeName = getOCLTypeName(cast( - VTH->getOperand(0))->getType(), mdconst::extract( - VTH->getOperand(1))->getZExtValue()); - emitRuntimeMDStringValue(OutStreamer, RuntimeMD::KeyVecTypeHint, - TypeName); - } - - // Emit KeyKernelEnd - OutStreamer->EmitIntValue(RuntimeMD::KeyKernelEnd, 1); -} diff --git a/lib/Target/AMDGPU/AMDGPUAsmPrinter.h b/lib/Target/AMDGPU/AMDGPUAsmPrinter.h index 7b04c539520d..9a4bafef3a25 100644 --- a/lib/Target/AMDGPU/AMDGPUAsmPrinter.h +++ b/lib/Target/AMDGPU/AMDGPUAsmPrinter.h @@ -15,10 +15,13 @@ #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUASMPRINTER_H #define LLVM_LIB_TARGET_AMDGPU_AMDGPUASMPRINTER_H +#include "AMDGPUMCInstLower.h" + #include "llvm/CodeGen/AsmPrinter.h" #include namespace llvm { +class MCOperand; class AMDGPUAsmPrinter final : public AsmPrinter { private: @@ -40,6 +43,8 @@ private: NumVGPR(0), NumSGPR(0), FlatUsed(false), + NumSGPRsForWavesPerEU(0), + NumVGPRsForWavesPerEU(0), ReservedVGPRFirst(0), ReservedVGPRCount(0), DebuggerWavefrontPrivateSegmentOffsetSGPR((uint16_t)-1), @@ -71,15 +76,23 @@ private: uint32_t LDSSize; bool FlatUsed; + // Number of SGPRs that meets number of waves per execution unit request. + uint32_t NumSGPRsForWavesPerEU; + + // Number of VGPRs that meets number of waves per execution unit request. + uint32_t NumVGPRsForWavesPerEU; + // If ReservedVGPRCount is 0 then must be 0. Otherwise, this is the first // fixed VGPR number reserved. uint16_t ReservedVGPRFirst; + // The number of consecutive VGPRs reserved. uint16_t ReservedVGPRCount; // Fixed SGPR number used to hold wave scratch offset for entire kernel // execution, or uint16_t(-1) if the register is not used or not known. uint16_t DebuggerWavefrontPrivateSegmentOffsetSGPR; + // Fixed SGPR number of the first 4 SGPRs used to hold scratch V# for entire // kernel execution, or uint16_t(-1) if the register is not used or not // known. @@ -108,9 +121,16 @@ public: bool runOnMachineFunction(MachineFunction &MF) override; - const char *getPassName() const override { - return "AMDGPU Assembly Printer"; - } + StringRef getPassName() const override; + + /// \brief Wrapper for MCInstLowering.lowerOperand() for the tblgen'erated + /// pseudo lowering. + bool lowerOperand(const MachineOperand &MO, MCOperand &MCOp) const; + + /// \brief tblgen'erated driver function for lowering simple MI->MC pseudo + /// instructions. + bool emitPseudoExpansionLowering(MCStreamer &OutStreamer, + const MachineInstr *MI); /// Implemented in AMDGPUMCInstLower.cpp void EmitInstruction(const MachineInstr *MI) override; @@ -123,14 +143,13 @@ public: void EmitStartOfAsmFile(Module &M) override; + bool isBlockOnlyReachableByFallthrough( + const MachineBasicBlock *MBB) const override; + bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo, unsigned AsmVariant, const char *ExtraCode, raw_ostream &O) override; - void emitStartOfRuntimeMetadata(const Module &M); - - void emitRuntimeMetadata(const Function &F); - protected: std::vector DisasmLines, HexLines; size_t DisasmLineMaxLen; diff --git a/lib/Target/AMDGPU/AMDGPUCallLowering.cpp b/lib/Target/AMDGPU/AMDGPUCallLowering.cpp index 1a1da8a254a7..d53cc153dc9a 100644 --- a/lib/Target/AMDGPU/AMDGPUCallLowering.cpp +++ b/lib/Target/AMDGPU/AMDGPUCallLowering.cpp @@ -1,4 +1,4 @@ -//===-- llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp - Call lowering ---===// +//===-- llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp - Call lowering -----===// // // The LLVM Compiler Infrastructure // @@ -34,9 +34,9 @@ bool AMDGPUCallLowering::lowerReturn(MachineIRBuilder &MIRBuilder, return true; } -bool AMDGPUCallLowering::lowerFormalArguments( - MachineIRBuilder &MIRBuilder, const Function::ArgumentListType &Args, - const SmallVectorImpl &VRegs) const { +bool AMDGPUCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder, + const Function &F, + ArrayRef VRegs) const { // TODO: Implement once there are generic loads/stores. return true; } diff --git a/lib/Target/AMDGPU/AMDGPUCallLowering.h b/lib/Target/AMDGPU/AMDGPUCallLowering.h index 61174bacdac3..9ae87c9397ab 100644 --- a/lib/Target/AMDGPU/AMDGPUCallLowering.h +++ b/lib/Target/AMDGPU/AMDGPUCallLowering.h @@ -27,10 +27,8 @@ class AMDGPUCallLowering: public CallLowering { bool lowerReturn(MachineIRBuilder &MIRBuiler, const Value *Val, unsigned VReg) const override; - bool - lowerFormalArguments(MachineIRBuilder &MIRBuilder, - const Function::ArgumentListType &Args, - const SmallVectorImpl &VRegs) const override; + bool lowerFormalArguments(MachineIRBuilder &MIRBuilder, const Function &F, + ArrayRef VRegs) const override; }; } // End of namespace llvm; #endif diff --git a/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp index b955e231699a..e6230547a9b3 100644 --- a/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp +++ b/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp @@ -39,6 +39,78 @@ class AMDGPUCodeGenPrepare : public FunctionPass, Module *Mod; bool HasUnsafeFPMath; + /// \brief Copies exact/nsw/nuw flags (if any) from binary operation \p I to + /// binary operation \p V. + /// + /// \returns Binary operation \p V. + Value *copyFlags(const BinaryOperator &I, Value *V) const; + + /// \returns \p T's base element bit width. + unsigned getBaseElementBitWidth(const Type *T) const; + + /// \returns Equivalent 32 bit integer type for given type \p T. For example, + /// if \p T is i7, then i32 is returned; if \p T is <3 x i12>, then <3 x i32> + /// is returned. + Type *getI32Ty(IRBuilder<> &B, const Type *T) const; + + /// \returns True if binary operation \p I is a signed binary operation, false + /// otherwise. + bool isSigned(const BinaryOperator &I) const; + + /// \returns True if the condition of 'select' operation \p I comes from a + /// signed 'icmp' operation, false otherwise. + bool isSigned(const SelectInst &I) const; + + /// \returns True if type \p T needs to be promoted to 32 bit integer type, + /// false otherwise. + bool needsPromotionToI32(const Type *T) const; + + /// \brief Promotes uniform binary operation \p I to equivalent 32 bit binary + /// operation. + /// + /// \details \p I's base element bit width must be greater than 1 and less + /// than or equal 16. Promotion is done by sign or zero extending operands to + /// 32 bits, replacing \p I with equivalent 32 bit binary operation, and + /// truncating the result of 32 bit binary operation back to \p I's original + /// type. Division operation is not promoted. + /// + /// \returns True if \p I is promoted to equivalent 32 bit binary operation, + /// false otherwise. + bool promoteUniformOpToI32(BinaryOperator &I) const; + + /// \brief Promotes uniform 'icmp' operation \p I to 32 bit 'icmp' operation. + /// + /// \details \p I's base element bit width must be greater than 1 and less + /// than or equal 16. Promotion is done by sign or zero extending operands to + /// 32 bits, and replacing \p I with 32 bit 'icmp' operation. + /// + /// \returns True. + bool promoteUniformOpToI32(ICmpInst &I) const; + + /// \brief Promotes uniform 'select' operation \p I to 32 bit 'select' + /// operation. + /// + /// \details \p I's base element bit width must be greater than 1 and less + /// than or equal 16. Promotion is done by sign or zero extending operands to + /// 32 bits, replacing \p I with 32 bit 'select' operation, and truncating the + /// result of 32 bit 'select' operation back to \p I's original type. + /// + /// \returns True. + bool promoteUniformOpToI32(SelectInst &I) const; + + /// \brief Promotes uniform 'bitreverse' intrinsic \p I to 32 bit 'bitreverse' + /// intrinsic. + /// + /// \details \p I's base element bit width must be greater than 1 and less + /// than or equal 16. Promotion is done by zero extending the operand to 32 + /// bits, replacing \p I with 32 bit 'bitreverse' intrinsic, shifting the + /// result of 32 bit 'bitreverse' intrinsic to the right with zero fill (the + /// shift amount is 32 minus \p I's base element bit width), and truncating + /// the result of the shift operation back to \p I's original type. + /// + /// \returns True. + bool promoteUniformBitreverseToI32(IntrinsicInst &I) const; + public: static char ID; AMDGPUCodeGenPrepare(const TargetMachine *TM = nullptr) : @@ -51,16 +123,18 @@ public: bool visitFDiv(BinaryOperator &I); - bool visitInstruction(Instruction &I) { - return false; - } + bool visitInstruction(Instruction &I) { return false; } + bool visitBinaryOperator(BinaryOperator &I); + bool visitICmpInst(ICmpInst &I); + bool visitSelectInst(SelectInst &I); + + bool visitIntrinsicInst(IntrinsicInst &I); + bool visitBitreverseIntrinsicInst(IntrinsicInst &I); bool doInitialization(Module &M) override; bool runOnFunction(Function &F) override; - const char *getPassName() const override { - return "AMDGPU IR optimizations"; - } + StringRef getPassName() const override { return "AMDGPU IR optimizations"; } void getAnalysisUsage(AnalysisUsage &AU) const override { AU.addRequired(); @@ -70,6 +144,171 @@ public: } // End anonymous namespace +Value *AMDGPUCodeGenPrepare::copyFlags( + const BinaryOperator &I, Value *V) const { + BinaryOperator *BinOp = dyn_cast(V); + if (!BinOp) // Possibly constant expression. + return V; + + if (isa(BinOp)) { + BinOp->setHasNoSignedWrap(I.hasNoSignedWrap()); + BinOp->setHasNoUnsignedWrap(I.hasNoUnsignedWrap()); + } else if (isa(BinOp)) + BinOp->setIsExact(I.isExact()); + + return V; +} + +unsigned AMDGPUCodeGenPrepare::getBaseElementBitWidth(const Type *T) const { + assert(needsPromotionToI32(T) && "T does not need promotion to i32"); + + if (T->isIntegerTy()) + return T->getIntegerBitWidth(); + return cast(T)->getElementType()->getIntegerBitWidth(); +} + +Type *AMDGPUCodeGenPrepare::getI32Ty(IRBuilder<> &B, const Type *T) const { + assert(needsPromotionToI32(T) && "T does not need promotion to i32"); + + if (T->isIntegerTy()) + return B.getInt32Ty(); + return VectorType::get(B.getInt32Ty(), cast(T)->getNumElements()); +} + +bool AMDGPUCodeGenPrepare::isSigned(const BinaryOperator &I) const { + return I.getOpcode() == Instruction::AShr || + I.getOpcode() == Instruction::SDiv || I.getOpcode() == Instruction::SRem; +} + +bool AMDGPUCodeGenPrepare::isSigned(const SelectInst &I) const { + return isa(I.getOperand(0)) ? + cast(I.getOperand(0))->isSigned() : false; +} + +bool AMDGPUCodeGenPrepare::needsPromotionToI32(const Type *T) const { + if (T->isIntegerTy() && T->getIntegerBitWidth() > 1 && + T->getIntegerBitWidth() <= 16) + return true; + if (!T->isVectorTy()) + return false; + return needsPromotionToI32(cast(T)->getElementType()); +} + +bool AMDGPUCodeGenPrepare::promoteUniformOpToI32(BinaryOperator &I) const { + assert(needsPromotionToI32(I.getType()) && + "I does not need promotion to i32"); + + if (I.getOpcode() == Instruction::SDiv || + I.getOpcode() == Instruction::UDiv) + return false; + + IRBuilder<> Builder(&I); + Builder.SetCurrentDebugLocation(I.getDebugLoc()); + + Type *I32Ty = getI32Ty(Builder, I.getType()); + Value *ExtOp0 = nullptr; + Value *ExtOp1 = nullptr; + Value *ExtRes = nullptr; + Value *TruncRes = nullptr; + + if (isSigned(I)) { + ExtOp0 = Builder.CreateSExt(I.getOperand(0), I32Ty); + ExtOp1 = Builder.CreateSExt(I.getOperand(1), I32Ty); + } else { + ExtOp0 = Builder.CreateZExt(I.getOperand(0), I32Ty); + ExtOp1 = Builder.CreateZExt(I.getOperand(1), I32Ty); + } + ExtRes = copyFlags(I, Builder.CreateBinOp(I.getOpcode(), ExtOp0, ExtOp1)); + TruncRes = Builder.CreateTrunc(ExtRes, I.getType()); + + I.replaceAllUsesWith(TruncRes); + I.eraseFromParent(); + + return true; +} + +bool AMDGPUCodeGenPrepare::promoteUniformOpToI32(ICmpInst &I) const { + assert(needsPromotionToI32(I.getOperand(0)->getType()) && + "I does not need promotion to i32"); + + IRBuilder<> Builder(&I); + Builder.SetCurrentDebugLocation(I.getDebugLoc()); + + Type *I32Ty = getI32Ty(Builder, I.getOperand(0)->getType()); + Value *ExtOp0 = nullptr; + Value *ExtOp1 = nullptr; + Value *NewICmp = nullptr; + + if (I.isSigned()) { + ExtOp0 = Builder.CreateSExt(I.getOperand(0), I32Ty); + ExtOp1 = Builder.CreateSExt(I.getOperand(1), I32Ty); + } else { + ExtOp0 = Builder.CreateZExt(I.getOperand(0), I32Ty); + ExtOp1 = Builder.CreateZExt(I.getOperand(1), I32Ty); + } + NewICmp = Builder.CreateICmp(I.getPredicate(), ExtOp0, ExtOp1); + + I.replaceAllUsesWith(NewICmp); + I.eraseFromParent(); + + return true; +} + +bool AMDGPUCodeGenPrepare::promoteUniformOpToI32(SelectInst &I) const { + assert(needsPromotionToI32(I.getType()) && + "I does not need promotion to i32"); + + IRBuilder<> Builder(&I); + Builder.SetCurrentDebugLocation(I.getDebugLoc()); + + Type *I32Ty = getI32Ty(Builder, I.getType()); + Value *ExtOp1 = nullptr; + Value *ExtOp2 = nullptr; + Value *ExtRes = nullptr; + Value *TruncRes = nullptr; + + if (isSigned(I)) { + ExtOp1 = Builder.CreateSExt(I.getOperand(1), I32Ty); + ExtOp2 = Builder.CreateSExt(I.getOperand(2), I32Ty); + } else { + ExtOp1 = Builder.CreateZExt(I.getOperand(1), I32Ty); + ExtOp2 = Builder.CreateZExt(I.getOperand(2), I32Ty); + } + ExtRes = Builder.CreateSelect(I.getOperand(0), ExtOp1, ExtOp2); + TruncRes = Builder.CreateTrunc(ExtRes, I.getType()); + + I.replaceAllUsesWith(TruncRes); + I.eraseFromParent(); + + return true; +} + +bool AMDGPUCodeGenPrepare::promoteUniformBitreverseToI32( + IntrinsicInst &I) const { + assert(I.getIntrinsicID() == Intrinsic::bitreverse && + "I must be bitreverse intrinsic"); + assert(needsPromotionToI32(I.getType()) && + "I does not need promotion to i32"); + + IRBuilder<> Builder(&I); + Builder.SetCurrentDebugLocation(I.getDebugLoc()); + + Type *I32Ty = getI32Ty(Builder, I.getType()); + Function *I32 = + Intrinsic::getDeclaration(Mod, Intrinsic::bitreverse, { I32Ty }); + Value *ExtOp = Builder.CreateZExt(I.getOperand(0), I32Ty); + Value *ExtRes = Builder.CreateCall(I32, { ExtOp }); + Value *LShrOp = + Builder.CreateLShr(ExtRes, 32 - getBaseElementBitWidth(I.getType())); + Value *TruncRes = + Builder.CreateTrunc(LShrOp, I.getType()); + + I.replaceAllUsesWith(TruncRes); + I.eraseFromParent(); + + return true; +} + static bool shouldKeepFDivF32(Value *Num, bool UnsafeDiv) { const ConstantFP *CNum = dyn_cast(Num); if (!CNum) @@ -85,7 +324,6 @@ static bool shouldKeepFDivF32(Value *Num, bool UnsafeDiv) { bool AMDGPUCodeGenPrepare::visitFDiv(BinaryOperator &FDiv) { Type *Ty = FDiv.getType(); - // TODO: Handle half if (!Ty->getScalarType()->isFloatTy()) return false; @@ -154,6 +392,55 @@ static bool hasUnsafeFPMath(const Function &F) { return Attr.getValueAsString() == "true"; } +bool AMDGPUCodeGenPrepare::visitBinaryOperator(BinaryOperator &I) { + bool Changed = false; + + if (ST->has16BitInsts() && needsPromotionToI32(I.getType()) && + DA->isUniform(&I)) + Changed |= promoteUniformOpToI32(I); + + return Changed; +} + +bool AMDGPUCodeGenPrepare::visitICmpInst(ICmpInst &I) { + bool Changed = false; + + if (ST->has16BitInsts() && needsPromotionToI32(I.getOperand(0)->getType()) && + DA->isUniform(&I)) + Changed |= promoteUniformOpToI32(I); + + return Changed; +} + +bool AMDGPUCodeGenPrepare::visitSelectInst(SelectInst &I) { + bool Changed = false; + + if (ST->has16BitInsts() && needsPromotionToI32(I.getType()) && + DA->isUniform(&I)) + Changed |= promoteUniformOpToI32(I); + + return Changed; +} + +bool AMDGPUCodeGenPrepare::visitIntrinsicInst(IntrinsicInst &I) { + switch (I.getIntrinsicID()) { + case Intrinsic::bitreverse: + return visitBitreverseIntrinsicInst(I); + default: + return false; + } +} + +bool AMDGPUCodeGenPrepare::visitBitreverseIntrinsicInst(IntrinsicInst &I) { + bool Changed = false; + + if (ST->has16BitInsts() && needsPromotionToI32(I.getType()) && + DA->isUniform(&I)) + Changed |= promoteUniformBitreverseToI32(I); + + return Changed; +} + bool AMDGPUCodeGenPrepare::doInitialization(Module &M) { Mod = &M; return false; diff --git a/lib/Target/AMDGPU/AMDGPUFrameLowering.cpp b/lib/Target/AMDGPU/AMDGPUFrameLowering.cpp index bbc28b885721..805fb7102a35 100644 --- a/lib/Target/AMDGPU/AMDGPUFrameLowering.cpp +++ b/lib/Target/AMDGPU/AMDGPUFrameLowering.cpp @@ -10,23 +10,22 @@ // Interface to describe a layout of a stack frame on a AMDGPU target machine. // //===----------------------------------------------------------------------===// + #include "AMDGPUFrameLowering.h" #include "AMDGPURegisterInfo.h" #include "AMDGPUSubtarget.h" - +#include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/MachineFrameInfo.h" -#include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/IR/Instructions.h" +#include "llvm/Support/MathExtras.h" using namespace llvm; AMDGPUFrameLowering::AMDGPUFrameLowering(StackDirection D, unsigned StackAl, int LAO, unsigned TransAl) : TargetFrameLowering(D, StackAl, LAO, TransAl) { } -AMDGPUFrameLowering::~AMDGPUFrameLowering() { } +AMDGPUFrameLowering::~AMDGPUFrameLowering() = default; unsigned AMDGPUFrameLowering::getStackWidth(const MachineFunction &MF) const { - // XXX: Hardcoding to 1 for now. // // I think the StackWidth should stored as metadata associated with the @@ -75,7 +74,7 @@ unsigned AMDGPUFrameLowering::getStackWidth(const MachineFunction &MF) const { int AMDGPUFrameLowering::getFrameIndexReference(const MachineFunction &MF, int FI, unsigned &FrameReg) const { - const MachineFrameInfo *MFI = MF.getFrameInfo(); + const MachineFrameInfo &MFI = MF.getFrameInfo(); const AMDGPURegisterInfo *RI = MF.getSubtarget().getRegisterInfo(); @@ -86,19 +85,18 @@ int AMDGPUFrameLowering::getFrameIndexReference(const MachineFunction &MF, // XXX: We should only do this when the shader actually uses this // information. unsigned OffsetBytes = 2 * (getStackWidth(MF) * 4); - int UpperBound = FI == -1 ? MFI->getNumObjects() : FI; + int UpperBound = FI == -1 ? MFI.getNumObjects() : FI; - for (int i = MFI->getObjectIndexBegin(); i < UpperBound; ++i) { - OffsetBytes = alignTo(OffsetBytes, MFI->getObjectAlignment(i)); - OffsetBytes += MFI->getObjectSize(i); + for (int i = MFI.getObjectIndexBegin(); i < UpperBound; ++i) { + OffsetBytes = alignTo(OffsetBytes, MFI.getObjectAlignment(i)); + OffsetBytes += MFI.getObjectSize(i); // Each register holds 4 bytes, so we must always align the offset to at // least 4 bytes, so that 2 frame objects won't share the same register. OffsetBytes = alignTo(OffsetBytes, 4); } if (FI != -1) - OffsetBytes = alignTo(OffsetBytes, MFI->getObjectAlignment(FI)); + OffsetBytes = alignTo(OffsetBytes, MFI.getObjectAlignment(FI)); return OffsetBytes / (getStackWidth(MF) * 4); } - diff --git a/lib/Target/AMDGPU/AMDGPUFrameLowering.h b/lib/Target/AMDGPU/AMDGPUFrameLowering.h index 513848a1d887..5d51351a00d2 100644 --- a/lib/Target/AMDGPU/AMDGPUFrameLowering.h +++ b/lib/Target/AMDGPU/AMDGPUFrameLowering.h @@ -11,6 +11,7 @@ /// \brief Interface to describe a layout of a stack frame on an AMDGPU target. // //===----------------------------------------------------------------------===// + #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUFRAMELOWERING_H #define LLVM_LIB_TARGET_AMDGPU_AMDGPUFRAMELOWERING_H @@ -27,7 +28,7 @@ class AMDGPUFrameLowering : public TargetFrameLowering { public: AMDGPUFrameLowering(StackDirection D, unsigned StackAl, int LAO, unsigned TransAl = 1); - virtual ~AMDGPUFrameLowering(); + ~AMDGPUFrameLowering() override; /// \returns The number of 32-bit sub-registers that are used when storing /// values to the stack. @@ -40,5 +41,7 @@ public: return false; } }; -} // namespace llvm -#endif + +} // end namespace llvm + +#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUFRAMELOWERING_H diff --git a/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp index 23c9352ce273..ef3b44f7c211 100644 --- a/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp +++ b/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp @@ -12,25 +12,48 @@ // //===----------------------------------------------------------------------===// +#include "AMDGPU.h" #include "AMDGPUInstrInfo.h" -#include "AMDGPUIntrinsicInfo.h" +#include "AMDGPURegisterInfo.h" #include "AMDGPUISelLowering.h" // For AMDGPUISD #include "AMDGPUSubtarget.h" +#include "SIDefines.h" +#include "SIInstrInfo.h" +#include "SIRegisterInfo.h" #include "SIISelLowering.h" #include "SIMachineFunctionInfo.h" +#include "llvm/ADT/APInt.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/StringRef.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/CodeGen/FunctionLoweringInfo.h" -#include "llvm/CodeGen/MachineFrameInfo.h" -#include "llvm/CodeGen/PseudoSourceValue.h" +#include "llvm/CodeGen/ISDOpcodes.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/MachineValueType.h" #include "llvm/CodeGen/SelectionDAG.h" #include "llvm/CodeGen/SelectionDAGISel.h" -#include "llvm/IR/DiagnosticInfo.h" +#include "llvm/CodeGen/SelectionDAGNodes.h" +#include "llvm/CodeGen/ValueTypes.h" +#include "llvm/IR/BasicBlock.h" +#include "llvm/IR/Instruction.h" +#include "llvm/MC/MCInstrDesc.h" +#include "llvm/Support/Casting.h" +#include "llvm/Support/CodeGen.h" +#include "llvm/Support/ErrorHandling.h" +#include "llvm/Support/MathExtras.h" +#include +#include +#include +#include using namespace llvm; namespace llvm { + class R600InstrInfo; -} + +} // end namespace llvm //===----------------------------------------------------------------------===// // Instruction Selector Implementation @@ -38,18 +61,6 @@ class R600InstrInfo; namespace { -static bool isCBranchSCC(const SDNode *N) { - assert(N->getOpcode() == ISD::BRCOND); - if (!N->hasOneUse()) - return false; - - SDValue Cond = N->getOperand(1); - if (Cond.getOpcode() == ISD::CopyToReg) - Cond = Cond.getOperand(2); - return Cond.getOpcode() == ISD::SETCC && - Cond.getOperand(0).getValueType() == MVT::i32 && Cond.hasOneUse(); -} - /// AMDGPU specific code to select AMDGPU machine instructions for /// SelectionDAG operations. class AMDGPUDAGToDAGISel : public SelectionDAGISel { @@ -58,16 +69,18 @@ class AMDGPUDAGToDAGISel : public SelectionDAGISel { const AMDGPUSubtarget *Subtarget; public: - AMDGPUDAGToDAGISel(TargetMachine &TM); - virtual ~AMDGPUDAGToDAGISel(); + explicit AMDGPUDAGToDAGISel(TargetMachine &TM, CodeGenOpt::Level OptLevel) + : SelectionDAGISel(TM, OptLevel) {} + ~AMDGPUDAGToDAGISel() override = default; + bool runOnMachineFunction(MachineFunction &MF) override; void Select(SDNode *N) override; - const char *getPassName() const override; - void PreprocessISelDAG() override; + StringRef getPassName() const override; void PostprocessISelDAG() override; private: - bool isInlineImmediate(SDNode *N) const; + SDValue foldFrameIndex(SDValue N) const; + bool isInlineImmediate(const SDNode *N) const; bool FoldOperand(SDValue &Src, SDValue &Sel, SDValue &Neg, SDValue &Abs, const R600InstrInfo *TII); bool FoldOperands(unsigned, const R600InstrInfo *, std::vector &); @@ -145,40 +158,46 @@ private: void SelectADD_SUB_I64(SDNode *N); void SelectDIV_SCALE(SDNode *N); + void SelectFMA_W_CHAIN(SDNode *N); + void SelectFMUL_W_CHAIN(SDNode *N); SDNode *getS_BFE(unsigned Opcode, const SDLoc &DL, SDValue Val, uint32_t Offset, uint32_t Width); void SelectS_BFEFromShifts(SDNode *N); void SelectS_BFE(SDNode *N); + bool isCBranchSCC(const SDNode *N) const; void SelectBRCOND(SDNode *N); void SelectATOMIC_CMP_SWAP(SDNode *N); // Include the pieces autogenerated from the target description. #include "AMDGPUGenDAGISel.inc" }; + } // end anonymous namespace /// \brief This pass converts a legalized DAG into a AMDGPU-specific // DAG, ready for instruction scheduling. -FunctionPass *llvm::createAMDGPUISelDag(TargetMachine &TM) { - return new AMDGPUDAGToDAGISel(TM); +FunctionPass *llvm::createAMDGPUISelDag(TargetMachine &TM, + CodeGenOpt::Level OptLevel) { + return new AMDGPUDAGToDAGISel(TM, OptLevel); } -AMDGPUDAGToDAGISel::AMDGPUDAGToDAGISel(TargetMachine &TM) - : SelectionDAGISel(TM) {} - bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) { Subtarget = &MF.getSubtarget(); return SelectionDAGISel::runOnMachineFunction(MF); } -AMDGPUDAGToDAGISel::~AMDGPUDAGToDAGISel() { -} +bool AMDGPUDAGToDAGISel::isInlineImmediate(const SDNode *N) const { + const SIInstrInfo *TII + = static_cast(Subtarget)->getInstrInfo(); + + if (const ConstantSDNode *C = dyn_cast(N)) + return TII->isInlineConstant(C->getAPIntValue()); -bool AMDGPUDAGToDAGISel::isInlineImmediate(SDNode *N) const { - const SITargetLowering *TL - = static_cast(getTargetLowering()); - return TL->analyzeImmediate(N) == 0; + if (const ConstantFPSDNode *C = dyn_cast(N)) + return TII->isInlineConstant(C->getValueAPF().bitcastToAPInt()); + + return false; } /// \brief Determine the register class for \p OpNo @@ -187,8 +206,21 @@ bool AMDGPUDAGToDAGISel::isInlineImmediate(SDNode *N) const { /// determined. const TargetRegisterClass *AMDGPUDAGToDAGISel::getOperandRegClass(SDNode *N, unsigned OpNo) const { - if (!N->isMachineOpcode()) + if (!N->isMachineOpcode()) { + if (N->getOpcode() == ISD::CopyToReg) { + unsigned Reg = cast(N->getOperand(1))->getReg(); + if (TargetRegisterInfo::isVirtualRegister(Reg)) { + MachineRegisterInfo &MRI = CurDAG->getMachineFunction().getRegInfo(); + return MRI.getRegClass(Reg); + } + + const SIRegisterInfo *TRI + = static_cast(Subtarget)->getRegisterInfo(); + return TRI->getPhysRegClass(Reg); + } + return nullptr; + } switch (N->getMachineOpcode()) { default: { @@ -244,7 +276,7 @@ SDNode *AMDGPUDAGToDAGISel::glueCopyToM0(SDNode *N) const { static unsigned selectSGPRVectorRegClassID(unsigned NumVectorElts) { switch (NumVectorElts) { case 1: - return AMDGPU::SReg_32RegClassID; + return AMDGPU::SReg_32_XM0RegClassID; case 2: return AMDGPU::SReg_64RegClassID; case 4: @@ -275,7 +307,11 @@ void AMDGPUDAGToDAGISel::Select(SDNode *N) { // DAG legalization, so we can fold some i64 ADDs used for address // calculation into the LOAD and STORE instructions. case ISD::ADD: - case ISD::SUB: { + case ISD::ADDC: + case ISD::ADDE: + case ISD::SUB: + case ISD::SUBC: + case ISD::SUBE: { if (N->getValueType(0) != MVT::i64 || Subtarget->getGeneration() < AMDGPUSubtarget::SOUTHERN_ISLANDS) break; @@ -283,6 +319,15 @@ void AMDGPUDAGToDAGISel::Select(SDNode *N) { SelectADD_SUB_I64(N); return; } + case AMDGPUISD::FMUL_W_CHAIN: { + SelectFMUL_W_CHAIN(N); + return; + } + case AMDGPUISD::FMA_W_CHAIN: { + SelectFMA_W_CHAIN(N); + return; + } + case ISD::SCALAR_TO_VECTOR: case AMDGPUISD::BUILD_VERTICAL_VECTOR: case ISD::BUILD_VECTOR: { @@ -498,7 +543,7 @@ bool AMDGPUDAGToDAGISel::isUniformBr(const SDNode *N) const { Term->getMetadata("structurizecfg.uniform"); } -const char *AMDGPUDAGToDAGISel::getPassName() const { +StringRef AMDGPUDAGToDAGISel::getPassName() const { return "AMDGPU DAG->DAG Pattern Instruction Selection"; } @@ -580,7 +625,12 @@ void AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) { SDValue LHS = N->getOperand(0); SDValue RHS = N->getOperand(1); - bool IsAdd = (N->getOpcode() == ISD::ADD); + unsigned Opcode = N->getOpcode(); + bool ConsumeCarry = (Opcode == ISD::ADDE || Opcode == ISD::SUBE); + bool ProduceCarry = + ConsumeCarry || Opcode == ISD::ADDC || Opcode == ISD::SUBC; + bool IsAdd = + (Opcode == ISD::ADD || Opcode == ISD::ADDC || Opcode == ISD::ADDE); SDValue Sub0 = CurDAG->getTargetConstant(AMDGPU::sub0, DL, MVT::i32); SDValue Sub1 = CurDAG->getTargetConstant(AMDGPU::sub1, DL, MVT::i32); @@ -596,25 +646,70 @@ void AMDGPUDAGToDAGISel::SelectADD_SUB_I64(SDNode *N) { DL, MVT::i32, RHS, Sub1); SDVTList VTList = CurDAG->getVTList(MVT::i32, MVT::Glue); - SDValue AddLoArgs[] = { SDValue(Lo0, 0), SDValue(Lo1, 0) }; unsigned Opc = IsAdd ? AMDGPU::S_ADD_U32 : AMDGPU::S_SUB_U32; unsigned CarryOpc = IsAdd ? AMDGPU::S_ADDC_U32 : AMDGPU::S_SUBB_U32; - SDNode *AddLo = CurDAG->getMachineNode( Opc, DL, VTList, AddLoArgs); - SDValue Carry(AddLo, 1); - SDNode *AddHi - = CurDAG->getMachineNode(CarryOpc, DL, MVT::i32, - SDValue(Hi0, 0), SDValue(Hi1, 0), Carry); + SDNode *AddLo; + if (!ConsumeCarry) { + SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0) }; + AddLo = CurDAG->getMachineNode(Opc, DL, VTList, Args); + } else { + SDValue Args[] = { SDValue(Lo0, 0), SDValue(Lo1, 0), N->getOperand(2) }; + AddLo = CurDAG->getMachineNode(CarryOpc, DL, VTList, Args); + } + SDValue AddHiArgs[] = { + SDValue(Hi0, 0), + SDValue(Hi1, 0), + SDValue(AddLo, 1) + }; + SDNode *AddHi = CurDAG->getMachineNode(CarryOpc, DL, VTList, AddHiArgs); - SDValue Args[5] = { + SDValue RegSequenceArgs[] = { CurDAG->getTargetConstant(AMDGPU::SReg_64RegClassID, DL, MVT::i32), SDValue(AddLo,0), Sub0, SDValue(AddHi,0), Sub1, }; - CurDAG->SelectNodeTo(N, AMDGPU::REG_SEQUENCE, MVT::i64, Args); + SDNode *RegSequence = CurDAG->getMachineNode(AMDGPU::REG_SEQUENCE, DL, + MVT::i64, RegSequenceArgs); + + if (ProduceCarry) { + // Replace the carry-use + CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 1), SDValue(AddHi, 1)); + } + + // Replace the remaining uses. + CurDAG->ReplaceAllUsesWith(N, RegSequence); + CurDAG->RemoveDeadNode(N); +} + +void AMDGPUDAGToDAGISel::SelectFMA_W_CHAIN(SDNode *N) { + SDLoc SL(N); + // src0_modifiers, src0, src1_modifiers, src1, src2_modifiers, src2, clamp, omod + SDValue Ops[10]; + + SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[6], Ops[7]); + SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]); + SelectVOP3Mods(N->getOperand(3), Ops[5], Ops[4]); + Ops[8] = N->getOperand(0); + Ops[9] = N->getOperand(4); + + CurDAG->SelectNodeTo(N, AMDGPU::V_FMA_F32, N->getVTList(), Ops); +} + +void AMDGPUDAGToDAGISel::SelectFMUL_W_CHAIN(SDNode *N) { + SDLoc SL(N); + // src0_modifiers, src0, src1_modifiers, src1, clamp, omod + SDValue Ops[8]; + + SelectVOP3Mods0(N->getOperand(1), Ops[1], Ops[0], Ops[4], Ops[5]); + SelectVOP3Mods(N->getOperand(2), Ops[3], Ops[2]); + Ops[6] = N->getOperand(0); + Ops[7] = N->getOperand(3); + + CurDAG->SelectNodeTo(N, AMDGPU::V_MUL_F32_e64, N->getVTList(), Ops); } // We need to handle this here because tablegen doesn't support matching @@ -779,6 +874,9 @@ bool AMDGPUDAGToDAGISel::SelectDS64Bit4ByteAligned(SDValue Addr, SDValue &Base, } // default case + + // FIXME: This is broken on SI where we still need to check if the base + // pointer is positive here. Base = Addr; Offset0 = CurDAG->getTargetConstant(0, DL, MVT::i8); Offset1 = CurDAG->getTargetConstant(1, DL, MVT::i8); @@ -825,7 +923,6 @@ bool AMDGPUDAGToDAGISel::SelectMUBUF(SDValue Addr, SDValue &Ptr, Ptr = N2; VAddr = N3; } else { - // (add N0, C1) -> offset VAddr = CurDAG->getTargetConstant(0, DL, MVT::i32); Ptr = N0; @@ -903,6 +1000,12 @@ bool AMDGPUDAGToDAGISel::SelectMUBUFAddr64(SDValue Addr, SDValue &SRsrc, return SelectMUBUFAddr64(Addr, SRsrc, VAddr, SOffset, Offset, GLC, SLC, TFE); } +SDValue AMDGPUDAGToDAGISel::foldFrameIndex(SDValue N) const { + if (auto FI = dyn_cast(N)) + return CurDAG->getTargetFrameIndex(FI->getIndex(), FI->getValueType(0)); + return N; +} + bool AMDGPUDAGToDAGISel::SelectMUBUFScratch(SDValue Addr, SDValue &Rsrc, SDValue &VAddr, SDValue &SOffset, SDValue &ImmOffset) const { @@ -922,14 +1025,14 @@ bool AMDGPUDAGToDAGISel::SelectMUBUFScratch(SDValue Addr, SDValue &Rsrc, // Offsets in vaddr must be positive. ConstantSDNode *C1 = cast(N1); if (isLegalMUBUFImmOffset(C1)) { - VAddr = N0; + VAddr = foldFrameIndex(N0); ImmOffset = CurDAG->getTargetConstant(C1->getZExtValue(), DL, MVT::i16); return true; } } // (node) - VAddr = Addr; + VAddr = foldFrameIndex(Addr); ImmOffset = CurDAG->getTargetConstant(0, DL, MVT::i16); return true; } @@ -1122,7 +1225,6 @@ bool AMDGPUDAGToDAGISel::SelectSMRDOffset(SDValue ByteOffsetNode, bool AMDGPUDAGToDAGISel::SelectSMRD(SDValue Addr, SDValue &SBase, SDValue &Offset, bool &Imm) const { - SDLoc SL(Addr); if (CurDAG->isBaseWithConstantOffset(Addr)) { SDValue N0 = Addr.getOperand(0); @@ -1327,36 +1429,53 @@ void AMDGPUDAGToDAGISel::SelectS_BFE(SDNode *N) { SelectCode(N); } +bool AMDGPUDAGToDAGISel::isCBranchSCC(const SDNode *N) const { + assert(N->getOpcode() == ISD::BRCOND); + if (!N->hasOneUse()) + return false; + + SDValue Cond = N->getOperand(1); + if (Cond.getOpcode() == ISD::CopyToReg) + Cond = Cond.getOperand(2); + + if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse()) + return false; + + MVT VT = Cond.getOperand(0).getSimpleValueType(); + if (VT == MVT::i32) + return true; + + if (VT == MVT::i64) { + auto ST = static_cast(Subtarget); + + ISD::CondCode CC = cast(Cond.getOperand(2))->get(); + return (CC == ISD::SETEQ || CC == ISD::SETNE) && ST->hasScalarCompareEq64(); + } + + return false; +} + void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) { SDValue Cond = N->getOperand(1); + if (Cond.isUndef()) { + CurDAG->SelectNodeTo(N, AMDGPU::SI_BR_UNDEF, MVT::Other, + N->getOperand(2), N->getOperand(0)); + return; + } + if (isCBranchSCC(N)) { // This brcond will use S_CBRANCH_SCC*, so let tablegen handle it. SelectCode(N); return; } - // The result of VOPC instructions is or'd against ~EXEC before it is - // written to vcc or another SGPR. This means that the value '1' is always - // written to the corresponding bit for results that are masked. In order - // to correctly check against vccz, we need to and VCC with the EXEC - // register in order to clear the value from the masked bits. - SDLoc SL(N); - SDNode *MaskedCond = - CurDAG->getMachineNode(AMDGPU::S_AND_B64, SL, MVT::i1, - CurDAG->getRegister(AMDGPU::EXEC, MVT::i1), - Cond); - SDValue VCC = CurDAG->getCopyToReg(N->getOperand(0), SL, AMDGPU::VCC, - SDValue(MaskedCond, 0), - SDValue()); // Passing SDValue() adds a - // glue output. + SDValue VCC = CurDAG->getCopyToReg(N->getOperand(0), SL, AMDGPU::VCC, Cond); CurDAG->SelectNodeTo(N, AMDGPU::S_CBRANCH_VCCNZ, MVT::Other, N->getOperand(2), // Basic Block - VCC.getValue(0), // Chain - VCC.getValue(1)); // Glue - return; + VCC.getValue(0)); } // This is here because there isn't a way to use the generated sub0_sub1 as the @@ -1427,7 +1546,6 @@ void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(SDNode *N) { bool AMDGPUDAGToDAGISel::SelectVOP3Mods(SDValue In, SDValue &Src, SDValue &SrcMods) const { - unsigned Mods = 0; Src = In; @@ -1491,62 +1609,6 @@ bool AMDGPUDAGToDAGISel::SelectVOP3Mods0Clamp0OMod(SDValue In, SDValue &Src, return SelectVOP3Mods(In, Src, SrcMods); } -void AMDGPUDAGToDAGISel::PreprocessISelDAG() { - MachineFrameInfo *MFI = CurDAG->getMachineFunction().getFrameInfo(); - - // Handle the perverse case where a frame index is being stored. We don't - // want to see multiple frame index operands on the same instruction since - // it complicates things and violates some assumptions about frame index - // lowering. - for (int I = MFI->getObjectIndexBegin(), E = MFI->getObjectIndexEnd(); - I != E; ++I) { - SDValue FI = CurDAG->getTargetFrameIndex(I, MVT::i32); - - // It's possible that we have a frame index defined in the function that - // isn't used in this block. - if (FI.use_empty()) - continue; - - // Skip over the AssertZext inserted during lowering. - SDValue EffectiveFI = FI; - auto It = FI->use_begin(); - if (It->getOpcode() == ISD::AssertZext && FI->hasOneUse()) { - EffectiveFI = SDValue(*It, 0); - It = EffectiveFI->use_begin(); - } - - for (auto It = EffectiveFI->use_begin(); !It.atEnd(); ) { - SDUse &Use = It.getUse(); - SDNode *User = Use.getUser(); - unsigned OpIdx = It.getOperandNo(); - ++It; - - if (MemSDNode *M = dyn_cast(User)) { - unsigned PtrIdx = M->getOpcode() == ISD::STORE ? 2 : 1; - if (OpIdx == PtrIdx) - continue; - - unsigned OpN = M->getNumOperands(); - SDValue NewOps[8]; - - assert(OpN < array_lengthof(NewOps)); - for (unsigned Op = 0; Op != OpN; ++Op) { - if (Op != OpIdx) { - NewOps[Op] = M->getOperand(Op); - continue; - } - - MachineSDNode *Mov = CurDAG->getMachineNode(AMDGPU::V_MOV_B32_e32, - SDLoc(M), MVT::i32, FI); - NewOps[Op] = SDValue(Mov, 0); - } - - CurDAG->UpdateNodeOperands(M, makeArrayRef(NewOps, OpN)); - } - } - } -} - void AMDGPUDAGToDAGISel::PostprocessISelDAG() { const AMDGPUTargetLowering& Lowering = *static_cast(getTargetLowering()); diff --git a/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/lib/Target/AMDGPU/AMDGPUISelLowering.cpp index 352423ed3ad6..a87204d46eae 100644 --- a/lib/Target/AMDGPU/AMDGPUISelLowering.cpp +++ b/lib/Target/AMDGPU/AMDGPUISelLowering.cpp @@ -37,7 +37,7 @@ static bool allocateKernArg(unsigned ValNo, MVT ValVT, MVT LocVT, MachineFunction &MF = State.getMachineFunction(); AMDGPUMachineFunction *MFI = MF.getInfo(); - uint64_t Offset = MFI->allocateKernArg(ValVT.getStoreSize(), + uint64_t Offset = MFI->allocateKernArg(LocVT.getStoreSize(), ArgFlags.getOrigAlign()); State.addLoc(CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT, LocInfo)); return true; @@ -55,14 +55,6 @@ EVT AMDGPUTargetLowering::getEquivalentMemType(LLVMContext &Ctx, EVT VT) { return EVT::getVectorVT(Ctx, MVT::i32, StoreSize / 32); } -EVT AMDGPUTargetLowering::getEquivalentBitType(LLVMContext &Ctx, EVT VT) { - unsigned StoreSize = VT.getStoreSizeInBits(); - if (StoreSize <= 32) - return EVT::getIntegerVT(Ctx, StoreSize); - - return EVT::getVectorVT(Ctx, MVT::i32, StoreSize / 32); -} - AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM, const AMDGPUSubtarget &STI) : TargetLowering(TM), Subtarget(&STI) { @@ -287,6 +279,7 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM, } setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand); + setOperationAction(ISD::FP_TO_FP16, MVT::f64, Custom); const MVT ScalarIntVTs[] = { MVT::i32, MVT::i64 }; for (MVT VT : ScalarIntVTs) { @@ -367,6 +360,8 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM, setOperationAction(ISD::FP_TO_SINT, VT, Expand); setOperationAction(ISD::FP_TO_UINT, VT, Expand); setOperationAction(ISD::MUL, VT, Expand); + setOperationAction(ISD::MULHU, VT, Expand); + setOperationAction(ISD::MULHS, VT, Expand); setOperationAction(ISD::OR, VT, Expand); setOperationAction(ISD::SHL, VT, Expand); setOperationAction(ISD::SRA, VT, Expand); @@ -440,22 +435,24 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM, setOperationAction(ISD::SELECT, MVT::v4f32, Promote); AddPromotedToType(ISD::SELECT, MVT::v4f32, MVT::v4i32); + // There are no libcalls of any kind. + for (int I = 0; I < RTLIB::UNKNOWN_LIBCALL; ++I) + setLibcallName(static_cast(I), nullptr); + setBooleanContents(ZeroOrNegativeOneBooleanContent); setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); setSchedulingPreference(Sched::RegPressure); setJumpIsExpensive(true); + setHasMultipleConditionRegisters(true); // SI at least has hardware support for floating point exceptions, but no way // of using or handling them is implemented. They are also optional in OpenCL // (Section 7.3) setHasFloatingPointExceptions(Subtarget->hasFPExceptions()); - setSelectIsExpensive(false); PredictableSelectIsExpensive = false; - setFsqrtIsCheap(true); - // We want to find all load dependencies for long chains of stores to enable // merging into very wide vectors. The problem is with vectors with > 4 // elements. MergeConsecutiveStores will attempt to merge these because x8/x16 @@ -472,11 +469,12 @@ AMDGPUTargetLowering::AMDGPUTargetLowering(const TargetMachine &TM, MaxStoresPerMemset = 4096; setTargetDAGCombine(ISD::BITCAST); - setTargetDAGCombine(ISD::AND); setTargetDAGCombine(ISD::SHL); setTargetDAGCombine(ISD::SRA); setTargetDAGCombine(ISD::SRL); setTargetDAGCombine(ISD::MUL); + setTargetDAGCombine(ISD::MULHU); + setTargetDAGCombine(ISD::MULHS); setTargetDAGCombine(ISD::SELECT); setTargetDAGCombine(ISD::SELECT_CC); setTargetDAGCombine(ISD::STORE); @@ -500,7 +498,8 @@ bool AMDGPUTargetLowering::isSelectSupported(SelectSupportKind SelType) const { // FIXME: Why are we reporting vectors of FP immediates as legal? bool AMDGPUTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT) const { EVT ScalarVT = VT.getScalarType(); - return (ScalarVT == MVT::f32 || ScalarVT == MVT::f64); + return (ScalarVT == MVT::f32 || ScalarVT == MVT::f64 || + (ScalarVT == MVT::f16 && Subtarget->has16BitInsts())); } // We don't want to shrink f64 / f32 constants. @@ -565,12 +564,12 @@ bool AMDGPUTargetLowering::isCheapToSpeculateCtlz() const { bool AMDGPUTargetLowering::isFAbsFree(EVT VT) const { assert(VT.isFloatingPoint()); - return VT == MVT::f32 || VT == MVT::f64; + return VT == MVT::f32 || VT == MVT::f64 || (Subtarget->has16BitInsts() && + VT == MVT::f16); } bool AMDGPUTargetLowering::isFNegFree(EVT VT) const { - assert(VT.isFloatingPoint()); - return VT == MVT::f32 || VT == MVT::f64; + return isFAbsFree(VT); } bool AMDGPUTargetLowering:: storeOfVectorConstantIsCheap(EVT MemVT, @@ -593,19 +592,32 @@ bool AMDGPUTargetLowering::aggressivelyPreferBuildVectorSources(EVT VecVT) const bool AMDGPUTargetLowering::isTruncateFree(EVT Source, EVT Dest) const { // Truncate is just accessing a subregister. - return Dest.bitsLT(Source) && (Dest.getSizeInBits() % 32 == 0); + + unsigned SrcSize = Source.getSizeInBits(); + unsigned DestSize = Dest.getSizeInBits(); + + return DestSize < SrcSize && DestSize % 32 == 0 ; } bool AMDGPUTargetLowering::isTruncateFree(Type *Source, Type *Dest) const { // Truncate is just accessing a subregister. - return Dest->getPrimitiveSizeInBits() < Source->getPrimitiveSizeInBits() && - (Dest->getPrimitiveSizeInBits() % 32 == 0); + + unsigned SrcSize = Source->getScalarSizeInBits(); + unsigned DestSize = Dest->getScalarSizeInBits(); + + if (DestSize== 16 && Subtarget->has16BitInsts()) + return SrcSize >= 32; + + return DestSize < SrcSize && DestSize % 32 == 0; } bool AMDGPUTargetLowering::isZExtFree(Type *Src, Type *Dest) const { unsigned SrcSize = Src->getScalarSizeInBits(); unsigned DestSize = Dest->getScalarSizeInBits(); + if (SrcSize == 16 && Subtarget->has16BitInsts()) + return DestSize >= 32; + return SrcSize == 32 && DestSize == 64; } @@ -614,6 +626,10 @@ bool AMDGPUTargetLowering::isZExtFree(EVT Src, EVT Dest) const { // practical purposes, the extra mov 0 to load a 64-bit is free. As used, // this will enable reducing 64-bit operations the 32-bit, which is always // good. + + if (Src == MVT::i16) + return Dest == MVT::i32 ||Dest == MVT::i64 ; + return Src == MVT::i32 && Dest == MVT::i64; } @@ -635,9 +651,105 @@ bool AMDGPUTargetLowering::isNarrowingProfitable(EVT SrcVT, EVT DestVT) const { // TargetLowering Callbacks //===---------------------------------------------------------------------===// -void AMDGPUTargetLowering::AnalyzeFormalArguments(CCState &State, +/// The SelectionDAGBuilder will automatically promote function arguments +/// with illegal types. However, this does not work for the AMDGPU targets +/// since the function arguments are stored in memory as these illegal types. +/// In order to handle this properly we need to get the original types sizes +/// from the LLVM IR Function and fixup the ISD:InputArg values before +/// passing them to AnalyzeFormalArguments() + +/// When the SelectionDAGBuilder computes the Ins, it takes care of splitting +/// input values across multiple registers. Each item in the Ins array +/// represents a single value that will be stored in regsters. Ins[x].VT is +/// the value type of the value that will be stored in the register, so +/// whatever SDNode we lower the argument to needs to be this type. +/// +/// In order to correctly lower the arguments we need to know the size of each +/// argument. Since Ins[x].VT gives us the size of the register that will +/// hold the value, we need to look at Ins[x].ArgVT to see the 'real' type +/// for the orignal function argument so that we can deduce the correct memory +/// type to use for Ins[x]. In most cases the correct memory type will be +/// Ins[x].ArgVT. However, this will not always be the case. If, for example, +/// we have a kernel argument of type v8i8, this argument will be split into +/// 8 parts and each part will be represented by its own item in the Ins array. +/// For each part the Ins[x].ArgVT will be the v8i8, which is the full type of +/// the argument before it was split. From this, we deduce that the memory type +/// for each individual part is i8. We pass the memory type as LocVT to the +/// calling convention analysis function and the register type (Ins[x].VT) as +/// the ValVT. +void AMDGPUTargetLowering::analyzeFormalArgumentsCompute(CCState &State, const SmallVectorImpl &Ins) const { + for (unsigned i = 0, e = Ins.size(); i != e; ++i) { + const ISD::InputArg &In = Ins[i]; + EVT MemVT; + + unsigned NumRegs = getNumRegisters(State.getContext(), In.ArgVT); + + if (!Subtarget->isAmdHsaOS() && + (In.ArgVT == MVT::i16 || In.ArgVT == MVT::i8 || In.ArgVT == MVT::f16)) { + // The ABI says the caller will extend these values to 32-bits. + MemVT = In.ArgVT.isInteger() ? MVT::i32 : MVT::f32; + } else if (NumRegs == 1) { + // This argument is not split, so the IR type is the memory type. + assert(!In.Flags.isSplit()); + if (In.ArgVT.isExtended()) { + // We have an extended type, like i24, so we should just use the register type + MemVT = In.VT; + } else { + MemVT = In.ArgVT; + } + } else if (In.ArgVT.isVector() && In.VT.isVector() && + In.ArgVT.getScalarType() == In.VT.getScalarType()) { + assert(In.ArgVT.getVectorNumElements() > In.VT.getVectorNumElements()); + // We have a vector value which has been split into a vector with + // the same scalar type, but fewer elements. This should handle + // all the floating-point vector types. + MemVT = In.VT; + } else if (In.ArgVT.isVector() && + In.ArgVT.getVectorNumElements() == NumRegs) { + // This arg has been split so that each element is stored in a separate + // register. + MemVT = In.ArgVT.getScalarType(); + } else if (In.ArgVT.isExtended()) { + // We have an extended type, like i65. + MemVT = In.VT; + } else { + unsigned MemoryBits = In.ArgVT.getStoreSizeInBits() / NumRegs; + assert(In.ArgVT.getStoreSizeInBits() % NumRegs == 0); + if (In.VT.isInteger()) { + MemVT = EVT::getIntegerVT(State.getContext(), MemoryBits); + } else if (In.VT.isVector()) { + assert(!In.VT.getScalarType().isFloatingPoint()); + unsigned NumElements = In.VT.getVectorNumElements(); + assert(MemoryBits % NumElements == 0); + // This vector type has been split into another vector type with + // a different elements size. + EVT ScalarVT = EVT::getIntegerVT(State.getContext(), + MemoryBits / NumElements); + MemVT = EVT::getVectorVT(State.getContext(), ScalarVT, NumElements); + } else { + llvm_unreachable("cannot deduce memory type."); + } + } + + // Convert one element vectors to scalar. + if (MemVT.isVector() && MemVT.getVectorNumElements() == 1) + MemVT = MemVT.getScalarType(); + if (MemVT.isExtended()) { + // This should really only happen if we have vec3 arguments + assert(MemVT.isVector() && MemVT.getVectorNumElements() == 3); + MemVT = MemVT.getPow2VectorType(State.getContext()); + } + + assert(MemVT.isSimple()); + allocateKernArg(i, In.VT, MemVT.getSimpleVT(), CCValAssign::Full, In.Flags, + State); + } +} + +void AMDGPUTargetLowering::AnalyzeFormalArguments(CCState &State, + const SmallVectorImpl &Ins) const { State.AnalyzeFormalArguments(Ins, CC_AMDGPU); } @@ -678,8 +790,10 @@ SDValue AMDGPUTargetLowering::LowerCall(CallLoweringInfo &CLI, Fn, "unsupported call to function " + FuncName, CLI.DL.getDebugLoc()); DAG.getContext()->diagnose(NoCalls); - for (unsigned I = 0, E = CLI.Ins.size(); I != E; ++I) - InVals.push_back(DAG.getUNDEF(CLI.Ins[I].VT)); + if (!CLI.IsTailCall) { + for (unsigned I = 0, E = CLI.Ins.size(); I != E; ++I) + InVals.push_back(DAG.getUNDEF(CLI.Ins[I].VT)); + } return DAG.getEntryNode(); } @@ -718,6 +832,7 @@ SDValue AMDGPUTargetLowering::LowerOperation(SDValue Op, case ISD::FFLOOR: return LowerFFLOOR(Op, DAG); case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG); case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG); + case ISD::FP_TO_FP16: return LowerFP_TO_FP16(Op, DAG); case ISD::FP_TO_SINT: return LowerFP_TO_SINT(Op, DAG); case ISD::FP_TO_UINT: return LowerFP_TO_UINT(Op, DAG); case ISD::CTLZ: @@ -745,94 +860,6 @@ void AMDGPUTargetLowering::ReplaceNodeResults(SDNode *N, } } -// FIXME: This implements accesses to initialized globals in the constant -// address space by copying them to private and accessing that. It does not -// properly handle illegal types or vectors. The private vector loads are not -// scalarized, and the illegal scalars hit an assertion. This technique will not -// work well with large initializers, and this should eventually be -// removed. Initialized globals should be placed into a data section that the -// runtime will load into a buffer before the kernel is executed. Uses of the -// global need to be replaced with a pointer loaded from an implicit kernel -// argument into this buffer holding the copy of the data, which will remove the -// need for any of this. -SDValue AMDGPUTargetLowering::LowerConstantInitializer(const Constant* Init, - const GlobalValue *GV, - const SDValue &InitPtr, - SDValue Chain, - SelectionDAG &DAG) const { - const DataLayout &TD = DAG.getDataLayout(); - SDLoc DL(InitPtr); - Type *InitTy = Init->getType(); - - if (const ConstantInt *CI = dyn_cast(Init)) { - EVT VT = EVT::getEVT(InitTy); - PointerType *PtrTy = PointerType::get(InitTy, AMDGPUAS::PRIVATE_ADDRESS); - return DAG.getStore(Chain, DL, DAG.getConstant(*CI, DL, VT), InitPtr, - MachinePointerInfo(UndefValue::get(PtrTy)), - TD.getPrefTypeAlignment(InitTy)); - } - - if (const ConstantFP *CFP = dyn_cast(Init)) { - EVT VT = EVT::getEVT(CFP->getType()); - PointerType *PtrTy = PointerType::get(CFP->getType(), 0); - return DAG.getStore(Chain, DL, DAG.getConstantFP(*CFP, DL, VT), InitPtr, - MachinePointerInfo(UndefValue::get(PtrTy)), - TD.getPrefTypeAlignment(CFP->getType())); - } - - if (StructType *ST = dyn_cast(InitTy)) { - const StructLayout *SL = TD.getStructLayout(ST); - - EVT PtrVT = InitPtr.getValueType(); - SmallVector Chains; - - for (unsigned I = 0, N = ST->getNumElements(); I != N; ++I) { - SDValue Offset = DAG.getConstant(SL->getElementOffset(I), DL, PtrVT); - SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, InitPtr, Offset); - - Constant *Elt = Init->getAggregateElement(I); - Chains.push_back(LowerConstantInitializer(Elt, GV, Ptr, Chain, DAG)); - } - - return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); - } - - if (SequentialType *SeqTy = dyn_cast(InitTy)) { - EVT PtrVT = InitPtr.getValueType(); - - unsigned NumElements; - if (ArrayType *AT = dyn_cast(SeqTy)) - NumElements = AT->getNumElements(); - else if (VectorType *VT = dyn_cast(SeqTy)) - NumElements = VT->getNumElements(); - else - llvm_unreachable("Unexpected type"); - - unsigned EltSize = TD.getTypeAllocSize(SeqTy->getElementType()); - SmallVector Chains; - for (unsigned i = 0; i < NumElements; ++i) { - SDValue Offset = DAG.getConstant(i * EltSize, DL, PtrVT); - SDValue Ptr = DAG.getNode(ISD::ADD, DL, PtrVT, InitPtr, Offset); - - Constant *Elt = Init->getAggregateElement(i); - Chains.push_back(LowerConstantInitializer(Elt, GV, Ptr, Chain, DAG)); - } - - return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); - } - - if (isa(Init)) { - EVT VT = EVT::getEVT(InitTy); - PointerType *PtrTy = PointerType::get(InitTy, AMDGPUAS::PRIVATE_ADDRESS); - return DAG.getStore(Chain, DL, DAG.getUNDEF(VT), InitPtr, - MachinePointerInfo(UndefValue::get(PtrTy)), - TD.getPrefTypeAlignment(InitTy)); - } - - Init->dump(); - llvm_unreachable("Unhandled constant initializer"); -} - static bool hasDefinedInitializer(const GlobalValue *GV) { const GlobalVariable *GVar = dyn_cast(GV); if (!GVar || !GVar->hasInitializer()) @@ -850,11 +877,6 @@ SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunction* MFI, const GlobalValue *GV = G->getGlobal(); switch (G->getAddressSpace()) { - case AMDGPUAS::CONSTANT_ADDRESS: { - MVT ConstPtrVT = getPointerTy(DL, AMDGPUAS::CONSTANT_ADDRESS); - SDValue GA = DAG.getTargetGlobalAddress(GV, SDLoc(G), ConstPtrVT); - return DAG.getNode(AMDGPUISD::CONST_DATA_PTR, SDLoc(G), ConstPtrVT, GA); - } case AMDGPUAS::LOCAL_ADDRESS: { // XXX: What does the value of G->getOffset() mean? assert(G->getOffset() == 0 && @@ -864,24 +886,8 @@ SDValue AMDGPUTargetLowering::LowerGlobalAddress(AMDGPUMachineFunction* MFI, if (hasDefinedInitializer(GV)) break; - unsigned Offset; - if (MFI->LocalMemoryObjects.count(GV) == 0) { - unsigned Align = GV->getAlignment(); - if (Align == 0) - Align = DL.getABITypeAlignment(GV->getValueType()); - - /// TODO: We should sort these to minimize wasted space due to alignment - /// padding. Currently the padding is decided by the first encountered use - /// during lowering. - Offset = MFI->LDSSize = alignTo(MFI->LDSSize, Align); - MFI->LocalMemoryObjects[GV] = Offset; - MFI->LDSSize += DL.getTypeAllocSize(GV->getValueType()); - } else { - Offset = MFI->LocalMemoryObjects[GV]; - } - - return DAG.getConstant(Offset, SDLoc(Op), - getPointerTy(DL, AMDGPUAS::LOCAL_ADDRESS)); + unsigned Offset = MFI->allocateLDSGlobal(DL, *GV); + return DAG.getConstant(Offset, SDLoc(Op), Op.getValueType()); } } @@ -1097,65 +1103,6 @@ SDValue AMDGPUTargetLowering::SplitVectorLoad(const SDValue Op, return DAG.getMergeValues(Ops, SL); } -// FIXME: This isn't doing anything for SI. This should be used in a target -// combine during type legalization. -SDValue AMDGPUTargetLowering::MergeVectorStore(const SDValue &Op, - SelectionDAG &DAG) const { - StoreSDNode *Store = cast(Op); - EVT MemVT = Store->getMemoryVT(); - unsigned MemBits = MemVT.getSizeInBits(); - - // Byte stores are really expensive, so if possible, try to pack 32-bit vector - // truncating store into an i32 store. - // XXX: We could also handle optimize other vector bitwidths. - if (!MemVT.isVector() || MemBits > 32) { - return SDValue(); - } - - SDLoc DL(Op); - SDValue Value = Store->getValue(); - EVT VT = Value.getValueType(); - EVT ElemVT = VT.getVectorElementType(); - SDValue Ptr = Store->getBasePtr(); - EVT MemEltVT = MemVT.getVectorElementType(); - unsigned MemEltBits = MemEltVT.getSizeInBits(); - unsigned MemNumElements = MemVT.getVectorNumElements(); - unsigned PackedSize = MemVT.getStoreSizeInBits(); - SDValue Mask = DAG.getConstant((1 << MemEltBits) - 1, DL, MVT::i32); - - assert(Value.getValueType().getScalarSizeInBits() >= 32); - - SDValue PackedValue; - for (unsigned i = 0; i < MemNumElements; ++i) { - SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ElemVT, Value, - DAG.getConstant(i, DL, MVT::i32)); - Elt = DAG.getZExtOrTrunc(Elt, DL, MVT::i32); - Elt = DAG.getNode(ISD::AND, DL, MVT::i32, Elt, Mask); // getZeroExtendInReg - - SDValue Shift = DAG.getConstant(MemEltBits * i, DL, MVT::i32); - Elt = DAG.getNode(ISD::SHL, DL, MVT::i32, Elt, Shift); - - if (i == 0) { - PackedValue = Elt; - } else { - PackedValue = DAG.getNode(ISD::OR, DL, MVT::i32, PackedValue, Elt); - } - } - - if (PackedSize < 32) { - EVT PackedVT = EVT::getIntegerVT(*DAG.getContext(), PackedSize); - return DAG.getTruncStore(Store->getChain(), DL, PackedValue, Ptr, - Store->getMemOperand()->getPointerInfo(), PackedVT, - Store->getAlignment(), - Store->getMemOperand()->getFlags()); - } - - return DAG.getStore(Store->getChain(), DL, PackedValue, Ptr, - Store->getMemOperand()->getPointerInfo(), - Store->getAlignment(), - Store->getMemOperand()->getFlags()); -} - SDValue AMDGPUTargetLowering::SplitVectorStore(SDValue Op, SelectionDAG &DAG) const { StoreSDNode *Store = cast(Op); @@ -1670,7 +1617,7 @@ SDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const { assert(Op.getValueType() == MVT::f64); - APFloat C1Val(APFloat::IEEEdouble, "0x1.0p+52"); + APFloat C1Val(APFloat::IEEEdouble(), "0x1.0p+52"); SDValue C1 = DAG.getConstantFP(C1Val, SL, MVT::f64); SDValue CopySign = DAG.getNode(ISD::FCOPYSIGN, SL, MVT::f64, C1, Src); @@ -1681,7 +1628,7 @@ SDValue AMDGPUTargetLowering::LowerFRINT(SDValue Op, SelectionDAG &DAG) const { SDValue Fabs = DAG.getNode(ISD::FABS, SL, MVT::f64, Src); - APFloat C2Val(APFloat::IEEEdouble, "0x1.fffffffffffffp+51"); + APFloat C2Val(APFloat::IEEEdouble(), "0x1.fffffffffffffp+51"); SDValue C2 = DAG.getConstantFP(C2Val, SL, MVT::f64); EVT SetCCVT = @@ -1988,14 +1935,26 @@ SDValue AMDGPUTargetLowering::LowerUINT_TO_FP(SDValue Op, assert(Op.getOperand(0).getValueType() == MVT::i64 && "operation should be legal"); + // TODO: Factor out code common with LowerSINT_TO_FP. + EVT DestVT = Op.getValueType(); - if (DestVT == MVT::f64) - return LowerINT_TO_FP64(Op, DAG, false); + if (Subtarget->has16BitInsts() && DestVT == MVT::f16) { + SDLoc DL(Op); + SDValue Src = Op.getOperand(0); + + SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src); + SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SDLoc(Op)); + SDValue FPRound = + DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag); + + return FPRound; + } if (DestVT == MVT::f32) return LowerINT_TO_FP32(Op, DAG, false); - return SDValue(); + assert(DestVT == MVT::f64); + return LowerINT_TO_FP64(Op, DAG, false); } SDValue AMDGPUTargetLowering::LowerSINT_TO_FP(SDValue Op, @@ -2003,14 +1962,26 @@ SDValue AMDGPUTargetLowering::LowerSINT_TO_FP(SDValue Op, assert(Op.getOperand(0).getValueType() == MVT::i64 && "operation should be legal"); + // TODO: Factor out code common with LowerUINT_TO_FP. + EVT DestVT = Op.getValueType(); + if (Subtarget->has16BitInsts() && DestVT == MVT::f16) { + SDLoc DL(Op); + SDValue Src = Op.getOperand(0); + + SDValue IntToFp32 = DAG.getNode(Op.getOpcode(), DL, MVT::f32, Src); + SDValue FPRoundFlag = DAG.getIntPtrConstant(0, SDLoc(Op)); + SDValue FPRound = + DAG.getNode(ISD::FP_ROUND, DL, MVT::f16, IntToFp32, FPRoundFlag); + + return FPRound; + } + if (DestVT == MVT::f32) return LowerINT_TO_FP32(Op, DAG, true); - if (DestVT == MVT::f64) - return LowerINT_TO_FP64(Op, DAG, true); - - return SDValue(); + assert(DestVT == MVT::f64); + return LowerINT_TO_FP64(Op, DAG, true); } SDValue AMDGPUTargetLowering::LowerFP64_TO_INT(SDValue Op, SelectionDAG &DAG, @@ -2042,10 +2013,118 @@ SDValue AMDGPUTargetLowering::LowerFP64_TO_INT(SDValue Op, SelectionDAG &DAG, return DAG.getNode(ISD::BITCAST, SL, MVT::i64, Result); } +SDValue AMDGPUTargetLowering::LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const { + + if (getTargetMachine().Options.UnsafeFPMath) { + // There is a generic expand for FP_TO_FP16 with unsafe fast math. + return SDValue(); + } + + SDLoc DL(Op); + SDValue N0 = Op.getOperand(0); + assert (N0.getSimpleValueType() == MVT::f64); + + // f64 -> f16 conversion using round-to-nearest-even rounding mode. + const unsigned ExpMask = 0x7ff; + const unsigned ExpBiasf64 = 1023; + const unsigned ExpBiasf16 = 15; + SDValue Zero = DAG.getConstant(0, DL, MVT::i32); + SDValue One = DAG.getConstant(1, DL, MVT::i32); + SDValue U = DAG.getNode(ISD::BITCAST, DL, MVT::i64, N0); + SDValue UH = DAG.getNode(ISD::SRL, DL, MVT::i64, U, + DAG.getConstant(32, DL, MVT::i64)); + UH = DAG.getZExtOrTrunc(UH, DL, MVT::i32); + U = DAG.getZExtOrTrunc(U, DL, MVT::i32); + SDValue E = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, + DAG.getConstant(20, DL, MVT::i64)); + E = DAG.getNode(ISD::AND, DL, MVT::i32, E, + DAG.getConstant(ExpMask, DL, MVT::i32)); + // Subtract the fp64 exponent bias (1023) to get the real exponent and + // add the f16 bias (15) to get the biased exponent for the f16 format. + E = DAG.getNode(ISD::ADD, DL, MVT::i32, E, + DAG.getConstant(-ExpBiasf64 + ExpBiasf16, DL, MVT::i32)); + + SDValue M = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, + DAG.getConstant(8, DL, MVT::i32)); + M = DAG.getNode(ISD::AND, DL, MVT::i32, M, + DAG.getConstant(0xffe, DL, MVT::i32)); + + SDValue MaskedSig = DAG.getNode(ISD::AND, DL, MVT::i32, UH, + DAG.getConstant(0x1ff, DL, MVT::i32)); + MaskedSig = DAG.getNode(ISD::OR, DL, MVT::i32, MaskedSig, U); + + SDValue Lo40Set = DAG.getSelectCC(DL, MaskedSig, Zero, Zero, One, ISD::SETEQ); + M = DAG.getNode(ISD::OR, DL, MVT::i32, M, Lo40Set); + + // (M != 0 ? 0x0200 : 0) | 0x7c00; + SDValue I = DAG.getNode(ISD::OR, DL, MVT::i32, + DAG.getSelectCC(DL, M, Zero, DAG.getConstant(0x0200, DL, MVT::i32), + Zero, ISD::SETNE), DAG.getConstant(0x7c00, DL, MVT::i32)); + + // N = M | (E << 12); + SDValue N = DAG.getNode(ISD::OR, DL, MVT::i32, M, + DAG.getNode(ISD::SHL, DL, MVT::i32, E, + DAG.getConstant(12, DL, MVT::i32))); + + // B = clamp(1-E, 0, 13); + SDValue OneSubExp = DAG.getNode(ISD::SUB, DL, MVT::i32, + One, E); + SDValue B = DAG.getNode(ISD::SMAX, DL, MVT::i32, OneSubExp, Zero); + B = DAG.getNode(ISD::SMIN, DL, MVT::i32, B, + DAG.getConstant(13, DL, MVT::i32)); + + SDValue SigSetHigh = DAG.getNode(ISD::OR, DL, MVT::i32, M, + DAG.getConstant(0x1000, DL, MVT::i32)); + + SDValue D = DAG.getNode(ISD::SRL, DL, MVT::i32, SigSetHigh, B); + SDValue D0 = DAG.getNode(ISD::SHL, DL, MVT::i32, D, B); + SDValue D1 = DAG.getSelectCC(DL, D0, SigSetHigh, One, Zero, ISD::SETNE); + D = DAG.getNode(ISD::OR, DL, MVT::i32, D, D1); + + SDValue V = DAG.getSelectCC(DL, E, One, D, N, ISD::SETLT); + SDValue VLow3 = DAG.getNode(ISD::AND, DL, MVT::i32, V, + DAG.getConstant(0x7, DL, MVT::i32)); + V = DAG.getNode(ISD::SRL, DL, MVT::i32, V, + DAG.getConstant(2, DL, MVT::i32)); + SDValue V0 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(3, DL, MVT::i32), + One, Zero, ISD::SETEQ); + SDValue V1 = DAG.getSelectCC(DL, VLow3, DAG.getConstant(5, DL, MVT::i32), + One, Zero, ISD::SETGT); + V1 = DAG.getNode(ISD::OR, DL, MVT::i32, V0, V1); + V = DAG.getNode(ISD::ADD, DL, MVT::i32, V, V1); + + V = DAG.getSelectCC(DL, E, DAG.getConstant(30, DL, MVT::i32), + DAG.getConstant(0x7c00, DL, MVT::i32), V, ISD::SETGT); + V = DAG.getSelectCC(DL, E, DAG.getConstant(1039, DL, MVT::i32), + I, V, ISD::SETEQ); + + // Extract the sign bit. + SDValue Sign = DAG.getNode(ISD::SRL, DL, MVT::i32, UH, + DAG.getConstant(16, DL, MVT::i32)); + Sign = DAG.getNode(ISD::AND, DL, MVT::i32, Sign, + DAG.getConstant(0x8000, DL, MVT::i32)); + + V = DAG.getNode(ISD::OR, DL, MVT::i32, Sign, V); + return DAG.getZExtOrTrunc(V, DL, Op.getValueType()); +} + SDValue AMDGPUTargetLowering::LowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) const { SDValue Src = Op.getOperand(0); + // TODO: Factor out code common with LowerFP_TO_UINT. + + EVT SrcVT = Src.getValueType(); + if (Subtarget->has16BitInsts() && SrcVT == MVT::f16) { + SDLoc DL(Op); + + SDValue FPExtend = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src); + SDValue FpToInt32 = + DAG.getNode(Op.getOpcode(), DL, MVT::i64, FPExtend); + + return FpToInt32; + } + if (Op.getValueType() == MVT::i64 && Src.getValueType() == MVT::f64) return LowerFP64_TO_INT(Op, DAG, true); @@ -2056,6 +2135,19 @@ SDValue AMDGPUTargetLowering::LowerFP_TO_UINT(SDValue Op, SelectionDAG &DAG) const { SDValue Src = Op.getOperand(0); + // TODO: Factor out code common with LowerFP_TO_SINT. + + EVT SrcVT = Src.getValueType(); + if (Subtarget->has16BitInsts() && SrcVT == MVT::f16) { + SDLoc DL(Op); + + SDValue FPExtend = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src); + SDValue FpToInt32 = + DAG.getNode(Op.getOpcode(), DL, MVT::i64, FPExtend); + + return FpToInt32; + } + if (Op.getValueType() == MVT::i64 && Src.getValueType() == MVT::f64) return LowerFP64_TO_INT(Op, DAG, false); @@ -2068,8 +2160,7 @@ SDValue AMDGPUTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op, MVT VT = Op.getSimpleValueType(); MVT ScalarVT = VT.getScalarType(); - if (!VT.isVector()) - return SDValue(); + assert(VT.isVector()); SDValue Src = Op.getOperand(0); SDLoc DL(Op); @@ -2108,17 +2199,20 @@ static bool isI24(SDValue Op, SelectionDAG &DAG) { (VT.getSizeInBits() - DAG.ComputeNumSignBits(Op)) < 24; } -static void simplifyI24(SDValue Op, TargetLowering::DAGCombinerInfo &DCI) { +static bool simplifyI24(SDNode *Node24, unsigned OpIdx, + TargetLowering::DAGCombinerInfo &DCI) { SelectionDAG &DAG = DCI.DAG; - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + SDValue Op = Node24->getOperand(OpIdx); EVT VT = Op.getValueType(); APInt Demanded = APInt::getLowBitsSet(VT.getSizeInBits(), 24); APInt KnownZero, KnownOne; TargetLowering::TargetLoweringOpt TLO(DAG, true, true); - if (TLI.SimplifyDemandedBits(Op, Demanded, KnownZero, KnownOne, TLO)) - DCI.CommitTargetLoweringOpt(TLO); + if (TLO.SimplifyDemandedBits(Node24, OpIdx, Demanded, DCI)) + return true; + + return false; } template @@ -2188,6 +2282,9 @@ SDValue AMDGPUTargetLowering::performLoadCombine(SDNode *N, // problems during legalization, the emitted instructions to pack and unpack // the bytes again are not eliminated in the case of an unaligned copy. if (!allowsMisalignedMemoryAccesses(VT, AS, Align, &IsFast)) { + if (VT.isVector()) + return scalarizeVectorLoad(LN, DAG); + SDValue Ops[2]; std::tie(Ops[0], Ops[1]) = expandUnalignedLoad(LN, DAG); return DAG.getMergeValues(Ops, SDLoc(N)); @@ -2236,8 +2333,12 @@ SDValue AMDGPUTargetLowering::performStoreCombine(SDNode *N, // order problems during legalization, the emitted instructions to pack and // unpack the bytes again are not eliminated in the case of an unaligned // copy. - if (!allowsMisalignedMemoryAccesses(VT, AS, Align, &IsFast)) + if (!allowsMisalignedMemoryAccesses(VT, AS, Align, &IsFast)) { + if (VT.isVector()) + return scalarizeVectorStore(SN, DAG); + return expandUnalignedStore(SN, DAG); + } if (!IsFast) return SDValue(); @@ -2262,38 +2363,21 @@ SDValue AMDGPUTargetLowering::performStoreCombine(SDNode *N, SN->getBasePtr(), SN->getMemOperand()); } -// TODO: Should repeat for other bit ops. -SDValue AMDGPUTargetLowering::performAndCombine(SDNode *N, - DAGCombinerInfo &DCI) const { - if (N->getValueType(0) != MVT::i64) - return SDValue(); - - // Break up 64-bit and of a constant into two 32-bit ands. This will typically - // happen anyway for a VALU 64-bit and. This exposes other 32-bit integer - // combine opportunities since most 64-bit operations are decomposed this way. - // TODO: We won't want this for SALU especially if it is an inline immediate. - const ConstantSDNode *RHS = dyn_cast(N->getOperand(1)); - if (!RHS) - return SDValue(); - - uint64_t Val = RHS->getZExtValue(); - if (Lo_32(Val) != 0 && Hi_32(Val) != 0 && !RHS->hasOneUse()) { - // If either half of the constant is 0, this is really a 32-bit and, so - // split it. If we can re-use the full materialized constant, keep it. - return SDValue(); - } - - SDLoc SL(N); +/// Split the 64-bit value \p LHS into two 32-bit components, and perform the +/// binary operation \p Opc to it with the corresponding constant operands. +SDValue AMDGPUTargetLowering::splitBinaryBitConstantOpImpl( + DAGCombinerInfo &DCI, const SDLoc &SL, + unsigned Opc, SDValue LHS, + uint32_t ValLo, uint32_t ValHi) const { SelectionDAG &DAG = DCI.DAG; - SDValue Lo, Hi; - std::tie(Lo, Hi) = split64BitValue(N->getOperand(0), DAG); + std::tie(Lo, Hi) = split64BitValue(LHS, DAG); - SDValue LoRHS = DAG.getConstant(Lo_32(Val), SL, MVT::i32); - SDValue HiRHS = DAG.getConstant(Hi_32(Val), SL, MVT::i32); + SDValue LoRHS = DAG.getConstant(ValLo, SL, MVT::i32); + SDValue HiRHS = DAG.getConstant(ValHi, SL, MVT::i32); - SDValue LoAnd = DAG.getNode(ISD::AND, SL, MVT::i32, Lo, LoRHS); - SDValue HiAnd = DAG.getNode(ISD::AND, SL, MVT::i32, Hi, HiRHS); + SDValue LoAnd = DAG.getNode(Opc, SL, MVT::i32, Lo, LoRHS); + SDValue HiAnd = DAG.getNode(Opc, SL, MVT::i32, Hi, HiRHS); // Re-visit the ands. It's possible we eliminated one of them and it could // simplify the vector. @@ -2408,11 +2492,40 @@ SDValue AMDGPUTargetLowering::performSrlCombine(SDNode *N, return DAG.getNode(ISD::BITCAST, SL, MVT::i64, BuildPair); } +// We need to specifically handle i64 mul here to avoid unnecessary conversion +// instructions. If we only match on the legalized i64 mul expansion, +// SimplifyDemandedBits will be unable to remove them because there will be +// multiple uses due to the separate mul + mulh[su]. +static SDValue getMul24(SelectionDAG &DAG, const SDLoc &SL, + SDValue N0, SDValue N1, unsigned Size, bool Signed) { + if (Size <= 32) { + unsigned MulOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24; + return DAG.getNode(MulOpc, SL, MVT::i32, N0, N1); + } + + // Because we want to eliminate extension instructions before the + // operation, we need to create a single user here (i.e. not the separate + // mul_lo + mul_hi) so that SimplifyDemandedBits will deal with it. + + unsigned MulOpc = Signed ? AMDGPUISD::MUL_LOHI_I24 : AMDGPUISD::MUL_LOHI_U24; + + SDValue Mul = DAG.getNode(MulOpc, SL, + DAG.getVTList(MVT::i32, MVT::i32), N0, N1); + + return DAG.getNode(ISD::BUILD_PAIR, SL, MVT::i64, + Mul.getValue(0), Mul.getValue(1)); +} + SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N, DAGCombinerInfo &DCI) const { EVT VT = N->getValueType(0); - if (VT.isVector() || VT.getSizeInBits() > 32) + unsigned Size = VT.getSizeInBits(); + if (VT.isVector() || Size > 64) + return SDValue(); + + // There are i16 integer mul/mad. + if (Subtarget->has16BitInsts() && VT.getScalarType().bitsLE(MVT::i16)) return SDValue(); SelectionDAG &DAG = DCI.DAG; @@ -2425,11 +2538,11 @@ SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N, if (Subtarget->hasMulU24() && isU24(N0, DAG) && isU24(N1, DAG)) { N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32); N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32); - Mul = DAG.getNode(AMDGPUISD::MUL_U24, DL, MVT::i32, N0, N1); + Mul = getMul24(DAG, DL, N0, N1, Size, false); } else if (Subtarget->hasMulI24() && isI24(N0, DAG) && isI24(N1, DAG)) { N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32); N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32); - Mul = DAG.getNode(AMDGPUISD::MUL_I24, DL, MVT::i32, N0, N1); + Mul = getMul24(DAG, DL, N0, N1, Size, true); } else { return SDValue(); } @@ -2439,6 +2552,77 @@ SDValue AMDGPUTargetLowering::performMulCombine(SDNode *N, return DAG.getSExtOrTrunc(Mul, DL, VT); } +SDValue AMDGPUTargetLowering::performMulhsCombine(SDNode *N, + DAGCombinerInfo &DCI) const { + EVT VT = N->getValueType(0); + + if (!Subtarget->hasMulI24() || VT.isVector()) + return SDValue(); + + SelectionDAG &DAG = DCI.DAG; + SDLoc DL(N); + + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + + if (!isI24(N0, DAG) || !isI24(N1, DAG)) + return SDValue(); + + N0 = DAG.getSExtOrTrunc(N0, DL, MVT::i32); + N1 = DAG.getSExtOrTrunc(N1, DL, MVT::i32); + + SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_I24, DL, MVT::i32, N0, N1); + DCI.AddToWorklist(Mulhi.getNode()); + return DAG.getSExtOrTrunc(Mulhi, DL, VT); +} + +SDValue AMDGPUTargetLowering::performMulhuCombine(SDNode *N, + DAGCombinerInfo &DCI) const { + EVT VT = N->getValueType(0); + + if (!Subtarget->hasMulU24() || VT.isVector() || VT.getSizeInBits() > 32) + return SDValue(); + + SelectionDAG &DAG = DCI.DAG; + SDLoc DL(N); + + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + + if (!isU24(N0, DAG) || !isU24(N1, DAG)) + return SDValue(); + + N0 = DAG.getZExtOrTrunc(N0, DL, MVT::i32); + N1 = DAG.getZExtOrTrunc(N1, DL, MVT::i32); + + SDValue Mulhi = DAG.getNode(AMDGPUISD::MULHI_U24, DL, MVT::i32, N0, N1); + DCI.AddToWorklist(Mulhi.getNode()); + return DAG.getZExtOrTrunc(Mulhi, DL, VT); +} + +SDValue AMDGPUTargetLowering::performMulLoHi24Combine( + SDNode *N, DAGCombinerInfo &DCI) const { + SelectionDAG &DAG = DCI.DAG; + + // Simplify demanded bits before splitting into multiple users. + if (simplifyI24(N, 0, DCI) || simplifyI24(N, 1, DCI)) + return SDValue(); + + SDValue N0 = N->getOperand(0); + SDValue N1 = N->getOperand(1); + + bool Signed = (N->getOpcode() == AMDGPUISD::MUL_LOHI_I24); + + unsigned MulLoOpc = Signed ? AMDGPUISD::MUL_I24 : AMDGPUISD::MUL_U24; + unsigned MulHiOpc = Signed ? AMDGPUISD::MULHI_I24 : AMDGPUISD::MULHI_U24; + + SDLoc SL(N); + + SDValue MulLo = DAG.getNode(MulLoOpc, SL, MVT::i32, N0, N1); + SDValue MulHi = DAG.getNode(MulHiOpc, SL, MVT::i32, N0, N1); + return DAG.getMergeValues({ MulLo, MulHi }, SL); +} + static bool isNegativeOne(SDValue Val) { if (ConstantSDNode *C = dyn_cast(Val)) return C->isAllOnesValue(); @@ -2449,23 +2633,21 @@ static bool isCtlzOpc(unsigned Opc) { return Opc == ISD::CTLZ || Opc == ISD::CTLZ_ZERO_UNDEF; } -// Get FFBH node if the incoming op may have been type legalized from a smaller -// type VT. -// Need to match pre-legalized type because the generic legalization inserts the -// add/sub between the select and compare. -static SDValue getFFBH_U32(const TargetLowering &TLI, SelectionDAG &DAG, - const SDLoc &SL, SDValue Op) { +SDValue AMDGPUTargetLowering::getFFBH_U32(SelectionDAG &DAG, + SDValue Op, + const SDLoc &DL) const { EVT VT = Op.getValueType(); - EVT LegalVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT); - if (LegalVT != MVT::i32) + EVT LegalVT = getTypeToTransformTo(*DAG.getContext(), VT); + if (LegalVT != MVT::i32 && (Subtarget->has16BitInsts() && + LegalVT != MVT::i16)) return SDValue(); if (VT != MVT::i32) - Op = DAG.getNode(ISD::ZERO_EXTEND, SL, MVT::i32, Op); + Op = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, Op); - SDValue FFBH = DAG.getNode(AMDGPUISD::FFBH_U32, SL, MVT::i32, Op); + SDValue FFBH = DAG.getNode(AMDGPUISD::FFBH_U32, DL, MVT::i32, Op); if (VT != MVT::i32) - FFBH = DAG.getNode(ISD::TRUNCATE, SL, VT, FFBH); + FFBH = DAG.getNode(ISD::TRUNCATE, DL, VT, FFBH); return FFBH; } @@ -2493,7 +2675,7 @@ SDValue AMDGPUTargetLowering::performCtlzCombine(const SDLoc &SL, SDValue Cond, isCtlzOpc(RHS.getOpcode()) && RHS.getOperand(0) == CmpLHS && isNegativeOne(LHS)) { - return getFFBH_U32(*this, DAG, SL, CmpLHS); + return getFFBH_U32(DAG, CmpLHS, SL); } // select (setcc x, 0, ne), (ctlz_zero_undef x), -1 -> ffbh_u32 x @@ -2501,7 +2683,7 @@ SDValue AMDGPUTargetLowering::performCtlzCombine(const SDLoc &SL, SDValue Cond, isCtlzOpc(LHS.getOpcode()) && LHS.getOperand(0) == CmpLHS && isNegativeOne(RHS)) { - return getFFBH_U32(*this, DAG, SL, CmpLHS); + return getFFBH_U32(DAG, CmpLHS, SL); } return SDValue(); @@ -2521,6 +2703,25 @@ SDValue AMDGPUTargetLowering::performSelectCombine(SDNode *N, SDValue True = N->getOperand(1); SDValue False = N->getOperand(2); + if (Cond.hasOneUse()) { // TODO: Look for multiple select uses. + SelectionDAG &DAG = DCI.DAG; + if ((DAG.isConstantValueOfAnyType(True) || + DAG.isConstantValueOfAnyType(True)) && + (!DAG.isConstantValueOfAnyType(False) && + !DAG.isConstantValueOfAnyType(False))) { + // Swap cmp + select pair to move constant to false input. + // This will allow using VOPC cndmasks more often. + // select (setcc x, y), k, x -> select (setcc y, x) x, x + + SDLoc SL(N); + ISD::CondCode NewCC = getSetCCInverse(cast(CC)->get(), + LHS.getValueType().isInteger()); + + SDValue NewCond = DAG.getSetCC(SL, Cond.getValueType(), LHS, RHS, NewCC); + return DAG.getNode(ISD::SELECT, SL, VT, NewCond, False, True); + } + } + if (VT == MVT::f32 && Cond.hasOneUse()) { SDValue MinMax = CombineFMinMaxLegacy(SDLoc(N), VT, LHS, RHS, True, False, CC, DCI); @@ -2543,6 +2744,33 @@ SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N, break; case ISD::BITCAST: { EVT DestVT = N->getValueType(0); + + // Push casts through vector builds. This helps avoid emitting a large + // number of copies when materializing floating point vector constants. + // + // vNt1 bitcast (vNt0 (build_vector t0:x, t0:y)) => + // vnt1 = build_vector (t1 (bitcast t0:x)), (t1 (bitcast t0:y)) + if (DestVT.isVector()) { + SDValue Src = N->getOperand(0); + if (Src.getOpcode() == ISD::BUILD_VECTOR) { + EVT SrcVT = Src.getValueType(); + unsigned NElts = DestVT.getVectorNumElements(); + + if (SrcVT.getVectorNumElements() == NElts) { + EVT DestEltVT = DestVT.getVectorElementType(); + + SmallVector CastedElts; + SDLoc SL(N); + for (unsigned I = 0, E = SrcVT.getVectorNumElements(); I != E; ++I) { + SDValue Elt = Src.getOperand(I); + CastedElts.push_back(DAG.getNode(ISD::BITCAST, DL, DestEltVT, Elt)); + } + + return DAG.getBuildVector(DestVT, SL, CastedElts); + } + } + } + if (DestVT.getSizeInBits() != 64 && !DestVT.isVector()) break; @@ -2591,22 +2819,24 @@ SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N, return performSraCombine(N, DCI); } - case ISD::AND: { - if (DCI.getDAGCombineLevel() < AfterLegalizeDAG) - break; - - return performAndCombine(N, DCI); - } case ISD::MUL: return performMulCombine(N, DCI); + case ISD::MULHS: + return performMulhsCombine(N, DCI); + case ISD::MULHU: + return performMulhuCombine(N, DCI); case AMDGPUISD::MUL_I24: - case AMDGPUISD::MUL_U24: { - SDValue N0 = N->getOperand(0); - SDValue N1 = N->getOperand(1); - simplifyI24(N0, DCI); - simplifyI24(N1, DCI); + case AMDGPUISD::MUL_U24: + case AMDGPUISD::MULHI_I24: + case AMDGPUISD::MULHI_U24: { + // If the first call to simplify is successfull, then N may end up being + // deleted, so we shouldn't call simplifyI24 again. + simplifyI24(N, 0, DCI) || simplifyI24(N, 1, DCI); return SDValue(); } + case AMDGPUISD::MUL_LOHI_I24: + case AMDGPUISD::MUL_LOHI_U24: + return performMulLoHi24Combine(N, DCI); case ISD::SELECT: return performSelectCombine(N, DCI); case AMDGPUISD::BFE_I32: @@ -2705,38 +2935,6 @@ SDValue AMDGPUTargetLowering::PerformDAGCombine(SDNode *N, // Helper functions //===----------------------------------------------------------------------===// -void AMDGPUTargetLowering::getOriginalFunctionArgs( - SelectionDAG &DAG, - const Function *F, - const SmallVectorImpl &Ins, - SmallVectorImpl &OrigIns) const { - - for (unsigned i = 0, e = Ins.size(); i < e; ++i) { - if (Ins[i].ArgVT == Ins[i].VT) { - OrigIns.push_back(Ins[i]); - continue; - } - - EVT VT; - if (Ins[i].ArgVT.isVector() && !Ins[i].VT.isVector()) { - // Vector has been split into scalars. - VT = Ins[i].ArgVT.getVectorElementType(); - } else if (Ins[i].VT.isVector() && Ins[i].ArgVT.isVector() && - Ins[i].ArgVT.getVectorElementType() != - Ins[i].VT.getVectorElementType()) { - // Vector elements have been promoted - VT = Ins[i].ArgVT; - } else { - // Vector has been spilt into smaller vectors. - VT = Ins[i].VT; - } - - ISD::InputArg Arg(Ins[i].Flags, VT, VT, Ins[i].Used, - Ins[i].OrigArgIndex, Ins[i].PartOffset); - OrigIns.push_back(Arg); - } -} - SDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG, const TargetRegisterClass *RC, unsigned Reg, EVT VT) const { @@ -2754,7 +2952,8 @@ SDValue AMDGPUTargetLowering::CreateLiveInRegister(SelectionDAG &DAG, uint32_t AMDGPUTargetLowering::getImplicitParameterOffset( const AMDGPUMachineFunction *MFI, const ImplicitParameter Param) const { - uint64_t ArgOffset = MFI->ABIArgOffset; + unsigned Alignment = Subtarget->getAlignmentForImplicitArgPtr(); + uint64_t ArgOffset = alignTo(MFI->getABIArgOffset(), Alignment); switch (Param) { case GRID_DIM: return ArgOffset; @@ -2779,6 +2978,10 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const { NODE_NAME_CASE(RETURN) NODE_NAME_CASE(DWORDADDR) NODE_NAME_CASE(FRACT) + NODE_NAME_CASE(SETCC) + NODE_NAME_CASE(SETREG) + NODE_NAME_CASE(FMA_W_CHAIN) + NODE_NAME_CASE(FMUL_W_CHAIN) NODE_NAME_CASE(CLAMP) NODE_NAME_CASE(COS_HW) NODE_NAME_CASE(SIN_HW) @@ -2800,7 +3003,9 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const { NODE_NAME_CASE(TRIG_PREOP) NODE_NAME_CASE(RCP) NODE_NAME_CASE(RSQ) + NODE_NAME_CASE(RCP_LEGACY) NODE_NAME_CASE(RSQ_LEGACY) + NODE_NAME_CASE(FMUL_LEGACY) NODE_NAME_CASE(RSQ_CLAMP) NODE_NAME_CASE(LDEXP) NODE_NAME_CASE(FP_CLASS) @@ -2812,12 +3017,19 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const { NODE_NAME_CASE(BFI) NODE_NAME_CASE(BFM) NODE_NAME_CASE(FFBH_U32) + NODE_NAME_CASE(FFBH_I32) NODE_NAME_CASE(MUL_U24) NODE_NAME_CASE(MUL_I24) + NODE_NAME_CASE(MULHI_U24) + NODE_NAME_CASE(MULHI_I24) + NODE_NAME_CASE(MUL_LOHI_U24) + NODE_NAME_CASE(MUL_LOHI_I24) NODE_NAME_CASE(MAD_U24) NODE_NAME_CASE(MAD_I24) NODE_NAME_CASE(TEXTURE_FETCH) NODE_NAME_CASE(EXPORT) + NODE_NAME_CASE(EXPORT_DONE) + NODE_NAME_CASE(R600_EXPORT) NODE_NAME_CASE(CONST_ADDRESS) NODE_NAME_CASE(REGISTER_LOAD) NODE_NAME_CASE(REGISTER_STORE) @@ -2833,6 +3045,7 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const { NODE_NAME_CASE(BUILD_VERTICAL_VECTOR) NODE_NAME_CASE(CONST_DATA_PTR) NODE_NAME_CASE(PC_ADD_REL_OFFSET) + NODE_NAME_CASE(KILL) case AMDGPUISD::FIRST_MEM_OPCODE_NUMBER: break; NODE_NAME_CASE(SENDMSG) NODE_NAME_CASE(INTERP_MOV) @@ -2844,16 +3057,18 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const { NODE_NAME_CASE(ATOMIC_CMP_SWAP) NODE_NAME_CASE(ATOMIC_INC) NODE_NAME_CASE(ATOMIC_DEC) + NODE_NAME_CASE(BUFFER_LOAD) + NODE_NAME_CASE(BUFFER_LOAD_FORMAT) case AMDGPUISD::LAST_AMDGPU_ISD_NUMBER: break; } return nullptr; } -SDValue AMDGPUTargetLowering::getRsqrtEstimate(SDValue Operand, - DAGCombinerInfo &DCI, - unsigned &RefinementSteps, - bool &UseOneConstNR) const { - SelectionDAG &DAG = DCI.DAG; +SDValue AMDGPUTargetLowering::getSqrtEstimate(SDValue Operand, + SelectionDAG &DAG, int Enabled, + int &RefinementSteps, + bool &UseOneConstNR, + bool Reciprocal) const { EVT VT = Operand.getValueType(); if (VT == MVT::f32) { @@ -2868,9 +3083,8 @@ SDValue AMDGPUTargetLowering::getRsqrtEstimate(SDValue Operand, } SDValue AMDGPUTargetLowering::getRecipEstimate(SDValue Operand, - DAGCombinerInfo &DCI, - unsigned &RefinementSteps) const { - SelectionDAG &DAG = DCI.DAG; + SelectionDAG &DAG, int Enabled, + int &RefinementSteps) const { EVT VT = Operand.getValueType(); if (VT == MVT::f32) { diff --git a/lib/Target/AMDGPU/AMDGPUISelLowering.h b/lib/Target/AMDGPU/AMDGPUISelLowering.h index c2c758592d1c..5cc5efb331e3 100644 --- a/lib/Target/AMDGPU/AMDGPUISelLowering.h +++ b/lib/Target/AMDGPU/AMDGPUISelLowering.h @@ -25,19 +25,19 @@ class AMDGPUSubtarget; class MachineRegisterInfo; class AMDGPUTargetLowering : public TargetLowering { +private: + /// \returns AMDGPUISD::FFBH_U32 node if the incoming \p Op may have been + /// legalized from a smaller type VT. Need to match pre-legalized type because + /// the generic legalization inserts the add/sub between the select and + /// compare. + SDValue getFFBH_U32(SelectionDAG &DAG, SDValue Op, const SDLoc &DL) const; + protected: const AMDGPUSubtarget *Subtarget; - SDValue LowerConstantInitializer(const Constant* Init, const GlobalValue *GV, - const SDValue &InitPtr, - SDValue Chain, - SelectionDAG &DAG) const; SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const; SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const; SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const; - /// \brief Lower vector stores by merging the vector elements into an integer - /// of the same bitwidth. - SDValue MergeVectorStore(const SDValue &Op, SelectionDAG &DAG) const; /// \brief Split a vector store into multiple scalar stores. /// \returns The resulting chain. @@ -60,6 +60,7 @@ protected: SDValue LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFP64_TO_INT(SDValue Op, SelectionDAG &DAG, bool Signed) const; + SDValue LowerFP_TO_FP16(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFP_TO_UINT(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFP_TO_SINT(SDValue Op, SelectionDAG &DAG) const; @@ -69,17 +70,22 @@ protected: bool shouldCombineMemoryType(EVT VT) const; SDValue performLoadCombine(SDNode *N, DAGCombinerInfo &DCI) const; SDValue performStoreCombine(SDNode *N, DAGCombinerInfo &DCI) const; - SDValue performAndCombine(SDNode *N, DAGCombinerInfo &DCI) const; + + SDValue splitBinaryBitConstantOpImpl(DAGCombinerInfo &DCI, const SDLoc &SL, + unsigned Opc, SDValue LHS, + uint32_t ValLo, uint32_t ValHi) const; SDValue performShlCombine(SDNode *N, DAGCombinerInfo &DCI) const; SDValue performSraCombine(SDNode *N, DAGCombinerInfo &DCI) const; SDValue performSrlCombine(SDNode *N, DAGCombinerInfo &DCI) const; SDValue performMulCombine(SDNode *N, DAGCombinerInfo &DCI) const; + SDValue performMulhsCombine(SDNode *N, DAGCombinerInfo &DCI) const; + SDValue performMulhuCombine(SDNode *N, DAGCombinerInfo &DCI) const; + SDValue performMulLoHi24Combine(SDNode *N, DAGCombinerInfo &DCI) const; SDValue performCtlzCombine(const SDLoc &SL, SDValue Cond, SDValue LHS, SDValue RHS, DAGCombinerInfo &DCI) const; SDValue performSelectCombine(SDNode *N, DAGCombinerInfo &DCI) const; static EVT getEquivalentMemType(LLVMContext &Context, EVT VT); - static EVT getEquivalentBitType(LLVMContext &Context, EVT VT); virtual SDValue LowerGlobalAddress(AMDGPUMachineFunction *MFI, SDValue Op, SelectionDAG &DAG) const; @@ -102,16 +108,8 @@ protected: SDValue LowerDIVREM24(SDValue Op, SelectionDAG &DAG, bool sign) const; void LowerUDIVREM64(SDValue Op, SelectionDAG &DAG, SmallVectorImpl &Results) const; - /// The SelectionDAGBuilder will automatically promote function arguments - /// with illegal types. However, this does not work for the AMDGPU targets - /// since the function arguments are stored in memory as these illegal types. - /// In order to handle this properly we need to get the origianl types sizes - /// from the LLVM IR Function and fixup the ISD:InputArg values before - /// passing them to AnalyzeFormalArguments() - void getOriginalFunctionArgs(SelectionDAG &DAG, - const Function *F, - const SmallVectorImpl &Ins, - SmallVectorImpl &OrigIns) const; + void analyzeFormalArgumentsCompute(CCState &State, + const SmallVectorImpl &Ins) const; void AnalyzeFormalArguments(CCState &State, const SmallVectorImpl &Ins) const; void AnalyzeReturn(CCState &State, @@ -171,13 +169,14 @@ public: const char* getTargetNodeName(unsigned Opcode) const override; - SDValue getRsqrtEstimate(SDValue Operand, - DAGCombinerInfo &DCI, - unsigned &RefinementSteps, - bool &UseOneConstNR) const override; - SDValue getRecipEstimate(SDValue Operand, - DAGCombinerInfo &DCI, - unsigned &RefinementSteps) const override; + bool isFsqrtCheap(SDValue Operand, SelectionDAG &DAG) const override { + return true; + } + SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, + int &RefinementSteps, bool &UseOneConstNR, + bool Reciprocal) const override; + SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, + int &RefinementSteps) const override; virtual SDNode *PostISelFolding(MachineSDNode *N, SelectionDAG &DAG) const = 0; @@ -228,6 +227,13 @@ enum NodeType : unsigned { DWORDADDR, FRACT, CLAMP, + // This is SETCC with the full mask result which is used for a compare with a + // result bit per item in the wavefront. + SETCC, + SETREG, + // FP ops with input and output chain. + FMA_W_CHAIN, + FMUL_W_CHAIN, // SIN_HW, COS_HW - f32 for SI, 1 ULP max error, valid from -100 pi to 100 pi. // Denormals handled on some parts. @@ -254,7 +260,9 @@ enum NodeType : unsigned { // For f64, max error 2^29 ULP, handles denormals. RCP, RSQ, + RCP_LEGACY, RSQ_LEGACY, + FMUL_LEGACY, RSQ_CLAMP, LDEXP, FP_CLASS, @@ -266,12 +274,19 @@ enum NodeType : unsigned { BFI, // (src0 & src1) | (~src0 & src2) BFM, // Insert a range of bits into a 32-bit word. FFBH_U32, // ctlz with -1 if input is zero. + FFBH_I32, MUL_U24, MUL_I24, + MULHI_U24, + MULHI_I24, MAD_U24, MAD_I24, + MUL_LOHI_I24, + MUL_LOHI_U24, TEXTURE_FETCH, - EXPORT, + EXPORT, // exp on SI+ + EXPORT_DONE, // exp on SI+ with done bit set + R600_EXPORT, CONST_ADDRESS, REGISTER_LOAD, REGISTER_STORE, @@ -302,6 +317,7 @@ enum NodeType : unsigned { INTERP_P1, INTERP_P2, PC_ADD_REL_OFFSET, + KILL, FIRST_MEM_OPCODE_NUMBER = ISD::FIRST_TARGET_MEMORY_OPCODE, STORE_MSKOR, LOAD_CONSTANT, @@ -309,6 +325,8 @@ enum NodeType : unsigned { ATOMIC_CMP_SWAP, ATOMIC_INC, ATOMIC_DEC, + BUFFER_LOAD, + BUFFER_LOAD_FORMAT, LAST_AMDGPU_ISD_NUMBER }; diff --git a/lib/Target/AMDGPU/AMDGPUInstrInfo.cpp b/lib/Target/AMDGPU/AMDGPUInstrInfo.cpp index 9a00ecb24ebe..e4dc6599e156 100644 --- a/lib/Target/AMDGPU/AMDGPUInstrInfo.cpp +++ b/lib/Target/AMDGPU/AMDGPUInstrInfo.cpp @@ -23,7 +23,6 @@ using namespace llvm; #define GET_INSTRINFO_CTOR_DTOR -#define GET_INSTRINFO_NAMED_OPS #define GET_INSTRMAP_INFO #include "AMDGPUGenInstrInfo.inc" @@ -33,10 +32,6 @@ void AMDGPUInstrInfo::anchor() {} AMDGPUInstrInfo::AMDGPUInstrInfo(const AMDGPUSubtarget &ST) : AMDGPUGenInstrInfo(-1, -1), ST(ST) {} -bool AMDGPUInstrInfo::enableClusterLoads() const { - return true; -} - // FIXME: This behaves strangely. If, for example, you have 32 load + stores, // the first 16 loads will be interleaved with the stores, and the next 16 will // be clustered as expected. It should really split into 2 16 store batches. diff --git a/lib/Target/AMDGPU/AMDGPUInstrInfo.h b/lib/Target/AMDGPU/AMDGPUInstrInfo.h index a59eafadeb93..bd8e389639f5 100644 --- a/lib/Target/AMDGPU/AMDGPUInstrInfo.h +++ b/lib/Target/AMDGPU/AMDGPUInstrInfo.h @@ -17,17 +17,12 @@ #define LLVM_LIB_TARGET_AMDGPU_AMDGPUINSTRINFO_H #include "llvm/Target/TargetInstrInfo.h" +#include "Utils/AMDGPUBaseInfo.h" #define GET_INSTRINFO_HEADER #define GET_INSTRINFO_ENUM -#define GET_INSTRINFO_OPERAND_ENUM #include "AMDGPUGenInstrInfo.inc" -#define OPCODE_IS_ZERO_INT AMDGPU::PRED_SETE_INT -#define OPCODE_IS_NOT_ZERO_INT AMDGPU::PRED_SETNE_INT -#define OPCODE_IS_ZERO AMDGPU::PRED_SETE -#define OPCODE_IS_NOT_ZERO AMDGPU::PRED_SETNE - namespace llvm { class AMDGPUSubtarget; @@ -44,8 +39,6 @@ private: public: explicit AMDGPUInstrInfo(const AMDGPUSubtarget &st); - bool enableClusterLoads() const override; - bool shouldScheduleLoadsNear(SDNode *Load1, SDNode *Load2, int64_t Offset1, int64_t Offset2, unsigned NumLoads) const override; @@ -59,15 +52,6 @@ public: /// equivalent opcode that writes \p Channels Channels. int getMaskedMIMGOp(uint16_t Opcode, unsigned Channels) const; }; - -namespace AMDGPU { - LLVM_READONLY - int16_t getNamedOperandIdx(uint16_t Opcode, uint16_t NamedIndex); -} // End namespace AMDGPU - } // End llvm namespace -#define AMDGPU_FLAG_REGISTER_LOAD (UINT64_C(1) << 63) -#define AMDGPU_FLAG_REGISTER_STORE (UINT64_C(1) << 62) - #endif diff --git a/lib/Target/AMDGPU/AMDGPUInstrInfo.td b/lib/Target/AMDGPU/AMDGPUInstrInfo.td index 2b13bb9079ea..e7b40016e272 100644 --- a/lib/Target/AMDGPU/AMDGPUInstrInfo.td +++ b/lib/Target/AMDGPU/AMDGPUInstrInfo.td @@ -40,6 +40,8 @@ def AMDGPUFmasOp : SDTypeProfile<1, 4, [SDTCisFP<0>, SDTCisSameAs<0, 1>, SDTCisSameAs<0, 2>, SDTCisSameAs<0, 3>, SDTCisInt<4>] >; +def AMDGPUKillSDT : SDTypeProfile<0, 1, [SDTCisInt<0>]>; + //===----------------------------------------------------------------------===// // AMDGPU DAG Nodes // @@ -65,6 +67,7 @@ def AMDGPUrcp : SDNode<"AMDGPUISD::RCP", SDTFPUnaryOp>; def AMDGPUrsq : SDNode<"AMDGPUISD::RSQ", SDTFPUnaryOp>; // out = 1.0 / sqrt(a) +def AMDGPUrcp_legacy : SDNode<"AMDGPUISD::RCP_LEGACY", SDTFPUnaryOp>; def AMDGPUrsq_legacy : SDNode<"AMDGPUISD::RSQ_LEGACY", SDTFPUnaryOp>; // out = 1.0 / sqrt(a) result clamped to +/- max_float. @@ -82,6 +85,10 @@ def AMDGPUfmax_legacy : SDNode<"AMDGPUISD::FMAX_LEGACY", SDTFPBinOp, [] >; +def AMDGPUfmul_legacy : SDNode<"AMDGPUISD::FMUL_LEGACY", SDTFPBinOp, + [SDNPCommutative, SDNPAssociative] +>; + def AMDGPUclamp : SDNode<"AMDGPUISD::CLAMP", SDTFPTernaryOp, []>; // out = max(a, b) a and b are signed ints @@ -137,6 +144,24 @@ def AMDGPUcarry : SDNode<"AMDGPUISD::CARRY", SDTIntBinOp, []>; // out = (src1 > src0) ? 1 : 0 def AMDGPUborrow : SDNode<"AMDGPUISD::BORROW", SDTIntBinOp, []>; +def AMDGPUSetCCOp : SDTypeProfile<1, 3, [ // setcc + SDTCisVT<0, i64>, SDTCisSameAs<1, 2>, SDTCisVT<3, OtherVT> +]>; + +def AMDGPUsetcc : SDNode<"AMDGPUISD::SETCC", AMDGPUSetCCOp>; + +def AMDGPUSetRegOp : SDTypeProfile<0, 2, [ + SDTCisInt<0>, SDTCisInt<1> +]>; + +def AMDGPUsetreg : SDNode<"AMDGPUISD::SETREG", AMDGPUSetRegOp, [ + SDNPHasChain, SDNPSideEffect, SDNPOptInGlue, SDNPOutGlue]>; + +def AMDGPUfma : SDNode<"AMDGPUISD::FMA_W_CHAIN", SDTFPTernaryOp, [ + SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>; + +def AMDGPUmul : SDNode<"AMDGPUISD::FMUL_W_CHAIN", SDTFPBinOp, [ + SDNPHasChain, SDNPOptInGlue, SDNPOutGlue]>; def AMDGPUcvt_f32_ubyte0 : SDNode<"AMDGPUISD::CVT_F32_UBYTE0", SDTIntToFPOp, []>; @@ -202,14 +227,22 @@ def AMDGPUbfi : SDNode<"AMDGPUISD::BFI", AMDGPUDTIntTernaryOp>; def AMDGPUbfm : SDNode<"AMDGPUISD::BFM", SDTIntBinOp>; def AMDGPUffbh_u32 : SDNode<"AMDGPUISD::FFBH_U32", SDTIntUnaryOp>; +def AMDGPUffbh_i32 : SDNode<"AMDGPUISD::FFBH_I32", SDTIntUnaryOp>; -// Signed and unsigned 24-bit mulitply. The highest 8-bits are ignore when -// performing the mulitply. The result is a 32-bit value. +// Signed and unsigned 24-bit multiply. The highest 8-bits are ignore +// when performing the mulitply. The result is a 32-bit value. def AMDGPUmul_u24 : SDNode<"AMDGPUISD::MUL_U24", SDTIntBinOp, - [SDNPCommutative] + [SDNPCommutative, SDNPAssociative] >; def AMDGPUmul_i24 : SDNode<"AMDGPUISD::MUL_I24", SDTIntBinOp, - [SDNPCommutative] + [SDNPCommutative, SDNPAssociative] +>; + +def AMDGPUmulhi_u24 : SDNode<"AMDGPUISD::MULHI_U24", SDTIntBinOp, + [SDNPCommutative, SDNPAssociative] +>; +def AMDGPUmulhi_i24 : SDNode<"AMDGPUISD::MULHI_I24", SDTIntBinOp, + [SDNPCommutative, SDNPAssociative] >; def AMDGPUmad_u24 : SDNode<"AMDGPUISD::MAD_U24", AMDGPUDTIntTernaryOp, @@ -245,6 +278,35 @@ def AMDGPUinterp_p2 : SDNode<"AMDGPUISD::INTERP_P2", SDTypeProfile<1, 4, [SDTCisFP<0>]>, [SDNPInGlue]>; + +def AMDGPUkill : SDNode<"AMDGPUISD::KILL", AMDGPUKillSDT, + [SDNPHasChain, SDNPSideEffect]>; + +// SI+ export +def AMDGPUExportOp : SDTypeProfile<0, 8, [ + SDTCisInt<0>, // i8 en + SDTCisInt<1>, // i1 vm + // skip done + SDTCisInt<2>, // i8 tgt + SDTCisSameAs<3, 1>, // i1 compr + SDTCisFP<4>, // f32 src0 + SDTCisSameAs<5, 4>, // f32 src1 + SDTCisSameAs<6, 4>, // f32 src2 + SDTCisSameAs<7, 4> // f32 src3 +]>; + +def AMDGPUexport: SDNode<"AMDGPUISD::EXPORT", AMDGPUExportOp, + [SDNPHasChain, SDNPMayStore]>; + +def AMDGPUexport_done: SDNode<"AMDGPUISD::EXPORT_DONE", AMDGPUExportOp, + [SDNPHasChain, SDNPMayLoad, SDNPMayStore]>; + + +def R600ExportOp : SDTypeProfile<0, 7, [SDTCisFP<0>, SDTCisInt<1>]>; + +def R600_EXPORT: SDNode<"AMDGPUISD::R600_EXPORT", R600ExportOp, + [SDNPHasChain, SDNPSideEffect]>; + //===----------------------------------------------------------------------===// // Flow Control Profile Types //===----------------------------------------------------------------------===// diff --git a/lib/Target/AMDGPU/AMDGPUInstructions.td b/lib/Target/AMDGPU/AMDGPUInstructions.td index 3944fdbd31e3..513df3a9cdf3 100644 --- a/lib/Target/AMDGPU/AMDGPUInstructions.td +++ b/lib/Target/AMDGPU/AMDGPUInstructions.td @@ -42,6 +42,7 @@ class AMDGPUShaderInst Inst = 0xffffffff; } +def FP16Denormals : Predicate<"Subtarget.hasFP16Denormals()">; def FP32Denormals : Predicate<"Subtarget.hasFP32Denormals()">; def FP64Denormals : Predicate<"Subtarget.hasFP64Denormals()">; def UnsafeFPMath : Predicate<"TM.Options.UnsafeFPMath">; @@ -49,13 +50,6 @@ def UnsafeFPMath : Predicate<"TM.Options.UnsafeFPMath">; def InstFlag : OperandWithDefaultOps ; def ADDRIndirect : ComplexPattern; -// 32-bit VALU immediate operand that uses the constant bus. -def u32kimm : Operand { - let OperandNamespace = "AMDGPU"; - let OperandType = "OPERAND_KIMM32"; - let PrintMethod = "printU32ImmOperand"; -} - let OperandType = "OPERAND_IMMEDIATE" in { def u32imm : Operand { @@ -172,6 +166,12 @@ class HasOneUseBinOp : PatFrag< [{ return N->hasOneUse(); }] >; +class HasOneUseTernaryOp : PatFrag< + (ops node:$src0, node:$src1, node:$src2), + (op $src0, $src1, $src2), + [{ return N->hasOneUse(); }] +>; + //===----------------------------------------------------------------------===// // Load/Store Pattern Fragments //===----------------------------------------------------------------------===// @@ -363,53 +363,54 @@ multiclass AtomicCmpSwapLocal { defm atomic_cmp_swap : AtomicCmpSwapLocal ; -def mskor_flat : PatFrag<(ops node:$val, node:$ptr), - (AMDGPUstore_mskor node:$val, node:$ptr), [{ - return cast(N)->getAddressSpace() == AMDGPUAS::FLAT_ADDRESS; -}]>; +multiclass global_binary_atomic_op { + def "" : PatFrag< + (ops node:$ptr, node:$value), + (atomic_op node:$ptr, node:$value), + [{return cast(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS;}]>; + + def _noret : PatFrag< + (ops node:$ptr, node:$value), + (atomic_op node:$ptr, node:$value), + [{return cast(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS && (SDValue(N, 0).use_empty());}]>; + + def _ret : PatFrag< + (ops node:$ptr, node:$value), + (atomic_op node:$ptr, node:$value), + [{return cast(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS && (!SDValue(N, 0).use_empty());}]>; +} -class global_binary_atomic_op : PatFrag< - (ops node:$ptr, node:$value), - (atomic_op node:$ptr, node:$value), - [{return cast(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS;}] ->; - -class flat_binary_atomic_op : PatFrag< - (ops node:$ptr, node:$value), - (atomic_op node:$ptr, node:$value), - [{return cast(N)->getAddressSpace() == AMDGPUAS::FLAT_ADDRESS;}] ->; - -def atomic_swap_global : global_binary_atomic_op; -def atomic_add_global : global_binary_atomic_op; -def atomic_and_global : global_binary_atomic_op; -def atomic_max_global : global_binary_atomic_op; -def atomic_min_global : global_binary_atomic_op; -def atomic_or_global : global_binary_atomic_op; -def atomic_sub_global : global_binary_atomic_op; -def atomic_umax_global : global_binary_atomic_op; -def atomic_umin_global : global_binary_atomic_op; -def atomic_xor_global : global_binary_atomic_op; - -def atomic_cmp_swap_global : global_binary_atomic_op; -def atomic_cmp_swap_global_nortn : PatFrag< - (ops node:$ptr, node:$value), - (atomic_cmp_swap_global node:$ptr, node:$value), - [{ return SDValue(N, 0).use_empty(); }] ->; - -def atomic_swap_flat : flat_binary_atomic_op; -def atomic_add_flat : flat_binary_atomic_op; -def atomic_and_flat : flat_binary_atomic_op; -def atomic_max_flat : flat_binary_atomic_op; -def atomic_min_flat : flat_binary_atomic_op; -def atomic_or_flat : flat_binary_atomic_op; -def atomic_sub_flat : flat_binary_atomic_op; -def atomic_umax_flat : flat_binary_atomic_op; -def atomic_umin_flat : flat_binary_atomic_op; -def atomic_xor_flat : flat_binary_atomic_op; - -def atomic_cmp_swap_flat : flat_binary_atomic_op; +defm atomic_swap_global : global_binary_atomic_op; +defm atomic_add_global : global_binary_atomic_op; +defm atomic_and_global : global_binary_atomic_op; +defm atomic_max_global : global_binary_atomic_op; +defm atomic_min_global : global_binary_atomic_op; +defm atomic_or_global : global_binary_atomic_op; +defm atomic_sub_global : global_binary_atomic_op; +defm atomic_umax_global : global_binary_atomic_op; +defm atomic_umin_global : global_binary_atomic_op; +defm atomic_xor_global : global_binary_atomic_op; + +//legacy +def AMDGPUatomic_cmp_swap_global : PatFrag< + (ops node:$ptr, node:$value), + (AMDGPUatomic_cmp_swap node:$ptr, node:$value), + [{return cast(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS;}]>; + +def atomic_cmp_swap_global : PatFrag< + (ops node:$ptr, node:$cmp, node:$value), + (atomic_cmp_swap node:$ptr, node:$cmp, node:$value), + [{return cast(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS;}]>; + +def atomic_cmp_swap_global_noret : PatFrag< + (ops node:$ptr, node:$cmp, node:$value), + (atomic_cmp_swap node:$ptr, node:$cmp, node:$value), + [{return cast(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS && (SDValue(N, 0).use_empty());}]>; + +def atomic_cmp_swap_global_ret : PatFrag< + (ops node:$ptr, node:$cmp, node:$value), + (atomic_cmp_swap node:$ptr, node:$cmp, node:$value), + [{return cast(N)->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS && (!SDValue(N, 0).use_empty());}]>; //===----------------------------------------------------------------------===// // Misc Pattern Fragments @@ -420,6 +421,7 @@ int TWO_PI = 0x40c90fdb; int PI = 0x40490fdb; int TWO_PI_INV = 0x3e22f983; int FP_UINT_MAX_PLUS_1 = 0x4f800000; // 1 << 32 in floating point encoding +int FP16_ONE = 0x3C00; int FP32_ONE = 0x3f800000; int FP32_NEG_ONE = 0xbf800000; int FP64_ONE = 0x3ff0000000000000; @@ -559,17 +561,26 @@ multiclass BFIPatterns ; def : Pat < (f64 (fcopysign f64:$src0, f64:$src1)), (REG_SEQUENCE RC64, (i32 (EXTRACT_SUBREG $src0, sub0)), sub0, - (BFI_INT (LoadImm32 0x7fffffff), + (BFI_INT (LoadImm32 (i32 0x7fffffff)), (i32 (EXTRACT_SUBREG $src0, sub1)), (i32 (EXTRACT_SUBREG $src1, sub1))), sub1) >; + + def : Pat < + (f64 (fcopysign f64:$src0, f32:$src1)), + (REG_SEQUENCE RC64, + (i32 (EXTRACT_SUBREG $src0, sub0)), sub0, + (BFI_INT (LoadImm32 (i32 0x7fffffff)), + (i32 (EXTRACT_SUBREG $src0, sub1)), + $src1), sub1) + >; } // SHA-256 Ma patterns @@ -618,11 +629,10 @@ def smax_oneuse : HasOneUseBinOp; def smin_oneuse : HasOneUseBinOp; def umax_oneuse : HasOneUseBinOp; def umin_oneuse : HasOneUseBinOp; +def sub_oneuse : HasOneUseBinOp; } // Properties = [SDNPCommutative, SDNPAssociative] - -// 24-bit arithmetic patterns -def umul24 : PatFrag <(ops node:$x, node:$y), (mul node:$x, node:$y)>; +def select_oneuse : HasOneUseTernaryOp